From 4ad7652540b572cc9152d4cf8104e36aa6c2859c Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 06:07:45 +0000 Subject: [PATCH 01/18] feat(sdk)!: unify explicit extension and resource configuration --- .github/workflows/ci.yml | 16 + .github/workflows/release.yml | 23 +- Cargo.lock | 13 +- Cargo.toml | 1 + docs/architecture/cluster-seeds-and-icu.md | 50 +- docs/architecture/database-storage.md | 16 +- docs/architecture/wasix-typescript-napi.md | 17 +- .../consumer-dx-release-blueprint.md | 713 +++--------------- .../maintainers/extension-packaging-policy.md | 2 +- docs/maintainers/release.md | 10 +- docs/maintainers/rust-sdk-policy.md | 10 +- docs/maintainers/sdk-api-surface.md | 186 +++-- docs/maintainers/sdk-parity-policy.md | 11 +- docs/maintainers/testing.md | 5 +- docs/maintainers/wasix-usage.md | 12 +- examples/electron/src/todos.ts | 4 +- examples/react-native-expo/README.md | 9 +- examples/react-native-expo/package.json | 1 + .../react-native-expo/src/SmokeDashboard.tsx | 36 +- .../src/generated/extension-smoke.ts | 324 +++++++- examples/tauri/src-tauri/Cargo.toml | 11 - examples/tauri/src-tauri/build.rs | 1 - examples/tauri/src-tauri/src/lib.rs | 5 +- pnpm-lock.yaml | 3 + .../crates/oliphaunt-wasix/Cargo.toml | 52 +- .../crates/oliphaunt-wasix/README.md | 39 +- .../crates/oliphaunt-wasix/src/async_api.rs | 34 +- .../src/bin/oliphaunt_wasix_dump.rs | 4 +- .../src/bin/oliphaunt_wasix_proxy.rs | 2 +- .../crates/oliphaunt-wasix/src/error.rs | 2 +- .../crates/oliphaunt-wasix/src/lib.rs | 11 +- .../oliphaunt-wasix/src/oliphaunt/aot.rs | 167 +++- .../oliphaunt-wasix/src/oliphaunt/assets.rs | 158 ++-- .../oliphaunt-wasix/src/oliphaunt/backend.rs | 8 +- .../oliphaunt-wasix/src/oliphaunt/base.rs | 27 +- .../oliphaunt-wasix/src/oliphaunt/builder.rs | 29 +- .../oliphaunt-wasix/src/oliphaunt/client.rs | 64 +- .../oliphaunt-wasix/src/oliphaunt/data_dir.rs | 6 +- .../src/oliphaunt/extensions.rs | 117 ++- .../generated_contrib_descriptors.rs | 162 ++++ .../src/oliphaunt/generated_extensions.rs | 234 +----- .../oliphaunt-wasix/src/oliphaunt/mod.rs | 2 +- .../src/oliphaunt/postgres_mod.rs | 16 +- .../oliphaunt-wasix/src/oliphaunt/server.rs | 47 +- .../oliphaunt-wasix/src/oliphaunt/tools.rs | 55 +- .../oliphaunt-wasix/tests/extensions_smoke.rs | 2 +- .../oliphaunt-wasix/tests/public_api.rs | 6 +- src/bindings/wasix-rust/moon.yml | 6 +- src/bindings/wasix-ts/README.md | 50 +- .../wasix-ts/src/__tests__/contrib-carrier.ts | 47 ++ .../src/__tests__/native-addon.test.ts | 1 + .../native-extension-packages.test.ts | 57 ++ .../src/__tests__/native-session.test.ts | 65 +- .../wasix-ts/src/contrib-carrier-shim.d.ts | 33 + src/bindings/wasix-ts/src/native-addon.ts | 30 +- .../wasix-ts/src/native-extension-packages.ts | 90 +++ src/bindings/wasix-ts/src/native-session.ts | 75 +- src/bindings/wasix-ts/src/public.ts | 1 + src/bindings/wasix-ts/tools/package.mjs | 1 + src/bindings/wasix-ts/tsconfig.json | 29 +- src/bindings/wasix-ts/vitest.config.ts | 3 + src/docs/content/learn/embedded-postgres.mdx | 7 +- src/docs/content/reference/extensions.mdx | 171 ++--- src/docs/content/sdk/index.mdx | 2 +- src/docs/content/sdk/kotlin/guide.mdx | 13 +- src/docs/content/sdk/kotlin/index.mdx | 17 +- .../content/sdk/react-native/architecture.mdx | 17 +- src/docs/content/sdk/react-native/guide.mdx | 33 +- src/docs/content/sdk/react-native/index.mdx | 32 +- src/docs/content/sdk/rust/guide.mdx | 15 +- src/docs/content/sdk/rust/index.mdx | 8 +- src/docs/content/sdk/swift/guide.mdx | 26 +- src/docs/content/sdk/swift/index.mdx | 21 +- src/docs/content/sdk/typescript/guide.mdx | 15 +- src/docs/content/sdk/typescript/index.mdx | 10 +- .../content/sdk/wasix-rust/dump-restore.mdx | 6 +- src/docs/content/sdk/wasix-rust/guide.mdx | 25 +- .../content/sdk/wasix-typescript/guide.mdx | 15 +- src/docs/src/components/home-interactive.tsx | 5 +- src/docs/src/components/oliphaunt.tsx | 22 +- src/docs/tools/check-docs-product.mjs | 12 +- .../external/pg_hashids/release.toml | 5 + src/extensions/external/pg_ivm/release.toml | 5 + .../external/pg_textsearch/release.toml | 5 + .../external/pg_uuidv7/release.toml | 5 + src/extensions/external/pgtap/release.toml | 1 + src/extensions/external/postgis/release.toml | 5 + src/extensions/external/vector/release.toml | 5 + src/extensions/tools/check-extension-model.py | 82 ++ src/runtimes/liboliphaunt/icu/Cargo.toml | 6 +- src/runtimes/liboliphaunt/icu/README.md | 19 +- src/runtimes/liboliphaunt/icu/build.rs | 175 ++++- src/runtimes/liboliphaunt/icu/src/lib.rs | 13 + .../native/icu-npm/OliphauntICU.podspec | 2 +- .../liboliphaunt/native/icu-npm/README.md | 36 +- .../liboliphaunt/native/icu-npm/index.d.ts | 8 + .../liboliphaunt/native/icu-npm/index.js | 7 + .../liboliphaunt/native/icu-npm/node.js | 7 + .../liboliphaunt/native/icu-npm/package.json | 19 +- .../liboliphaunt/native/include/oliphaunt.h | 8 +- src/runtimes/liboliphaunt/native/moon.yml | 3 + .../liboliphaunt_static_extension_registry.c | 19 +- .../src/liboliphaunt_static_extensions.c | 86 +-- .../liboliphaunt/wasix/crates/assets/build.rs | 19 +- src/runtimes/liboliphaunt/wasix/release.toml | 4 + src/runtimes/wasix-napi/Cargo.toml | 89 +-- src/runtimes/wasix-napi/build.rs | 32 +- src/runtimes/wasix-napi/moon.yml | 5 +- .../wasix-napi/src/extension_package.rs | 463 ++++++++++++ src/runtimes/wasix-napi/src/icu_data.rs | 87 +++ src/runtimes/wasix-napi/src/lib.rs | 105 ++- src/runtimes/wasix-napi/tools/build-native.sh | 27 +- .../wasix-napi/tools/check-build-inputs.mjs | 48 +- .../wasix-napi/tools/package-platform.mjs | 3 +- src/sdks/js/README.md | 44 +- src/sdks/js/package.json | 4 + src/sdks/js/src/__tests__/client.test.ts | 6 + src/sdks/js/src/__tests__/config.test.ts | 12 +- .../js/src/__tests__/native-bindings.test.ts | 8 +- src/sdks/js/src/__tests__/resources.test.ts | 40 + src/sdks/js/src/client.ts | 5 +- src/sdks/js/src/config.ts | 16 +- src/sdks/js/src/extensions.ts | 22 + src/sdks/js/src/index.ts | 3 + src/sdks/js/src/native/assets-deno.ts | 53 +- src/sdks/js/src/native/assets-node.ts | 91 ++- src/sdks/js/src/native/deno.ts | 73 +- src/sdks/js/src/native/node.ts | 14 +- src/sdks/js/src/native/types.ts | 4 + src/sdks/js/src/runtime/broker.ts | 61 +- src/sdks/js/src/runtime/direct.ts | 2 + src/sdks/js/src/runtime/server.ts | 41 +- src/sdks/js/src/storage/node.ts | 13 + src/sdks/js/src/types.ts | 5 +- src/sdks/kotlin/README.md | 85 ++- src/sdks/kotlin/gradle/libs.versions.toml | 1 + .../build.gradle.kts | 5 + .../android/OliphauntAndroidPlugin.java | 255 ++----- .../ResolveOliphauntAndroidAssetsTask.java | 15 + .../build.gradle.kts | 74 +- .../src/androidMain/cpp/include/oliphaunt.h | 8 +- .../oliphaunt/AndroidNativeDirectEngine.kt | 2 + .../kotlin/dev/oliphaunt/OliphauntAndroid.kt | 43 +- .../OliphauntAndroidRuntimeAssets.kt | 83 +- .../kotlin/dev/oliphaunt/OliphauntJava.kt | 28 + .../OliphauntAndroidRuntimeAssetsTest.kt | 4 +- .../dev/oliphaunt/OliphauntAndroidTest.kt | 40 +- .../dev/oliphaunt/GeneratedExtensions.kt | 146 +++- .../kotlin/dev/oliphaunt/Oliphaunt.kt | 5 +- .../kotlin/dev/oliphaunt/Resources.kt | 69 ++ .../consumer/JavaPublicApiConsumer.java | 28 + .../oliphaunt/consumer/PublicApiConsumer.kt | 5 +- src/sdks/react-native/README.md | 60 +- .../src/main/cpp/OliphauntJsiBindings.cpp | 8 - .../android/src/main/cpp/include/oliphaunt.h | 8 +- .../oliphaunt/reactnative/OliphauntModule.kt | 39 +- src/sdks/react-native/app.plugin.js | 68 +- src/sdks/react-native/ios/Oliphaunt.mm | 23 +- src/sdks/react-native/ios/OliphauntAdapter.h | 1 - .../react-native/ios/OliphauntAdapter.swift | 85 +-- src/sdks/react-native/package.json | 3 +- src/sdks/react-native/release.toml | 5 + .../react-native/src/__tests__/client.test.ts | 25 +- .../src/__tests__/config-plugin.test.ts | 36 + .../src/__tests__/storage.test.ts | 31 + src/sdks/react-native/src/client.ts | 45 +- src/sdks/react-native/src/extensions.ts | 22 + src/sdks/react-native/src/index.ts | 5 + src/sdks/react-native/src/jsiTransport.ts | 6 +- .../react-native/src/specs/NativeOliphaunt.ts | 4 +- src/sdks/react-native/src/storage.ts | 31 + .../react-native/tools/expo-android-runner.sh | 10 +- .../react-native/tools/expo-ios-runner.sh | 32 +- .../tools/expo-runner-ios-installed-app.sh | 10 +- .../tools/expo-runner-reporting.sh | 6 +- .../tools/expo-runner-runtime-resources.sh | 9 +- .../tools/native-resource-closure.mjs | 10 +- src/sdks/react-native/tools/stage-ios-app.mjs | 26 +- .../react-native/tools/stage-ios-app.test.mjs | 6 + .../react-native/tools/verify-ios-package.mjs | 15 +- src/sdks/rust/Cargo.toml | 9 +- src/sdks/rust/README.md | 52 +- src/sdks/rust/build.rs | 8 + .../rust/crates/oliphaunt-build/src/lib.rs | 124 ++- .../crates/oliphaunt-resources/Cargo.toml | 13 + .../rust/crates/oliphaunt-resources/README.md | 3 + .../crates/oliphaunt-resources/src/lib.rs | 121 +++ src/sdks/rust/moon.yml | 8 +- src/sdks/rust/release.toml | 2 +- src/sdks/rust/src/broker.rs | 21 +- src/sdks/rust/src/broker_support.rs | 2 + src/sdks/rust/src/build_resources.rs | 291 ++++++- src/sdks/rust/src/builder.rs | 60 +- src/sdks/rust/src/config.rs | 4 + src/sdks/rust/src/direct.rs | 28 +- src/sdks/rust/src/extension.rs | 60 ++ src/sdks/rust/src/extensions.rs | 290 +++++++ src/sdks/rust/src/lib.rs | 6 +- src/sdks/rust/src/liboliphaunt/ffi.rs | 21 +- src/sdks/rust/src/liboliphaunt/root.rs | 4 +- .../rust/src/liboliphaunt/root/runtime.rs | 46 +- .../src/liboliphaunt/root/runtime/locate.rs | 22 +- src/sdks/rust/tools/package-source.mjs | 6 +- src/sdks/swift/README.md | 297 ++------ src/sdks/swift/Sources/COliphaunt/bridge.c | 9 + .../Sources/COliphaunt/include/COliphaunt.h | 2 + .../Sources/COliphaunt/include/oliphaunt.h | 8 +- .../swift/Sources/Oliphaunt/Oliphaunt.swift | 39 +- .../OliphauntExtensionResources.swift | 36 +- .../Oliphaunt/OliphauntExtensions.swift | 79 ++ .../Oliphaunt/OliphauntNativeDirect.swift | 18 +- .../Oliphaunt/OliphauntResources.swift | 95 +++ .../Oliphaunt/OliphauntRuntimeResources.swift | 44 +- .../OliphauntStaticExtensionRegistry.swift | 114 +++ .../OliphauntExtensionSupport.swift | 102 +-- .../ExtensionResourceCompositionTests.swift | 16 + .../Tests/OliphauntTests/OliphauntTests.swift | 16 +- .../OliphauntPublicApiConsumer/main.swift | 2 +- .../swift/tools/render-extension-products.mjs | 81 +- .../swift/tools/swift-carrier-resolver.mjs | 3 +- .../tools/swift-carrier-resolver.test.mjs | 33 +- src/shared/js-core/package.json | 39 +- src/shared/js-core/src/resources.ts | 105 +++ src/shared/js-core/tools/stage-package.mjs | 4 + tools/policy/generate-sdk-api-surface.mjs | 22 +- .../bootstrap-publication-capsule.test.mjs | 5 +- .../release/build-cargo-sdk-ci-artifacts.mjs | 6 +- .../build-cargo-sdk-ci-artifacts.test.mjs | 6 +- .../build-linux-wasix-napi-baseline.sh | 2 - .../release/build_maven_artifact_manifest.mjs | 31 +- .../release/carrier-license-contract.test.mjs | 4 +- .../check-cargo-package-test-closure.mjs | 14 +- .../check-cargo-package-test-closure.test.mjs | 15 + .../check-liboliphaunt-release-assets.mjs | 37 +- tools/release/check-release-metadata.mjs | 6 +- tools/release/check-staged-artifacts.mjs | 15 +- tools/release/check-staged-artifacts.test.mjs | 1 + tools/release/check_artifact_targets.mjs | 6 + tools/release/example-cargo-policy.mjs | 13 +- tools/release/example-cargo-policy.test.mjs | 7 +- tools/release/extension-registry-packages.mjs | 23 +- .../extension-wasix-npm-packages.test.mjs | 35 +- tools/release/ios-carrier-manifest.mjs | 4 +- tools/release/ios-carrier-manifest.test.mjs | 8 +- ...rtifact-manifest-publication-lock.test.mjs | 2 +- tools/release/maven-artifact-staging.mjs | 104 ++- tools/release/maven-artifact-staging.test.mjs | 32 + tools/release/native-icu-seeds.mjs | 78 ++ tools/release/native-icu-seeds.test.mjs | 70 ++ .../native-runtime-carrier-contract.mjs | 6 +- .../package-extension-cargo-facades.mjs | 55 +- .../package-extension-cargo-facades.test.mjs | 37 +- .../package-extension-release-carriers.mjs | 129 +++- .../package-liboliphaunt-cargo-artifacts.mjs | 13 +- ...kage-liboliphaunt-cargo-artifacts.test.mjs | 55 +- .../package-liboliphaunt-icu-data.test.mjs | 2 +- .../package-liboliphaunt-linux-assets.sh | 4 + .../package-liboliphaunt-macos-assets.sh | 4 + .../package-liboliphaunt-mobile-assets.sh | 3 + ...iboliphaunt-wasix-cargo-artifacts.test.mjs | 5 +- .../package-liboliphaunt-windows-assets.ps1 | 3 + tools/release/package-release-carriers.mjs | 42 +- ...age_liboliphaunt_wasix_cargo_artifacts.mjs | 72 +- .../package_oliphaunt_wasix_sdk_crate.mjs | 3 +- .../release/preflight-swiftpm-source-tag.mjs | 21 +- tools/release/prepare-rust-release-source.mjs | 7 +- tools/release/public-consumer-smoke.mjs | 43 +- tools/release/public-consumer-smoke.test.mjs | 24 + tools/release/publication-catalog.test.mjs | 4 +- tools/release/publication-lock.mjs | 33 +- tools/release/publication-lock.test.mjs | 27 +- tools/release/publish_swiftpm_source_tag.mjs | 57 ++ .../publish_swiftpm_source_tag.test.mjs | 37 + tools/release/release-artifact-targets.mjs | 8 + tools/release/release-notices.mjs | 4 +- tools/release/release-publish.mjs | 24 + .../render_swiftpm_release_package.mjs | 85 ++- .../render_swiftpm_release_package.test.mjs | 14 +- tools/release/sdk-artifacts/rust.mjs | 7 +- tools/release/sdk-artifacts/swift.mjs | 2 +- tools/release/source-only-sdk-package.mjs | 10 + .../release/source-only-sdk-package.test.mjs | 1 + .../stage-react-native-resource-packages.mjs | 82 ++ ...ge-react-native-resource-packages.test.mjs | 46 ++ ...extension-release-consumer-inputs.test.mjs | 1 + .../release/swift-source-carrier-contract.mjs | 1 + tools/release/verify-release-commit.mjs | 20 +- tools/release/wasix-extension-features.mjs | 4 +- .../release/wasix-extension-features.test.mjs | 8 +- .../wasix-runtime-npm-carrier.test.mjs | 22 +- tools/release/wasix-tools-aot-npm.test.mjs | 43 ++ tools/release/wasix-tools-npm-carrier.mjs | 52 ++ tools/release/wasix-typescript-package.mjs | 4 +- .../release/wasix-typescript-package.test.mjs | 1 + .../create-liboliphaunt-release-fixture.mjs | 37 +- tools/xtask/src/extension_catalog.rs | 115 ++- 296 files changed, 7955 insertions(+), 3339 deletions(-) create mode 100644 src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/generated_contrib_descriptors.rs create mode 100644 src/bindings/wasix-ts/src/__tests__/contrib-carrier.ts create mode 100644 src/bindings/wasix-ts/src/__tests__/native-extension-packages.test.ts create mode 100644 src/bindings/wasix-ts/src/contrib-carrier-shim.d.ts create mode 100644 src/bindings/wasix-ts/src/native-extension-packages.ts create mode 100644 src/runtimes/liboliphaunt/native/icu-npm/index.d.ts create mode 100644 src/runtimes/liboliphaunt/native/icu-npm/index.js create mode 100644 src/runtimes/liboliphaunt/native/icu-npm/node.js create mode 100644 src/runtimes/wasix-napi/src/extension_package.rs create mode 100644 src/runtimes/wasix-napi/src/icu_data.rs create mode 100644 src/sdks/js/src/__tests__/resources.test.ts create mode 100644 src/sdks/js/src/extensions.ts create mode 100644 src/sdks/js/src/storage/node.ts create mode 100644 src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntJava.kt create mode 100644 src/sdks/kotlin/oliphaunt/src/commonMain/kotlin/dev/oliphaunt/Resources.kt create mode 100644 src/sdks/kotlin/tests/public-api-consumer/src/main/java/dev/oliphaunt/consumer/JavaPublicApiConsumer.java create mode 100644 src/sdks/react-native/src/__tests__/storage.test.ts create mode 100644 src/sdks/react-native/src/extensions.ts create mode 100644 src/sdks/react-native/src/storage.ts create mode 100644 src/sdks/rust/crates/oliphaunt-resources/Cargo.toml create mode 100644 src/sdks/rust/crates/oliphaunt-resources/README.md create mode 100644 src/sdks/rust/crates/oliphaunt-resources/src/lib.rs create mode 100644 src/sdks/rust/src/extensions.rs create mode 100644 src/sdks/swift/Sources/Oliphaunt/OliphauntExtensions.swift create mode 100644 src/sdks/swift/Sources/Oliphaunt/OliphauntResources.swift create mode 100644 src/sdks/swift/Sources/Oliphaunt/OliphauntStaticExtensionRegistry.swift create mode 100644 src/shared/js-core/src/resources.ts create mode 100644 tools/release/native-icu-seeds.mjs create mode 100644 tools/release/native-icu-seeds.test.mjs create mode 100644 tools/release/stage-react-native-resource-packages.mjs create mode 100644 tools/release/stage-react-native-resource-packages.test.mjs create mode 100644 tools/release/wasix-tools-aot-npm.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c70dcfd2d..37506c616 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2547,6 +2547,14 @@ jobs: exit 1 fi closure="$(dirname "$1")" + set -- + while IFS= read -r archive; do + [ -n "$archive" ] && set -- "$@" "$archive" + done < <(find target/liboliphaunt/android-runtime-assets -type f -name 'liboliphaunt-*-icu-seed-android-datum64.tar.gz' | LC_ALL=C sort) + [ "$#" -eq 1 ] || { echo "Expected one optional android-datum64 ICU seed" >&2; exit 1; } + mkdir -p "$closure/cluster-seed-icu" + tar -xzf "$1" -C "$closure/cluster-seed-icu" + # JavaScript template interpolation belongs inside the single-quoted Bun program. # shellcheck disable=SC2016 tools/dev/bun.sh -e ' @@ -2698,6 +2706,14 @@ jobs: exit 1 fi closure="$(dirname "$1")" + set -- + while IFS= read -r archive; do + [ -n "$archive" ] && set -- "$@" "$archive" + done < <(find target/liboliphaunt/release-assets -type f -name 'liboliphaunt-*-icu-seed-ios-datum64.tar.gz' | LC_ALL=C sort) + [ "$#" -eq 1 ] || { echo "Expected one optional ios-datum64 ICU seed" >&2; exit 1; } + mkdir -p "$closure/cluster-seed-icu" + tar -xzf "$1" -C "$closure/cluster-seed-icu" + # JavaScript template interpolation belongs inside the single-quoted Bun program. # shellcheck disable=SC2016 tools/dev/bun.sh -e ' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 180e3d9d6..b5ca6a2b3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1016,8 +1016,8 @@ jobs: - name: Prove the exact SwiftPM source tag is remotely collision-free id: preflight_swift_source_tag - if: ${{ steps.release_plan.outputs.has_release_changes == 'true' && inputs.operation == 'publish' && contains(fromJson(steps.release_plan.outputs.products_json), 'oliphaunt-swift') }} - timeout-minutes: 2 + if: ${{ steps.release_plan.outputs.has_release_changes == 'true' && inputs.operation == 'publish' }} + timeout-minutes: 10 run: | tools/dev/bun.sh tools/release/preflight-swiftpm-source-tag.mjs \ --publication-lock "$PUBLICATION_LOCK_PATH" \ @@ -1220,6 +1220,25 @@ jobs: target/liboliphaunt/release-assets/*.sha256 target/extension-artifacts/liboliphaunt-native/oliphaunt-extension-contrib-pg18/release-assets/* + - name: Create standalone SwiftPM package token + id: swift_package_token + if: ${{ steps.preflight_swift_source_tag.outputs.repositories != '' && inputs.operation == 'publish' }} + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 + with: + client-id: ${{ secrets.RELEASE_TAG_APP_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_TAG_APP_PRIVATE_KEY }} + owner: f0rr0 + repositories: ${{ steps.preflight_swift_source_tag.outputs.repositories }} + permission-contents: write + + - name: Publish exact standalone SwiftPM package tags + if: ${{ steps.preflight_swift_source_tag.outputs.repositories != '' && inputs.operation == 'publish' }} + timeout-minutes: 45 + env: + GH_TOKEN: ${{ steps.swift_package_token.outputs.token }} + PRODUCTS_JSON: ${{ steps.release_plan.outputs.products_json }} + run: tools/dev/bun.sh tools/release/release-publish.mjs publish --step swift-package-source-tag --products-json "$PRODUCTS_JSON" --head-ref "$RELEASE_HEAD_SHA" --publication-lock "$PUBLICATION_LOCK_PATH" + - name: Create fresh SwiftPM tag token id: swift_tag_token if: ${{ steps.release_plan.outputs.has_release_changes == 'true' && inputs.operation == 'publish' && contains(fromJson(steps.release_plan.outputs.products_json), 'oliphaunt-swift') }} diff --git a/Cargo.lock b/Cargo.lock index ca62f66da..aa4f9803c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2446,7 +2446,10 @@ version = "0.2.0" dependencies = [ "fs2", "getrandom 0.3.4", + "libc", "libloading 0.8.9", + "oliphaunt-build", + "oliphaunt-resources", "serde", "serde_json", "sha2 0.10.9", @@ -2474,6 +2477,7 @@ dependencies = [ name = "oliphaunt-icu" version = "0.2.0" dependencies = [ + "oliphaunt-resources", "sha2 0.10.9", "tar", "zstd", @@ -2531,6 +2535,10 @@ dependencies = [ "tokio-postgres", ] +[[package]] +name = "oliphaunt-resources" +version = "0.2.0" + [[package]] name = "oliphaunt-tools" version = "0.2.0" @@ -2552,7 +2560,7 @@ dependencies = [ "liboliphaunt-wasix-aot-x86_64-pc-windows-msvc", "liboliphaunt-wasix-aot-x86_64-unknown-linux-gnu", "liboliphaunt-wasix-portable", - "oliphaunt-icu", + "oliphaunt-resources", "oliphaunt-wasix-tools", "oliphaunt-wasix-tools-aot-aarch64-apple-darwin", "oliphaunt-wasix-tools-aot-aarch64-unknown-linux-gnu", @@ -2589,10 +2597,9 @@ dependencies = [ "napi", "napi-build", "napi-derive", - "oliphaunt-icu", "oliphaunt-wasix", - "oliphaunt-wasix-tools", "rustix", + "serde_json", "sha2 0.10.9", ] diff --git a/Cargo.toml b/Cargo.toml index 1cb30801a..83d18ef05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "src/bindings/wasix-rust/crates/oliphaunt-wasix", "src/sdks/rust/crates/oliphaunt-build", + "src/sdks/rust/crates/oliphaunt-resources", "src/sdks/rust", "src/runtimes/liboliphaunt/native/crates/tools", "src/runtimes/broker", diff --git a/docs/architecture/cluster-seeds-and-icu.md b/docs/architecture/cluster-seeds-and-icu.md index d0b1ae9ca..207bbf1ac 100644 --- a/docs/architecture/cluster-seeds-and-icu.md +++ b/docs/architecture/cluster-seeds-and-icu.md @@ -62,7 +62,7 @@ There is no public initialization-mode enum and no raw seed/archive injection API. Package-managed SDKs resolve the correct seed transitively. - Installing/selecting the ordinary runtime resolves the `standard` seed. -- Selecting the language-native ICU package or feature resolves `icu-data` and +- Selecting the language-native ICU descriptor resolves `icu-data` and the matching `icu` seed as one checked closure. - Every seed and `initdb` fallback creates PostgreSQL's fixed `postgres` bootstrap role. Public `username` options consistently select an existing @@ -96,12 +96,12 @@ Other SDKs retain language-native package selection: | SDK | Ordinary selection | ICU selection | | --- | --- | --- | -| Native Rust | target runtime artifact selected by `oliphaunt-build` | Cargo ICU feature/artifact stages `oliphaunt-icu` | -| Native TypeScript | target runtime npm package | optional `@oliphaunt/icu` package | -| Swift | ordinary runtime resources | `OliphauntICU` SwiftPM/CocoaPods resources | -| Kotlin | ordinary Maven runtime resources | Gradle ICU dependency/selection | -| React Native | ordinary generated native carrier | `@oliphaunt/icu` native resource carrier | -| Rust WASIX | portable runtime artifact | Cargo ICU feature/artifact | +| Native Rust | SDK selects its target runtime internally | `oliphaunt-icu` dependency and `.icu(oliphaunt_icu::ICU)` | +| Native TypeScript | target runtime npm package | import `@oliphaunt/icu`, pass `{ icu }` | +| Swift | base SwiftPM runtime resources | independent `OliphauntICU` package and `icu: OliphauntICU.descriptor` | +| Kotlin / Java | ordinary Maven runtime resources | ICU dependency and explicit `ICU.data` configuration | +| React Native | ordinary generated native resources | import `@oliphaunt/icu`, pass `{ icu }` | +| Rust WASIX | portable runtime artifact | `oliphaunt-icu` dependency and `.icu(oliphaunt_icu::ICU)` | | WASIX TypeScript | default runtime descriptor | explicit `@oliphaunt/wasix-icu` descriptor | This is semantic parity, not identical signatures. @@ -152,12 +152,11 @@ There is one logical `icu-data` artifact. Ecosystem wrappers may differ: data behind WASIX-specific descriptors/archives; and - every wrapper must prove the same logical tree digest. -The shared Rust `oliphaunt-icu` crate and other platform-neutral native ICU -wrappers remain data-only. They cannot safely carry one native physical seed -for every operating system and architecture. Each target-specific native -runtime carrier transports its own small matching `icu` seed as -`cluster-seed-icu`; the runtime resolver pairs it with the independently staged -`icu-data` artifact. +Optional native ICU wrappers carry seeds under `native-seeds/` alongside +the common data. The shared Rust crate embeds only the build target's native +seed; its WASIX descriptor also carries the WASIX seed. Native release producers +publish each seed as a separate `liboliphaunt--icu-seed-.tar.gz` +asset. Base native runtime packages contain only `cluster-seed`. The data-only carrier exposes one canonical `manifest.properties` containing only its `oliphaunt-icu-data-v1` schema, `icu-data` role, ICU version/form, and @@ -169,7 +168,8 @@ runtime and ICU seed manifests after both release families have been assembled; single-family focused builds do not manufacture a cross-family proof. The native ICU release asset records only `icu-data`. Each target runtime report -records `cluster-seed` and `cluster-seed-icu` separately from runtime bytes. +records the standard seed separately from runtime bytes, with zero ICU seed bytes +in the base distribution. Cargo and npm package limits continue to apply to the final carrier bytes. ## Seed compatibility @@ -335,21 +335,13 @@ The repository implementation must keep every item below true: emit extension-free, clean-shutdown seeds. - [x] WASIX runtime manifests use format v2 and carry both seed descriptors and archives under `cluster-seeds/`. -- [x] Native target release assets carry target-qualified `standard` and `icu` - seeds. -- [x] Target-specific native runtime assets carry their matching `icu` seed; - platform-neutral native ICU data assets stay data-only and carry an exact - logical tree binding. -- [x] The WASIX ICU carrier carries shared ICU data plus the WASIX `icu` seed; - it never carries a native seed. -- [x] Every native carrier declares `clusterSeedTarget` and the fixed sibling - paths `cluster-seed` and `cluster-seed-icu`. A SwiftPM application receives - the closure embedded in its selected XCFramework slice; React Native stages - the one app-selected closure and removes embedded copies from its staged base - framework. -- [x] Native Cargo target carriers aggregate both native seeds while the shared - `oliphaunt-icu` crate and every other platform-neutral ICU wrapper stay - data-only. +- [x] Native base runtime assets contain the target-qualified standard seed. +- [x] Optional native ICU wrappers own the data and target-qualified ICU seeds. +- [x] The WASIX npm ICU carrier owns the WASIX ICU seed and shared data. +- [x] Native producer seed sidecars have independent byte hashes, target identities, + and exact ICU data tree bindings. +- [x] SwiftPM loads the selected ICU package's seed. React Native merges an ICU + sidecar only when the optional package is selected for the app. - [x] npm, Cargo, SwiftPM, Maven, Kotlin, React Native, Rust, native TypeScript, Rust WASIX, and WASIX TypeScript carrier/resolver paths reject missing or wrong-profile closure members. diff --git a/docs/architecture/database-storage.md b/docs/architecture/database-storage.md index e3677c503..7863405b4 100644 --- a/docs/architecture/database-storage.md +++ b/docs/architecture/database-storage.md @@ -54,16 +54,14 @@ configuration schema. | TypeScript | omitted `storage` | `{ kind: 'directory', path }` | | Swift | `.temporaryDirectory` | `.directory(url)` | | Kotlin | `DatabaseStorage.TemporaryDirectory` | `DatabaseStorage.Directory(path)` | -| React Native | omitted `storage` | `{ kind: 'directory', path }` or `{ kind: 'applicationData', name }` | +| React Native | omitted `storage` | `directory(pathOrFileURI)` | | Rust WASIX | `DatabaseStorage::Memory` | `DatabaseStorage::Directory(path)` | | WASIX TypeScript | omitted `storage` | `indexedDB(name)` or `opfs(name)` in browsers; `directory(path)` on Node, Bun, Deno, and Electron | -The React Native `applicationData` case is intentional. JavaScript has no -portable API for constructing an iOS/Android app-sandbox path, so the native -adapter resolves one portable name. Swift and Kotlin callers already have URL -and File APIs and do not need a second path abstraction. Rust WASIX callers -likewise resolve temporary or application-data paths with their preferred host -crate and pass the result through `Directory(path)`. +React Native callers obtain an app sandbox directory from their filesystem +library, such as Expo's `Paths.document`, and pass its path or local file URI +to `directory()`. Swift and Kotlin use their platform URL/File APIs. Rust +callers can use their preferred host crate to locate application directories. WASIX TypeScript does not expose a browser `temporaryDirectory` case: omitted storage already gives the cheapest anonymous lifetime without host I/O. Its @@ -212,8 +210,8 @@ hide real semantics or add indirection to a hot path. memory filesystem. - IndexedDB and OPFS are TypeScript browser providers. Rust WASIX does not grow browser-shaped APIs that its ecosystem cannot use naturally. -- React Native keeps `applicationData(name)` because JavaScript cannot resolve - mobile sandbox paths portably; native Swift and Kotlin callers use URL/File. +- React Native accepts a path or local file URI from a mobile filesystem library; + Swift and Kotlin callers use their platform URL/File APIs. - Native direct storage and Rust WASIX host storage are direct filesystem I/O. WASIX TypeScript uses both direct OPFS I/O and asynchronous providers, so publication failure state is a real part of that SDK's error API. diff --git a/docs/architecture/wasix-typescript-napi.md b/docs/architecture/wasix-typescript-napi.md index 4a6e0b811..82c4ee747 100644 --- a/docs/architecture/wasix-typescript-napi.md +++ b/docs/architecture/wasix-typescript-napi.md @@ -307,16 +307,13 @@ Electron versions rather than inventing a Node maximum unrelated to evidence. Linux carriers follow the repository's existing glibc build and maximum-symbol policy; this project does not redefine that baseline. -The release topology is one addon binary per target. It contains both qualified -standard and ICU seed/data profiles plus the release's frozen extension and -tool catalog; opening a database selects the profile without loading a second -addon. Profile selection is immutable per builder/database, including server -builders, and every reusable runtime, seed, and materialization cache is keyed -by profile. This removes duplicate runtime/code/catalog payload while -preserving the existing descriptor-based application API. Do not redesign -native extension loading in this migration. Independent native extension -carriers are a future package-size optimization and need their own measurements -and threat model. +The release topology is one addon binary per target, containing the runtime, +standard seed, and supported contrib. External extensions, ICU data and matching +seeds, and frontend tools are separate packages. TypeScript resolves installed +descriptors and passes their payloads to Rust; Rust validates owner, version, +target, runtime identity, and hashes before loading. Profile and extension +selection remain immutable per database, and reusable caches include the +selected resource identities. `@oliphaunt/wasix-ts` deliberately remains one universal browser-and-server npm package. The published tarball includes the patched browser Wasmer host and its diff --git a/docs/maintainers/consumer-dx-release-blueprint.md b/docs/maintainers/consumer-dx-release-blueprint.md index e2c1cbf7a..09907fef0 100644 --- a/docs/maintainers/consumer-dx-release-blueprint.md +++ b/docs/maintainers/consumer-dx-release-blueprint.md @@ -1,622 +1,91 @@ -# Consumer DX Release Implementation Plan - -Status: archived design plan; non-normative. Do not use package names, carrier -counts, workflow steps, or release policy from this file. They are historical -snapshots and may intentionally be stale. See `docs/maintainers/README.md`, -`docs/maintainers/release.md`, and `docs/maintainers/extension-packaging-policy.md` -for the current operational contract. - -## Decisions - -1. Ship no runtime artifact downloads in public SDK paths. -2. Ship no consumer install lifecycle scripts. -3. Remove CLI-first consumer flows from public docs. -4. Keep `oliphaunt-resources`, extension artifact tools, and release download - helpers as maintainer and CI tools. -5. Publish byte-carrying artifacts through the package manager used by each - ecosystem. -6. Publish every byte-carrying PostgreSQL extension as an explicit dependency or - product. Do not hide contrib extension bytes in base packages. -7. Keep the base runtime free of selected extension payloads and full ICU data. -8. Publish full ICU data as runtime-owned sidecar packages. Do not publish an - `icu-full` package. Model ICU as a runtime/data capability, not as a SQL - extension. -9. Generate every consumer package descriptor from the repo product graph, - release metadata, and extension catalog evidence. -10. Fail builds at package/install/build time on missing artifacts. Never defer - artifact errors to first database open. - -## Grounding - -- npm supports platform package selection with `os`, `cpu`, `libc`, and - `optionalDependencies`; npm also supports omitting dependencies from that - field. Resolver code must treat missing selected platform packages as install - errors. - Source: [npm package.json docs](https://docs.npmjs.com/files/package.json/). -- crates.io has a 10 MB compressed `.crate` limit. Oliphaunt publishes WASIX - runtime and AOT bytes through direct public artifact crates and fails release - packaging when a generated `.crate` exceeds that limit. Source: - [Cargo publishing docs](https://doc.rust-lang.org/cargo/reference/publishing.html). -- SwiftPM binary artifacts are declared in `Package.swift` with URL and checksum. - Plugins do not add new products after package resolution. Source: - [SwiftPM binary dependencies](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0272-swiftpm-binary-dependencies.md), - [SwiftPM plugin docs](https://github.com/swiftlang/swift-package-manager/blob/main/Sources/PackageManagerDocs/Documentation.docc/Plugins.md). -- iOS apps are self-contained bundles and do not download executable code that - changes functionality after review. Source: - [Apple App Review Guideline 2.5.2](https://developer.apple.com/app-store/review/guidelines/). -- Android and Gradle support variant-aware publication and dependency - verification. Source: - [Android publication variants](https://developer.android.com/build/publish-library/configure-pub-variants), - [Gradle dependency verification](https://docs.gradle.org/current/userguide/dependency_verification.html). - -## Current Code To Remove Or Replace - -### JavaScript - -Remove runtime GitHub downloads from: - -- `src/sdks/js/src/native/assets-node.ts` -- `src/sdks/js/src/native/assets-deno.ts` -- `src/sdks/js/src/native/node-addon.ts` -- `src/sdks/js/src/runtime/broker.ts` - -Replace those paths with package-local artifact resolution. Delete public use of: - -- `OLIPHAUNT_RELEASE_BASE_URL` -- `OLIPHAUNT_LIBOLIPHAUNT_ASSET_DIR` -- `OLIPHAUNT_NODE_DIRECT_RELEASE_BASE_URL` -- runtime cache installation as a normal public path - -Keep fixture directories for tests under test-owned names. Do not expose them as -consumer configuration. - -### Rust Native - -Remove CLI-first consumer install guidance from: - -- `src/sdks/rust/README.md` -- `docs/maintainers/sdk-products-policy.md` -- release metadata checks that treat `--resolve-release-assets` as the consumer - path - -Keep `oliphaunt-resources` for CI and maintainer packaging. Add the consumer -path through `oliphaunt-build`. - -### Rust WASIX - -Remove the public env-var archive path from consumer docs: - -- `OLIPHAUNT_WASM_RUNTIME_ARCHIVE` -- `OLIPHAUNT_WASM_AOT_ARCHIVE` - -Delete the inert `bundled` feature contract that advertises a feature and then -hard-fails. Replace it with a build-produced asset manifest contract. - -### Android And Kotlin - -Remove direct GitHub asset downloads from: - -- `src/sdks/kotlin/oliphaunt-android-gradle-plugin/src/main/java/dev/oliphaunt/android/ResolveOliphauntAndroidAssetsTask.java` -- the duplicate resolver in `src/sdks/kotlin/oliphaunt/build.gradle.kts` - -Replace downloads with Gradle/Maven artifact dependencies and Gradle dependency -verification. - -### React Native - -Replace the iOS config-only path in: - -- `src/sdks/react-native/app.plugin.js` -- `src/sdks/react-native/OliphauntReactNative.podspec` -- `src/sdks/react-native/ios/podspecs/*.podspec` - -The config plugin writes a locked native artifact plan. The native build consumes -real local package artifacts installed by npm packages. - -### Swift - -Extend the generated SwiftPM release manifest in: - -- `tools/release/render_swiftpm_release_package.mjs` - -Generate extension products and checksum-pinned binary targets. Do not use a -plugin to add dependencies. - -## Target Consumer DX - -### JavaScript: Node And Bun - -Base install: - -```sh -pnpm add @oliphaunt/ts -``` - -Runtime artifacts: - -- `@oliphaunt/ts` declares platform packages for `liboliphaunt`: - `@oliphaunt/liboliphaunt-darwin-arm64`, - `@oliphaunt/liboliphaunt-linux-x64-gnu`, - `@oliphaunt/liboliphaunt-linux-arm64-gnu`, and - `@oliphaunt/liboliphaunt-win32-x64-msvc`. -- Each `@oliphaunt/liboliphaunt-*` platform package includes the matching - `liboliphaunt` library and PostgreSQL runtime tree for that platform. -- `@oliphaunt/ts` declares platform packages for `oliphaunt-broker`: - `@oliphaunt/broker-darwin-arm64`, `@oliphaunt/broker-linux-x64-gnu`, - `@oliphaunt/broker-linux-arm64-gnu`, and - `@oliphaunt/broker-win32-x64-msvc`. -- `@oliphaunt/ts` keeps the existing platform packages for - `oliphaunt-node-direct`. -- Platform packages use `os`, `cpu`, and `libc`. -- Runtime resolution uses literal package names and an exported artifact - manifest. -- Missing selected platform packages fail with an install message. -- Runtime resolution never fetches GitHub assets. - -Extension install: - -```sh -pnpm add @oliphaunt/ts @oliphaunt/extension-vector -``` - -Usage: - -```ts -import { Oliphaunt } from '@oliphaunt/ts'; -import { vector } from '@oliphaunt/extension-vector'; - -const db = await Oliphaunt.open({ - storage: { kind: 'directory', path: '.oliphaunt' }, - extensions: [vector], -}); -``` - -Contrib uses the same explicit package shape: - -```sh -pnpm add @oliphaunt/ts @oliphaunt/extension-pg-trgm -``` - -```ts -import { pgTrgm } from '@oliphaunt/extension-pg-trgm'; -``` - -Package layout: - -- `@oliphaunt/extension-` is a descriptor package. -- Descriptor packages declare native platform artifact packages. -- `@oliphaunt/extension--wasix` is the separately imported host-neutral - WASIX descriptor package. It carries browser bytes; Node, Bun, Deno, and - Electron validate the descriptor and use the matching artifact embedded in - their Node-API addon. It shares - the unsuffixed product's version and does not introduce a `-native` alias. -- `@oliphaunt/liboliphaunt-wasix` is the generated, internal runtime carrier - selected by the WASIX binding. Applications do not supply ordinary asset - URLs. Its manifest is the exact core-only projection; extension leaves carry - their own frozen install contracts and remain independently versioned. -- Other npm byte packages cover native desktop, Android ABI, and iOS - XCFramework target classes. -- Maven byte packages use - `dev.oliphaunt.extensions:-android-arm64-v8a` and - `dev.oliphaunt.extensions:-android-x86_64`. -- SwiftPM binary targets are generated into the release manifest. -- Platform artifact packages carry exact extension bytes for one target family. -- Descriptor exports include SQL name, product version, runtime compatibility, - target package names, checksums, byte sizes, dependencies, and required - preload libraries. - -The intended WASIX consumer surface stays selective and does not expose the -runtime carrier's package-relative assets: - -```ts -import Oliphaunt from '@oliphaunt/wasix-ts'; -import pgtap from '@oliphaunt/extension-pgtap-wasix'; - -const db = await Oliphaunt.open({ extensions: [pgtap] }); -``` - -Browser, Node, Bun, Deno, and Electron WASIX hosts share the same `-wasix` -descriptor. Browsers materialize its bytes; native hosts validate its identity -and select addon-embedded artifacts. Host selection belongs to `@oliphaunt/wasix-ts`; native -JavaScript remains on the established unsuffixed surface. - -### JavaScript: Deno And npm - -Deno native users install the same package as Node and Bun through npm: - -```ts -import { Oliphaunt } from 'npm:@oliphaunt/ts'; -``` - -The Deno native path uses the same npm-installed platform packages as Node and -Bun. - -Deno WASIX users install the separately versioned full runtime facade from npm: - -```ts -import Oliphaunt from 'npm:@oliphaunt/wasix-ts'; -import { directory } from 'npm:@oliphaunt/wasix-ts/storage/deno'; -``` - -That carrier pins `npm:@oliphaunt/liboliphaunt-wasix` and `npm:fzstd` at exact -versions, publishes the patched host bytes, and owns the -`oliphaunt-wasix-ts` product version. It remains separate from native -`@oliphaunt/ts`. - -### Rust Native - -Base install: - -```sh -cargo add oliphaunt -cargo add --build oliphaunt-build -``` - -Extension install: - -```sh -cargo add oliphaunt-extension-vector -cargo add oliphaunt-extension-contrib-pg18 -``` - -ICU install: - -```sh -cargo add oliphaunt-icu -``` - -Application manifest: - -```toml -[package.metadata.oliphaunt] -runtime = "liboliphaunt-native" -runtime-version = "0.1.0" -extensions = ["vector", "pg_trgm"] -``` - -The contrib dependency is a shared distribution carrier. The metadata list -still selects exact SQL members; it does not stage every contrib extension. - -Add `icu = true` under `[package.metadata.oliphaunt]` after adding the -`oliphaunt-icu` dependency. - -Build script: - -```rust -fn main() { - oliphaunt_build::configure(); -} -``` - -`oliphaunt-build` is a build-dependency crate released with `oliphaunt-rust`. -It is the Rust application integration point for local runtime packaging. -Cargo resolves the target-specific artifact crates; `oliphaunt-build` only -stages the already-resolved files for the current application build. It performs -all app packaging work: - -- reads `[package.metadata.oliphaunt]`; -- reads generated product metadata; -- reads the resolved Cargo package graph; -- resolves native target from Cargo target variables; -- locates Cargo-resolved native runtime, broker, ICU, and exact extension - artifact crates; -- performs no network I/O; -- adds no dependencies to the Cargo graph; -- writes `OUT_DIR/oliphaunt/oliphaunt-assets.lock`; -- validates the lockfile and manifests; -- writes generated Rust constants and Cargo env vars; -- copies the exact runtime, broker helper, ICU data, and selected extension - artifacts into `OUT_DIR/oliphaunt/resources`; -- writes no generated files outside `OUT_DIR`; -- emits `cargo:rerun-if-changed=Cargo.toml`; -- fails on missing Cargo dependencies, missing checksums, unknown targets, - missing extension artifacts, selected ICU without `oliphaunt-icu`, and - unselected extension leakage. - -Cargo owns artifact selection: - -- Direct application dependencies expose artifact manifests through Cargo - `links` metadata. `oliphaunt-build` consumes only the metadata that Cargo - passes to the application build script. -- `oliphaunt` owns the base runtime and broker selector build script. It reads - its target-specific artifact dependencies and re-emits the selected manifests - as `oliphaunt-build` inputs. -- `oliphaunt-extension-*` crates own exact extension selector build scripts. - They read their target-specific native and WASIX artifact dependencies and - re-emit selected manifests as `oliphaunt-build` inputs. -- `oliphaunt-icu` is the Cargo ICU data artifact crate. It owns the ICU data - selector build script and re-emits selected manifests as `oliphaunt-build` - inputs. -- Raw target artifact crates are private implementation dependencies. Consumers - do not add target artifact crates directly. -- `oliphaunt-build` stages only artifacts exposed by direct application - dependencies. It never relies on transitive Cargo metadata. - -Consumer code uses the SDK without environment variables: - -```rust -let mut db = oliphaunt::Oliphaunt::builder() - .storage(oliphaunt::DatabaseStorage::Directory(".oliphaunt".into())) - .direct() - .extension(oliphaunt::Extension::VECTOR) - .open()?; -``` - -Tauri documentation shows the one bundler stanza that copies the generated -Oliphaunt resource directory into the app image. - -### Rust WASIX - -Base install: - -```sh -cargo add oliphaunt-wasix -cargo add --build oliphaunt-build -``` - -Extension install: - -```sh -cargo add oliphaunt-extension-vector -``` - -Application manifest: - -```toml -[package.metadata.oliphaunt] -runtime = "liboliphaunt-wasix" -runtime-version = "0.1.0" -extensions = ["vector"] -``` - -Build script: - -```rust -fn main() { - oliphaunt_build::configure(); -} -``` - -WASIX uses Cargo-selected runtime artifacts. The public `oliphaunt-wasix` crate -depends on `liboliphaunt-wasix-portable` and target-specific `liboliphaunt-wasix-aot-*` -artifact crates. Release packaging generates and packages those public artifact -crates directly from staged WASIX release assets. Each generated `.crate` must -fit the crates.io 10 MB package limit. Release packaging publishes the artifact -crates, then publishes `oliphaunt-wasix`. Consumers add only -`oliphaunt-wasix`; Cargo selects the matching target dependency. The crate does -not expose a `bundled` feature and does not read runtime/AOT archive env vars in -the normal path. - -Delete source-only/private WASIX asset and AOT fallbacks from the consumer path. - -### Android And Kotlin - -Base install: - -```gradle -plugins { - id("com.android.application") - id("dev.oliphaunt.android") version "0.1.0" -} - -dependencies { - implementation("dev.oliphaunt:oliphaunt-android:0.1.0") -} - -oliphaunt { - liboliphauntVersion.set("0.1.0") - selectedExtensions.add("vector") -} -``` - -Plugin behavior: - -- maps selected ABIs to Maven runtime artifacts; -- adds runtime artifact dependencies; -- adds exact extension artifact dependencies; -- writes generated Android assets and `jniLibs`; -- verifies dependencies through Gradle dependency verification; -- fails on missing artifacts, checksum mismatches, unknown extensions, and - unselected extension leakage. - -Published artifacts: - -- `dev.oliphaunt.runtime:liboliphaunt-android-arm64-v8a` -- `dev.oliphaunt.runtime:liboliphaunt-android-x86_64` -- `dev.oliphaunt.extensions:-android-arm64-v8a` -- `dev.oliphaunt.extensions:-android-x86_64` -- `dev.oliphaunt.runtime:oliphaunt-icu` - -### Swift - -Base install: - -```swift -.package(url: "https://github.com/f0rr0/oliphaunt.git", exact: "0.6.0") -``` - -Base product: - -```swift -.product(name: "Oliphaunt", package: "oliphaunt") -``` - -Extension product: - -```swift -.product(name: "OliphauntExtensionVector", package: "oliphaunt") -``` - -Generated release manifest: - -- declares `Oliphaunt`; -- declares one product per exact extension; -- declares `OliphauntICU`; -- declares checksum-pinned binary targets for base runtime artifacts; -- declares checksum-pinned binary targets for exact extension artifacts; -- links selected extension products before app archive. - -Runtime open validates that every configured SQL extension has a linked product. -Missing extension products fail before database open. iOS never downloads -runtime, extension, or ICU executable payloads after app archive. - -### React Native - -Install: - -```sh -pnpm add @oliphaunt/react-native @oliphaunt/extension-vector -``` - -Expo config: - -```json -{ - "expo": { - "plugins": [ - ["@oliphaunt/react-native", { "extensions": ["vector"] }] - ] - } -} -``` - -Config plugin behavior: - -- reads installed `@oliphaunt/extension-*` descriptors; -- writes one locked native artifact plan for iOS and Android; -- wires Android through the Kotlin Gradle plugin; -- runs `oliphaunt-react-native-stage-ios` before CocoaPods; -- materializes `ios/frameworks`, `ios/extension-frameworks`, `ios/resources`, - and `ios/generated/static-registry` from npm-installed extension packages; -- patches Podfile/podspec entries to use staged local artifacts; -- fails during prebuild on missing npm packages, version mismatch, missing iOS - artifacts, missing Android artifacts, and unselected extension leakage. - -React Native owns no independent runtime resolver. - -### ICU - -ICU product model: - -- base runtime excludes full ICU data; -- ICU-enabled runtime capability is explicit in release metadata; -- full ICU data ships as runtime-owned sidecar packages; -- do not publish `icu-full`; -- do not publish a Cargo crate named `icu`; -- package surfaces: - - npm: `@oliphaunt/icu` - - Maven: `dev.oliphaunt.runtime:oliphaunt-icu` - - SwiftPM: `OliphauntICU` - - Rust native: `oliphaunt-icu` plus `[package.metadata.oliphaunt] icu = true` - - Rust WASIX: `oliphaunt-icu` behind the `oliphaunt-wasix/icu` feature -- builders fail on ICU selection without a matching ICU data artifact. - -## Release Graph - -Keep source ownership split and explicit: - -- Moon owns tasks, dependency edges, and artifact target membership. -- `release.toml` owns publish targets and registry package names. -- Release Please owns versions, changelogs, and the generated release PR; the - protected workflow owns exact-SHA tags and draft GitHub releases. -- extension catalog/evidence/recipes own extension identity, readiness, - dependencies, and capability data. - -Add release surfaces, not aggregate products: - -- `liboliphaunt-native` owns JS base runtime platform packages, Cargo native - runtime artifact crates, Android Maven runtime payloads, SwiftPM base binary - metadata, and native runtime package-size budgets. -- `oliphaunt-broker` owns JS broker helper platform packages and Cargo broker - artifact crates. -- `oliphaunt-node-direct` keeps the existing npm addon platform packages and - drops runtime download code. -- each existing `oliphaunt-extension-*` product owns generated npm descriptor - packages, native npm platform byte packages, one host-neutral portable WASIX - npm leaf, Cargo descriptor and artifact crates, and Maven payload packages. -- `oliphaunt-swift` generates SwiftPM extension products from exact extension - product metadata. -- `oliphaunt-rust` owns the derived `crates:oliphaunt-build` crate, SDK crate, - target-specific Cargo dependency wiring, and version file. -- `liboliphaunt-native` owns native ICU release assets plus `@oliphaunt/icu`, - `dev.oliphaunt.runtime:oliphaunt-icu`, and `OliphauntICU` packaging. -- `liboliphaunt-wasix` owns the WASIX ICU release asset and the - `crates:oliphaunt-icu` artifact crate. - -Do not add a second catalog. Generate package descriptors from existing product -metadata and extension evidence. Commit the generated descriptor snapshots used -by release checks. - -## Minimal Release Gates - -Keep these gates: - -1. Metadata parity: Moon, `release.toml`, release-please, and extension generated - catalogs agree. -2. Package-size gate: every npm tarball, crate, Maven artifact, Swift binary - archive, ICU archive, and extension package stays within its declared budget. -3. No runtime download gate: public SDK packages contain no `fetch(`, - GitHub-release URL construction, or runtime asset cache installer in normal - runtime paths. -4. Real package smoke: install from packed artifacts in scratch projects for npm - across Node, Bun, Deno, and Electron, plus Cargo native, Cargo WASIX, Android Gradle, - SwiftPM, and React Native. -5. Network-off smoke: after package install, disable network and open a database - with one contrib extension and one external extension. -6. Exact extension gate: selected extension artifacts enter the app; unselected - extension artifacts stay out. -7. Publish idempotency gate: already-published registry artifacts with matching - version/provenance skip; GitHub assets with matching checksum skip; mismatched - bytes fail. -8. Release-plan gate: contrib source/runtime-contract changes release contrib - products only; external extension source changes release that external - extension only; runtime ABI changes release compatible runtime and extension - products. - -Enforcement ownership: - -- Release metadata and artifact-target checks validate package manifests, - registry carriers, platform selectors, and exact-extension targets. -- Product `package` tasks own carrier assembly/inspection; `smoke` and `e2e` - own installed runtime behavior. `qualify` is the explicit aggregate when a - candidate needs both. Candidate CI executes the independent tasks on the - runner and artifact combinations they require. -- The Rust package-shape check builds a scratch Cargo application from the - generated `oliphaunt` release source, calls `oliphaunt-build`, and verifies - that native-runtime and broker-helper artifacts are staged into the generated - lockfile. - -Remove these gates and policies: - -- checks requiring TypeScript runtime GitHub downloads; -- checks requiring Android GitHub asset downloads; -- checks treating `oliphaunt-resources --resolve-release-assets` as consumer DX; -- WASIX checks that keep an inert `bundled` feature; -- docs that present env vars as normal consumer setup; -- duplicate Kotlin asset resolver code outside the Android Gradle plugin. -- Swift metadata gates that forbid generated extension products. - -## Implementation Order - -1. Replace public docs with the target DX above. -2. Add generated descriptor schema for runtime, extension, ICU, and platform - artifacts. -3. Add JS `liboliphaunt` and broker platform packages. -4. Remove JS runtime GitHub download paths and update tests to expect missing - package errors. -5. Generate npm extension descriptor/platform packages for all exact extension - products. -6. Add JS bundler/package smokes. -7. Add Cargo native runtime, broker, extension, and ICU artifact crates. -8. Add the derived `oliphaunt-build` crate and Rust native build integration. -9. Switch Rust WASIX to Cargo-selected artifact crates, - `oliphaunt-build` manifest loading, and no public env-var archive setup. -10. Publish Android runtime and extension artifacts to Maven and replace Gradle - downloads with dependency resolution. -11. Generate SwiftPM extension products and `OliphauntICU` in the release - manifest. -12. Implement React Native iOS staging from installed npm packages. -13. Add ICU data packages and capability validation. -14. Update release CI to the minimal gates above. -15. Delete obsolete release metadata checks, docs, tests, and environment - overrides tied to removed paths. +# SDK resource packaging contract + +Applications install a base SDK and explicitly select descriptors when opening a +database. The base SDK includes PostgreSQL, supported contrib extensions, and +the standard cluster seed. External extensions, ICU data with its matching seed, +and frontend tools are optional dependencies. + +## Consumer setup + +| SDK | External vector dependency | Per-database selection | +| --- | --- | --- | +| Native Rust | `oliphaunt-extension-vector` | `.extensions([oliphaunt_extension_vector::VECTOR, oliphaunt::extensions::HSTORE])` | +| WASIX Rust | `oliphaunt-extension-vector-wasix` | `.extensions([oliphaunt_extension_vector_wasix::VECTOR, oliphaunt_wasix::extensions::HSTORE])` | +| Native Node / Bun / Deno | `@oliphaunt/extension-vector` | `extensions: [vector, extensions.hstore]` | +| WASIX Node / Bun / Deno | `@oliphaunt/extension-vector-wasix` | `extensions: [vector, extensions.hstore]` | +| Kotlin / Java | `dev.oliphaunt.extensions:oliphaunt-extension-vector` | `Vector.descriptor`, `Extensions.HSTORE` | +| React Native | `@oliphaunt/extension-vector` | `extensions: [vector, extensions.hstore]` | +| Swift | `OliphauntExtensionVector` product | `OliphauntExtensionVector.descriptor`, `OliphauntExtensions.hstore` | + +Dependency declarations and lockfiles choose external package versions. The SDK +validates compatibility with its runtime. Installing an extension does not select +it for every database. Applications use SQL migrations to run `CREATE EXTENSION`. +Required extension dependencies are included in the selected resource closure. + +See the SDK READMEs for executable setup examples: + +- [Native Rust](../../src/sdks/rust/README.md) +- [WASIX Rust](../../src/bindings/wasix-rust/crates/oliphaunt-wasix/README.md) +- [Native JavaScript](../../src/sdks/js/README.md) +- [WASIX TypeScript](../../src/bindings/wasix-ts/README.md) +- [Kotlin and Java](../../src/sdks/kotlin/README.md) +- [React Native](../../src/sdks/react-native/README.md) +- [Swift](../../src/sdks/swift/README.md) + +## Package ownership + +The native Rust SDK performs its build-time artifact selection internally. An +ordinary application needs no `build.rs`, `oliphaunt-build` dependency, application +runtime metadata, or resource registration. The SDK materializes verified embedded +files into a reusable cache. Advanced preassembled-resource and signed application +paths remain available. + +WASIX Rust uses independent external extension crates. Per-extension SDK features +are absent. Internal artifact crates retain features used by source qualification; +those are not the consumer API. Package-owned descriptors identify portable bytes +and host AOT artifacts. + +JavaScript descriptors bind native package resolution to the imported package's +location, including npm aliases and nested installations. The native addon receives +only explicitly selected external packages. WASIX N-API contains the core runtime, +standard seed, and contrib payloads. It obtains external AOT modules, ICU resources, +and frontend tool modules from optional packages. + +Kotlin's Gradle plugin resolves the variant dependency graph to determine which +resources to package. React Native's Expo plugin reads installed dependencies and +uses the same native extension packages as Node. Applications do not maintain a +second version map or package-selection list. Both still require explicit +per-database descriptors. Java exposes a blocking `AutoCloseable` facade for worker +threads. + +SwiftPM's base product includes contrib. External products and ICU have independent +package trees. Their descriptors hide native registration and resource locations. +Release tooling generates these packages from verified carrier inputs; consumers +use SwiftPM dependencies and products. + +## ICU and storage + +The default runtime includes only the standard cluster seed. Optional ICU packages +include ICU data and compatible seeds. Native seeds have explicit target identities +and bind the ICU logical data tree digest; WASIX seeds retain the WASIX physical +format. Seeds are implementation resources, not extra consumer configuration. + +Startup PostgreSQL settings use Rust setters, JavaScript objects, Kotlin/Java maps, +and Swift dictionaries. Filesystem locations use platform conventions: paths in +Rust and JavaScript, URLs in Swift, and File/path APIs in Kotlin and Java. React +Native's `directory()` accepts a native path or local file URI supplied by a mobile +filesystem library. It does not define an application-data storage kind. + +## Validation boundaries + +Validate installed package ownership, version, runtime compatibility, target, +archive layout, and hashes before loading payloads. Native AOT validation also +checks compiler/runtime identity and the source fingerprint before deserialization. +An arbitrary caller-supplied path and adjacent checksum are not package provenance. +Native package installation is a trust boundary equivalent to installing native +code; package verification is not a claim of cryptographic publisher signatures. + +Release checks inspect produced packages, enforce package-size limits and legal +notices, and reject external extension, ICU seed, or tool payloads leaking into the +base package. SDK tests exercise explicit selection and reject incompatible or +conflicting descriptors. Platform qualification must use the produced artifacts. diff --git a/docs/maintainers/extension-packaging-policy.md b/docs/maintainers/extension-packaging-policy.md index fee9fe88d..a685ab9fb 100644 --- a/docs/maintainers/extension-packaging-policy.md +++ b/docs/maintainers/extension-packaging-policy.md @@ -487,7 +487,7 @@ an empty one requires `not-required`. These are exact identities, not selection aliases or catalog expansions. SDK availability checks use `selectedExtensions`; they must never use the narrower createable `extensions` field to decide whether module-only resources exist. -SDKs reject `open(... extensions: ["vector"])` when the selected runtime does +SDKs reject `open({ extensions: [vector] })` when the selected runtime does not advertise `vector` in `selectedExtensions`. The size report is exact-extension based: diff --git a/docs/maintainers/release.md b/docs/maintainers/release.md index 56279a75d..4b4a2e705 100644 --- a/docs/maintainers/release.md +++ b/docs/maintainers/release.md @@ -533,7 +533,15 @@ before transport. A paced mutation receives its complete configured transport timeout only when that timeout still fits the job deadline. Read transports recompute and clamp their attempt timeout after journal reservation, and never start if the read deadline was exhausted while acquiring the journal lock. The -SwiftPM source-tag push is likewise +Independent SwiftPM extension packages publish semantic tags to their matching +`f0rr0/oliphaunt-extension-*` repositories; ICU publishes to +`f0rr0/oliphaunt-icu`. Provision these public repositories and grant the release +GitHub App contents access before enabling their first release. The workflow +requests a separate token scoped to the selected repositories. Each tag contains +only its frozen package source, and the anonymous consumer gate checks that +source against the publication lock and parses its SwiftPM manifest. + +The SwiftPM source-tag push is likewise noninteractive and bounded; success, rejection, disconnect, and timeout are all resolved by an exact remote tag/SHA read before the result is accepted. Before any release mutation, a lock-derived SwiftPM preflight constructs the exact diff --git a/docs/maintainers/rust-sdk-policy.md b/docs/maintainers/rust-sdk-policy.md index 7ef9edb83..5ac6997bd 100644 --- a/docs/maintainers/rust-sdk-policy.md +++ b/docs/maintainers/rust-sdk-policy.md @@ -90,12 +90,10 @@ runtime-specific implementation errors or extension catalogs: `postgres_error()`, `transaction_rollback_errors()`, and `transaction_callback_database_errors()` accessors retain detailed causes; WASIX adds `tool_error()` only with its `tools` feature. -- `Extension` is an opaque `Copy + Eq + Hash + Ord` selector with uppercase - associated constants, `Extension::ALL`, `Extension::by_sql_name`, and - `sql_name`. Native `ALL` contains the packaged PostgreSQL 18 catalog. WASIX - exposes the type and builder methods under `extensions` and includes only - Cargo-feature-enabled extensions. Neither product exposes free/module - constants or PascalCase compatibility aliases. +- Both products accept explicit descriptors from `extensions` for bundled + contrib and from independently versioned external extension crates. WASIX + external crates use the `-wasix` suffix. External versions are chosen through + Cargo dependencies; the SDK has no per-extension Cargo features. Selecting an extension makes its runtime artifact, dependencies, and required pre-start preload/GUC settings available. It never executes `CREATE EXTENSION`, diff --git a/docs/maintainers/sdk-api-surface.md b/docs/maintainers/sdk-api-surface.md index dcad09643..d5e29495e 100644 --- a/docs/maintainers/sdk-api-surface.md +++ b/docs/maintainers/sdk-api-surface.md @@ -36,6 +36,7 @@ node tools/policy/generate-sdk-api-surface.mjs --write - `oliphaunt::AsyncOliphauntBuilder.direct()` - `oliphaunt::AsyncOliphauntBuilder.extension()` - `oliphaunt::AsyncOliphauntBuilder.extensions()` +- `oliphaunt::AsyncOliphauntBuilder.icu()` - `oliphaunt::AsyncOliphauntBuilder.new()` - `oliphaunt::AsyncOliphauntBuilder.open()` - `oliphaunt::AsyncOliphauntBuilder.startup_guc()` @@ -51,6 +52,7 @@ node tools/policy/generate-sdk-api-surface.mjs --write - `oliphaunt::AsyncOliphauntServerBuilder.database()` - `oliphaunt::AsyncOliphauntServerBuilder.extension()` - `oliphaunt::AsyncOliphauntServerBuilder.extensions()` +- `oliphaunt::AsyncOliphauntServerBuilder.icu()` - `oliphaunt::AsyncOliphauntServerBuilder.listen()` - `oliphaunt::AsyncOliphauntServerBuilder.new()` - `oliphaunt::AsyncOliphauntServerBuilder.server_executable()` @@ -84,6 +86,7 @@ node tools/policy/generate-sdk-api-surface.mjs --write - `oliphaunt::CommandResult.row_count()` - `oliphaunt::DatabaseStorage` - `oliphaunt::DecodeError` +- `oliphaunt::EmbeddedResource` - `oliphaunt::Error` - `oliphaunt::Error.kind()` - `oliphaunt::Error.postgres_error()` @@ -136,9 +139,11 @@ node tools/policy/generate-sdk-api-surface.mjs --write - `oliphaunt::Extension.VECTOR` - `oliphaunt::Extension.by_sql_name()` - `oliphaunt::Extension.sql_name()` +- `oliphaunt::ExtensionDescriptor` - `oliphaunt::FromSql` - `oliphaunt::FromSql.check_type()` - `oliphaunt::FromSql.from_sql()` +- `oliphaunt::IcuData` - `oliphaunt::IntoParameter` - `oliphaunt::IntoParameter.TYPE_OID` - `oliphaunt::IntoParameter.into_parameter()` @@ -168,6 +173,7 @@ node tools/policy/generate-sdk-api-surface.mjs --write - `oliphaunt::OliphauntBuilder.direct()` - `oliphaunt::OliphauntBuilder.extension()` - `oliphaunt::OliphauntBuilder.extensions()` +- `oliphaunt::OliphauntBuilder.icu()` - `oliphaunt::OliphauntBuilder.new()` - `oliphaunt::OliphauntBuilder.open()` - `oliphaunt::OliphauntBuilder.startup_guc()` @@ -183,6 +189,7 @@ node tools/policy/generate-sdk-api-surface.mjs --write - `oliphaunt::OliphauntServerBuilder.database()` - `oliphaunt::OliphauntServerBuilder.extension()` - `oliphaunt::OliphauntServerBuilder.extensions()` +- `oliphaunt::OliphauntServerBuilder.icu()` - `oliphaunt::OliphauntServerBuilder.listen()` - `oliphaunt::OliphauntServerBuilder.new()` - `oliphaunt::OliphauntServerBuilder.server_executable()` @@ -420,6 +427,7 @@ The unpublished workspace packaging tool enables `internal-native-packaging` and - `oliphaunt_build::BuildOutput.resources_dir` - `oliphaunt_build::Error` - `oliphaunt_build::configure()` +- `oliphaunt_build::embed_resolved_artifacts()` - `oliphaunt_build::try_configure()` ## Native Rust tools: oliphaunt-tools @@ -469,6 +477,7 @@ These symbols require no optional Cargo feature. Target-gated symbols (for examp - `oliphaunt_wasix::AsyncOliphaunt.transaction()` - `oliphaunt_wasix::AsyncOliphauntBuilder` - `oliphaunt_wasix::AsyncOliphauntBuilder.database()` +- `oliphaunt_wasix::AsyncOliphauntBuilder.icu()` - `oliphaunt_wasix::AsyncOliphauntBuilder.new()` - `oliphaunt_wasix::AsyncOliphauntBuilder.open()` - `oliphaunt_wasix::AsyncOliphauntBuilder.startup_guc()` @@ -482,6 +491,7 @@ These symbols require no optional Cargo feature. Target-gated symbols (for examp - `oliphaunt_wasix::AsyncOliphauntServer.is_closed()` - `oliphaunt_wasix::AsyncOliphauntServerBuilder` - `oliphaunt_wasix::AsyncOliphauntServerBuilder.database()` +- `oliphaunt_wasix::AsyncOliphauntServerBuilder.icu()` - `oliphaunt_wasix::AsyncOliphauntServerBuilder.listen()` - `oliphaunt_wasix::AsyncOliphauntServerBuilder.new()` - `oliphaunt_wasix::AsyncOliphauntServerBuilder.start()` @@ -525,6 +535,7 @@ These symbols require no optional Cargo feature. Target-gated symbols (for examp - `oliphaunt_wasix::FromSql` - `oliphaunt_wasix::FromSql.check_type()` - `oliphaunt_wasix::FromSql.from_sql()` +- `oliphaunt_wasix::IcuData` - `oliphaunt_wasix::IntoParameter` - `oliphaunt_wasix::IntoParameter.TYPE_OID` - `oliphaunt_wasix::IntoParameter.into_parameter()` @@ -547,6 +558,7 @@ These symbols require no optional Cargo feature. Target-gated symbols (for examp - `oliphaunt_wasix::Oliphaunt.transaction()` - `oliphaunt_wasix::OliphauntBuilder` - `oliphaunt_wasix::OliphauntBuilder.database()` +- `oliphaunt_wasix::OliphauntBuilder.icu()` - `oliphaunt_wasix::OliphauntBuilder.new()` - `oliphaunt_wasix::OliphauntBuilder.open()` - `oliphaunt_wasix::OliphauntBuilder.startup_guc()` @@ -560,6 +572,7 @@ These symbols require no optional Cargo feature. Target-gated symbols (for examp - `oliphaunt_wasix::OliphauntServer.is_closed()` - `oliphaunt_wasix::OliphauntServerBuilder` - `oliphaunt_wasix::OliphauntServerBuilder.database()` +- `oliphaunt_wasix::OliphauntServerBuilder.icu()` - `oliphaunt_wasix::OliphauntServerBuilder.listen()` - `oliphaunt_wasix::OliphauntServerBuilder.new()` - `oliphaunt_wasix::OliphauntServerBuilder.start()` @@ -759,6 +772,7 @@ These symbols require no optional Cargo feature. Target-gated symbols (for examp - `oliphaunt_wasix::ValueRef.format()` - `oliphaunt_wasix::ValueRef.is_null()` - `oliphaunt_wasix::ValueRef.type_oid()` +- `oliphaunt_wasix::WasixExtensionDescriptor` ### `extensions` feature @@ -768,8 +782,48 @@ These symbols require no optional Cargo feature. Target-gated symbols (for examp - `oliphaunt_wasix::AsyncOliphauntServerBuilder.extensions()` - `oliphaunt_wasix::Extension` - `oliphaunt_wasix::Extension.ALL` +- `oliphaunt_wasix::Extension.AMCHECK` +- `oliphaunt_wasix::Extension.AUTO_EXPLAIN` +- `oliphaunt_wasix::Extension.BLOOM` +- `oliphaunt_wasix::Extension.BTREE_GIN` +- `oliphaunt_wasix::Extension.BTREE_GIST` +- `oliphaunt_wasix::Extension.CITEXT` +- `oliphaunt_wasix::Extension.CUBE` +- `oliphaunt_wasix::Extension.DICT_INT` +- `oliphaunt_wasix::Extension.DICT_XSYN` +- `oliphaunt_wasix::Extension.EARTHDISTANCE` +- `oliphaunt_wasix::Extension.FILE_FDW` +- `oliphaunt_wasix::Extension.FUZZYSTRMATCH` +- `oliphaunt_wasix::Extension.HSTORE` +- `oliphaunt_wasix::Extension.INTARRAY` +- `oliphaunt_wasix::Extension.ISN` +- `oliphaunt_wasix::Extension.LO` +- `oliphaunt_wasix::Extension.LTREE` +- `oliphaunt_wasix::Extension.PAGEINSPECT` +- `oliphaunt_wasix::Extension.PGCRYPTO` +- `oliphaunt_wasix::Extension.PGTAP` +- `oliphaunt_wasix::Extension.PG_BUFFERCACHE` +- `oliphaunt_wasix::Extension.PG_FREESPACEMAP` +- `oliphaunt_wasix::Extension.PG_HASHIDS` +- `oliphaunt_wasix::Extension.PG_IVM` +- `oliphaunt_wasix::Extension.PG_SURGERY` +- `oliphaunt_wasix::Extension.PG_TEXTSEARCH` +- `oliphaunt_wasix::Extension.PG_TRGM` +- `oliphaunt_wasix::Extension.PG_UUIDV7` +- `oliphaunt_wasix::Extension.PG_VISIBILITY` +- `oliphaunt_wasix::Extension.PG_WALINSPECT` +- `oliphaunt_wasix::Extension.POSTGIS` +- `oliphaunt_wasix::Extension.SEG` +- `oliphaunt_wasix::Extension.TABLEFUNC` +- `oliphaunt_wasix::Extension.TCN` +- `oliphaunt_wasix::Extension.TSM_SYSTEM_ROWS` +- `oliphaunt_wasix::Extension.TSM_SYSTEM_TIME` +- `oliphaunt_wasix::Extension.UNACCENT` +- `oliphaunt_wasix::Extension.UUID_OSSP` +- `oliphaunt_wasix::Extension.VECTOR` - `oliphaunt_wasix::Extension.by_sql_name()` - `oliphaunt_wasix::Extension.sql_name()` +- `oliphaunt_wasix::ExtensionPackage` - `oliphaunt_wasix::OliphauntBuilder.extension()` - `oliphaunt_wasix::OliphauntBuilder.extensions()` - `oliphaunt_wasix::OliphauntServerBuilder.extension()` @@ -805,50 +859,8 @@ These symbols require no optional Cargo feature. Target-gated symbols (for examp - `oliphaunt_wasix::tools::PsqlOptions.command()` - `oliphaunt_wasix::tools::PsqlOptions.new()` - `oliphaunt_wasix::tools::PsqlOptions.script()` - -### Individual `extension-*` features - -Each leaf feature also enables `extensions`; the constant below additionally requires the feature shown. - -- `extension-amcheck`: `oliphaunt_wasix::Extension.AMCHECK` -- `extension-auto-explain`: `oliphaunt_wasix::Extension.AUTO_EXPLAIN` -- `extension-bloom`: `oliphaunt_wasix::Extension.BLOOM` -- `extension-btree-gin`: `oliphaunt_wasix::Extension.BTREE_GIN` -- `extension-btree-gist`: `oliphaunt_wasix::Extension.BTREE_GIST` -- `extension-citext`: `oliphaunt_wasix::Extension.CITEXT` -- `extension-cube`: `oliphaunt_wasix::Extension.CUBE` -- `extension-dict-int`: `oliphaunt_wasix::Extension.DICT_INT` -- `extension-dict-xsyn`: `oliphaunt_wasix::Extension.DICT_XSYN` -- `extension-earthdistance`: `oliphaunt_wasix::Extension.EARTHDISTANCE` -- `extension-file-fdw`: `oliphaunt_wasix::Extension.FILE_FDW` -- `extension-fuzzystrmatch`: `oliphaunt_wasix::Extension.FUZZYSTRMATCH` -- `extension-hstore`: `oliphaunt_wasix::Extension.HSTORE` -- `extension-intarray`: `oliphaunt_wasix::Extension.INTARRAY` -- `extension-isn`: `oliphaunt_wasix::Extension.ISN` -- `extension-lo`: `oliphaunt_wasix::Extension.LO` -- `extension-ltree`: `oliphaunt_wasix::Extension.LTREE` -- `extension-pageinspect`: `oliphaunt_wasix::Extension.PAGEINSPECT` -- `extension-pg-buffercache`: `oliphaunt_wasix::Extension.PG_BUFFERCACHE` -- `extension-pg-freespacemap`: `oliphaunt_wasix::Extension.PG_FREESPACEMAP` -- `extension-pg-hashids`: `oliphaunt_wasix::Extension.PG_HASHIDS` -- `extension-pg-ivm`: `oliphaunt_wasix::Extension.PG_IVM` -- `extension-pg-surgery`: `oliphaunt_wasix::Extension.PG_SURGERY` -- `extension-pg-textsearch`: `oliphaunt_wasix::Extension.PG_TEXTSEARCH` -- `extension-pg-trgm`: `oliphaunt_wasix::Extension.PG_TRGM` -- `extension-pg-uuidv7`: `oliphaunt_wasix::Extension.PG_UUIDV7` -- `extension-pg-visibility`: `oliphaunt_wasix::Extension.PG_VISIBILITY` -- `extension-pg-walinspect`: `oliphaunt_wasix::Extension.PG_WALINSPECT` -- `extension-pgcrypto`: `oliphaunt_wasix::Extension.PGCRYPTO` -- `extension-pgtap`: `oliphaunt_wasix::Extension.PGTAP` -- `extension-postgis`: `oliphaunt_wasix::Extension.POSTGIS` -- `extension-seg`: `oliphaunt_wasix::Extension.SEG` -- `extension-tablefunc`: `oliphaunt_wasix::Extension.TABLEFUNC` -- `extension-tcn`: `oliphaunt_wasix::Extension.TCN` -- `extension-tsm-system-rows`: `oliphaunt_wasix::Extension.TSM_SYSTEM_ROWS` -- `extension-tsm-system-time`: `oliphaunt_wasix::Extension.TSM_SYSTEM_TIME` -- `extension-unaccent`: `oliphaunt_wasix::Extension.UNACCENT` -- `extension-uuid-ossp`: `oliphaunt_wasix::Extension.UUID_OSSP` -- `extension-vector`: `oliphaunt_wasix::Extension.VECTOR` +- `oliphaunt_wasix::tools::installed_tool_wasm()` +- `oliphaunt_wasix::tools::register_installed_package()` ## Native C ABI: liboliphaunt @@ -904,6 +916,7 @@ Each leaf feature also enables `extensions`; the constant below additionally req - `OliphauntCommandResult.rowCount` - `OliphauntConfiguration.database` - `OliphauntConfiguration.extensions` +- `OliphauntConfiguration.icu` - `OliphauntConfiguration.init` - `OliphauntConfiguration.startupGUCs` - `OliphauntConfiguration.storage` @@ -925,6 +938,45 @@ Each leaf feature also enables `extensions`; the constant below additionally req - `OliphauntExecResult.init` - `OliphauntExecResult.notices` - `OliphauntExecResult.statements` +- `OliphauntExtension.init` +- `OliphauntExtension.product` +- `OliphauntExtension.sqlName` +- `OliphauntExtension.version` +- `OliphauntExtensions.amcheck` +- `OliphauntExtensions.autoExplain` +- `OliphauntExtensions.bloom` +- `OliphauntExtensions.btreeGin` +- `OliphauntExtensions.btreeGist` +- `OliphauntExtensions.citext` +- `OliphauntExtensions.cube` +- `OliphauntExtensions.dictInt` +- `OliphauntExtensions.dictXsyn` +- `OliphauntExtensions.earthdistance` +- `OliphauntExtensions.fileFdw` +- `OliphauntExtensions.fuzzystrmatch` +- `OliphauntExtensions.hstore` +- `OliphauntExtensions.intarray` +- `OliphauntExtensions.isn` +- `OliphauntExtensions.lo` +- `OliphauntExtensions.ltree` +- `OliphauntExtensions.pageinspect` +- `OliphauntExtensions.pgBuffercache` +- `OliphauntExtensions.pgFreespacemap` +- `OliphauntExtensions.pgSurgery` +- `OliphauntExtensions.pgTrgm` +- `OliphauntExtensions.pgVisibility` +- `OliphauntExtensions.pgWalinspect` +- `OliphauntExtensions.pgcrypto` +- `OliphauntExtensions.seg` +- `OliphauntExtensions.tablefunc` +- `OliphauntExtensions.tcn` +- `OliphauntExtensions.tsmSystemRows` +- `OliphauntExtensions.tsmSystemTime` +- `OliphauntExtensions.unaccent` +- `OliphauntExtensions.uuidOssp` +- `OliphauntIcuData.init` +- `OliphauntIcuData.resourceDirectory` +- `OliphauntIcuData.version` - `OliphauntPostgresDecodable.decodePostgres()` - `OliphauntPostgresDiagnostic.columnName` - `OliphauntPostgresDiagnostic.constraintName` @@ -1065,9 +1117,6 @@ Each leaf feature also enables `extensions`; the constant below additionally req - `OliphauntQueryRow.text()` - `OliphauntQueryRow.value()` - `OliphauntQueryRow.values` -- `OliphauntStartupGUC.init` -- `OliphauntStartupGUC.name` -- `OliphauntStartupGUC.value` - `OliphauntTransaction.describe()` - `OliphauntTransaction.exec()` - `OliphauntTransaction.execute()` @@ -1083,6 +1132,7 @@ Each leaf feature also enables `extensions`; the constant below additionally req - `actor OliphauntDatabase` - `enum OliphauntDatabaseStorage` - `enum OliphauntError` +- `enum OliphauntExtensions` - `enum OliphauntQueryFormat` - `enum OliphauntStatementResult` - `enum OliphauntValueFormat` @@ -1092,6 +1142,8 @@ Each leaf feature also enables `extensions`; the constant below additionally req - `struct OliphauntCommandResult` - `struct OliphauntConfiguration` - `struct OliphauntExecResult` +- `struct OliphauntExtension` +- `struct OliphauntIcuData` - `struct OliphauntPostgresDiagnostic` - `struct OliphauntPostgresError` - `struct OliphauntPostgresErrorField` @@ -1101,7 +1153,6 @@ Each leaf feature also enables `extensions`; the constant below additionally req - `struct OliphauntQueryParam` - `struct OliphauntQueryResult` - `struct OliphauntQueryRow` -- `struct OliphauntStartupGUC` - `struct OliphauntTransaction` - `struct OliphauntTransactionDatabaseError` - `struct OliphauntTransactionRollbackError` @@ -1123,6 +1174,10 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `CommandResult.rowCount` - `ExecResult.notices` - `ExecResult.statements` +- `ExtensionDescriptor.product` +- `ExtensionDescriptor.sqlName` +- `ExtensionDescriptor.version` +- `IcuData.version` - `OliphauntDatabase.backup()` - `OliphauntDatabase.cancel()` - `OliphauntDatabase.close()` @@ -1270,8 +1325,6 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `PostgresOid.varcharArray` - `PostgresOid.xml` - `PostgresOid.xmlArray` -- `PostgresStartupGuc.name` -- `PostgresStartupGuc.value` - `QueryDescription.fields` - `QueryDescription.notices` - `QueryDescription.parameterTypes` @@ -1314,6 +1367,8 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `StatementResult.Rows.result` - `class CommandResult` - `class ExecResult` +- `class ExtensionDescriptor` +- `class IcuData` - `class OliphauntDatabase` - `class OliphauntException` - `class OliphauntTransaction` @@ -1324,7 +1379,6 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `class PostgresErrorField` - `class PostgresException` - `class PostgresNotice` -- `class PostgresStartupGuc` - `class QueryDescription` - `class QueryField` - `class QueryFormat` @@ -1339,6 +1393,7 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `enum class ValueFormat` - `fun interface PostgresDecoder` - `interface StatementResult` +- `object Extensions` - `object PostgresDecoders` - `object QueryFormat.Binary` - `object QueryFormat.Text` @@ -1347,19 +1402,38 @@ This version-locked carrier seam is consumed by generated Swift extension produc ### androidMain +- `BlockingOliphauntDatabase.backup()` +- `BlockingOliphauntDatabase.cancel()` +- `BlockingOliphauntDatabase.exec()` +- `BlockingOliphauntDatabase.execute()` +- `BlockingOliphauntDatabase.query()` - `DatabaseStorage.Directory.path` - `Oliphaunt.open()` - `Oliphaunt.restore()` +- `OliphauntConfig.Builder.build()` +- `OliphauntConfig.Builder.database()` +- `OliphauntConfig.Builder.extensions()` +- `OliphauntConfig.Builder.icu()` +- `OliphauntConfig.Builder.startupGuc()` +- `OliphauntConfig.Builder.startupGucs()` +- `OliphauntConfig.Builder.storage()` +- `OliphauntConfig.Builder.username()` +- `OliphauntConfig.builder()` - `OliphauntConfig.database` - `OliphauntConfig.extensions` +- `OliphauntConfig.icu` - `OliphauntConfig.startupGucs` - `OliphauntConfig.storage` - `OliphauntConfig.username` +- `OliphauntJava.open()` +- `class BlockingOliphauntDatabase` - `class DatabaseStorage.Directory` - `class OliphauntConfig` +- `class OliphauntConfig.Builder` - `interface DatabaseStorage` - `object DatabaseStorage.TemporaryDirectory` - `object Oliphaunt` +- `object OliphauntJava` ### jvmMain @@ -1392,6 +1466,8 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `EncodedQueryParameter` - `ExecResult` - `InferQueryRow` +- `NativeExtensionDescriptor` +- `NativeIcuDescriptor` - `NullQueryParameter` - `OliphauntClient` - `OliphauntDatabase` @@ -1425,6 +1501,8 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `PostgresError` - `array` - `binary` +- `directory` +- `extensions` - `json` - `postgresOids` - `text` @@ -1469,6 +1547,7 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `OliphauntTransaction.rollback()` - `OpenConfig.database` - `OpenConfig.extensions` +- `OpenConfig.icu` - `OpenConfig.startupGUCs` - `OpenConfig.storage` - `OpenConfig.username` @@ -1552,6 +1631,7 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `. = {"types":"./lib/index.d.ts","default":"./lib/index.js"}` - `./package.json = {"default":"./package.json"}` +- `./storage/node = {"types":"./lib/storage/node.d.ts","default":"./lib/storage/node.js"}` ### Types @@ -1563,6 +1643,8 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `EncodedQueryParameter` - `ExecResult` - `InferQueryRow` +- `NativeExtensionDescriptor` +- `NativeIcuDescriptor` - `NullQueryParameter` - `OliphauntClient` - `OliphauntDatabase` @@ -1599,6 +1681,7 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `PostgresError` - `array` - `binary` +- `extensions` - `json` - `postgresOids` - `text` @@ -1649,6 +1732,7 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `OpenConfig.brokerExecutable` - `OpenConfig.database` - `OpenConfig.extensions` +- `OpenConfig.icu` - `OpenConfig.libraryPath` - `OpenConfig.runtimeDirectory` - `OpenConfig.startupGUCs` diff --git a/docs/maintainers/sdk-parity-policy.md b/docs/maintainers/sdk-parity-policy.md index ed4c5645c..18e3931ec 100644 --- a/docs/maintainers/sdk-parity-policy.md +++ b/docs/maintainers/sdk-parity-policy.md @@ -89,12 +89,11 @@ These are language-native deltas, not parity failures: failures. `Error::kind()` is the stable category boundary. PostgreSQL, structured WASIX storage, and composite transaction detail remains available through dedicated accessors without exposing runtime-specific causes. -- Both Rust products use an opaque root `Extension` with uppercase associated - constants, `Extension::ALL`, `Extension::by_sql_name`, and `sql_name`. - Native `ALL` is the packaged PostgreSQL 18 catalog; WASIX exposes only - Cargo-feature-enabled extensions and gates the type and builder methods on - its `extensions` feature. Free/module constants and PascalCase aliases are - not public compatibility surfaces. +- Both Rust products accept explicit descriptors through `.extensions(...)`. + Supported contrib ships with the base SDK and uses `extensions::HSTORE`-style + constants. External crates own their versions and bytes; native and WASIX + have separate external crate names. Installing a crate does not select it + for a database. SQL migrations still own `CREATE EXTENSION`. - Swift uses actors, `URL`, `Data`, and `OliphauntPostgresDecodable`. - Kotlin uses coroutines, sealed storage types, `ByteArray`, and `PostgresDecoder`. diff --git a/docs/maintainers/testing.md b/docs/maintainers/testing.md index 24052cf42..433489813 100644 --- a/docs/maintainers/testing.md +++ b/docs/maintainers/testing.md @@ -215,8 +215,9 @@ lane has already established that correctness evidence. Doctest coverage itself requires nightly rustdoc flags, so it is not part of the default stable LCOV gate. WASIX library unit coverage intentionally uses `--no-default-features`. WASIX doctests run with the `tools` feature because the README contains -tools-gated examples. The `public_api` lane separately enables one exact leaf -extension feature to compile-check its root selector. Runtime Postgres/WASIX +tools-gated examples. The `public_api` lane separately enables descriptor support to compile-check +explicit contrib selection. External package fixtures compile real consumer +crates against the generated independently versioned descriptors. Runtime Postgres/WASIX execution stays in `smoke` and `regression`, where missing runtime assets must fail or skip explicitly according to the lane policy. diff --git a/docs/maintainers/wasix-usage.md b/docs/maintainers/wasix-usage.md index 83238ce13..0430d2bdf 100644 --- a/docs/maintainers/wasix-usage.md +++ b/docs/maintainers/wasix-usage.md @@ -58,12 +58,12 @@ server handles are `Send + Sync`, while `AsyncTransaction` is `Send` but not `Sync` and requires exclusive mutable access. These traits expose the real Wasmer/runtime owner instead of implying concurrency that does not exist. -Each exact `extension-*` leaf feature enables the common extension selector -machinery and its matching uppercase associated constant. `Extension::ALL` -and `Extension::by_sql_name` contain only the enabled leaf set, while -`sql_name` returns the selected extension's PostgreSQL name. Reopening a -database whose catalog uses an extension requires the receiving host to select -and provide that runtime code again. +Select contrib explicitly through `extensions::HSTORE` (and other contrib +descriptors). External extensions use independently versioned crates, for example +`oliphaunt-extension-vector-wasix` and its `VECTOR` descriptor. Pass both to +`.extensions([VECTOR, extensions::HSTORE])`. The SDK has no per-extension Cargo +features. Reopening a database whose catalog uses an extension requires the +receiving host to select and provide that runtime code again. ## TypeScript host diff --git a/examples/electron/src/todos.ts b/examples/electron/src/todos.ts index 6f4010ec9..e1df05cf0 100644 --- a/examples/electron/src/todos.ts +++ b/examples/electron/src/todos.ts @@ -1,6 +1,6 @@ import { join } from "node:path"; -import { Oliphaunt, type OliphauntServer } from "@oliphaunt/ts"; +import { Oliphaunt, extensions, type OliphauntServer } from "@oliphaunt/ts"; import { pgDump, psql } from "@oliphaunt/tools"; import { Kysely, PostgresDialect, sql, type Generated } from "kysely"; import pg from "pg"; @@ -68,7 +68,7 @@ export function getDatabase(userData: string) { async function openDatabase(userData: string): Promise { const native = await Oliphaunt.openServer({ storage: { kind: "directory", path: join(userData, "oliphaunt-native-todos") }, - extensions: ["hstore", "pg_trgm", "unaccent"], + extensions: [extensions.hstore, extensions.pg_trgm, extensions.unaccent], }); const connectionString = native.connectionString; const db = new Kysely({ diff --git a/examples/react-native-expo/README.md b/examples/react-native-expo/README.md index fbefaa478..183c628ef 100644 --- a/examples/react-native-expo/README.md +++ b/examples/react-native-expo/README.md @@ -72,7 +72,8 @@ pnpm run crash:ios The runners choose isolated persistent storage by default. Set `OLIPHAUNT_EXPO_ANDROID_CRASH_STORAGE` or `OLIPHAUNT_EXPO_IOS_CRASH_STORAGE` to override it; the iOS runner accepts an -`app-data:` selector for the public `applicationData` storage case. +`app-directory:` harness selector, resolved with Expo FileSystem under +`Paths.document` and passed to `directory()`. The smoke script: @@ -157,9 +158,9 @@ pnpm run bench:ios Set `OLIPHAUNT_EXPO_IOS_DEVICE_ID` to pick a specific paired device, and `OLIPHAUNT_EXPO_IOS_METRO_URL` if the device cannot reach the host address that -the harness auto-detects. Device crash-recovery runs default to the public -`{ kind: 'applicationData', name }` storage case, which the platform SDK -resolves inside the app sandbox and which survives process death. +the harness auto-detects. Device crash-recovery runs use an Expo FileSystem +directory under `Paths.document`, passed to `directory()`, which survives +process death. Physical-device runs require a working Apple Development signing setup. The harness first checks that the paired phone has Developer Mode and Developer Disk diff --git a/examples/react-native-expo/package.json b/examples/react-native-expo/package.json index 26679b3d7..5abda6758 100644 --- a/examples/react-native-expo/package.json +++ b/examples/react-native-expo/package.json @@ -6,6 +6,7 @@ "@oliphaunt/react-native": "workspace:*", "expo": "~56.0.15", "expo-dev-client": "~56.0.22", + "expo-file-system": "~56.0.8", "expo-splash-screen": "~56.0.12", "expo-sqlite": "~56.0.5", "expo-system-ui": "~56.0.5", diff --git a/examples/react-native-expo/src/SmokeDashboard.tsx b/examples/react-native-expo/src/SmokeDashboard.tsx index 3c663bf53..755b5d353 100644 --- a/examples/react-native-expo/src/SmokeDashboard.tsx +++ b/examples/react-native-expo/src/SmokeDashboard.tsx @@ -1,10 +1,14 @@ import { Oliphaunt, + directory, type DatabaseStorage, + type NativeIcuDescriptor, type OliphauntDatabase, type QueryResult, } from '@oliphaunt/react-native'; +import { Directory, Paths } from 'expo-file-system'; import { + GENERATED_MOBILE_ICU, GENERATED_MOBILE_EXTENSION_METADATA_SHA256, GENERATED_MOBILE_EXTENSION_PLAN, GENERATED_MOBILE_EXTENSION_SMOKE, @@ -78,6 +82,7 @@ type SmokeGlobalState = { }; type OpenTuning = { + icu?: NativeIcuDescriptor; startupGUCs?: Readonly>; storage?: DatabaseStorage; storageLabel?: string; @@ -927,9 +932,13 @@ async function openDatabase( const started = now(); const { storage, storageLabel: _storageLabel, ...tuning } = openTuning; const config = { - storage: storage ?? ({ kind: 'applicationData', name: defaultSmokeStorageName } as const), + storage: storage ?? directory(new Directory(Paths.document, defaultSmokeStorageName).uri), ...tuning, - extensions, + extensions: extensions.map(sqlName => { + const selected = GENERATED_MOBILE_EXTENSION_PLAN.find(row => row.sqlName === sqlName); + if (!selected) throw new Error(`unknown qualification extension ${sqlName}`); + return selected.descriptor; + }), username: 'postgres', database: 'postgres', } satisfies Parameters[0]; @@ -961,6 +970,7 @@ async function resolveOpenTuning(): Promise { const startupGUCs = parseStartupGUCs(rawStartupGUCs); return { startupGUCs: Object.keys(startupGUCs).length > 0 ? startupGUCs : undefined, + icu: packagedCatalogProfile === 'icu' ? GENERATED_MOBILE_ICU : undefined, ...resolveHarnessStorage(url), }; } @@ -978,24 +988,22 @@ function parseStartupGUCs(value: string): Record { } function resolveHarnessStorage(url: string | null): Pick { - const applicationData = optionalNonBlankString( - process.env.EXPO_PUBLIC_OLIPHAUNT_APPLICATION_DATA ?? - extractQueryParam(url, 'liboliphauntApplicationData'), - 'liboliphauntApplicationData', + const name = optionalNonBlankString( + process.env.EXPO_PUBLIC_OLIPHAUNT_DIRECTORY_NAME ?? extractQueryParam(url, 'liboliphauntDirectoryName'), + 'liboliphauntDirectoryName', ); - if (applicationData) { - return { - storage: { kind: 'applicationData', name: applicationData }, - storageLabel: `applicationData:${applicationData}`, - }; + if (name) { + if (name === '.' || name === '..' || /[\\/\0]/.test(name)) throw new Error('database directory name must be one path component'); + const storage = directory(new Directory(Paths.document, name).uri); + return { storage, storageLabel: storage.path }; } - const directory = optionalNonBlankString( + const location = optionalNonBlankString( process.env.EXPO_PUBLIC_OLIPHAUNT_STORAGE_DIRECTORY ?? extractQueryParam(url, 'liboliphauntStorageDirectory'), 'liboliphauntStorageDirectory', ); - return directory - ? { storage: { kind: 'directory', path: directory }, storageLabel: directory } + return location + ? { storage: directory(location), storageLabel: location } : {}; } diff --git a/examples/react-native-expo/src/generated/extension-smoke.ts b/examples/react-native-expo/src/generated/extension-smoke.ts index 7cc43c923..08721dbcb 100644 --- a/examples/react-native-expo/src/generated/extension-smoke.ts +++ b/examples/react-native-expo/src/generated/extension-smoke.ts @@ -1,211 +1,457 @@ // This file is generated by src/extensions/tools/check-extension-model.mjs. // Do not edit by hand. It belongs only to installed mobile qualification. +import type { NativeExtensionDescriptor } from '@oliphaunt/react-native'; + export type GeneratedMobileExtensionProof = { readonly sqlName: string; + readonly descriptor: NativeExtensionDescriptor; readonly createsExtension: boolean; readonly selectedExtensionDependencies: readonly string[]; }; +export const GENERATED_MOBILE_ICU = { schema: 'oliphaunt-native-icu-v1', packageName: '@oliphaunt/icu', version: "0.2.0" } as const; + export const GENERATED_MOBILE_EXTENSION_METADATA_SHA256 = "c1d2e09905d7ecc0172173b34b9e4104dad58987503dd3dab7af4ae78890d9f3" as const; export const GENERATED_MOBILE_EXTENSION_PLAN = [ { "sqlName": "amcheck", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "amcheck", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "auto_explain", "createsExtension": false, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "auto_explain", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "bloom", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "bloom", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "btree_gin", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "btree_gin", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "btree_gist", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "btree_gist", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "citext", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "citext", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "cube", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "cube", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "dict_int", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "dict_int", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "dict_xsyn", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "dict_xsyn", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "earthdistance", "createsExtension": true, "selectedExtensionDependencies": [ "cube" - ] + ], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "earthdistance", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "file_fdw", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "file_fdw", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "fuzzystrmatch", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "fuzzystrmatch", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "hstore", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "hstore", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "intarray", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "intarray", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "isn", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "isn", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "lo", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "lo", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "ltree", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "ltree", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "pageinspect", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "pageinspect", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "pg_buffercache", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "pg_buffercache", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "pg_freespacemap", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "pg_freespacemap", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "pg_hashids", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "pg_hashids", + "product": "oliphaunt-extension-pg-hashids", + "packageName": "@oliphaunt/extension-pg-hashids", + "version": "0.2.0" + } }, { "sqlName": "pg_ivm", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "pg_ivm", + "product": "oliphaunt-extension-pg-ivm", + "packageName": "@oliphaunt/extension-pg-ivm", + "version": "0.2.0" + } }, { "sqlName": "pg_surgery", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "pg_surgery", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "pg_textsearch", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "pg_textsearch", + "product": "oliphaunt-extension-pg-textsearch", + "packageName": "@oliphaunt/extension-pg-textsearch", + "version": "0.2.0" + } }, { "sqlName": "pg_trgm", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "pg_trgm", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "pg_uuidv7", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "pg_uuidv7", + "product": "oliphaunt-extension-pg-uuidv7", + "packageName": "@oliphaunt/extension-pg-uuidv7", + "version": "0.2.0" + } }, { "sqlName": "pg_visibility", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "pg_visibility", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "pg_walinspect", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "pg_walinspect", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "pgcrypto", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "pgcrypto", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "pgtap", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "pgtap", + "product": "oliphaunt-extension-pgtap", + "packageName": "@oliphaunt/extension-pgtap", + "version": "0.2.0" + } }, { "sqlName": "postgis", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "postgis", + "product": "oliphaunt-extension-postgis", + "packageName": "@oliphaunt/extension-postgis", + "version": "0.2.0" + } }, { "sqlName": "seg", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "seg", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "tablefunc", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "tablefunc", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "tcn", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "tcn", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "tsm_system_rows", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "tsm_system_rows", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "tsm_system_time", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "tsm_system_time", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "unaccent", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "unaccent", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "uuid-ossp", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "uuid-ossp", + "product": "oliphaunt-extension-contrib-pg18", + "packageName": "@oliphaunt/extension-contrib-pg18" + } }, { "sqlName": "vector", "createsExtension": true, - "selectedExtensionDependencies": [] + "selectedExtensionDependencies": [], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": "vector", + "product": "oliphaunt-extension-vector", + "packageName": "@oliphaunt/extension-vector", + "version": "0.2.0" + } } ] as const satisfies readonly GeneratedMobileExtensionProof[]; diff --git a/examples/tauri/src-tauri/Cargo.toml b/examples/tauri/src-tauri/Cargo.toml index 3356fdea4..f3850827a 100644 --- a/examples/tauri/src-tauri/Cargo.toml +++ b/examples/tauri/src-tauri/Cargo.toml @@ -11,13 +11,7 @@ publish = false name = "oliphaunt_example_tauri_lib" crate-type = ["staticlib", "cdylib", "rlib"] -[package.metadata.oliphaunt] -runtime = "liboliphaunt-native" -runtime-version = "0.2.0" -extensions = ["hstore", "pg_trgm", "unaccent"] - [build-dependencies] -oliphaunt-build = { version = "=0.2.0" } tauri-build = { version = "2", features = [] } [dependencies] @@ -26,8 +20,3 @@ oliphaunt = { version = "=0.2.0" } serde = { version = "1", features = ["derive"] } tauri = { version = "2", features = [] } thiserror = "2" - -[target.'cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))'.dependencies] -liboliphaunt-native-linux-x64-gnu = { version = "=0.2.0" } -oliphaunt-broker-linux-x64-gnu = { version = "=0.2.0" } -oliphaunt-extension-contrib-pg18-linux-x64-gnu = { version = "=0.2.0" } diff --git a/examples/tauri/src-tauri/build.rs b/examples/tauri/src-tauri/build.rs index c26929e0f..261851f6b 100644 --- a/examples/tauri/src-tauri/build.rs +++ b/examples/tauri/src-tauri/build.rs @@ -1,4 +1,3 @@ fn main() { - oliphaunt_build::configure(); tauri_build::build(); } diff --git a/examples/tauri/src-tauri/src/lib.rs b/examples/tauri/src-tauri/src/lib.rs index c75ebb3f6..d0d46a695 100644 --- a/examples/tauri/src-tauri/src/lib.rs +++ b/examples/tauri/src-tauri/src/lib.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use oliphaunt::{AsyncOliphaunt, DatabaseStorage}; -use oliphaunt::{Extension, QueryResult}; +use oliphaunt::{extensions, QueryResult}; use serde::ser::Serializer; use serde::{Deserialize, Serialize}; use tauri::Manager; @@ -123,10 +123,9 @@ impl From for CommandError { } async fn open_database(directory: PathBuf) -> anyhow::Result { - oliphaunt::register_build_resources!()?; let db = AsyncOliphaunt::builder() .storage(DatabaseStorage::Directory(directory)) - .extensions([Extension::HSTORE, Extension::PG_TRGM, Extension::UNACCENT]) + .extensions([extensions::HSTORE, extensions::PG_TRGM, extensions::UNACCENT]) .open() .await?; db.execute(SCHEMA).await?; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c76e571bb..3d48d4cef 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -58,6 +58,9 @@ importers: expo-dev-client: specifier: ~56.0.22 version: 56.0.22(expo@56.0.15(@babel/core@7.29.0)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@react-native-community/cli@20.2.0(typescript@6.0.3))(@react-native/metro-config@0.85.3)(@types/react@19.2.16)(react@19.2.3))(react@19.2.3)(typescript@6.0.3))(react-native@0.85.3(@react-native-community/cli@20.2.0(typescript@6.0.3))(@react-native/metro-config@0.85.3)(@types/react@19.2.16)(react@19.2.3)) + expo-file-system: + specifier: ~56.0.8 + version: 56.0.8(expo@56.0.15(@babel/core@7.29.0)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@react-native-community/cli@20.2.0(typescript@6.0.3))(@react-native/metro-config@0.85.3)(@types/react@19.2.16)(react@19.2.3))(react@19.2.3)(typescript@6.0.3))(react-native@0.85.3(@react-native-community/cli@20.2.0(typescript@6.0.3))(@react-native/metro-config@0.85.3)(@types/react@19.2.16)(react@19.2.3)) expo-splash-screen: specifier: ~56.0.12 version: 56.0.12(expo@56.0.15(@babel/core@7.29.0)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@react-native-community/cli@20.2.0(typescript@6.0.3))(@react-native/metro-config@0.85.3)(@types/react@19.2.16)(react@19.2.3))(react@19.2.3)(typescript@6.0.3))(typescript@6.0.3) diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/Cargo.toml b/src/bindings/wasix-rust/crates/oliphaunt-wasix/Cargo.toml index dce698dd1..8328934f1 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/Cargo.toml +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/Cargo.toml @@ -22,60 +22,18 @@ exclude = [ runtime-version = "0.2.0" [features] -default = [] +default = ["extensions"] __internal-napi = [] extensions = [] +__internal-tools = [] tools = [ + "__internal-tools", "dep:oliphaunt-wasix-tools", "dep:oliphaunt-wasix-tools-aot-aarch64-apple-darwin", "dep:oliphaunt-wasix-tools-aot-aarch64-unknown-linux-gnu", "dep:oliphaunt-wasix-tools-aot-x86_64-pc-windows-msvc", "dep:oliphaunt-wasix-tools-aot-x86_64-unknown-linux-gnu", ] -extension-amcheck = ["extensions", "liboliphaunt-wasix-portable/extension-amcheck"] -extension-auto-explain = ["extensions", "liboliphaunt-wasix-portable/extension-auto-explain"] -extension-bloom = ["extensions", "liboliphaunt-wasix-portable/extension-bloom"] -extension-btree-gin = ["extensions", "liboliphaunt-wasix-portable/extension-btree-gin"] -extension-btree-gist = ["extensions", "liboliphaunt-wasix-portable/extension-btree-gist"] -extension-citext = ["extensions", "liboliphaunt-wasix-portable/extension-citext"] -extension-cube = ["extensions", "liboliphaunt-wasix-portable/extension-cube"] -extension-dict-int = ["extensions", "liboliphaunt-wasix-portable/extension-dict-int"] -extension-dict-xsyn = ["extensions", "liboliphaunt-wasix-portable/extension-dict-xsyn"] -extension-earthdistance = [ - "extensions", - "extension-cube", - "liboliphaunt-wasix-portable/extension-earthdistance", -] -extension-file-fdw = ["extensions", "liboliphaunt-wasix-portable/extension-file-fdw"] -extension-fuzzystrmatch = ["extensions", "liboliphaunt-wasix-portable/extension-fuzzystrmatch"] -extension-hstore = ["extensions", "liboliphaunt-wasix-portable/extension-hstore"] -extension-intarray = ["extensions", "liboliphaunt-wasix-portable/extension-intarray"] -extension-isn = ["extensions", "liboliphaunt-wasix-portable/extension-isn"] -extension-lo = ["extensions", "liboliphaunt-wasix-portable/extension-lo"] -extension-ltree = ["extensions", "liboliphaunt-wasix-portable/extension-ltree"] -extension-pageinspect = ["extensions", "liboliphaunt-wasix-portable/extension-pageinspect"] -extension-pg-buffercache = ["extensions", "liboliphaunt-wasix-portable/extension-pg-buffercache"] -extension-pg-freespacemap = ["extensions", "liboliphaunt-wasix-portable/extension-pg-freespacemap"] -extension-pg-hashids = ["extensions", "liboliphaunt-wasix-portable/extension-pg-hashids"] -extension-pg-ivm = ["extensions", "liboliphaunt-wasix-portable/extension-pg-ivm"] -extension-pg-surgery = ["extensions", "liboliphaunt-wasix-portable/extension-pg-surgery"] -extension-pg-textsearch = ["extensions", "liboliphaunt-wasix-portable/extension-pg-textsearch"] -extension-pg-trgm = ["extensions", "liboliphaunt-wasix-portable/extension-pg-trgm"] -extension-pg-uuidv7 = ["extensions", "liboliphaunt-wasix-portable/extension-pg-uuidv7"] -extension-pg-visibility = ["extensions", "liboliphaunt-wasix-portable/extension-pg-visibility"] -extension-pg-walinspect = ["extensions", "liboliphaunt-wasix-portable/extension-pg-walinspect"] -extension-pgcrypto = ["extensions", "liboliphaunt-wasix-portable/extension-pgcrypto"] -extension-pgtap = ["extensions", "liboliphaunt-wasix-portable/extension-pgtap"] -extension-postgis = ["extensions", "liboliphaunt-wasix-portable/extension-postgis"] -extension-seg = ["extensions", "liboliphaunt-wasix-portable/extension-seg"] -extension-tablefunc = ["extensions", "liboliphaunt-wasix-portable/extension-tablefunc"] -extension-tcn = ["extensions", "liboliphaunt-wasix-portable/extension-tcn"] -extension-tsm-system-rows = ["extensions", "liboliphaunt-wasix-portable/extension-tsm-system-rows"] -extension-tsm-system-time = ["extensions", "liboliphaunt-wasix-portable/extension-tsm-system-time"] -extension-unaccent = ["extensions", "liboliphaunt-wasix-portable/extension-unaccent"] -extension-uuid-ossp = ["extensions", "liboliphaunt-wasix-portable/extension-uuid-ossp"] -extension-vector = ["extensions", "liboliphaunt-wasix-portable/extension-vector"] -icu = ["dep:oliphaunt-icu"] [package.metadata.oliphaunt-wasix.assets] postgres-version = "18.4" @@ -90,6 +48,7 @@ cluster-seed-icu-archive-sha256 = "5036966275469d39b665d99ccd59d208fe6e77bcb2509 initdb-wasix-sha256 = "c2438dd844943811da9de5dd5c2786c7b2cb1b03df41d8b71108dce1d632758b" [dependencies] +oliphaunt-resources = { version = "0.2.0", path = "../../../../sdks/rust/crates/oliphaunt-resources" } anyhow = "1" async-trait = "0.1" cap-fs-ext = "4" @@ -105,9 +64,8 @@ tempfile = "3" sha2 = "0.10" dunce = "1" filetime = "0.2" -liboliphaunt-wasix-portable = { version = "*", path = "../../../../runtimes/liboliphaunt/wasix/crates/assets" } +liboliphaunt-wasix-portable = { version = "*", path = "../../../../runtimes/liboliphaunt/wasix/crates/assets", features = ["extension-amcheck", "extension-auto-explain", "extension-bloom", "extension-btree-gin", "extension-btree-gist", "extension-citext", "extension-cube", "extension-dict-int", "extension-dict-xsyn", "extension-earthdistance", "extension-file-fdw", "extension-fuzzystrmatch", "extension-hstore", "extension-intarray", "extension-isn", "extension-lo", "extension-ltree", "extension-pageinspect", "extension-pg-buffercache", "extension-pg-freespacemap", "extension-pg-surgery", "extension-pg-trgm", "extension-pg-visibility", "extension-pg-walinspect", "extension-pgcrypto", "extension-seg", "extension-tablefunc", "extension-tcn", "extension-tsm-system-rows", "extension-tsm-system-time", "extension-unaccent", "extension-uuid-ossp"] } oliphaunt-wasix-tools = { version = "*", path = "../../../../runtimes/liboliphaunt/wasix/crates/tools", optional = true } -oliphaunt-icu = { version = "*", path = "../../../../runtimes/liboliphaunt/icu", optional = true } tokio = { version = "1", features = ["io-util", "rt-multi-thread", "sync"] } wasmer = { version = "=7.2.1", default-features = false, features = [ "sys", diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/README.md b/src/bindings/wasix-rust/crates/oliphaunt-wasix/README.md index fa5dc3128..fb9fb6fa3 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/README.md +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/README.md @@ -113,14 +113,37 @@ close-only, and a retained callback panic is not resumed into an unknown session state. WASIX query cancellation is intentionally absent until the guest runtime can interrupt execution and prove protocol recovery. -The builder also supports `username`, `database`, `startup_gucs`, and bundled -`extension`/`extensions` when the corresponding crate features are enabled. -Selecting an extension makes its artifact and required pre-start configuration -available; it never runs `CREATE EXTENSION`, `LOAD`, or migration SQL. Install -database-local objects explicitly through your normal migrations. Each -associated selector is compiled only by its matching `extension-*` feature; -`Extension::ALL` and `Extension::by_sql_name` therefore describe exactly the -artifacts enabled in the current Cargo build, not the full packaging catalog. +## Extensions and ICU + +The base dependency includes the supported contrib distribution. Select contrib +explicitly using values from `oliphaunt_wasix::extensions`. External extensions +are independently versioned Cargo dependencies: + +```sh +cargo add oliphaunt-extension-vector-wasix +``` + +```rust,ignore +use oliphaunt_wasix::{extensions, Oliphaunt}; +use oliphaunt_extension_vector_wasix::VECTOR; + +let mut database = Oliphaunt::builder() + .extensions([VECTOR, extensions::HSTORE]) + .open()?; +database.execute("CREATE EXTENSION vector")?; +database.execute("CREATE EXTENSION hstore")?; +database.close()?; +``` + +Cargo selects the extension package's matching host AOT artifact. No application +build script or SDK `extension-vector` feature is required. The builder accepts +`username`, `database`, and PostgreSQL `startup_guc` settings as usual. Extension +selection prepares artifacts and startup requirements; migrations remain +application-owned. + +For ICU collations, add `oliphaunt-icu` and pass `.icu(oliphaunt_icu::ICU)`. +The optional package supplies ICU data and the matching initialization seed. +Omitting it selects standard initialization. ## Storage and physical backup diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/async_api.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/async_api.rs index b39b3ffde..19ede7937 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/async_api.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/async_api.rs @@ -1647,7 +1647,7 @@ impl AsyncOliphaunt { } /// Run packaged `pg_dump` against this database on its owner thread. - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub async fn pg_dump(&self, options: crate::oliphaunt::tools::PgDumpOptions) -> Result { self.owner .call(None, move |database| { @@ -1658,7 +1658,7 @@ impl AsyncOliphaunt { /// Submit packaged `pg_dump` and return exact stdout/stderr bytes without /// creating or polling a Rust future. - #[cfg(all(feature = "tools", any(feature = "__internal-napi", test)))] + #[cfg(all(feature = "__internal-tools", any(feature = "__internal-napi", test)))] #[doc(hidden)] pub fn pg_dump_output_with_completion( &self, @@ -1675,7 +1675,7 @@ impl AsyncOliphaunt { } /// Run packaged non-interactive `psql` against this database on its owner thread. - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub async fn psql(&self, options: crate::oliphaunt::tools::PsqlOptions) -> Result { self.owner .call(None, move |database| { @@ -1686,7 +1686,7 @@ impl AsyncOliphaunt { /// Submit packaged `psql` and return exact stdout/stderr bytes without /// creating or polling a Rust future. - #[cfg(all(feature = "tools", any(feature = "__internal-napi", test)))] + #[cfg(all(feature = "__internal-tools", any(feature = "__internal-napi", test)))] #[doc(hidden)] pub fn psql_output_with_completion( &self, @@ -1766,6 +1766,12 @@ impl AsyncOliphauntBuilder { self } + /// Select ICU data from the optional `oliphaunt-icu` package. + pub fn icu(mut self, data: oliphaunt_resources::IcuData) -> Self { + self.inner = self.inner.icu(data); + self + } + /// Set one PostgreSQL startup GUC. pub fn startup_guc(mut self, name: impl Into, value: impl Into) -> Self { self.inner = self.inner.startup_guc(name, value); @@ -1797,7 +1803,7 @@ impl AsyncOliphauntBuilder { #[cfg(feature = "extensions")] /// Make one bundled PostgreSQL extension artifact available to the database. /// Database-local installation remains the application's migration concern. - pub fn extension(mut self, extension: Extension) -> Self { + pub fn extension(mut self, extension: impl Into) -> Self { self.inner = self.inner.extension(extension); self } @@ -1805,7 +1811,10 @@ impl AsyncOliphauntBuilder { #[cfg(feature = "extensions")] /// Make bundled PostgreSQL extension artifacts available to the database. /// Database-local installation remains the application's migration concern. - pub fn extensions(mut self, extensions: impl IntoIterator) -> Self { + pub fn extensions>( + mut self, + extensions: impl IntoIterator, + ) -> Self { self.inner = self.inner.extensions(extensions); self } @@ -2393,6 +2402,12 @@ impl AsyncOliphauntServerBuilder { self } + /// Select ICU data from the optional `oliphaunt-icu` package. + pub fn icu(mut self, data: oliphaunt_resources::IcuData) -> Self { + self.inner = self.inner.icu(data); + self + } + /// Select loopback TCP on any supported host or a PostgreSQL Unix-domain /// socket on a Unix host. pub fn listen(mut self, listen: ServerListen) -> Self { @@ -2431,7 +2446,7 @@ impl AsyncOliphauntServerBuilder { #[cfg(feature = "extensions")] /// Make one bundled PostgreSQL extension artifact available to clients. /// Database-local installation remains the application's migration concern. - pub fn extension(mut self, extension: Extension) -> Self { + pub fn extension(mut self, extension: impl Into) -> Self { self.inner = self.inner.extension(extension); self } @@ -2439,7 +2454,10 @@ impl AsyncOliphauntServerBuilder { #[cfg(feature = "extensions")] /// Make bundled PostgreSQL extension artifacts available to clients. /// Database-local installation remains the application's migration concern. - pub fn extensions(mut self, extensions: impl IntoIterator) -> Self { + pub fn extensions>( + mut self, + extensions: impl IntoIterator, + ) -> Self { self.inner = self.inner.extensions(extensions); self } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/bin/oliphaunt_wasix_dump.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/bin/oliphaunt_wasix_dump.rs index 30125df6e..0f4d54250 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/bin/oliphaunt_wasix_dump.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/bin/oliphaunt_wasix_dump.rs @@ -133,7 +133,9 @@ fn print_usage() { eprintln!(" --directory PATH Use this database root. Defaults to ./.oliphaunt"); eprintln!(" --database NAME Select the database to dump. Defaults to postgres"); eprintln!(" --username NAME Select the database user. Defaults to postgres"); - eprintln!(" --extension NAME Select an extension artifact by SQL name; repeat as needed"); + eprintln!( + " --extension NAME Select a bundled contrib extension by SQL name; repeat as needed" + ); eprintln!("Example: oliphaunt-wasix-dump --directory ./.oliphaunt -- --schema-only"); } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/bin/oliphaunt_wasix_proxy.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/bin/oliphaunt_wasix_proxy.rs index a1c5bce87..bd26d8b1a 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/bin/oliphaunt_wasix_proxy.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/bin/oliphaunt_wasix_proxy.rs @@ -149,5 +149,5 @@ fn print_usage() { eprintln!(" --print-uri Print the PostgreSQL connection URI to stdout"); eprintln!(" --startup-guc NAME=VALUE"); eprintln!(" Set a PostgreSQL startup GUC on the embedded backend"); - eprintln!(" --extension NAME Select an extension artifact by SQL name"); + eprintln!(" --extension NAME Select a bundled contrib extension by SQL name"); } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/error.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/error.rs index 4f9329bed..96dcd9686 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/error.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/error.rs @@ -443,7 +443,7 @@ impl Error { } /// Return structured frontend-program failure details for `pg_dump` or `psql`. - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub fn tool_error(&self) -> Option<&crate::tools::PostgresToolError> { self.inner.downcast_ref() } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/lib.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/lib.rs index f1ff833e6..11f17f2aa 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/lib.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/lib.rs @@ -6,7 +6,12 @@ mod error; mod oliphaunt; #[cfg(feature = "extensions")] -pub use oliphaunt::extensions::Extension; +pub use oliphaunt::extensions::{Extension, ExtensionPackage}; +pub use oliphaunt_resources::{IcuData, WasixExtensionDescriptor as ExtensionDescriptor}; +/// Explicit selection of PostgreSQL contrib extensions included with the runtime. +pub mod extensions { + include!("oliphaunt/generated_contrib_descriptors.rs"); +} pub use async_api::{ AsyncOliphaunt, AsyncOliphauntBuilder, AsyncOliphauntServer, AsyncOliphauntServerBuilder, @@ -29,9 +34,11 @@ pub use oliphaunt::{ }; /// Options and structured errors for packaged PostgreSQL frontend programs. -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] pub mod tools { pub use crate::oliphaunt::tools::{ PgDumpOptions, PostgresToolError, PostgresToolOutput, PsqlOptions, }; + #[doc(hidden)] + pub use crate::oliphaunt::tools::{installed_tool_wasm, register_installed_package}; } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/aot.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/aot.rs index 785819eb0..db3164df8 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/aot.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/aot.rs @@ -100,13 +100,144 @@ pub(crate) fn load_artifact_module(engine: &Engine, artifact_name: &str) -> Resu Ok(module) } -#[cfg(feature = "tools")] +#[cfg(any(feature = "extensions", feature = "__internal-tools"))] +fn validate_package_aot_header(manifest: &AotManifest) -> Result<()> { + ensure!( + manifest.format_version == Some(1), + "package AOT format version mismatch" + ); + ensure!( + manifest.source_lane.as_deref() == Some("stable"), + "package AOT source lane mismatch" + ); + ensure!( + manifest + .source_fingerprint + .as_deref() + .is_some_and(|value| value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())), + "package AOT must identify its runtime source fingerprint" + ); + ensure!( + manifest.target_triple == target_triple(), + "package AOT target mismatch" + ); + ensure!( + manifest.engine == EXPECTED_AOT_ENGINE, + "package AOT engine mismatch" + ); + ensure!( + manifest.wasmer_version == EXPECTED_WASMER_VERSION, + "package AOT Wasmer version mismatch" + ); + ensure!( + manifest.wasmer_wasix_version == EXPECTED_WASMER_WASIX_VERSION, + "package AOT WASIX engine version mismatch" + ); + Ok(()) +} + +/// Load AOT from a trusted package descriptor, checking its exact module identity. +#[cfg(any(feature = "extensions", feature = "__internal-tools"))] +#[allow(unsafe_code)] +pub(crate) fn load_package_module( + engine: &Engine, + package: &oliphaunt_resources::WasixPackage, + name: &str, + wasm: &[u8], +) -> Result { + let manifest: AotManifest = serde_json::from_str(package.aot_manifest()) + .context("parse selected extension AOT manifest")?; + validate_package_aot_header(&manifest)?; + let runtime = assets::asset_manifest_metadata()?; + ensure!( + manifest.source_lane == runtime.source_lane, + "package AOT source lane mismatch" + ); + ensure!( + manifest.source_fingerprint == runtime.source_fingerprint, + "extension AOT runtime source fingerprint mismatch" + ); + ensure!( + manifest.postgres_version.as_deref() == Some(runtime.postgres_version.as_str()), + "extension AOT PostgreSQL version mismatch" + ); + let matches = manifest + .artifacts + .iter() + .filter(|artifact| artifact.name == name) + .collect::>(); + ensure!( + matches.len() == 1, + "extension AOT must contain exactly one artifact {name}" + ); + let artifact = matches[0]; + ensure!( + artifact.module_sha256 == sha256_hex(wasm), + "extension AOT WebAssembly module identity mismatch" + ); + let matches = package + .aot_artifacts() + .iter() + .filter(|(artifact_name, _)| *artifact_name == name) + .collect::>(); + ensure!( + matches.len() == 1, + "extension package must contain exactly one AOT payload {name}" + ); + let bytes = matches[0].1; + validate_compressed_artifact_manifest(name, artifact, bytes)?; + let key = format!("package:{name}:{}", artifact.sha256); + let mut modules = MODULE_CACHE + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .expect("AOT module cache poisoned"); + if let Some(module) = modules.get(&key) { + return Ok(module.clone()); + } + let size = artifact + .raw_size + .context("extension AOT is missing raw-size")?; + let mut raw = Vec::new(); + if bytes.starts_with(ZSTD_MAGIC) { + ZstdDecoder::new(Cursor::new(bytes))? + .take( + size.checked_add(1) + .context("extension AOT raw-size overflow")?, + ) + .read_to_end(&mut raw)?; + } else { + raw.extend_from_slice(bytes); + } + expected_raw_hash(name, artifact, &raw, AotVerifyMode::Full)?; + // SAFETY: ExtensionPackage can only be constructed through its unsafe trusted + // release constructor. We additionally verify the engine/target/runtime and + // both compressed and raw identities, and deserialize these same owned bytes. + let module = unsafe { Module::deserialize(engine, raw) } + .context("deserialize selected extension AOT module")?; + modules.insert(key, module.clone()); + Ok(module) +} + +#[cfg(feature = "__internal-tools")] pub(crate) fn load_pg_dump_module(engine: &Engine) -> Result { + if let Some(package) = super::tools::installed_tool_package() { + let wasm = super::tools::installed_tool_wasm("pg_dump") + .context("installed package has no pg_dump")?; + return load_package_module(engine, package, "tool:pg_dump", wasm); + } load_artifact_module(engine, "tool:pg_dump") } -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] pub(crate) fn load_psql_module(engine: &Engine) -> Result { + if let Some(package) = super::tools::installed_tool_package() { + let wasm = + super::tools::installed_tool_wasm("psql").context("installed package has no psql")?; + return load_package_module(engine, package, "tool:psql", wasm); + } load_artifact_module(engine, "tool:psql") } @@ -931,6 +1062,10 @@ fn target_tools_aot_manifest_json_for_crate() -> Option<&'static str> { #[derive(Debug, Deserialize)] #[serde(rename_all = "kebab-case")] struct AotManifest { + #[allow(dead_code)] + format_version: Option, + #[allow(dead_code)] + source_lane: Option, source_fingerprint: Option, postgres_version: Option, target_triple: String, @@ -1084,3 +1219,31 @@ mod tests { panic!("WASIX toolchain manifest has toolchain.{key}"); } } + +#[cfg(all(test, any(feature = "extensions", feature = "__internal-tools")))] +#[test] +fn package_aot_header_rejects_foreign_or_incomplete_executable_identities() { + let valid = serde_json::json!({ + "format-version": 1, "source-lane": "stable", "source-fingerprint": "a".repeat(64), + "postgres-version": "18.4", "target-triple": target_triple(), "engine": EXPECTED_AOT_ENGINE, + "wasmer-version": EXPECTED_WASMER_VERSION, "wasmer-wasix-version": EXPECTED_WASMER_WASIX_VERSION, + "artifacts": [] + }); + validate_package_aot_header(&serde_json::from_value(valid.clone()).unwrap()).unwrap(); + for (field, value) in [ + ("format-version", serde_json::json!(2)), + ("source-lane", serde_json::json!("other")), + ("source-fingerprint", serde_json::Value::Null), + ("target-triple", serde_json::json!("other")), + ("engine", serde_json::json!("other")), + ("wasmer-version", serde_json::json!("other")), + ("wasmer-wasix-version", serde_json::json!("other")), + ] { + let mut invalid = valid.clone(); + invalid[field] = value; + assert!( + validate_package_aot_header(&serde_json::from_value(invalid).unwrap()).is_err(), + "{field}" + ); + } +} diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/assets.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/assets.rs index 5e1f881c5..e29d45aed 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/assets.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/assets.rs @@ -32,9 +32,9 @@ impl CatalogProfile { } pub(crate) fn validate_available(self) -> Result<()> { - if self == Self::Icu && !cfg!(feature = "icu") { + if self == Self::Icu && SELECTED_ICU.get().is_none() { return Err(crate::error::invalid_configuration( - "the ICU catalog profile requires the oliphaunt-wasix `icu` feature", + "the ICU catalog profile requires explicitly selected ICU package data", )); } Ok(()) @@ -48,11 +48,70 @@ impl Default for CatalogProfile { } pub(crate) const fn default_catalog_profile() -> CatalogProfile { - if cfg!(feature = "icu") { - CatalogProfile::Icu - } else { - CatalogProfile::Standard - } + CatalogProfile::Standard +} + +// One exact ICU release serves this runtime generation. Database selection stays +// on each builder; publishing data here never changes another builder's profile. +static SELECTED_ICU: std::sync::OnceLock = std::sync::OnceLock::new(); + +pub(crate) fn register_icu(data: oliphaunt_resources::IcuData) -> Result<()> { + use sha2::{Digest, Sha256}; + ensure!( + data.runtime_version == liboliphaunt_wasix_portable::PACKAGE_VERSION, + "ICU package is incompatible with the selected WASIX runtime" + ); + let archive = data + .wasix_archive + .context("selected ICU package has no WASIX archive")?; + let expected = data + .wasix_archive_sha256 + .context("selected ICU package has no archive digest")?; + ensure!( + format!("{:x}", Sha256::digest(archive)) == expected, + "ICU package archive hash mismatch" + ); + let tree = data + .wasix_data_tree_sha256 + .context("selected ICU package has no logical tree digest")?; + let manifest = liboliphaunt_wasix_portable::manifest()?; + let seed = data + .wasix_seed_archive + .context("selected ICU package has no cluster seed")?; + let seed_manifest = data + .wasix_seed_manifest + .context("selected ICU package has no seed manifest")?; + let expected_seed = manifest + .cluster_seeds + .get("icu") + .context("runtime has no ICU seed identity")?; + ensure!( + format!("{:x}", Sha256::digest(seed)) == expected_seed.sha256, + "ICU cluster seed archive hash mismatch" + ); + let parsed: serde_json::Value = serde_json::from_slice(seed_manifest)?; + ensure!( + parsed["catalogProfile"] == "icu" + && parsed["runtime"]["version"] == data.runtime_version + && parsed["icu"]["dataTreeSha256"] == tree, + "ICU seed manifest does not match selected package" + ); + ensure!( + manifest + .cluster_seeds + .get("icu") + .and_then(|seed| seed.icu_data_tree_sha256.as_deref()) + == Some(tree), + "selected ICU package does not match the runtime's ICU seed" + ); + let selected = SELECTED_ICU.get_or_init(|| data); + ensure!( + selected.version == data.version + && selected.wasix_archive_sha256 == data.wasix_archive_sha256 + && selected.wasix_data_tree_sha256 == data.wasix_data_tree_sha256, + "conflicting ICU packages for one WASIX runtime" + ); + Ok(()) } pub fn asset_manifest_metadata() -> Result { @@ -120,79 +179,60 @@ pub(crate) fn expected_runtime_archive_sha256() -> Result { pub(crate) fn cluster_seed_archive(profile: CatalogProfile) -> Option<&'static [u8]> { match profile { CatalogProfile::Standard => liboliphaunt_wasix_portable::standard_cluster_seed_archive(), - CatalogProfile::Icu => { - #[cfg(feature = "icu")] - { - liboliphaunt_wasix_portable::icu_cluster_seed_archive() - } - #[cfg(not(feature = "icu"))] - { - None - } - } + CatalogProfile::Icu => SELECTED_ICU.get().and_then(|data| data.wasix_seed_archive), } } pub(crate) fn cluster_seed_manifest(profile: CatalogProfile) -> Option<&'static [u8]> { match profile { CatalogProfile::Standard => liboliphaunt_wasix_portable::standard_cluster_seed_manifest(), - CatalogProfile::Icu => { - #[cfg(feature = "icu")] - { - liboliphaunt_wasix_portable::icu_cluster_seed_manifest() - } - #[cfg(not(feature = "icu"))] - { - None - } - } + CatalogProfile::Icu => SELECTED_ICU.get().and_then(|data| data.wasix_seed_manifest), } } -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] pub(crate) fn pg_dump_wasm() -> Option<&'static [u8]> { - oliphaunt_wasix_tools::pg_dump_wasm() + if let Some(bytes) = super::tools::installed_tool_wasm("pg_dump") { + return Some(bytes); + } + #[cfg(feature = "tools")] + { + return oliphaunt_wasix_tools::pg_dump_wasm(); + } + #[allow(unreachable_code)] + None } -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] pub(crate) fn psql_wasm() -> Option<&'static [u8]> { - oliphaunt_wasix_tools::psql_wasm() + if let Some(bytes) = super::tools::installed_tool_wasm("psql") { + return Some(bytes); + } + #[cfg(feature = "tools")] + { + return oliphaunt_wasix_tools::psql_wasm(); + } + #[allow(unreachable_code)] + None } pub(crate) fn icu_data_archive(profile: CatalogProfile) -> Option<&'static [u8]> { if profile == CatalogProfile::Standard { return None; } - #[cfg(feature = "icu")] - { - oliphaunt_icu::icu_data_archive() - } - #[cfg(not(feature = "icu"))] - { - None - } + SELECTED_ICU.get().and_then(|data| data.wasix_archive) } pub(crate) fn expected_icu_data_archive_sha256() -> Option<&'static str> { - #[cfg(feature = "icu")] - { - oliphaunt_icu::ICU_DATA_ARCHIVE_SHA256 - } - #[cfg(not(feature = "icu"))] - { - None - } + SELECTED_ICU + .get() + .and_then(|data| data.wasix_archive_sha256) } pub(crate) fn expected_icu_data_tree_sha256() -> Option<&'static str> { - #[cfg(feature = "icu")] - { - oliphaunt_icu::ICU_DATA_TREE_SHA256 - } - #[cfg(not(feature = "icu"))] - { - None - } + SELECTED_ICU + .get() + .and_then(|data| data.wasix_data_tree_sha256) } #[cfg(feature = "extensions")] @@ -232,16 +272,12 @@ mod tests { #[test] fn asset_helpers_expose_a_consistent_feature_contract() { - let default_profile = if cfg!(feature = "icu") { - CatalogProfile::Icu - } else { - CatalogProfile::Standard - }; + let default_profile = CatalogProfile::Standard; assert_eq!(CatalogProfile::default(), default_profile); CatalogProfile::Standard.validate_available().unwrap(); assert_eq!( CatalogProfile::Icu.validate_available().is_ok(), - cfg!(feature = "icu") + super::SELECTED_ICU.get().is_some() ); let metadata = asset_manifest_metadata().unwrap(); diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/backend.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/backend.rs index 4b3f3b03a..a9a785247 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/backend.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/backend.rs @@ -139,12 +139,12 @@ impl WasixBackendSession { self.pg.start_protocol_with_startup_packet(message) } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub(crate) fn existing_startup_response(&self) -> Option> { self.pg.existing_startup_response() } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub(crate) fn startup_config(&self) -> &StartupConfig { self.pg.startup_config() } @@ -238,12 +238,12 @@ impl BackendSession { self.0.startup_with_packet(message) } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub(crate) fn existing_startup_response(&self) -> Option> { self.0.existing_startup_response() } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub(crate) fn startup_config(&self) -> &StartupConfig { self.0.startup_config() } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/base.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/base.rs index e34d21624..56b20c23f 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/base.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/base.rs @@ -1849,6 +1849,27 @@ pub(crate) fn install_missing_extension_archives( extensions: &[Extension], ) -> Result<()> { for extension in extensions { + if let Some(package) = extension.package() { + let (_, bytes, expected) = package + .archives() + .iter() + .find(|(name, _, _)| *name == extension.sql_name()) + .with_context(|| { + format!( + "{}@{} has no archive for {}", + package.product(), + package.version(), + extension.sql_name() + ) + })?; + ensure!( + sha256_hex(bytes) == *expected, + "extension {} archive hash mismatch", + extension.sql_name() + ); + install_extension_reader(&outcome.runtime_layout.mutable_root, Cursor::new(bytes))?; + continue; + } let bytes = assets::extension_archive(extension.sql_name()).ok_or_else(|| { crate::error::invalid_configuration(format!( "extension asset '{}' is not bundled in this oliphaunt-wasix build", @@ -2317,7 +2338,6 @@ fn copy_runtime_file_if_exists(src: PathBuf, dest: PathBuf) -> Result<()> { mod tests { use super::*; - #[cfg(feature = "icu")] #[derive(Debug)] struct PreparedProfileSnapshot { profile: CatalogProfile, @@ -2328,7 +2348,6 @@ mod tests { has_icu_data: bool, } - #[cfg(feature = "icu")] fn both_catalog_profiles_are_packaged() -> bool { assets::runtime_archive().is_some() && [CatalogProfile::Standard, CatalogProfile::Icu] @@ -2340,7 +2359,6 @@ mod tests { && assets::icu_data_archive(CatalogProfile::Icu).is_some() } - #[cfg(feature = "icu")] fn prepare_profile_snapshot(profile: CatalogProfile) -> Result { let prepared = prepare_database( DatabasePlan::new(DatabaseStorage::Memory, profile), @@ -2375,7 +2393,6 @@ mod tests { }) } - #[cfg(feature = "icu")] fn assert_profile_snapshots_do_not_contaminate( standard: &PreparedProfileSnapshot, icu: &PreparedProfileSnapshot, @@ -2542,7 +2559,6 @@ mod tests { ); } - #[cfg(feature = "icu")] #[test] fn database_profiles_remain_isolated_in_both_construction_orders() -> Result<()> { if !both_catalog_profiles_are_packaged() { @@ -2563,7 +2579,6 @@ mod tests { Ok(()) } - #[cfg(feature = "icu")] #[test] fn database_profiles_materialize_concurrently_without_contamination() -> Result<()> { if !both_catalog_profiles_are_packaged() { diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs index 31590041e..a7c513f7a 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs @@ -17,6 +17,7 @@ use crate::oliphaunt::storage::DatabaseStorage; pub struct OliphauntBuilder { storage: DatabaseStorage, catalog_profile: CatalogProfile, + icu: Option, postgres_config: PostgresConfig, startup_config: StartupConfig, #[cfg(feature = "extensions")] @@ -28,6 +29,7 @@ impl Default for OliphauntBuilder { Self { storage: DatabaseStorage::Memory, catalog_profile: default_catalog_profile(), + icu: None, postgres_config: PostgresConfig::default(), startup_config: StartupConfig::default(), #[cfg(feature = "extensions")] @@ -57,6 +59,13 @@ impl OliphauntBuilder { self } + /// Select ICU data from the optional `oliphaunt-icu` package. + pub fn icu(mut self, data: oliphaunt_resources::IcuData) -> Self { + self.icu = Some(data); + self.catalog_profile = CatalogProfile::Icu; + self + } + /// Set a PostgreSQL startup GUC for this embedded backend. pub fn startup_guc(mut self, name: impl Into, value: impl Into) -> Self { self.postgres_config.insert(name, value); @@ -92,16 +101,20 @@ impl OliphauntBuilder { /// Make one bundled PostgreSQL extension artifact available to the database. /// Database-local installation remains the application's migration concern. #[cfg(feature = "extensions")] - pub fn extension(mut self, extension: Extension) -> Self { - self.extensions.push(extension); + pub fn extension(mut self, extension: impl Into) -> Self { + self.extensions.push(extension.into()); self } /// Make bundled PostgreSQL extension artifacts available to the database. /// Database-local installation remains the application's migration concern. #[cfg(feature = "extensions")] - pub fn extensions(mut self, extensions: impl IntoIterator) -> Self { - self.extensions.extend(extensions); + pub fn extensions>( + mut self, + extensions: impl IntoIterator, + ) -> Self { + self.extensions + .extend(extensions.into_iter().map(Into::into)); self } @@ -117,6 +130,9 @@ impl OliphauntBuilder { let postgres_config = self.postgres_config.clone(); postgres_config.validate()?; self.storage.validate()?; + if let Some(data) = self.icu { + crate::oliphaunt::assets::register_icu(data)?; + } self.startup_config.validate()?; let plan = DatabasePlan::new(self.storage.clone(), self.catalog_profile); let prepared = prepare_database(plan, &self.startup_config.username)?; @@ -191,7 +207,6 @@ mod storage_tests { assert_eq!(icu.catalog_profile, CatalogProfile::Icu); } - #[cfg(not(feature = "icu"))] #[test] fn unavailable_icu_profile_is_rejected_before_storage_mutation() { let parent = tempfile::tempdir().expect("temporary parent"); @@ -201,7 +216,7 @@ mod storage_tests { .catalog_profile(CatalogProfile::Icu) .open() .err() - .expect("ICU profile requires its packaging feature"); + .expect("ICU profile requires explicit package data"); assert_eq!(error.kind(), crate::ErrorKind::InvalidConfiguration); assert!(error.to_string().contains("requires")); @@ -239,7 +254,7 @@ mod storage_tests { } } -#[cfg(all(test, feature = "extension-pg-textsearch"))] +#[cfg(all(test, feature = "extensions"))] mod tests { use super::*; use crate::oliphaunt::extensions::Extension; diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/client.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/client.rs index b01e49fe0..59517b9c8 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/client.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/client.rs @@ -6,13 +6,13 @@ use std::sync::{Arc, Mutex}; use anyhow::{Context, Result, bail, ensure}; use tempfile::TempDir; -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] use tokio::io::AsyncWriteExt; -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] use tokio::runtime::Runtime; -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] use wasmer_wasix::virtual_net::tcp_pair::TcpSocketHalfRx; -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] use wasmer_wasix::virtual_net::tcp_pair::TcpSocketHalfTx; use crate::oliphaunt::backend::BackendSession; @@ -39,16 +39,16 @@ use crate::oliphaunt::query::{ use crate::oliphaunt::storage::PgDataStorage; #[cfg(all(feature = "extensions", test))] use crate::oliphaunt::storage::StorageRoot; -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] use crate::oliphaunt::tools::{ DirectToolSocket, PgDumpOptions, PostgresToolOutput, PsqlOptions, decode_tool_output, is_direct_tool_outcome_unknown, run_direct_pg_dump_output, run_direct_psql_output, }; -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] use crate::oliphaunt::wire::{FrontendFrameKind, FrontendFrameReader, classify_frontend_message}; const PROTOCOL_CALLBACK_CHUNK_BYTES: usize = 64 * 1024; -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] const DIRECT_TOOL_READ_BUFFER_BYTES: usize = 64 * 1024; /// Direct, single-session Oliphaunt WASIX database. @@ -385,7 +385,7 @@ struct CallbackProtocolState { callback: Option, error: Option, panic: Option, - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] tool_io: Option, } @@ -395,7 +395,7 @@ struct CallbackProtocolStream { impl Read for CallbackProtocolStream { fn read(&mut self, buffer: &mut [u8]) -> io::Result { - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] { let mut state = self .state @@ -416,7 +416,7 @@ impl Write for CallbackProtocolStream { .state .lock() .map_err(|_| io::Error::other("WASIX protocol callback lock poisoned"))?; - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] if let Some(tool_io) = state.tool_io.as_mut() { return tool_io.write(buffer); } @@ -449,7 +449,7 @@ impl Write for CallbackProtocolStream { } fn flush(&mut self) -> io::Result<()> { - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] { let mut state = self .state @@ -465,7 +465,7 @@ impl Write for CallbackProtocolStream { impl ProtocolStream for CallbackProtocolStream { fn read_ready(&mut self) -> io::Result { - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] { let state = self .state @@ -1022,13 +1022,13 @@ impl Oliphaunt { Ok(()) } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub(crate) fn run_pg_dump_tool(&mut self, options: PgDumpOptions) -> Result { self.run_pg_dump_tool_output(options) .and_then(|output| decode_tool_output("pg_dump", output)) } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub(crate) fn run_pg_dump_tool_output( &mut self, options: PgDumpOptions, @@ -1043,13 +1043,13 @@ impl Oliphaunt { self.finish_tool_session(result) } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub(crate) fn run_psql_tool(&mut self, options: PsqlOptions) -> Result { self.run_psql_tool_output(options) .and_then(|output| decode_tool_output("psql", output)) } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub(crate) fn run_psql_tool_output( &mut self, options: PsqlOptions, @@ -1064,7 +1064,7 @@ impl Oliphaunt { self.finish_tool_session(result) } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] fn prepare_tool_session(&mut self) -> Result<()> { self.check_ready()?; if self.in_transaction { @@ -1076,7 +1076,7 @@ impl Oliphaunt { .context("prepare embedded session for WASIX tool") } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] fn finish_tool_session(&mut self, result: Result) -> Result { let outcome_unknown = result .as_ref() @@ -1102,7 +1102,7 @@ impl Oliphaunt { } } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] fn reset_tool_session(&mut self) -> Result<()> { self.execute_inner("ROLLBACK") .context("roll back embedded session")?; @@ -1119,7 +1119,7 @@ impl Oliphaunt { Ok(()) } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] fn serve_direct_tool_protocol(&mut self, socket: DirectToolSocket) -> Result<()> { self.ensure_protocol_stream_attached()?; { @@ -1148,7 +1148,7 @@ impl Oliphaunt { result.and(cleanup) } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] fn serve_direct_tool_protocol_inner(&mut self) -> Result<()> { let mut reader = FrontendFrameReader::default(); let mut buffer = [0u8; DIRECT_TOOL_READ_BUFFER_BYTES]; @@ -1189,12 +1189,12 @@ impl Oliphaunt { } } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] fn write_direct_tool_protocol(&self, bytes: &[u8]) -> Result<()> { self.with_direct_tool_io(|tool_io| tool_io.write_all(bytes)) } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] fn with_direct_tool_io( &self, operation: impl FnOnce(&mut DirectToolProtocolIo) -> io::Result, @@ -1218,25 +1218,25 @@ impl Oliphaunt { } /// Run packaged `pg_dump` directly against this database. - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub fn pg_dump(&mut self, options: PgDumpOptions) -> crate::Result { crate::error::public_result(self.run_pg_dump_tool(options)) } /// Run packaged `pg_dump` and return exact stdout/stderr bytes. - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub fn pg_dump_output(&mut self, options: PgDumpOptions) -> crate::Result { crate::error::public_result(self.run_pg_dump_tool_output(options)) } /// Run packaged non-interactive `psql` directly against this database. - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub fn psql(&mut self, options: PsqlOptions) -> crate::Result { crate::error::public_result(self.run_psql_tool(options)) } /// Run packaged non-interactive `psql` and return exact stdout/stderr bytes. - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub fn psql_output(&mut self, options: PsqlOptions) -> crate::Result { crate::error::public_result(self.run_psql_tool_output(options)) } @@ -1562,7 +1562,7 @@ impl Oliphaunt { } } -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] fn finish_direct_tool_frontend(reader: &FrontendFrameReader) -> Result<()> { ensure!( reader.pending().is_empty(), @@ -1745,14 +1745,14 @@ fn combine_backup_failures( primary.context(format!("{cleanup_label}: {cleanup:#}")) } -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] struct DirectToolProtocolIo { runtime: Runtime, writer: TcpSocketHalfTx, reader: TcpSocketHalfRx, } -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] impl DirectToolProtocolIo { fn new(socket: DirectToolSocket) -> Result { let (writer, reader) = socket.split(); @@ -1767,7 +1767,7 @@ impl DirectToolProtocolIo { } } -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] impl Read for DirectToolProtocolIo { fn read(&mut self, buffer: &mut [u8]) -> io::Result { self.runtime.block_on(async { @@ -1791,7 +1791,7 @@ impl Read for DirectToolProtocolIo { } } -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] impl Write for DirectToolProtocolIo { fn write(&mut self, bytes: &[u8]) -> io::Result { self.runtime.block_on(self.writer.write_all(bytes))?; diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/data_dir.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/data_dir.rs index 9a6cab123..47e6aa6ab 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/data_dir.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/data_dir.rs @@ -13,9 +13,9 @@ use tar::{Archive, Builder, EntryType, Header}; use wasmer_wasix::virtual_fs::FileSystem as VirtualFileSystem; use super::base::DirectoryLock; -use super::database_root_descriptor::{ - PGDATA_DIRECTORY, PHYSICAL_FORMAT, POSTGRES_MAJOR, write_database_root_descriptor, -}; +use liboliphaunt_wasix_portable::{PHYSICAL_FORMAT, POSTGRES_MAJOR}; + +use super::database_root_descriptor::{PGDATA_DIRECTORY, write_database_root_descriptor}; use crate::oliphaunt::storage::{PgDataStorage, vfs_read}; use crate::{StorageCommitState, StorageErrorCode, StorageErrorPhase}; diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/extensions.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/extensions.rs index 234dfa8c1..a2d2a151a 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/extensions.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/extensions.rs @@ -1,7 +1,7 @@ -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use anyhow::Result; -#[cfg(all(test, feature = "extension-pg-textsearch"))] +#[cfg(test)] use anyhow::bail; use crate::oliphaunt::config::PostgresConfig; @@ -27,6 +27,9 @@ impl ExtensionNativeModule { } } +use oliphaunt_resources::WasixExtensionDescriptor; +pub use oliphaunt_resources::WasixPackage as ExtensionPackage; + /// A bundled PostgreSQL extension artifact that Oliphaunt can make available. /// /// Selecting an extension does not run `CREATE EXTENSION`, `LOAD`, or other @@ -39,9 +42,21 @@ pub struct Extension { aot_name: Option<&'static str>, dependencies: &'static [&'static str], startup_config: &'static [&'static str], + package: Option<&'static ExtensionPackage>, } impl Extension { + /// Bind extension metadata to the exact package selected by Cargo. + #[doc(hidden)] + pub const fn with_package(mut self, package: &'static ExtensionPackage) -> Self { + self.package = Some(package); + self + } + + pub(crate) const fn package(self) -> Option<&'static ExtensionPackage> { + self.package + } + /// SQL extension name used in `CREATE EXTENSION`. pub const fn sql_name(self) -> &'static str { self.sql_name @@ -76,14 +91,65 @@ impl Extension { } } +impl From for Extension { + fn from(descriptor: WasixExtensionDescriptor) -> Self { + let mut extension = Self::by_sql_name(descriptor.sql_name).unwrap_or(Self { + sql_name: descriptor.sql_name, + native_support_modules: &[], + native_module_file: None, + aot_name: None, + dependencies: &[], + startup_config: &[], + package: None, + }); + extension.package = descriptor.package; + extension + } +} + pub(crate) fn resolve_extension_set(extensions: &[Extension]) -> Result> { let mut visiting = BTreeSet::new(); let mut visited = BTreeSet::new(); let mut resolved = Vec::new(); let mut requested = extensions.to_vec(); requested.sort_by_key(|extension| extension.sql_name()); + for pair in requested.windows(2) { + if pair[0].sql_name() == pair[1].sql_name() && pair[0] != pair[1] { + return Err(crate::error::invalid_configuration(format!( + "conflicting packages selected for extension '{}'", + pair[0].sql_name() + ))); + } + } + let selected: BTreeMap<_, _> = requested + .iter() + .map(|extension| (extension.sql_name(), *extension)) + .collect(); for extension in requested { - visit_extension(extension, &mut visiting, &mut visited, &mut resolved)?; + if Extension::by_sql_name(extension.sql_name()).is_none() { + return Err(crate::error::invalid_configuration(format!( + "unsupported extension '{}'", + extension.sql_name() + ))); + } + if let Some(package) = extension.package { + if package.runtime_version() != liboliphaunt_wasix_portable::PACKAGE_VERSION { + return Err(crate::error::invalid_configuration(format!( + "{}@{} requires WASIX runtime {}, selected {}", + package.product(), + package.version(), + package.runtime_version(), + liboliphaunt_wasix_portable::PACKAGE_VERSION + ))); + } + } + visit_extension( + extension, + &selected, + &mut visiting, + &mut visited, + &mut resolved, + )?; } Ok(resolved) } @@ -143,7 +209,7 @@ pub(crate) fn postgres_config_with_extension_startup( Ok(postgres_config) } -#[cfg(all(test, feature = "extension-pg-textsearch"))] +#[cfg(test)] pub(crate) fn ensure_extension_startup_config_is_active( postgres_config: &PostgresConfig, extension: Extension, @@ -215,11 +281,21 @@ fn comma_separated_values(value: &str) -> impl Iterator { fn visit_extension( extension: Extension, + selected: &BTreeMap<&'static str, Extension>, visiting: &mut BTreeSet<&'static str>, visited: &mut BTreeSet<&'static str>, resolved: &mut Vec, ) -> Result<()> { if visited.contains(extension.sql_name()) { + if resolved + .iter() + .any(|prior| prior.sql_name() == extension.sql_name() && *prior != extension) + { + return Err(crate::error::invalid_configuration(format!( + "conflicting packages selected for extension '{}'", + extension.sql_name() + ))); + } return Ok(()); } if !visiting.insert(extension.sql_name()) { @@ -229,14 +305,31 @@ fn visit_extension( ))); } for dependency in extension.dependencies() { - let dependency_extension = Extension::by_sql_name(dependency).ok_or_else(|| { + let mut dependency_extension = Extension::by_sql_name(dependency).ok_or_else(|| { crate::error::invalid_configuration(format!( "selected extension '{}' depends on missing catalog extension '{}'", extension.sql_name(), dependency )) })?; - visit_extension(dependency_extension, visiting, visited, resolved)?; + if let Some(package) = extension.package { + if package + .archives() + .iter() + .any(|(name, _, _)| name == dependency) + { + dependency_extension = dependency_extension.with_package(package); + } + } + if let Some(explicit) = selected.get(dependency) { + if dependency_extension.package.is_some() && dependency_extension != *explicit { + return Err(crate::error::invalid_configuration(format!( + "conflicting packages selected for extension '{dependency}'" + ))); + } + dependency_extension = *explicit; + } + visit_extension(dependency_extension, selected, visiting, visited, resolved)?; } visiting.remove(extension.sql_name()); visited.insert(extension.sql_name()); @@ -264,7 +357,7 @@ fn extension_activation_sql_for_test(extension: Extension) -> Result Result<()> { let activation = extension_activation_sql_for_test(Extension::EARTHDISTANCE)?; assert_eq!(activation.len(), 2); @@ -326,24 +419,24 @@ mod extension_tests { } #[test] - #[cfg(feature = "extension-uuid-ossp")] + fn uuid_ossp_aot_direct_and_restart_smoke() -> Result<()> { run_direct_and_restart_smoke_set(&[Extension::UUID_OSSP]) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - #[cfg(feature = "extension-uuid-ossp")] + async fn uuid_ossp_aot_server_smoke() -> Result<()> { run_server_smoke_set(&[Extension::UUID_OSSP]).await } #[test] - #[cfg(feature = "extension-uuid-ossp")] + fn uuid_ossp_aot_materialization_smoke() -> Result<()> { run_lifecycle_materialization_set(&[Extension::UUID_OSSP]) } - #[cfg(all(feature = "tools", feature = "extension-uuid-ossp"))] + #[cfg(feature = "__internal-tools")] #[test] fn uuid_ossp_aot_dump_restore_smoke() -> Result<()> { use crate::tools::{PgDumpOptions, PsqlOptions}; diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/generated_contrib_descriptors.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/generated_contrib_descriptors.rs new file mode 100644 index 000000000..5be334c2f --- /dev/null +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/generated_contrib_descriptors.rs @@ -0,0 +1,162 @@ +// @generated by `cargo run -p xtask -- extensions generate` + +/// Select `amcheck`. +pub const AMCHECK: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "amcheck", + package: None, +}; +/// Select `auto_explain`. +pub const AUTO_EXPLAIN: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "auto_explain", + package: None, +}; +/// Select `bloom`. +pub const BLOOM: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "bloom", + package: None, +}; +/// Select `btree_gin`. +pub const BTREE_GIN: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "btree_gin", + package: None, +}; +/// Select `btree_gist`. +pub const BTREE_GIST: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "btree_gist", + package: None, +}; +/// Select `citext`. +pub const CITEXT: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "citext", + package: None, +}; +/// Select `cube`. +pub const CUBE: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "cube", + package: None, +}; +/// Select `dict_int`. +pub const DICT_INT: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "dict_int", + package: None, +}; +/// Select `dict_xsyn`. +pub const DICT_XSYN: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "dict_xsyn", + package: None, +}; +/// Select `earthdistance`. +pub const EARTHDISTANCE: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "earthdistance", + package: None, +}; +/// Select `file_fdw`. +pub const FILE_FDW: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "file_fdw", + package: None, +}; +/// Select `fuzzystrmatch`. +pub const FUZZYSTRMATCH: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "fuzzystrmatch", + package: None, +}; +/// Select `hstore`. +pub const HSTORE: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "hstore", + package: None, +}; +/// Select `intarray`. +pub const INTARRAY: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "intarray", + package: None, +}; +/// Select `isn`. +pub const ISN: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "isn", + package: None, +}; +/// Select `lo`. +pub const LO: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "lo", + package: None, +}; +/// Select `ltree`. +pub const LTREE: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "ltree", + package: None, +}; +/// Select `pageinspect`. +pub const PAGEINSPECT: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pageinspect", + package: None, +}; +/// Select `pg_buffercache`. +pub const PG_BUFFERCACHE: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pg_buffercache", + package: None, +}; +/// Select `pg_freespacemap`. +pub const PG_FREESPACEMAP: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pg_freespacemap", + package: None, +}; +/// Select `pg_surgery`. +pub const PG_SURGERY: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pg_surgery", + package: None, +}; +/// Select `pg_trgm`. +pub const PG_TRGM: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pg_trgm", + package: None, +}; +/// Select `pg_visibility`. +pub const PG_VISIBILITY: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pg_visibility", + package: None, +}; +/// Select `pg_walinspect`. +pub const PG_WALINSPECT: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pg_walinspect", + package: None, +}; +/// Select `pgcrypto`. +pub const PGCRYPTO: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pgcrypto", + package: None, +}; +/// Select `seg`. +pub const SEG: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "seg", + package: None, +}; +/// Select `tablefunc`. +pub const TABLEFUNC: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "tablefunc", + package: None, +}; +/// Select `tcn`. +pub const TCN: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "tcn", + package: None, +}; +/// Select `tsm_system_rows`. +pub const TSM_SYSTEM_ROWS: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "tsm_system_rows", + package: None, +}; +/// Select `tsm_system_time`. +pub const TSM_SYSTEM_TIME: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "tsm_system_time", + package: None, +}; +/// Select `unaccent`. +pub const UNACCENT: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "unaccent", + package: None, +}; +/// Select `uuid-ossp`. +pub const UUID_OSSP: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "uuid-ossp", + package: None, +}; diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/generated_extensions.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/generated_extensions.rs index 2b8089676..3117cf52c 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/generated_extensions.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/generated_extensions.rs @@ -2,7 +2,6 @@ use super::Extension; -#[cfg(feature = "extension-amcheck")] const DEFINITION_AMCHECK: Extension = Extension { sql_name: "amcheck", native_support_modules: &[], @@ -10,9 +9,9 @@ const DEFINITION_AMCHECK: Extension = Extension { aot_name: Some("extension:amcheck"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-auto-explain")] const DEFINITION_AUTO_EXPLAIN: Extension = Extension { sql_name: "auto_explain", native_support_modules: &[], @@ -20,9 +19,9 @@ const DEFINITION_AUTO_EXPLAIN: Extension = Extension { aot_name: Some("extension:auto_explain"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-bloom")] const DEFINITION_BLOOM: Extension = Extension { sql_name: "bloom", native_support_modules: &[], @@ -30,9 +29,9 @@ const DEFINITION_BLOOM: Extension = Extension { aot_name: Some("extension:bloom"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-btree-gin")] const DEFINITION_BTREE_GIN: Extension = Extension { sql_name: "btree_gin", native_support_modules: &[], @@ -40,9 +39,9 @@ const DEFINITION_BTREE_GIN: Extension = Extension { aot_name: Some("extension:btree_gin"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-btree-gist")] const DEFINITION_BTREE_GIST: Extension = Extension { sql_name: "btree_gist", native_support_modules: &[], @@ -50,9 +49,9 @@ const DEFINITION_BTREE_GIST: Extension = Extension { aot_name: Some("extension:btree_gist"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-citext")] const DEFINITION_CITEXT: Extension = Extension { sql_name: "citext", native_support_modules: &[], @@ -60,9 +59,9 @@ const DEFINITION_CITEXT: Extension = Extension { aot_name: Some("extension:citext"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-cube")] const DEFINITION_CUBE: Extension = Extension { sql_name: "cube", native_support_modules: &[], @@ -70,9 +69,9 @@ const DEFINITION_CUBE: Extension = Extension { aot_name: Some("extension:cube"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-dict-int")] const DEFINITION_DICT_INT: Extension = Extension { sql_name: "dict_int", native_support_modules: &[], @@ -80,9 +79,9 @@ const DEFINITION_DICT_INT: Extension = Extension { aot_name: Some("extension:dict_int"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-dict-xsyn")] const DEFINITION_DICT_XSYN: Extension = Extension { sql_name: "dict_xsyn", native_support_modules: &[], @@ -90,9 +89,9 @@ const DEFINITION_DICT_XSYN: Extension = Extension { aot_name: Some("extension:dict_xsyn"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-earthdistance")] const DEFINITION_EARTHDISTANCE: Extension = Extension { sql_name: "earthdistance", native_support_modules: &[], @@ -100,9 +99,9 @@ const DEFINITION_EARTHDISTANCE: Extension = Extension { aot_name: Some("extension:earthdistance"), dependencies: &["cube"], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-file-fdw")] const DEFINITION_FILE_FDW: Extension = Extension { sql_name: "file_fdw", native_support_modules: &[], @@ -110,9 +109,9 @@ const DEFINITION_FILE_FDW: Extension = Extension { aot_name: Some("extension:file_fdw"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-fuzzystrmatch")] const DEFINITION_FUZZYSTRMATCH: Extension = Extension { sql_name: "fuzzystrmatch", native_support_modules: &[], @@ -120,9 +119,9 @@ const DEFINITION_FUZZYSTRMATCH: Extension = Extension { aot_name: Some("extension:fuzzystrmatch"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-hstore")] const DEFINITION_HSTORE: Extension = Extension { sql_name: "hstore", native_support_modules: &[], @@ -130,9 +129,9 @@ const DEFINITION_HSTORE: Extension = Extension { aot_name: Some("extension:hstore"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-intarray")] const DEFINITION_INTARRAY: Extension = Extension { sql_name: "intarray", native_support_modules: &[], @@ -140,9 +139,9 @@ const DEFINITION_INTARRAY: Extension = Extension { aot_name: Some("extension:intarray"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-isn")] const DEFINITION_ISN: Extension = Extension { sql_name: "isn", native_support_modules: &[], @@ -150,9 +149,9 @@ const DEFINITION_ISN: Extension = Extension { aot_name: Some("extension:isn"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-lo")] const DEFINITION_LO: Extension = Extension { sql_name: "lo", native_support_modules: &[], @@ -160,9 +159,9 @@ const DEFINITION_LO: Extension = Extension { aot_name: Some("extension:lo"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-ltree")] const DEFINITION_LTREE: Extension = Extension { sql_name: "ltree", native_support_modules: &[], @@ -170,9 +169,9 @@ const DEFINITION_LTREE: Extension = Extension { aot_name: Some("extension:ltree"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-pageinspect")] const DEFINITION_PAGEINSPECT: Extension = Extension { sql_name: "pageinspect", native_support_modules: &[], @@ -180,9 +179,9 @@ const DEFINITION_PAGEINSPECT: Extension = Extension { aot_name: Some("extension:pageinspect"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-pg-buffercache")] const DEFINITION_PG_BUFFERCACHE: Extension = Extension { sql_name: "pg_buffercache", native_support_modules: &[], @@ -190,9 +189,9 @@ const DEFINITION_PG_BUFFERCACHE: Extension = Extension { aot_name: Some("extension:pg_buffercache"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-pg-freespacemap")] const DEFINITION_PG_FREESPACEMAP: Extension = Extension { sql_name: "pg_freespacemap", native_support_modules: &[], @@ -200,9 +199,9 @@ const DEFINITION_PG_FREESPACEMAP: Extension = Extension { aot_name: Some("extension:pg_freespacemap"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-pg-hashids")] const DEFINITION_PG_HASHIDS: Extension = Extension { sql_name: "pg_hashids", native_support_modules: &[], @@ -210,9 +209,9 @@ const DEFINITION_PG_HASHIDS: Extension = Extension { aot_name: Some("extension:pg_hashids"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-pg-ivm")] const DEFINITION_PG_IVM: Extension = Extension { sql_name: "pg_ivm", native_support_modules: &[], @@ -220,9 +219,9 @@ const DEFINITION_PG_IVM: Extension = Extension { aot_name: Some("extension:pg_ivm"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-pg-surgery")] const DEFINITION_PG_SURGERY: Extension = Extension { sql_name: "pg_surgery", native_support_modules: &[], @@ -230,9 +229,9 @@ const DEFINITION_PG_SURGERY: Extension = Extension { aot_name: Some("extension:pg_surgery"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-pg-textsearch")] const DEFINITION_PG_TEXTSEARCH: Extension = Extension { sql_name: "pg_textsearch", native_support_modules: &[], @@ -240,9 +239,9 @@ const DEFINITION_PG_TEXTSEARCH: Extension = Extension { aot_name: Some("extension:pg_textsearch"), dependencies: &[], startup_config: &["shared_preload_libraries=pg_textsearch"], + package: None, }; -#[cfg(feature = "extension-pg-trgm")] const DEFINITION_PG_TRGM: Extension = Extension { sql_name: "pg_trgm", native_support_modules: &[], @@ -250,9 +249,9 @@ const DEFINITION_PG_TRGM: Extension = Extension { aot_name: Some("extension:pg_trgm"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-pg-uuidv7")] const DEFINITION_PG_UUIDV7: Extension = Extension { sql_name: "pg_uuidv7", native_support_modules: &[], @@ -260,9 +259,9 @@ const DEFINITION_PG_UUIDV7: Extension = Extension { aot_name: Some("extension:pg_uuidv7"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-pg-visibility")] const DEFINITION_PG_VISIBILITY: Extension = Extension { sql_name: "pg_visibility", native_support_modules: &[], @@ -270,9 +269,9 @@ const DEFINITION_PG_VISIBILITY: Extension = Extension { aot_name: Some("extension:pg_visibility"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-pg-walinspect")] const DEFINITION_PG_WALINSPECT: Extension = Extension { sql_name: "pg_walinspect", native_support_modules: &[], @@ -280,9 +279,9 @@ const DEFINITION_PG_WALINSPECT: Extension = Extension { aot_name: Some("extension:pg_walinspect"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-pgcrypto")] const DEFINITION_PGCRYPTO: Extension = Extension { sql_name: "pgcrypto", native_support_modules: &[], @@ -290,9 +289,9 @@ const DEFINITION_PGCRYPTO: Extension = Extension { aot_name: Some("extension:pgcrypto"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-pgtap")] const DEFINITION_PGTAP: Extension = Extension { sql_name: "pgtap", native_support_modules: &[], @@ -300,9 +299,9 @@ const DEFINITION_PGTAP: Extension = Extension { aot_name: None, dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-postgis")] const DEFINITION_POSTGIS: Extension = Extension { sql_name: "postgis", native_support_modules: &[super::ExtensionNativeModule { @@ -313,9 +312,9 @@ const DEFINITION_POSTGIS: Extension = Extension { aot_name: Some("extension:postgis"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-seg")] const DEFINITION_SEG: Extension = Extension { sql_name: "seg", native_support_modules: &[], @@ -323,9 +322,9 @@ const DEFINITION_SEG: Extension = Extension { aot_name: Some("extension:seg"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-tablefunc")] const DEFINITION_TABLEFUNC: Extension = Extension { sql_name: "tablefunc", native_support_modules: &[], @@ -333,9 +332,9 @@ const DEFINITION_TABLEFUNC: Extension = Extension { aot_name: Some("extension:tablefunc"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-tcn")] const DEFINITION_TCN: Extension = Extension { sql_name: "tcn", native_support_modules: &[], @@ -343,9 +342,9 @@ const DEFINITION_TCN: Extension = Extension { aot_name: Some("extension:tcn"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-tsm-system-rows")] const DEFINITION_TSM_SYSTEM_ROWS: Extension = Extension { sql_name: "tsm_system_rows", native_support_modules: &[], @@ -353,9 +352,9 @@ const DEFINITION_TSM_SYSTEM_ROWS: Extension = Extension { aot_name: Some("extension:tsm_system_rows"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-tsm-system-time")] const DEFINITION_TSM_SYSTEM_TIME: Extension = Extension { sql_name: "tsm_system_time", native_support_modules: &[], @@ -363,9 +362,9 @@ const DEFINITION_TSM_SYSTEM_TIME: Extension = Extension { aot_name: Some("extension:tsm_system_time"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-unaccent")] const DEFINITION_UNACCENT: Extension = Extension { sql_name: "unaccent", native_support_modules: &[], @@ -373,9 +372,9 @@ const DEFINITION_UNACCENT: Extension = Extension { aot_name: Some("extension:unaccent"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-uuid-ossp")] const DEFINITION_UUID_OSSP: Extension = Extension { sql_name: "uuid-ossp", native_support_modules: &[], @@ -383,9 +382,9 @@ const DEFINITION_UUID_OSSP: Extension = Extension { aot_name: Some("extension:uuid-ossp"), dependencies: &[], startup_config: &[], + package: None, }; -#[cfg(feature = "extension-vector")] const DEFINITION_VECTOR: Extension = Extension { sql_name: "vector", native_support_modules: &[], @@ -393,206 +392,129 @@ const DEFINITION_VECTOR: Extension = Extension { aot_name: Some("extension:vector"), dependencies: &[], startup_config: &[], + package: None, }; impl Extension { /// Select the `amcheck` artifact. - #[cfg(feature = "extension-amcheck")] pub const AMCHECK: Self = DEFINITION_AMCHECK; /// Select the `auto_explain` artifact. - #[cfg(feature = "extension-auto-explain")] pub const AUTO_EXPLAIN: Self = DEFINITION_AUTO_EXPLAIN; /// Select the `bloom` artifact. - #[cfg(feature = "extension-bloom")] pub const BLOOM: Self = DEFINITION_BLOOM; /// Select the `btree_gin` artifact. - #[cfg(feature = "extension-btree-gin")] pub const BTREE_GIN: Self = DEFINITION_BTREE_GIN; /// Select the `btree_gist` artifact. - #[cfg(feature = "extension-btree-gist")] pub const BTREE_GIST: Self = DEFINITION_BTREE_GIST; /// Select the `citext` artifact. - #[cfg(feature = "extension-citext")] pub const CITEXT: Self = DEFINITION_CITEXT; /// Select the `cube` artifact. - #[cfg(feature = "extension-cube")] pub const CUBE: Self = DEFINITION_CUBE; /// Select the `dict_int` artifact. - #[cfg(feature = "extension-dict-int")] pub const DICT_INT: Self = DEFINITION_DICT_INT; /// Select the `dict_xsyn` artifact. - #[cfg(feature = "extension-dict-xsyn")] pub const DICT_XSYN: Self = DEFINITION_DICT_XSYN; /// Select the `earthdistance` artifact. - #[cfg(feature = "extension-earthdistance")] pub const EARTHDISTANCE: Self = DEFINITION_EARTHDISTANCE; /// Select the `file_fdw` artifact. - #[cfg(feature = "extension-file-fdw")] pub const FILE_FDW: Self = DEFINITION_FILE_FDW; /// Select the `fuzzystrmatch` artifact. - #[cfg(feature = "extension-fuzzystrmatch")] pub const FUZZYSTRMATCH: Self = DEFINITION_FUZZYSTRMATCH; /// Select the `hstore` artifact. - #[cfg(feature = "extension-hstore")] pub const HSTORE: Self = DEFINITION_HSTORE; /// Select the `intarray` artifact. - #[cfg(feature = "extension-intarray")] pub const INTARRAY: Self = DEFINITION_INTARRAY; /// Select the `isn` artifact. - #[cfg(feature = "extension-isn")] pub const ISN: Self = DEFINITION_ISN; /// Select the `lo` artifact. - #[cfg(feature = "extension-lo")] pub const LO: Self = DEFINITION_LO; /// Select the `ltree` artifact. - #[cfg(feature = "extension-ltree")] pub const LTREE: Self = DEFINITION_LTREE; /// Select the `pageinspect` artifact. - #[cfg(feature = "extension-pageinspect")] pub const PAGEINSPECT: Self = DEFINITION_PAGEINSPECT; /// Select the `pg_buffercache` artifact. - #[cfg(feature = "extension-pg-buffercache")] pub const PG_BUFFERCACHE: Self = DEFINITION_PG_BUFFERCACHE; /// Select the `pg_freespacemap` artifact. - #[cfg(feature = "extension-pg-freespacemap")] pub const PG_FREESPACEMAP: Self = DEFINITION_PG_FREESPACEMAP; /// Select the `pg_hashids` artifact. - #[cfg(feature = "extension-pg-hashids")] pub const PG_HASHIDS: Self = DEFINITION_PG_HASHIDS; /// Select the `pg_ivm` artifact. - #[cfg(feature = "extension-pg-ivm")] pub const PG_IVM: Self = DEFINITION_PG_IVM; /// Select the `pg_surgery` artifact. - #[cfg(feature = "extension-pg-surgery")] pub const PG_SURGERY: Self = DEFINITION_PG_SURGERY; /// Select the `pg_textsearch` artifact. - #[cfg(feature = "extension-pg-textsearch")] pub const PG_TEXTSEARCH: Self = DEFINITION_PG_TEXTSEARCH; /// Select the `pg_trgm` artifact. - #[cfg(feature = "extension-pg-trgm")] pub const PG_TRGM: Self = DEFINITION_PG_TRGM; /// Select the `pg_uuidv7` artifact. - #[cfg(feature = "extension-pg-uuidv7")] pub const PG_UUIDV7: Self = DEFINITION_PG_UUIDV7; /// Select the `pg_visibility` artifact. - #[cfg(feature = "extension-pg-visibility")] pub const PG_VISIBILITY: Self = DEFINITION_PG_VISIBILITY; /// Select the `pg_walinspect` artifact. - #[cfg(feature = "extension-pg-walinspect")] pub const PG_WALINSPECT: Self = DEFINITION_PG_WALINSPECT; /// Select the `pgcrypto` artifact. - #[cfg(feature = "extension-pgcrypto")] pub const PGCRYPTO: Self = DEFINITION_PGCRYPTO; /// Select the `pgtap` artifact. - #[cfg(feature = "extension-pgtap")] pub const PGTAP: Self = DEFINITION_PGTAP; /// Select the `postgis` artifact. - #[cfg(feature = "extension-postgis")] pub const POSTGIS: Self = DEFINITION_POSTGIS; /// Select the `seg` artifact. - #[cfg(feature = "extension-seg")] pub const SEG: Self = DEFINITION_SEG; /// Select the `tablefunc` artifact. - #[cfg(feature = "extension-tablefunc")] pub const TABLEFUNC: Self = DEFINITION_TABLEFUNC; /// Select the `tcn` artifact. - #[cfg(feature = "extension-tcn")] pub const TCN: Self = DEFINITION_TCN; /// Select the `tsm_system_rows` artifact. - #[cfg(feature = "extension-tsm-system-rows")] pub const TSM_SYSTEM_ROWS: Self = DEFINITION_TSM_SYSTEM_ROWS; /// Select the `tsm_system_time` artifact. - #[cfg(feature = "extension-tsm-system-time")] pub const TSM_SYSTEM_TIME: Self = DEFINITION_TSM_SYSTEM_TIME; /// Select the `unaccent` artifact. - #[cfg(feature = "extension-unaccent")] pub const UNACCENT: Self = DEFINITION_UNACCENT; /// Select the `uuid-ossp` artifact. - #[cfg(feature = "extension-uuid-ossp")] pub const UUID_OSSP: Self = DEFINITION_UUID_OSSP; /// Select the `vector` artifact. - #[cfg(feature = "extension-vector")] pub const VECTOR: Self = DEFINITION_VECTOR; /// Extension artifacts enabled in this Cargo build. pub const ALL: &'static [Self] = &[ - #[cfg(feature = "extension-amcheck")] Self::AMCHECK, - #[cfg(feature = "extension-auto-explain")] Self::AUTO_EXPLAIN, - #[cfg(feature = "extension-bloom")] Self::BLOOM, - #[cfg(feature = "extension-btree-gin")] Self::BTREE_GIN, - #[cfg(feature = "extension-btree-gist")] Self::BTREE_GIST, - #[cfg(feature = "extension-citext")] Self::CITEXT, - #[cfg(feature = "extension-cube")] Self::CUBE, - #[cfg(feature = "extension-dict-int")] Self::DICT_INT, - #[cfg(feature = "extension-dict-xsyn")] Self::DICT_XSYN, - #[cfg(feature = "extension-earthdistance")] Self::EARTHDISTANCE, - #[cfg(feature = "extension-file-fdw")] Self::FILE_FDW, - #[cfg(feature = "extension-fuzzystrmatch")] Self::FUZZYSTRMATCH, - #[cfg(feature = "extension-hstore")] Self::HSTORE, - #[cfg(feature = "extension-intarray")] Self::INTARRAY, - #[cfg(feature = "extension-isn")] Self::ISN, - #[cfg(feature = "extension-lo")] Self::LO, - #[cfg(feature = "extension-ltree")] Self::LTREE, - #[cfg(feature = "extension-pageinspect")] Self::PAGEINSPECT, - #[cfg(feature = "extension-pg-buffercache")] Self::PG_BUFFERCACHE, - #[cfg(feature = "extension-pg-freespacemap")] Self::PG_FREESPACEMAP, - #[cfg(feature = "extension-pg-hashids")] Self::PG_HASHIDS, - #[cfg(feature = "extension-pg-ivm")] Self::PG_IVM, - #[cfg(feature = "extension-pg-surgery")] Self::PG_SURGERY, - #[cfg(feature = "extension-pg-textsearch")] Self::PG_TEXTSEARCH, - #[cfg(feature = "extension-pg-trgm")] Self::PG_TRGM, - #[cfg(feature = "extension-pg-uuidv7")] Self::PG_UUIDV7, - #[cfg(feature = "extension-pg-visibility")] Self::PG_VISIBILITY, - #[cfg(feature = "extension-pg-walinspect")] Self::PG_WALINSPECT, - #[cfg(feature = "extension-pgcrypto")] Self::PGCRYPTO, - #[cfg(feature = "extension-pgtap")] Self::PGTAP, - #[cfg(feature = "extension-postgis")] Self::POSTGIS, - #[cfg(feature = "extension-seg")] Self::SEG, - #[cfg(feature = "extension-tablefunc")] Self::TABLEFUNC, - #[cfg(feature = "extension-tcn")] Self::TCN, - #[cfg(feature = "extension-tsm-system-rows")] Self::TSM_SYSTEM_ROWS, - #[cfg(feature = "extension-tsm-system-time")] Self::TSM_SYSTEM_TIME, - #[cfg(feature = "extension-unaccent")] Self::UNACCENT, - #[cfg(feature = "extension-uuid-ossp")] Self::UUID_OSSP, - #[cfg(feature = "extension-vector")] Self::VECTOR, ]; } @@ -600,83 +522,44 @@ impl Extension { #[cfg(test)] pub(super) fn creates_database_object_for_test(extension: Extension) -> bool { match extension.sql_name() { - #[cfg(feature = "extension-amcheck")] "amcheck" => true, - #[cfg(feature = "extension-auto-explain")] "auto_explain" => false, - #[cfg(feature = "extension-bloom")] "bloom" => true, - #[cfg(feature = "extension-btree-gin")] "btree_gin" => true, - #[cfg(feature = "extension-btree-gist")] "btree_gist" => true, - #[cfg(feature = "extension-citext")] "citext" => true, - #[cfg(feature = "extension-cube")] "cube" => true, - #[cfg(feature = "extension-dict-int")] "dict_int" => true, - #[cfg(feature = "extension-dict-xsyn")] "dict_xsyn" => true, - #[cfg(feature = "extension-earthdistance")] "earthdistance" => true, - #[cfg(feature = "extension-file-fdw")] "file_fdw" => true, - #[cfg(feature = "extension-fuzzystrmatch")] "fuzzystrmatch" => true, - #[cfg(feature = "extension-hstore")] "hstore" => true, - #[cfg(feature = "extension-intarray")] "intarray" => true, - #[cfg(feature = "extension-isn")] "isn" => true, - #[cfg(feature = "extension-lo")] "lo" => true, - #[cfg(feature = "extension-ltree")] "ltree" => true, - #[cfg(feature = "extension-pageinspect")] "pageinspect" => true, - #[cfg(feature = "extension-pg-buffercache")] "pg_buffercache" => true, - #[cfg(feature = "extension-pg-freespacemap")] "pg_freespacemap" => true, - #[cfg(feature = "extension-pg-hashids")] "pg_hashids" => true, - #[cfg(feature = "extension-pg-ivm")] "pg_ivm" => true, - #[cfg(feature = "extension-pg-surgery")] "pg_surgery" => true, - #[cfg(feature = "extension-pg-textsearch")] "pg_textsearch" => true, - #[cfg(feature = "extension-pg-trgm")] "pg_trgm" => true, - #[cfg(feature = "extension-pg-uuidv7")] "pg_uuidv7" => true, - #[cfg(feature = "extension-pg-visibility")] "pg_visibility" => true, - #[cfg(feature = "extension-pg-walinspect")] "pg_walinspect" => true, - #[cfg(feature = "extension-pgcrypto")] "pgcrypto" => true, - #[cfg(feature = "extension-pgtap")] "pgtap" => true, - #[cfg(feature = "extension-postgis")] "postgis" => true, - #[cfg(feature = "extension-seg")] "seg" => true, - #[cfg(feature = "extension-tablefunc")] "tablefunc" => true, - #[cfg(feature = "extension-tcn")] "tcn" => true, - #[cfg(feature = "extension-tsm-system-rows")] "tsm_system_rows" => true, - #[cfg(feature = "extension-tsm-system-time")] "tsm_system_time" => true, - #[cfg(feature = "extension-unaccent")] "unaccent" => true, - #[cfg(feature = "extension-uuid-ossp")] "uuid-ossp" => true, - #[cfg(feature = "extension-vector")] "vector" => true, _ => false, } @@ -685,124 +568,85 @@ pub(super) fn creates_database_object_for_test(extension: Extension) -> bool { #[cfg(test)] pub(super) fn activation_sql_for_test(extension: Extension) -> &'static [&'static str] { match extension.sql_name() { - #[cfg(feature = "extension-amcheck")] "amcheck" => &["CREATE EXTENSION IF NOT EXISTS \"amcheck\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-auto-explain")] "auto_explain" => &[ "LOAD 'auto_explain';", "SET auto_explain.log_min_duration = '0';", "SET auto_explain.log_analyze = 'true';", "SET auto_explain.log_level = 'NOTICE';", ], - #[cfg(feature = "extension-bloom")] "bloom" => &["CREATE EXTENSION IF NOT EXISTS \"bloom\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-btree-gin")] "btree_gin" => { &["CREATE EXTENSION IF NOT EXISTS \"btree_gin\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-btree-gist")] "btree_gist" => { &["CREATE EXTENSION IF NOT EXISTS \"btree_gist\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-citext")] "citext" => &["CREATE EXTENSION IF NOT EXISTS \"citext\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-cube")] "cube" => &["CREATE EXTENSION IF NOT EXISTS \"cube\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-dict-int")] "dict_int" => &["CREATE EXTENSION IF NOT EXISTS \"dict_int\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-dict-xsyn")] "dict_xsyn" => { &["CREATE EXTENSION IF NOT EXISTS \"dict_xsyn\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-earthdistance")] "earthdistance" => { &["CREATE EXTENSION IF NOT EXISTS \"earthdistance\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-file-fdw")] "file_fdw" => &["CREATE EXTENSION IF NOT EXISTS \"file_fdw\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-fuzzystrmatch")] "fuzzystrmatch" => { &["CREATE EXTENSION IF NOT EXISTS \"fuzzystrmatch\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-hstore")] "hstore" => &["CREATE EXTENSION IF NOT EXISTS \"hstore\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-intarray")] "intarray" => &["CREATE EXTENSION IF NOT EXISTS \"intarray\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-isn")] "isn" => &["CREATE EXTENSION IF NOT EXISTS \"isn\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-lo")] "lo" => &["CREATE EXTENSION IF NOT EXISTS \"lo\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-ltree")] "ltree" => &["CREATE EXTENSION IF NOT EXISTS \"ltree\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-pageinspect")] "pageinspect" => { &["CREATE EXTENSION IF NOT EXISTS \"pageinspect\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-pg-buffercache")] "pg_buffercache" => { &["CREATE EXTENSION IF NOT EXISTS \"pg_buffercache\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-pg-freespacemap")] "pg_freespacemap" => { &["CREATE EXTENSION IF NOT EXISTS \"pg_freespacemap\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-pg-hashids")] "pg_hashids" => { &["CREATE EXTENSION IF NOT EXISTS \"pg_hashids\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-pg-ivm")] "pg_ivm" => &["CREATE EXTENSION IF NOT EXISTS \"pg_ivm\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-pg-surgery")] "pg_surgery" => { &["CREATE EXTENSION IF NOT EXISTS \"pg_surgery\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-pg-textsearch")] "pg_textsearch" => { &["CREATE EXTENSION IF NOT EXISTS \"pg_textsearch\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-pg-trgm")] "pg_trgm" => &["CREATE EXTENSION IF NOT EXISTS \"pg_trgm\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-pg-uuidv7")] "pg_uuidv7" => { &["CREATE EXTENSION IF NOT EXISTS \"pg_uuidv7\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-pg-visibility")] "pg_visibility" => { &["CREATE EXTENSION IF NOT EXISTS \"pg_visibility\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-pg-walinspect")] "pg_walinspect" => { &["CREATE EXTENSION IF NOT EXISTS \"pg_walinspect\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-pgcrypto")] "pgcrypto" => &["CREATE EXTENSION IF NOT EXISTS \"pgcrypto\";"], - #[cfg(feature = "extension-pgtap")] "pgtap" => &["CREATE EXTENSION IF NOT EXISTS \"pgtap\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-postgis")] "postgis" => &["CREATE EXTENSION IF NOT EXISTS \"postgis\";"], - #[cfg(feature = "extension-seg")] "seg" => &["CREATE EXTENSION IF NOT EXISTS \"seg\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-tablefunc")] "tablefunc" => { &["CREATE EXTENSION IF NOT EXISTS \"tablefunc\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-tcn")] "tcn" => &["CREATE EXTENSION IF NOT EXISTS \"tcn\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-tsm-system-rows")] "tsm_system_rows" => { &["CREATE EXTENSION IF NOT EXISTS \"tsm_system_rows\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-tsm-system-time")] "tsm_system_time" => { &["CREATE EXTENSION IF NOT EXISTS \"tsm_system_time\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-unaccent")] "unaccent" => &["CREATE EXTENSION IF NOT EXISTS \"unaccent\" WITH SCHEMA \"pg_catalog\";"], - #[cfg(feature = "extension-uuid-ossp")] "uuid-ossp" => { &["CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\" WITH SCHEMA \"pg_catalog\";"] } - #[cfg(feature = "extension-vector")] "vector" => &["CREATE EXTENSION IF NOT EXISTS \"vector\" WITH SCHEMA \"pg_catalog\";"], _ => &[], } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/mod.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/mod.rs index aedd91bd3..54d712be0 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/mod.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/mod.rs @@ -22,7 +22,7 @@ pub(crate) mod storage; pub(crate) mod sync_host_fs; #[cfg(test)] pub(crate) mod test_fixtures; -#[cfg(feature = "tools")] +#[cfg(feature = "__internal-tools")] pub mod tools; pub(crate) mod transport; pub(crate) mod wire; diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/postgres_mod.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/postgres_mod.rs index 10b039d5e..869711db5 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/postgres_mod.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/postgres_mod.rs @@ -421,6 +421,7 @@ impl PostgresMod { &self.runtime_storage, module.runtime_path(), module.aot_name(), + extension.package(), &format!( "extension '{}' support module '{}'", extension.sql_name(), @@ -439,6 +440,7 @@ impl PostgresMod { &self.runtime_storage, &format!("lib/postgresql/{module_file}"), extension.aot_name(), + extension.package(), &format!("extension '{}'", extension.sql_name()), )?; Ok(()) @@ -755,12 +757,12 @@ impl PostgresMod { }) } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub(crate) fn existing_startup_response(&self) -> Option> { self.startup_response.clone() } - #[cfg(feature = "tools")] + #[cfg(feature = "__internal-tools")] pub(crate) fn startup_config(&self) -> &StartupConfig { &self.startup_config } @@ -1363,6 +1365,7 @@ fn seed_extension_side_module( runtime_root: &StorageRoot, runtime_path: &str, aot_name: Option<&'static str>, + package: Option<&super::extensions::ExtensionPackage>, label: &str, ) -> Result<()> { let Some(aot_name) = aot_name else { @@ -1372,6 +1375,15 @@ fn seed_extension_side_module( let wasm = runtime_root .read(&path) .with_context(|| format!("{label} is not installed at {}", path.display()))?; + if let Some(package) = package { + let module = aot::load_package_module(engine, package, aot_name, &wasm)?; + block_on_tokio_runtime( + runtime, + module_cache.save(ModuleHash::new(&wasm), engine, &module), + ) + .with_context(|| format!("seed Wasmer module cache for {label}"))?; + return Ok(()); + } seed_wasix_module_cache_bytes(runtime, engine, module_cache, &wasm, aot_name, label) } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/server.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/server.rs index 94aff0bee..c968be304 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/server.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/server.rs @@ -51,9 +51,9 @@ pub struct OliphauntServer { active_connection: Arc, handle: Option>>, close_result: Option, - #[cfg(all(test, feature = "icu"))] + #[cfg(test)] catalog_profile: CatalogProfile, - #[cfg(all(test, feature = "icu"))] + #[cfg(test)] runtime_root: PathBuf, #[cfg(unix)] owned_unix_socket: Option, @@ -195,9 +195,7 @@ pub(crate) fn server_with_worker_result_for_test( active_connection: Arc::new(ActiveConnection::default()), handle: Some(thread::spawn(move || result)), close_result: None, - #[cfg(feature = "icu")] catalog_profile: CatalogProfile::default(), - #[cfg(feature = "icu")] runtime_root: PathBuf::new(), #[cfg(unix)] owned_unix_socket: None, @@ -209,6 +207,7 @@ pub(crate) fn server_with_worker_result_for_test( pub struct OliphauntServerBuilder { storage: DatabaseStorage, catalog_profile: CatalogProfile, + icu: Option, listen: ServerListen, postgres_config: PostgresConfig, startup_config: StartupConfig, @@ -273,6 +272,7 @@ impl Default for OliphauntServerBuilder { Self { storage: DatabaseStorage::Memory, catalog_profile: default_catalog_profile(), + icu: None, listen: ServerListen::tcp(), postgres_config: PostgresConfig::default(), startup_config: StartupConfig::default(), @@ -310,6 +310,13 @@ impl OliphauntServerBuilder { self } + /// Select ICU data from the optional `oliphaunt-icu` package. + pub fn icu(mut self, data: oliphaunt_resources::IcuData) -> Self { + self.icu = Some(data); + self.catalog_profile = CatalogProfile::Icu; + self + } + /// Set a PostgreSQL startup GUC for the embedded backend used by this /// server. pub fn startup_guc(mut self, name: impl Into, value: impl Into) -> Self { @@ -345,16 +352,20 @@ impl OliphauntServerBuilder { /// Make one bundled PostgreSQL extension artifact available to clients. /// Database-local installation remains the application's migration concern. #[cfg(feature = "extensions")] - pub fn extension(mut self, extension: Extension) -> Self { - self.extensions.push(extension); + pub fn extension(mut self, extension: impl Into) -> Self { + self.extensions.push(extension.into()); self } /// Make bundled PostgreSQL extension artifacts available to clients. /// Database-local installation remains the application's migration concern. #[cfg(feature = "extensions")] - pub fn extensions(mut self, extensions: impl IntoIterator) -> Self { - self.extensions.extend(extensions); + pub fn extensions>( + mut self, + extensions: impl IntoIterator, + ) -> Self { + self.extensions + .extend(extensions.into_iter().map(Into::into)); self } @@ -383,6 +394,9 @@ impl OliphauntServerBuilder { let postgres_config = self.postgres_config.clone(); postgres_config.validate()?; self.storage.validate()?; + if let Some(data) = self.icu { + crate::oliphaunt::assets::register_icu(data)?; + } self.startup_config.validate()?; let startup_config = self.startup_config.clone(); @@ -395,9 +409,9 @@ impl OliphauntServerBuilder { directory_lock, outcome, } = prepared_database; - #[cfg(all(test, feature = "icu"))] + #[cfg(test)] let catalog_profile = outcome.runtime_layout.catalog_profile; - #[cfg(all(test, feature = "icu"))] + #[cfg(test)] let runtime_root = outcome.runtime_layout.module_root.clone(); let shutdown = Arc::new(AtomicBool::new(false)); @@ -450,9 +464,9 @@ impl OliphauntServerBuilder { active_connection, handle: Some(handle), close_result: None, - #[cfg(all(test, feature = "icu"))] + #[cfg(test)] catalog_profile, - #[cfg(all(test, feature = "icu"))] + #[cfg(test)] runtime_root, #[cfg(unix)] owned_unix_socket, @@ -785,10 +799,10 @@ fn percent_encode_bytes(value: &[u8]) -> String { #[cfg(test)] mod tests { use super::*; - #[cfg(feature = "extension-pg-textsearch")] + + #[cfg(feature = "extensions")] use crate::oliphaunt::extensions::Extension; - #[cfg(feature = "icu")] fn both_catalog_profiles_are_packaged() -> bool { crate::oliphaunt::assets::runtime_archive().is_some() && [CatalogProfile::Standard, CatalogProfile::Icu] @@ -800,7 +814,6 @@ mod tests { && crate::oliphaunt::assets::icu_data_archive(CatalogProfile::Icu).is_some() } - #[cfg(feature = "icu")] fn assert_server_profile(server: &OliphauntServer, expected: CatalogProfile) { assert_eq!(server.catalog_profile, expected); assert_eq!( @@ -996,7 +1009,6 @@ mod tests { assert_eq!(builder.catalog_profile, CatalogProfile::default()); } - #[cfg(feature = "icu")] #[test] fn server_profiles_remain_isolated_in_both_construction_orders() -> Result<()> { if !both_catalog_profiles_are_packaged() { @@ -1026,7 +1038,6 @@ mod tests { Ok(()) } - #[cfg(feature = "icu")] #[test] fn server_profiles_start_concurrently_without_contamination() -> Result<()> { if !both_catalog_profiles_are_packaged() { @@ -1144,8 +1155,8 @@ mod tests { assert!(error.to_string().contains("omit it to allocate one")); } - #[cfg(feature = "extension-pg-textsearch")] #[test] + #[cfg(feature = "extensions")] fn server_path_merges_pg_textsearch_preload_once_before_start() { let builder = OliphauntServerBuilder::new() .startup_guc("shared_preload_libraries", "auto_explain,pg_textsearch") diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/tools.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/tools.rs index aa39c6d80..98c21a84b 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/tools.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/tools.rs @@ -423,6 +423,60 @@ fn disallowed_flag( None } +static INSTALLED_TOOLS: std::sync::OnceLock = + std::sync::OnceLock::new(); + +/// Register an installed, trusted tools package for the native binding. +#[doc(hidden)] +pub fn register_installed_package(package: oliphaunt_resources::WasixPackage) -> Result<()> { + anyhow::ensure!( + package.runtime_version() == liboliphaunt_wasix_portable::PACKAGE_VERSION, + "installed tools package runtime mismatch" + ); + for name in ["pg_dump", "psql"] { + let matches: Vec<_> = package + .archives() + .iter() + .filter(|(entry, _, _)| *entry == name) + .collect(); + anyhow::ensure!( + matches.len() == 1, + "tools package must contain exactly one {name}" + ); + use sha2::{Digest, Sha256}; + anyhow::ensure!( + format!("{:x}", Sha256::digest(matches[0].1)) == matches[0].2, + "tools package {name} hash mismatch" + ); + aot::load_package_module( + &aot::headless_engine(), + &package, + &format!("tool:{name}"), + matches[0].1, + )?; + } + let selected = INSTALLED_TOOLS.get_or_init(|| package); + anyhow::ensure!( + *selected == package, + "conflicting tools packages for one runtime" + ); + Ok(()) +} + +pub(crate) fn installed_tool_package() -> Option<&'static oliphaunt_resources::WasixPackage> { + INSTALLED_TOOLS.get() +} + +#[doc(hidden)] +pub fn installed_tool_wasm(name: &str) -> Option<&'static [u8]> { + INSTALLED_TOOLS + .get()? + .archives() + .iter() + .find(|(entry, _, _)| *entry == name) + .map(|(_, bytes, _)| *bytes) +} + fn pg_dump_wasm_asset() -> Result<&'static [u8]> { assets::pg_dump_wasm() .filter(|bytes| !bytes.is_empty()) @@ -1289,7 +1343,6 @@ mod tests { )) } - #[cfg(feature = "extension-pgtap")] #[test] fn public_tools_round_trip_shared_logical_fixture() -> crate::Result<()> { let seed = crate::oliphaunt::test_fixtures::text("postgres/logical-tools-seed.sql"); diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/tests/extensions_smoke.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/tests/extensions_smoke.rs index 4cef5ca59..41525f23c 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/tests/extensions_smoke.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/tests/extensions_smoke.rs @@ -1,4 +1,4 @@ -#![cfg(feature = "extension-vector")] +#![cfg(feature = "extensions")] use anyhow::Result; use oliphaunt_wasix::{AsyncOliphauntServer, Extension, Oliphaunt}; diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/tests/public_api.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/tests/public_api.rs index b1429e95e..5e4c477b1 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/tests/public_api.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/tests/public_api.rs @@ -578,7 +578,7 @@ fn typed_and_fluent_database_api_is_public() { let _: fn(&AsyncOliphauntServer) = async_server_surface; } -#[cfg(feature = "extension-vector")] +#[cfg(feature = "extensions")] #[test] fn extensions_expose_only_the_selection_contract() { use oliphaunt_wasix::Extension; @@ -592,9 +592,9 @@ fn extensions_expose_only_the_selection_contract() { assert!(Extension::ALL.contains(&extension)); } -#[cfg(feature = "extension-earthdistance")] +#[cfg(feature = "extensions")] #[test] -fn extension_features_expose_required_dependency_selectors() { +fn extensions_expose_required_dependency_selectors() { use oliphaunt_wasix::Extension; assert_eq!( diff --git a/src/bindings/wasix-rust/moon.yml b/src/bindings/wasix-rust/moon.yml index 6d9884277..5e27beabd 100644 --- a/src/bindings/wasix-rust/moon.yml +++ b/src/bindings/wasix-rust/moon.yml @@ -56,7 +56,7 @@ tasks: set -e cargo test -p oliphaunt-wasix --doc --locked cargo test -p oliphaunt-wasix --doc --locked --features tools - cargo nextest run -p oliphaunt-wasix --locked --profile ci --no-default-features --features extensions,tools,extension-vector --test public_api --no-tests=fail --test-threads=1 + cargo nextest run -p oliphaunt-wasix --locked --profile ci --no-default-features --features extensions,tools --test public_api --no-tests=fail --test-threads=1 cargo test -p oliphaunt-wasix --locked --no-default-features --features extensions --test runtime_smoke --test client_compat --no-run env: CARGO_TARGET_DIR: "target/moon/oliphaunt-wasix-rust/test" @@ -110,8 +110,8 @@ tasks: set -e rm -rf target/oliphaunt-wasix-rust/package node src/bindings/wasix-rust/tools/package-source.mjs target/oliphaunt-wasix-rust/package/source - cargo package --manifest-path target/oliphaunt-wasix-rust/package/source/Cargo.toml --allow-dirty --no-verify cargo package --manifest-path target/oliphaunt-wasix-rust/package/source/Cargo.toml --allow-dirty --list > target/oliphaunt-wasix-rust/package/oliphaunt-wasix.package-files.txt + tools/dev/bun.sh tools/release/package_oliphaunt_wasix_sdk_crate.mjs --output-dir target/oliphaunt-wasix-rust/package/artifacts env: CARGO_TARGET_DIR: "target/moon/oliphaunt-wasix-rust/package" inputs: @@ -125,6 +125,8 @@ tasks: group: "crates" - "/src/sources/toolchains/wasix.toml" - "/src/bindings/wasix-rust/tools/package-source.mjs" + - "/tools/release/package_oliphaunt_wasix_sdk_crate.mjs" + - "@group(release-archive-contract)" outputs: - "/target/oliphaunt-wasix-rust/package/**/*" options: diff --git a/src/bindings/wasix-ts/README.md b/src/bindings/wasix-ts/README.md index 095a6f9ff..373a52610 100644 --- a/src/bindings/wasix-ts/README.md +++ b/src/bindings/wasix-ts/README.md @@ -20,8 +20,9 @@ The published SDK is one universal browser-and-server package. Its browser host files and exact `@oliphaunt/liboliphaunt-wasix` dependency are therefore installed on Node.js, Bun, Deno, and Electron too, although native export conditions never load them. The matching target-filtered optional platform -package embeds the runtime, both cluster profiles, tools, and qualified -extension catalog used on those hosts. Carrier packages have no install scripts +package embeds PostgreSQL, standard initialization assets, and contrib. +External extensions, ICU data with its matching seed, and frontend tools are +provided by their separately installed packages. Carrier packages have no install scripts and do not download a binary at install or first use. Applications do not configure raw runtime assets. @@ -63,11 +64,9 @@ import icu from '@oliphaunt/wasix-icu'; await using database = await Oliphaunt.open({ icu }); ``` -Browser conditions load the ICU assets from their portable carrier. Each -native platform carrier contains one addon with both `standard` and `icu` -profiles, and the existing `icu` option selects the database profile. The loader checks -the exact SDK/carrier version, WASIX runtime version, addon ABI, Node-API level, -target, and ICU profile before running native code. +Both browser and native conditions load ICU data and the matching seed from the +selected optional package. The native adapter passes the data through N-API; +Rust verifies runtime, seed, and ICU tree identities before initialization. ## Query PostgreSQL @@ -254,27 +253,32 @@ thread, and `/worker` uses a temporary package-owned Worker. ## Extensions -Import package-authored WASIX extension descriptors and pass them at open: +Install vector independently of the SDK: + +```sh +npm install @oliphaunt/wasix-ts @oliphaunt/extension-vector-wasix +``` ```ts -import Oliphaunt from '@oliphaunt/wasix-ts'; -import pgtap from '@oliphaunt/extension-pgtap-wasix'; +import Oliphaunt, { extensions } from '@oliphaunt/wasix-ts'; +import { directory } from '@oliphaunt/wasix-ts/storage/node'; +import vector from '@oliphaunt/extension-vector-wasix'; -await using database = await Oliphaunt.open({ extensions: [pgtap] }); -await database.execute('CREATE EXTENSION pgtap'); -const version = await database.query('select pgtap_version()'); +await using database = await Oliphaunt.open({ + storage: directory('./postgres'), + extensions: [vector, extensions.hstore], +}); +await database.execute('CREATE EXTENSION vector'); +await database.execute('CREATE EXTENSION hstore'); ``` -The call shape and lifecycle ownership are host-independent. A browser verifies -the selected carrier and its dependency closure, installs its artifacts before -startup, and applies required startup/preload settings. Node.js, Bun, Deno, and Electron -validate the same descriptor but resolve its SQL name against the extension -catalog compiled into the platform addon. Release addons contain the complete -currently supported extension catalog; they do not load arbitrary side-module -bytes from npm at runtime. Adding or upgrading a server extension therefore -requires a matching N-API carrier release. This increases the carrier size in -exchange for eliminating runtime archive expansion and dynamic linking on the -native path. +Contrib is supplied by the base dependency and selected explicitly through +`extensions`. External descriptors bind the imported package's version and +resources. Browser hosts load portable artifacts. Node.js, Bun, and Deno resolve +the installed package and matching host AOT dependency, then pass those package +locations through N-API. Rust validates package ownership, runtime compatibility, +engine and host identity, and payload hashes before loading executable code. +A compatible extension update does not require another base addon release. Neither host runs database-local `CREATE EXTENSION`, `LOAD`, schema, post-create, upgrade, or migration SQL. Applications and ORM migrations own diff --git a/src/bindings/wasix-ts/src/__tests__/contrib-carrier.ts b/src/bindings/wasix-ts/src/__tests__/contrib-carrier.ts new file mode 100644 index 000000000..b25ca4bff --- /dev/null +++ b/src/bindings/wasix-ts/src/__tests__/contrib-carrier.ts @@ -0,0 +1,47 @@ +import { defineWasixExtension } from '../extension-descriptor.js'; + +export const hstore = defineWasixExtension({ + schema: 'oliphaunt-wasix-extension-v1', + runtime: 'wasix', + product: 'oliphaunt-extension-contrib-pg18', + version: '0.1.1', + sqlName: 'hstore', + compatibility: { + extensionRuntimeContract: 'oliphaunt-extension-runtime-contract-v1', + postgresMajor: '18', + wasixRuntimeProduct: 'liboliphaunt-wasix', + wasixRuntimeVersion: '0.1.1', + }, + carriers: [ + { + product: 'oliphaunt-extension-contrib-pg18', + version: '0.1.1', + sqlName: 'hstore', + archive: 'extensions/hstore.tar.zst', + source: '/extensions/hstore.tar.zst', + sha256: '2'.repeat(64), + size: 100, + install: { + schema: 'oliphaunt-wasix-extension-install-v1', + name: 'hstore', + nativeModule: null, + nativeModules: [], + dependencies: [], + coreExportsRequired: [], + loadOrder: [], + unresolvedImports: [], + installedFiles: ['share/postgresql/extension/hstore.control'], + lifecycle: { + createExtension: true, + createSchema: 'pg_catalog', + loadSql: [], + postCreateSql: [], + startupConfig: [], + preloadRequired: false, + restartRequired: false, + sharedMemoryRequired: false, + }, + }, + }, + ], +}); diff --git a/src/bindings/wasix-ts/src/__tests__/native-addon.test.ts b/src/bindings/wasix-ts/src/__tests__/native-addon.test.ts index b367046d3..62149c39c 100644 --- a/src/bindings/wasix-ts/src/__tests__/native-addon.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/native-addon.test.ts @@ -167,6 +167,7 @@ function addonFixture(): NativeWasixAddon { payloadIdentity: () => `${'a'.repeat(64)}:1`, extensionIdentity: () => `${'a'.repeat(64)}:1`, toolIdentity: () => `${'a'.repeat(64)}:1`, + registerTools: () => {}, }; } diff --git a/src/bindings/wasix-ts/src/__tests__/native-extension-packages.test.ts b/src/bindings/wasix-ts/src/__tests__/native-extension-packages.test.ts new file mode 100644 index 000000000..b60dc5ac5 --- /dev/null +++ b/src/bindings/wasix-ts/src/__tests__/native-extension-packages.test.ts @@ -0,0 +1,57 @@ +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { afterEach, expect, it } from 'vitest'; +import { nativeExtensionPackages } from '../native-extension-packages.js'; +import { workerOpenOptions } from './worker-helpers.js'; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +it('resolves an imported package by its file identity, including npm aliases, and rejects substituted versions', () => { + const root = mkdtempSync(join(tmpdir(), 'oliphaunt-extension-')); + roots.push(root); + const owner = join(root, 'node_modules', 'my-pgtap-version'); + const path = 'extensions/pgtap/extension.tar.zst'; + mkdirSync(join(owner, 'extensions', 'pgtap'), { recursive: true }); + writeFileSync(join(owner, path), 'fixture'); + const options = workerOpenOptions(); + const carrier = { + sqlName: 'pgtap', + product: 'oliphaunt-extension-pgtap', + version: '1.3.4', + sha256: 'a'.repeat(64), + size: 7, + source: pathToFileURL(join(owner, path)).href, + }; + // The serializer validates the complete install descriptor before this resolver. + options.extensionCarriers.pgtap = carrier as (typeof options.extensionCarriers)[string]; + options.extensions = ['pgtap']; + const manifest = { + name: '@oliphaunt/extension-pgtap-wasix', + version: carrier.version, + oliphaunt: { + product: carrier.product, + kind: 'exact-extension-wasix', + wasixRuntimeVersion: options.runtime.version, + carriers: { pgtap: { path, sha256: carrier.sha256, size: 7, requiresAot: false } }, + }, + }; + writeFileSync(join(owner, 'package.json'), JSON.stringify(manifest)); + expect(nativeExtensionPackages(options)).toEqual([ + { + sqlName: 'pgtap', + product: carrier.product, + version: carrier.version, + packageJson: join(owner, 'package.json'), + }, + ]); + manifest.version = '1.3.5'; + writeFileSync(join(owner, 'package.json'), JSON.stringify(manifest)); + expect(() => nativeExtensionPackages(options)).toThrow('does not match its installed package'); + carrier.source = 'https://example.com/untrusted.tar.zst'; + expect(() => nativeExtensionPackages(options)).toThrow('requires an installed package file URL'); +}); diff --git a/src/bindings/wasix-ts/src/__tests__/native-session.test.ts b/src/bindings/wasix-ts/src/__tests__/native-session.test.ts index 69b1374da..fb5ece5b1 100644 --- a/src/bindings/wasix-ts/src/__tests__/native-session.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/native-session.test.ts @@ -19,6 +19,15 @@ const nativeMocks = vi.hoisted(() => ({ pgDump: vi.fn(), psql: vi.fn(), toolIdentity: vi.fn(), + registerTools: vi.fn(), +})); + +vi.mock('../native-extension-packages.js', async (importOriginal) => ({ + ...(await importOriginal()), + nativeToolPackage: () => ({ + packageJson: '/installed/tools/package.json', + aotPackageJson: '/installed/tools-aot/package.json', + }), })); vi.mock('../native-addon.js', () => ({ @@ -73,23 +82,29 @@ describe('WASIX native embedded payload compatibility', () => { ); }); - it('accepts an extension descriptor only when its exact archive is embedded', () => { + it('accepts a contrib descriptor only when its exact archive is embedded', () => { const options = workerOpenOptions(); - options.extensionCarriers.pgtap = extensionCarrier('pgtap'); + options.extensionCarriers.hstore = { + ...extensionCarrier('hstore'), + product: 'oliphaunt-extension-contrib-pg18', + }; expect(requireCompatibleNativeWasixAddon(options)).toBe( nativeMocks.loadAddon.mock.results[0]?.value, ); - expect(nativeMocks.extensionIdentity).toHaveBeenCalledWith('pgtap'); + expect(nativeMocks.extensionIdentity).toHaveBeenCalledWith('hstore'); }); - it('rejects an extension descriptor whose archive differs from the embedded archive', () => { + it('rejects a contrib descriptor whose archive differs from the embedded archive', () => { const options = workerOpenOptions(); - options.extensionCarriers.pgtap = extensionCarrier('pgtap'); + options.extensionCarriers.hstore = { + ...extensionCarrier('hstore'), + product: 'oliphaunt-extension-contrib-pg18', + }; nativeMocks.extensionIdentity.mockReturnValue(`${'b'.repeat(64)}:7`); expect(() => requireCompatibleNativeWasixAddon(options)).toThrow( - 'WASIX extension pgtap descriptor does not match the archive embedded in the native addon', + 'WASIX extension hstore descriptor does not match the archive embedded in the native addon', ); }); @@ -103,9 +118,7 @@ describe('WASIX native embedded payload compatibility', () => { tool: { name: 'pg_dump', sha256: digest, size: 7, source: 'embedded' }, args: pgDumpArguments(), }), - ).rejects.toThrow( - 'WASIX pg_dump descriptor does not match the tool embedded in the native addon', - ); + ).rejects.toThrow('WASIX pg_dump descriptor does not match the tool in the installed package'); expect(nativeMocks.pgDump).not.toHaveBeenCalled(); }); @@ -188,7 +201,38 @@ describe('WASIX native embedded payload compatibility', () => { ); expect( nativeWasixOpenOptions( - { ...options, icu: {} as NonNullable }, + { + ...options, + icu: { + schema: 'oliphaunt-wasix-icu-v1', + runtime: 'wasix', + product: 'oliphaunt-icu', + version: '0.1.1', + compatibility: { + runtimeProduct: 'liboliphaunt-wasix', + runtimeVersion: '0.1.1', + postgresMajor: '18', + physicalFormat: 'wasix-pg18-v1', + compatibilityKey: 'wasix-pg18-datum32-v1', + dataVersion: '76.1', + dataForm: 'files-le', + dataTreeSha256: digest, + }, + dataArchive: { + archive: 'icu.tar.zst', + sha256: digest, + size: 1, + source: Uint8Array.of(1), + }, + clusterSeedArchive: { + archive: 'seed.tar.zst', + sha256: digest, + size: 1, + source: Uint8Array.of(2), + }, + clusterSeedManifest: { sha256: digest, size: 1, source: Uint8Array.of(3) }, + }, + }, { kind: 'memory' }, ).profile, ).toBe('icu'); @@ -352,6 +396,7 @@ function addon(): NativeWasixAddon { payloadIdentity: nativeMocks.payloadIdentity, extensionIdentity: nativeMocks.extensionIdentity, toolIdentity: nativeMocks.toolIdentity, + registerTools: nativeMocks.registerTools, }; } diff --git a/src/bindings/wasix-ts/src/contrib-carrier-shim.d.ts b/src/bindings/wasix-ts/src/contrib-carrier-shim.d.ts new file mode 100644 index 000000000..12c4fef67 --- /dev/null +++ b/src/bindings/wasix-ts/src/contrib-carrier-shim.d.ts @@ -0,0 +1,33 @@ +// Generated by check-extension-model.py. +export declare const amcheck: import('./types.js').WasixExtensionDescriptor; +export declare const auto_explain: import('./types.js').WasixExtensionDescriptor; +export declare const bloom: import('./types.js').WasixExtensionDescriptor; +export declare const btree_gin: import('./types.js').WasixExtensionDescriptor; +export declare const btree_gist: import('./types.js').WasixExtensionDescriptor; +export declare const citext: import('./types.js').WasixExtensionDescriptor; +export declare const cube: import('./types.js').WasixExtensionDescriptor; +export declare const dict_int: import('./types.js').WasixExtensionDescriptor; +export declare const dict_xsyn: import('./types.js').WasixExtensionDescriptor; +export declare const earthdistance: import('./types.js').WasixExtensionDescriptor; +export declare const file_fdw: import('./types.js').WasixExtensionDescriptor; +export declare const fuzzystrmatch: import('./types.js').WasixExtensionDescriptor; +export declare const hstore: import('./types.js').WasixExtensionDescriptor; +export declare const intarray: import('./types.js').WasixExtensionDescriptor; +export declare const isn: import('./types.js').WasixExtensionDescriptor; +export declare const lo: import('./types.js').WasixExtensionDescriptor; +export declare const ltree: import('./types.js').WasixExtensionDescriptor; +export declare const pageinspect: import('./types.js').WasixExtensionDescriptor; +export declare const pg_buffercache: import('./types.js').WasixExtensionDescriptor; +export declare const pg_freespacemap: import('./types.js').WasixExtensionDescriptor; +export declare const pg_surgery: import('./types.js').WasixExtensionDescriptor; +export declare const pg_trgm: import('./types.js').WasixExtensionDescriptor; +export declare const pg_visibility: import('./types.js').WasixExtensionDescriptor; +export declare const pg_walinspect: import('./types.js').WasixExtensionDescriptor; +export declare const pgcrypto: import('./types.js').WasixExtensionDescriptor; +export declare const seg: import('./types.js').WasixExtensionDescriptor; +export declare const tablefunc: import('./types.js').WasixExtensionDescriptor; +export declare const tcn: import('./types.js').WasixExtensionDescriptor; +export declare const tsm_system_rows: import('./types.js').WasixExtensionDescriptor; +export declare const tsm_system_time: import('./types.js').WasixExtensionDescriptor; +export declare const unaccent: import('./types.js').WasixExtensionDescriptor; +export declare const uuid_ossp: import('./types.js').WasixExtensionDescriptor; diff --git a/src/bindings/wasix-ts/src/native-addon.ts b/src/bindings/wasix-ts/src/native-addon.ts index e36f35b95..7d74580bf 100644 --- a/src/bindings/wasix-ts/src/native-addon.ts +++ b/src/bindings/wasix-ts/src/native-addon.ts @@ -17,6 +17,26 @@ export type NativeWasixOpenOptions = Readonly<{ database: string; startupGucs: Record; extensions: string[]; + extensionPackages?: NativeExtensionPackage[]; + icu?: Readonly<{ + version: string; + runtimeVersion: string; + archive: Uint8Array; + archiveSha256: string; + dataTreeSha256: string; + seedArchive: Uint8Array; + seedArchiveSha256: string; + seedManifest: Uint8Array; + seedManifestSha256: string; + }>; +}>; + +export type NativeExtensionPackage = Readonly<{ + sqlName: string; + product: string; + version: string; + packageJson: string; + aotPackageJson?: string; }>; export type NativeWasixServerListen = @@ -84,16 +104,11 @@ export type NativeWasixAddon = { runtimeVersion(): string; supportedProfiles(): readonly NativeProfile[]; payloadIdentity( - component: - | 'runtimeArchive' - | 'standardSeedArchive' - | 'standardSeedManifest' - | 'icuDataArchive' - | 'icuSeedArchive' - | 'icuSeedManifest', + component: 'runtimeArchive' | 'standardSeedArchive' | 'standardSeedManifest', ): string; extensionIdentity(sqlName: string): string; toolIdentity(name: 'pg_dump' | 'psql'): string; + registerTools(options: { packageJson: string; aotPackageJson: string }): void; }; type WasixPackageMetadata = Readonly<{ @@ -362,6 +377,7 @@ export function validateNativeWasixAddon( typeof addon.supportedProfiles !== 'function' || typeof addon.payloadIdentity !== 'function' || typeof addon.extensionIdentity !== 'function' || + typeof addon.registerTools !== 'function' || typeof addon.toolIdentity !== 'function' ) { throw new Error(`Oliphaunt WASIX native addon ${path} has an invalid export surface`); diff --git a/src/bindings/wasix-ts/src/native-extension-packages.ts b/src/bindings/wasix-ts/src/native-extension-packages.ts new file mode 100644 index 000000000..8c8f195af --- /dev/null +++ b/src/bindings/wasix-ts/src/native-extension-packages.ts @@ -0,0 +1,90 @@ +import { readFileSync, realpathSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { arch, platform } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { nativeTarget, type NativeExtensionPackage } from './native-addon.js'; +import type { SerializedOpenOptions } from './rpc.js'; +import type { WasixToolProcessOptions } from './tool-runtime.js'; + +export function nativeToolPackage(tool: WasixToolProcessOptions['tool']): { + packageJson: string; + aotPackageJson: string; +} { + if (typeof tool.source !== 'string' || !tool.source.startsWith('file:')) { + throw new Error('WASIX native tools require an installed package file URL'); + } + const modulePath = realpathSync(fileURLToPath(tool.source)); + const packageJson = join(dirname(dirname(modulePath)), 'package.json'); + const manifest = JSON.parse(readFileSync(packageJson, 'utf8')); + const name = '@oliphaunt/liboliphaunt-wasix-tools'; + const payload = manifest.oliphaunt?.tools?.[tool.name]; + if ( + manifest.name !== name || + manifest.oliphaunt?.kind !== 'wasix-tools' || + payload?.path !== `assets/${tool.name}.wasix.wasm` || + payload.sha256 !== tool.sha256 || + payload.size !== tool.size + ) { + throw new Error('WASIX tool descriptor does not match its installed package'); + } + const aotName = `${name}-${nativeTarget(platform(), arch()).id}`; + if (manifest.optionalDependencies?.[aotName] !== manifest.version) { + throw new Error('WASIX tools package has no exact host AOT dependency'); + } + return { + packageJson, + aotPackageJson: createRequire(packageJson).resolve(`${aotName}/package.json`), + }; +} + +/** Installed native dependencies have the same trust as the application's imports. */ +export function nativeExtensionPackages(options: SerializedOpenOptions): NativeExtensionPackage[] { + return Object.values(options.extensionCarriers) + .filter((carrier) => carrier.product !== 'oliphaunt-extension-contrib-pg18') + .map((carrier) => { + if (typeof carrier.source !== 'string' || !carrier.source.startsWith('file:')) { + throw new Error( + `WASIX native extension ${carrier.sqlName} requires an installed package file URL`, + ); + } + const archive = realpathSync(fileURLToPath(carrier.source)); + const root = dirname(dirname(dirname(archive))); + const packageJson = join(root, 'package.json'); + const manifest = JSON.parse(readFileSync(packageJson, 'utf8')); + const packageName = `@oliphaunt/${carrier.product.slice('oliphaunt-'.length)}-wasix`; + const payload = manifest.oliphaunt?.carriers?.[carrier.sqlName]; + if ( + manifest.name !== packageName || + manifest.version !== carrier.version || + manifest.oliphaunt?.product !== carrier.product || + manifest.oliphaunt?.kind !== 'exact-extension-wasix' || + manifest.oliphaunt?.wasixRuntimeVersion !== options.runtime.version || + payload?.path !== `extensions/${carrier.sqlName}/extension.tar.zst` || + payload.sha256 !== carrier.sha256 || + payload.size !== carrier.size || + typeof payload.requiresAot !== 'boolean' || + archive !== realpathSync(join(root, payload.path)) + ) { + throw new Error( + `WASIX extension ${carrier.sqlName} descriptor does not match its installed package`, + ); + } + let aotPackageJson: string | undefined; + if (payload.requiresAot) { + const target = nativeTarget(platform(), arch()); + const aotName = `${packageName}-${target.id}`; + if (manifest.optionalDependencies?.[aotName] !== carrier.version) { + throw new Error(`WASIX extension ${carrier.sqlName} has no exact host AOT dependency`); + } + aotPackageJson = createRequire(packageJson).resolve(`${aotName}/package.json`); + } + return { + sqlName: carrier.sqlName, + product: carrier.product, + version: carrier.version, + packageJson, + ...(aotPackageJson === undefined ? {} : { aotPackageJson }), + }; + }); +} diff --git a/src/bindings/wasix-ts/src/native-session.ts b/src/bindings/wasix-ts/src/native-session.ts index 4ef002bc4..7fc5a1c35 100644 --- a/src/bindings/wasix-ts/src/native-session.ts +++ b/src/bindings/wasix-ts/src/native-session.ts @@ -1,3 +1,5 @@ +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; import { WasixStorageError, type WasixStorageCommitState, @@ -28,6 +30,7 @@ import type { WasixToolProcessResult, } from './tool-runtime.js'; import { validateWasixToolDescriptor } from './tool-runtime.js'; +import { nativeExtensionPackages, nativeToolPackage } from './native-extension-packages.js'; /** @internal A synchronous Rust Oliphaunt owned by the importing JavaScript realm. */ export class NativeWasixSession implements WasixDatabaseSession { @@ -135,18 +138,7 @@ export class NativeWasixSession implements WasixDatabaseSession { async runTool(options: WasixToolProcessOptions): Promise { this.#assertOpen(); - if (options.runtimeVersion !== '' && options.runtimeVersion !== this.#runtimeVersion) { - throw new Error( - `WASIX tools runtime ${options.runtimeVersion} is incompatible with database runtime ${this.#runtimeVersion}`, - ); - } - validateWasixToolDescriptor(options.tool); - const expectedIdentity = `${options.tool.sha256}:${options.tool.size}`; - if (this.#addon.toolIdentity(options.tool.name) !== expectedIdentity) { - throw new Error( - `WASIX ${options.tool.name} descriptor does not match the tool embedded in the native addon`, - ); - } + validateNativeToolCall(this.#addon, this.#runtimeVersion, options); if (options.tool.name === 'pg_dump') { try { return toolProcessResult( @@ -471,30 +463,11 @@ export function requireCompatibleNativeWasixAddon( options.runtime.standardSeedManifest, 'standard cluster seed manifest', ); - if (options.icu !== undefined) { - requireEmbeddedPayloadIdentity( - addon, - 'icuDataArchive', - options.icu.dataArchive, - 'ICU data archive', - ); - requireEmbeddedPayloadIdentity( - addon, - 'icuSeedArchive', - options.icu.clusterSeedArchive, - 'ICU cluster seed archive', - ); - requireEmbeddedPayloadIdentity( - addon, - 'icuSeedManifest', - options.icu.clusterSeedManifest, - 'ICU cluster seed manifest', - ); - } for (const [sqlName, carrier] of Object.entries(options.extensionCarriers)) { if (carrier.sqlName !== sqlName) { throw new Error(`WASIX extension carrier key ${sqlName} does not match ${carrier.sqlName}`); } + if (carrier.product !== 'oliphaunt-extension-contrib-pg18') continue; const expectedIdentity = `${carrier.sha256}:${carrier.size}`; if (addon.extensionIdentity(sqlName) !== expectedIdentity) { throw new Error( @@ -524,14 +497,41 @@ export function nativeWasixOpenOptions( const identity = normalizeWasixDatabaseIdentity(options.username, options.database); return { profile: options.icu === undefined ? 'standard' : 'icu', + ...(options.icu === undefined + ? {} + : { + icu: { + version: options.icu.version, + runtimeVersion: options.icu.compatibility.runtimeVersion, + archive: nativeIcuBytes(options.icu.dataArchive.source), + archiveSha256: options.icu.dataArchive.sha256, + dataTreeSha256: options.icu.compatibility.dataTreeSha256, + seedArchive: nativeIcuBytes(options.icu.clusterSeedArchive.source), + seedArchiveSha256: options.icu.clusterSeedArchive.sha256, + seedManifest: nativeIcuBytes(options.icu.clusterSeedManifest.source), + seedManifestSha256: options.icu.clusterSeedManifest.sha256, + }, + }), storage, username: identity.username, database: identity.database, startupGucs: { ...options.startupGUCs }, extensions: [...options.extensions], + ...(Object.values(options.extensionCarriers).some( + (carrier) => carrier.product !== 'oliphaunt-extension-contrib-pg18', + ) + ? { extensionPackages: nativeExtensionPackages(options) } + : {}), }; } +function nativeIcuBytes(source: string | Uint8Array): Uint8Array { + if (source instanceof Uint8Array) return Buffer.from(source); + if (!source.startsWith('file:')) + throw new TypeError('WASIX native ICU data requires an installed file URL or bytes'); + return readFileSync(fileURLToPath(source)); +} + function nativeStorage(options: SerializedOpenOptions): NativeWasixOpenOptions['storage'] { if (options.storage.kind === 'memory') return { kind: 'memory' }; if (options.storage.kind === 'directory') { @@ -541,6 +541,8 @@ function nativeStorage(options: SerializedOpenOptions): NativeWasixOpenOptions[' throw new TypeError(`@oliphaunt/wasix-ts ${provider} storage is browser-only`); } +const registeredTools = new WeakMap>(); + function validateNativeToolCall( addon: NativeWasixAddon, runtimeVersion: string, @@ -552,10 +554,17 @@ function validateNativeToolCall( ); } validateWasixToolDescriptor(options.tool); + const key = `${options.tool.name}:${options.tool.sha256}:${options.tool.source}`; + const registered = registeredTools.get(addon) ?? new Set(); + if (!registered.has(key)) { + addon.registerTools(nativeToolPackage(options.tool)); + registered.add(key); + registeredTools.set(addon, registered); + } const expectedIdentity = `${options.tool.sha256}:${options.tool.size}`; if (addon.toolIdentity(options.tool.name) !== expectedIdentity) { throw new Error( - `WASIX ${options.tool.name} descriptor does not match the tool embedded in the native addon`, + `WASIX ${options.tool.name} descriptor does not match the tool in the installed package`, ); } } diff --git a/src/bindings/wasix-ts/src/public.ts b/src/bindings/wasix-ts/src/public.ts index fc7607ec2..13372568a 100644 --- a/src/bindings/wasix-ts/src/public.ts +++ b/src/bindings/wasix-ts/src/public.ts @@ -1,3 +1,4 @@ +export * as extensions from '@oliphaunt/extension-contrib-pg18-wasix'; export { type WasixStorageCommitState, WasixStorageError, diff --git a/src/bindings/wasix-ts/tools/package.mjs b/src/bindings/wasix-ts/tools/package.mjs index 97cc7cb11..e3b1af75b 100755 --- a/src/bindings/wasix-ts/tools/package.mjs +++ b/src/bindings/wasix-ts/tools/package.mjs @@ -43,6 +43,7 @@ export function prepareWasixTypescriptPackage(packageDir) { ...(manifest.dependencies ?? {}), [JS_CORE_PACKAGE]: coreManifest.version, [RUNTIME]: runtimeVersion, + '@oliphaunt/extension-contrib-pg18-wasix': runtimeVersion, }).sort(), ); manifest.optionalDependencies = Object.fromEntries(NATIVE.map((name) => [name, nativeVersion])); diff --git a/src/bindings/wasix-ts/tsconfig.json b/src/bindings/wasix-ts/tsconfig.json index a840cb7c3..85d50269e 100644 --- a/src/bindings/wasix-ts/tsconfig.json +++ b/src/bindings/wasix-ts/tsconfig.json @@ -2,7 +2,13 @@ "compilerOptions": { "declaration": true, "declarationMap": false, - "lib": ["ES2023", "ESNext.Disposable", "DOM", "DOM.Iterable", "WebWorker"], + "lib": [ + "ES2023", + "ESNext.Disposable", + "DOM", + "DOM.Iterable", + "WebWorker" + ], "module": "NodeNext", "moduleResolution": "NodeNext", "noEmit": true, @@ -11,14 +17,27 @@ "noUncheckedIndexedAccess": true, "outDir": "lib", "paths": { - "@oliphaunt/liboliphaunt-wasix": ["./src/runtime-carrier-shim.d.ts"] + "@oliphaunt/liboliphaunt-wasix": [ + "./src/runtime-carrier-shim.d.ts" + ], + "@oliphaunt/extension-contrib-pg18-wasix": [ + "./src/contrib-carrier-shim.d.ts" + ] }, "rootDir": "src", "skipLibCheck": true, "strict": true, "target": "ES2022", - "types": ["node", "vitest/globals"] + "types": [ + "node", + "vitest/globals" + ] }, - "include": ["src/**/*"], - "exclude": ["lib", "node_modules"] + "include": [ + "src/**/*" + ], + "exclude": [ + "lib", + "node_modules" + ] } diff --git a/src/bindings/wasix-ts/vitest.config.ts b/src/bindings/wasix-ts/vitest.config.ts index a46fbc5fd..0dc32f536 100644 --- a/src/bindings/wasix-ts/vitest.config.ts +++ b/src/bindings/wasix-ts/vitest.config.ts @@ -4,6 +4,9 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ resolve: { alias: { + "@oliphaunt/extension-contrib-pg18-wasix": fileURLToPath( + new URL("./src/__tests__/contrib-carrier.ts", import.meta.url), + ), "@oliphaunt/liboliphaunt-wasix": fileURLToPath( new URL("./src/__tests__/runtime-carrier.ts", import.meta.url), ), diff --git a/src/docs/content/learn/embedded-postgres.mdx b/src/docs/content/learn/embedded-postgres.mdx index f38fd65d7..19068ee90 100644 --- a/src/docs/content/learn/embedded-postgres.mdx +++ b/src/docs/content/learn/embedded-postgres.mdx @@ -57,10 +57,9 @@ process or local server is the better runtime shape. ## Extension Selection -Extensions are selected exactly before packaging or opening the database. -Native SDKs use SQL-name selectors, Rust WASIX uses exact Cargo features and -typed values, and WASIX TypeScript imports exact `-wasix` descriptors. App -artifacts include only the selected carrier closure. +Contrib ships with the base SDK, and external extensions come from independent +packages. Every SDK selects explicit contrib and external descriptors when +opening a database. Installed dependencies determine which artifacts ship. `CREATE EXTENSION` succeeds only when the selected runtime resources include that extension for the target platform. See the diff --git a/src/docs/content/reference/extensions.mdx b/src/docs/content/reference/extensions.mdx index 3a006146a..e7628d221 100644 --- a/src/docs/content/reference/extensions.mdx +++ b/src/docs/content/reference/extensions.mdx @@ -1,129 +1,112 @@ --- title: Extensions -description: Select exact PostgreSQL extensions through each SDK's native API and verify the carriers that enter an app. +description: Install external packages independently and explicitly select external and contrib descriptors in each SDK. --- # Extensions -Oliphaunt uses exact, opt-in PostgreSQL extension selection. Native SDKs accept -exact SQL names, Rust WASIX exposes exact Cargo features and typed values, and -WASIX TypeScript accepts selectively imported portable descriptors. Browser -artifacts contain only the selected extensions plus mandatory dependencies -declared by extension metadata. The Node/Bun/Deno/Electron Node-API carrier embeds the -qualified catalog once and validates each selected descriptor against those -exact bytes. - -There are no extension packs, aliases, or grouped selectors. Selection remains -exact even when a native carrier physically contains the full catalog. +The base SDK includes PostgreSQL, supported contrib extensions, and the standard +cluster seed. External extensions come from independently versioned packages. +Both contrib and external extensions require explicit descriptors in database +configuration. Installing a package makes its bytes available; migrations own +`CREATE EXTENSION`, `LOAD`, and upgrades. -## Native selection +## Native Rust -Select extensions before opening the database: +```toml +[dependencies] +oliphaunt = "0.2" +oliphaunt-extension-vector = "0.2" +``` ```rust -use oliphaunt::{Extension, Oliphaunt}; +use oliphaunt::{extensions, Oliphaunt}; +use oliphaunt_extension_vector::VECTOR; -# fn demo() -> oliphaunt::Result<()> { let mut db = Oliphaunt::builder() - .direct() - .extension(Extension::VECTOR) + .extensions([VECTOR, extensions::HSTORE]) .open()?; - -db.execute("CREATE EXTENSION vector")?; -# Ok(()) -# } +db.execute("CREATE EXTENSION vector; CREATE EXTENSION hstore")?; ``` -`CREATE EXTENSION` succeeds when the selected runtime resources contain that -extension for the target platform. The SDK loads only the selected extension -artifacts and their declared dependencies. - -## Rust WASIX selection +The SDK handles target artifacts and embedding internally. Ordinary applications +need no build script, build dependency, resource registration, or package metadata. -Enable the exact extension feature and pass its typed value to the WASIX -builder. The builder makes the artifact available; application migrations must -still install database-local objects explicitly: +## Rust WASIX ```toml [dependencies] -oliphaunt-wasix = { version = "0.1", features = ["extension-pgtap"] } +oliphaunt-wasix = "0.2" +oliphaunt-extension-vector-wasix = "0.2" ``` ```rust -use oliphaunt_wasix::{Extension, Oliphaunt}; +use oliphaunt_wasix::{extensions, Oliphaunt}; +use oliphaunt_extension_vector_wasix::VECTOR; -let mut database = Oliphaunt::builder().extension(Extension::PGTAP).open()?; -database.execute("CREATE EXTENSION pgtap")?; +let mut db = Oliphaunt::builder() + .extensions([VECTOR, extensions::HSTORE]) + .open()?; +db.execute("CREATE EXTENSION vector; CREATE EXTENSION hstore")?; ``` -## WASIX TypeScript selection +External versions are ordinary Cargo dependency versions. There are no +per-extension SDK Cargo features. + +## TypeScript -Import only the portable descriptors the browser, Node, Bun, Deno, or Electron application uses: +For native Node, Bun, or Deno, install `@oliphaunt/ts` and +`@oliphaunt/extension-vector`: ```ts -import Oliphaunt from '@oliphaunt/wasix-ts'; -import pgtap from '@oliphaunt/extension-pgtap-wasix'; +import { Oliphaunt, extensions } from '@oliphaunt/ts'; +import vector from '@oliphaunt/extension-vector'; -const database = await Oliphaunt.open({ extensions: [pgtap] }); +const db = await Oliphaunt.open({ extensions: [vector, extensions.hstore] }); ``` -The descriptor carries the exact dependency and compatibility metadata needed -by the selected runtime owner. SQL-name strings are intentionally not accepted -by this API. Use only extensions published for the selected WASIX host. - -## Platform Behavior - -| Platform | Expected behavior | -| --- | --- | -| Rust/Tauri desktop | SDK resolves selected runtime extension artifacts for the target | -| iOS/macOS Swift | App bundle includes selected extension artifacts and dependencies only | -| Android Kotlin | Android package includes selected extension artifacts and dependencies only | -| React Native | Config plugin delegates selection to Swift/Kotlin packaging | -| TypeScript | SDK resolves selected native artifacts or helper-process resources | -| Rust WASIX | Exact Cargo features carry selected portable artifacts; typed values select them at open | -| WASIX TypeScript | Selective `-wasix` imports carry exact descriptors and browser bytes; Node, Bun, Deno, and Electron validate those descriptors against extensions embedded in the matching Node-API carrier | - -## Dependencies - -Some PostgreSQL extensions depend on other extensions or runtime files. Those -dependencies are explicit metadata. If `earthdistance` declares `cube` as a -dependency, selecting `earthdistance` may include `cube`; selecting `vector` -includes `vector` and its declared dependencies only. - -## External Extensions - -External extensions are distributed as exact extension artifacts or indexes. -Native consumers select SQL names; WASIX consumers use the runtime-specific -typed value or descriptor published for that extension. - -Each public external extension has its own product tag, changelog, and package -version. The PostgreSQL contrib bundle is only a logical distribution: its -native and WASIX carriers inherit the corresponding runtime product version. -Exact compatibility metadata pins a consumer to a published dependency version -without causing either product to release. External extension packages own -independent packaging SemVer; their immutable upstream version/commit and -compatible Oliphaunt runtime versions are separate metadata. -Do not assume an external package version matches either its upstream project -version or the runtime version. - -The WASIX carrier uses the explicit `-wasix` identifier while the native/default -package keeps its existing name. Both carriers belong to the same extension -product version stream; `-wasix` does not create an unrelated second extension -release line. - -## Verifying App Artifacts - -Before release, app tooling reports: - -- selected SQL extension names; -- included extension files; -- mandatory dependencies; -- package-size contribution per extension; -- target platform and architecture. - -That report lets developers confirm that an app using only `vector` ships -`vector` and its declared dependencies, without unrelated extension artifacts. +For WASIX, install `@oliphaunt/wasix-ts` and +`@oliphaunt/extension-vector-wasix`: + +```ts +import { Oliphaunt, extensions } from '@oliphaunt/wasix-ts'; +import vector from '@oliphaunt/extension-vector-wasix'; + +const db = await Oliphaunt.open({ extensions: [vector, extensions.hstore] }); +``` + +On native hosts, TypeScript resolves the installed package and passes its +payload to Rust through N-API. Rust validates compatibility, target, and hashes +before loading it. The base addon contains runtime and contrib bytes; external +extension, ICU, and frontend tool bytes remain in their optional packages. + +## Mobile packaging + +SwiftPM products and Android Gradle dependencies determine which external +packages ship with an application. React Native uses the same native npm +extension packages as Node; its Expo plugin discovers installed dependencies +and arranges platform resources. Each database still selects external and +contrib descriptors explicitly. Rebuild mobile apps after changing native +dependencies. + +See the [Swift](/docs/sdk/swift/guide), [Kotlin and Java](/docs/sdk/kotlin/guide), +and [React Native](/docs/sdk/react-native/guide) guides for installation examples. + +## Dependencies and versions + +Mandatory extension dependencies are part of the declared package contract. +Selecting `earthdistance`, for example, also makes its required `cube` +resources available. SQL migrations still create the corresponding objects. + +Each external extension has its own product tag, changelog, and package version. +The native package and its `-wasix` counterpart share that version stream. +Contrib carriers inherit the corresponding runtime version. Package versions, +upstream extension versions, and compatible runtime versions are separate +identities. + +ICU is optional and owns both its data and matching cluster seeds. Installing it +does not select it for every database; pass the ICU descriptor explicitly. diff --git a/src/docs/content/sdk/index.mdx b/src/docs/content/sdk/index.mdx index d53686a01..542a59da4 100644 --- a/src/docs/content/sdk/index.mdx +++ b/src/docs/content/sdk/index.mdx @@ -56,7 +56,7 @@ subpaths add a local endpoint on Node, Bun, Deno, and Electron. | Which package do I install? | Package identity determines native versus WASIX runtime ownership. | | Which host runs PostgreSQL? | Native process, Rust WASIX host, browser realm, Rust owner, direct realm, and Worker placements have different lifecycle behavior. | | What is the storage default? | Sensible defaults avoid configuration while explicit storage opts into persistence. | -| How do I select extensions? | Native SDKs select exact SQL names; WASIX TypeScript imports exact portable descriptors. | +| How do I select extensions? | SDKs explicitly select contrib and external descriptors; external packages are independently versioned. | | Which APIs are actually present? | Server, tools, transaction-helper, and data-movement support differs by product. | ## Where To Go Next diff --git a/src/docs/content/sdk/kotlin/guide.mdx b/src/docs/content/sdk/kotlin/guide.mdx index 3dc07cbc5..86b185da7 100644 --- a/src/docs/content/sdk/kotlin/guide.mdx +++ b/src/docs/content/sdk/kotlin/guide.mdx @@ -33,11 +33,9 @@ plugins { dependencies { implementation("dev.oliphaunt:oliphaunt-android:0.2.0") + implementation("dev.oliphaunt.extensions:oliphaunt-extension-vector:0.2.0") } -oliphaunt { - selectedExtensions.add("vector") -} ``` Storage defaults to an SDK-owned temporary directory. Choose an app-private @@ -54,6 +52,9 @@ coroutines. ```kotlin +import dev.oliphaunt.* +import dev.oliphaunt.extensions.vector.Vector + val database = Oliphaunt.open( context = context, @@ -61,7 +62,7 @@ val database = storage = DatabaseStorage.Directory( context.filesDir.resolve("main.oliphaunt").absolutePath, ), - extensions = listOf("vector"), + extensions = listOf(Vector.descriptor, Extensions.HSTORE), ), ) @@ -181,8 +182,8 @@ safely rollbackable is not automatically wrapped in either composite exception. ### Select extensions -Select exact SQL extension names before opening the database. Android artifacts -contain only those selected extensions and mandatory dependencies. +Declare external packages as Gradle dependencies. The plugin packages them with +base contrib; each database explicitly selects descriptors in its configuration. diff --git a/src/docs/content/sdk/kotlin/index.mdx b/src/docs/content/sdk/kotlin/index.mdx index 1a4826cce..a880b2b33 100644 --- a/src/docs/content/sdk/kotlin/index.mdx +++ b/src/docs/content/sdk/kotlin/index.mdx @@ -25,22 +25,21 @@ plugins { dependencies { implementation("dev.oliphaunt:oliphaunt-android:0.2.0") + implementation("dev.oliphaunt.extensions:oliphaunt-extension-vector:0.2.0") } -oliphaunt { - selectedExtensions.add("vector") -} ``` -The Gradle plugin verifies and packages selected native runtime artifacts, -Android ABIs, and exact extension files. The app ships only the selected -extensions plus declared dependencies. +The Gradle plugin reads resolved dependencies for each Android variant. The app +ships PostgreSQL and contrib with the SDK, plus external packages you install. ## Open And Query Open a database from a coroutine: ```kotlin +import dev.oliphaunt.Extensions +import dev.oliphaunt.extensions.vector.Vector import dev.oliphaunt.DatabaseStorage import dev.oliphaunt.Oliphaunt import dev.oliphaunt.OliphauntConfig @@ -51,7 +50,7 @@ val database = Oliphaunt.open( storage = DatabaseStorage.Directory( applicationContext.filesDir.resolve("main.oliphaunt").absolutePath, ), - extensions = listOf("vector"), + extensions = listOf(Vector.descriptor, Extensions.HSTORE), ), ) @@ -73,8 +72,8 @@ first open. - Store persistent data in app-private storage unless the app deliberately exports a backup. -- Select exact SQL extension names at build/configuration time so the APK or AAB - contains only selected extension artifacts and declared dependencies. +- Use Gradle dependencies to choose shipped external packages, then select + external and contrib descriptors for each database. - Use SDK backup and restore APIs for archive validation and destination materialization. diff --git a/src/docs/content/sdk/react-native/architecture.mdx b/src/docs/content/sdk/react-native/architecture.mdx index d7d9fbc2f..f005f76c9 100644 --- a/src/docs/content/sdk/react-native/architecture.mdx +++ b/src/docs/content/sdk/react-native/architecture.mdx @@ -26,10 +26,12 @@ same direct-mode semantics on each mobile OS. ## JavaScript surface ```ts -import Oliphaunt from '@oliphaunt/react-native'; +import Oliphaunt, { directory, extensions } from '@oliphaunt/react-native'; +import vector from '@oliphaunt/extension-vector'; +import { Paths } from 'expo-file-system'; await using database = await Oliphaunt.open({ - storage: { kind: 'applicationData', name: 'main' }, + storage: directory(Paths.document.uri + 'main.oliphaunt'), startupGUCs: { application_name: 'mobile-app' }, }); @@ -66,9 +68,8 @@ payloads binary end to end. ## Storage and lifecycle -Omitted storage uses an SDK-owned temporary directory. `applicationData(name)` -lets native platform code resolve an app-private path; `directory(path)` is for -applications that already own one. Each path names a managed root, not PGDATA. +Omitted storage uses an SDK-owned temporary directory. Pass an absolute native +path or local file URI from your filesystem library to `directory()` for persistence. Each path names a managed root, not PGDATA. The bridge owns one mobile direct database at a time. It rejects another open while one is pending, active, or closing. App lifecycle policy calls `cancel()` @@ -79,7 +80,7 @@ a new or existing-empty destination and does not replace data. ## Extensions and packaging -The config plugin selects exact extension artifacts at build time. `open()` -repeats the selected SQL names at runtime so the platform SDK validates that -the installed app contains them. PostgreSQL `CREATE EXTENSION` remains the SQL +The config plugin reads installed dependencies to package base contrib and +external artifacts. `open()` accepts explicit descriptors for each database and +validates that the app contains their matching artifacts. PostgreSQL `CREATE EXTENSION` remains the SQL activation step. diff --git a/src/docs/content/sdk/react-native/guide.mdx b/src/docs/content/sdk/react-native/guide.mdx index 3e0366f5f..25b490255 100644 --- a/src/docs/content/sdk/react-native/guide.mdx +++ b/src/docs/content/sdk/react-native/guide.mdx @@ -31,7 +31,8 @@ resources before JavaScript calls `Oliphaunt.open()`. Expo apps: ```sh -npx expo install @oliphaunt/react-native +npx expo install @oliphaunt/react-native expo-file-system +npm install @oliphaunt/extension-vector npx expo prebuild npx expo run:ios npx expo run:android @@ -43,14 +44,7 @@ artifacts are included in the app build. ```json { "expo": { - "plugins": [ - [ - "@oliphaunt/react-native", - { - "extensions": ["vector"] - } - ] - ] + "plugins": ["@oliphaunt/react-native"] } } ``` @@ -71,11 +65,13 @@ needs the handle. ```ts -import Oliphaunt from '@oliphaunt/react-native'; +import Oliphaunt, { directory, extensions } from '@oliphaunt/react-native'; +import vector from '@oliphaunt/extension-vector'; +import { Paths } from 'expo-file-system'; const db = await Oliphaunt.open({ - storage: { kind: 'applicationData', name: 'main' }, - extensions: ['vector'], + storage: directory(Paths.document.uri + 'main.oliphaunt'), + extensions: [vector, extensions.hstore], }); const rows = await db.query('SELECT 1::text AS value'); @@ -136,16 +132,15 @@ PostgreSQL wire messages directly. ### Configure Configure storage, selected exact extensions, PostgreSQL startup GUCs, and -startup identity through the JS API and config plugin. Build-time extension selection -controls what ships in the app bundle; runtime storage controls where app data -lives. +startup identity through the JS API. Installed dependencies determine which +resources ship in the app; the config plugin discovers them automatically. Keep build-time and runtime settings separate. The config plugin controls native artifacts in the installed app. `Oliphaunt.open()` controls storage, startup identity, GUCs, and extension activation for that app run. Omit `storage` for an SDK-owned temporary directory. Use -`{ kind: 'applicationData', name: 'main' }` for normal persistent mobile data, -or `{ kind: 'directory', path }` when the app already owns a platform path. +`directory(Paths.document.uri + 'main.oliphaunt')` with Expo FileSystem for persistent mobile data, +or `directory(path)` with another filesystem library. @@ -183,8 +178,8 @@ Direct mobile mode can logically reopen inside one resident app process. ### Select extensions -Select exact SQL extension names in configuration. The native app packages -include only selected extensions plus declared dependencies. `CREATE EXTENSION` +Install external packages as app dependencies. Base contrib ships with the SDK; +select both external and contrib descriptors explicitly in database configuration. `CREATE EXTENSION` succeeds when the selected runtime resources contain that extension for the target platform. diff --git a/src/docs/content/sdk/react-native/index.mdx b/src/docs/content/sdk/react-native/index.mdx index cde28ada8..da72eabe5 100644 --- a/src/docs/content/sdk/react-native/index.mdx +++ b/src/docs/content/sdk/react-native/index.mdx @@ -20,32 +20,26 @@ installed-app integration. Install the package and build a development client or native app binary: ```sh -npx expo install @oliphaunt/react-native +npx expo install @oliphaunt/react-native expo-file-system +npm install @oliphaunt/extension-vector ``` Oliphaunt includes native Swift and Kotlin code, so React Native apps run it from an Expo development build or a native app binary. The config plugin selects the native runtime and exact extension artifacts that ship in the app. -Configure selected extensions in app config: +Enable the plugin; it discovers native resource packages from installed dependencies: ```json { "expo": { - "plugins": [ - [ - "@oliphaunt/react-native", - { - "extensions": ["vector"] - } - ] - ] + "plugins": ["@oliphaunt/react-native"] } } ``` Rebuild the development client or native app binary after changing native -runtime or extension selections. +runtime or extension dependencies. ## Open And Query @@ -53,11 +47,13 @@ Open from TypeScript and keep the handle in app state, a data service, or a provider that matches your navigation lifetime: ```ts -import Oliphaunt from '@oliphaunt/react-native'; +import Oliphaunt, { directory, extensions } from '@oliphaunt/react-native'; +import vector from '@oliphaunt/extension-vector'; +import { Paths } from 'expo-file-system'; const db = await Oliphaunt.open({ - storage: { kind: 'applicationData', name: 'main' }, - extensions: ['vector'], + storage: directory(Paths.document.uri + 'main.oliphaunt'), + extensions: [vector, extensions.hstore], }); const rows = await db.query('SELECT 1::text AS value'); @@ -80,10 +76,10 @@ are queued through the platform SDK. ## App Responsibilities - Build with a native app binary or development client. -- Omit `storage` for temporary work, or select `applicationData` for persistent - app data without constructing platform-specific paths. -- Select exact SQL extension names so only selected extension artifacts and - declared dependencies enter the iOS or Android app artifact. +- Omit `storage` for temporary work, or pass an app-private path from your + filesystem library to `directory()` for persistent app data. +- Install external packages as dependencies, then explicitly select external + and contrib descriptors for each database. - Use SDK backup and restore APIs for user-visible export/import flows instead of copying platform storage directories from JavaScript. diff --git a/src/docs/content/sdk/rust/guide.mdx b/src/docs/content/sdk/rust/guide.mdx index 6a2e0c9d6..38254df9f 100644 --- a/src/docs/content/sdk/rust/guide.mdx +++ b/src/docs/content/sdk/rust/guide.mdx @@ -34,7 +34,8 @@ through configuration: ```toml [dependencies] -oliphaunt = "0.1" +oliphaunt = "0.2" +oliphaunt-extension-vector = "0.2" ``` @@ -48,13 +49,14 @@ close. ```rust -use oliphaunt::{DatabaseStorage, Extension, Oliphaunt}; +use oliphaunt::{extensions, DatabaseStorage, Oliphaunt}; +use oliphaunt_extension_vector::VECTOR; fn open_database() -> oliphaunt::Result<()> { let mut db = Oliphaunt::builder() .storage(DatabaseStorage::Directory(".oliphaunt".into())) .direct() - .extension(Extension::VECTOR) + .extensions([VECTOR, extensions::HSTORE]) .open()?; let rows = db.query("SELECT 1::text AS value")?; @@ -227,9 +229,10 @@ session reusable. Independent transport or recovery failure is ### Select extensions -Select exact SQL extension names before open. There are no packs, aliases, or -implicit selectors. If you select `vector`, the generated artifacts include -`vector` and only its declared dependencies. +Select explicit extension descriptors before open. There are no selection aliases or +implicit selectors. Selecting `vector` makes its package resources and declared +dependencies available to that database. The base SDK also carries supported contrib +resources; select their descriptors separately when a database needs them. diff --git a/src/docs/content/sdk/rust/index.mdx b/src/docs/content/sdk/rust/index.mdx index edad01128..b493663d4 100644 --- a/src/docs/content/sdk/rust/index.mdx +++ b/src/docs/content/sdk/rust/index.mdx @@ -23,7 +23,8 @@ Add the crate to your Rust app: ```toml [dependencies] -oliphaunt = "0.1" +oliphaunt = "0.2" +oliphaunt-extension-vector = "0.2" ``` ## Open And Query @@ -31,13 +32,14 @@ oliphaunt = "0.1" Then open a database in app-owned storage: ```rust -use oliphaunt::{DatabaseStorage, Extension, Oliphaunt}; +use oliphaunt::{extensions, DatabaseStorage, Oliphaunt}; +use oliphaunt_extension_vector::VECTOR; fn open_database() -> oliphaunt::Result<()> { let mut db = Oliphaunt::builder() .storage(DatabaseStorage::Directory("./app-data/main.oliphaunt".into())) .direct() - .extension(Extension::VECTOR) + .extensions([VECTOR, extensions::HSTORE]) .open()?; let rows = db.query("SELECT 1::text AS value")?; diff --git a/src/docs/content/sdk/swift/guide.mdx b/src/docs/content/sdk/swift/guide.mdx index b9d77f0a7..e3be78322 100644 --- a/src/docs/content/sdk/swift/guide.mdx +++ b/src/docs/content/sdk/swift/guide.mdx @@ -24,9 +24,19 @@ Add the Swift package in Xcode or `Package.swift`. The package includes the Swift API plus the platform runtime artifacts required for the selected target. ```swift -dependencies: [ - .package(url: "https://github.com/f0rr0/oliphaunt.git", from: "0.7.0") -] +// Package.swift +let package = Package( + name: "Example", + platforms: [.iOS(.v17), .macOS(.v14)], + dependencies: [ + .package(url: "https://github.com/f0rr0/oliphaunt.git", from: "0.7.0"), + .package(url: "https://github.com/f0rr0/oliphaunt-extension-vector.git", from: "0.2.0") + ], + targets: [.executableTarget(name: "Example", dependencies: [ + .product(name: "Oliphaunt", package: "oliphaunt"), + .product(name: "OliphauntExtensionVector", package: "oliphaunt-extension-vector") + ])] +) ``` Persistent storage lives under your app container. App users install your app; the @@ -43,6 +53,10 @@ close when the app no longer needs the handle. ```swift +import Foundation +import Oliphaunt +import OliphauntExtensionVector + let appSupport = FileManager.default.urls( for: .applicationSupportDirectory, in: .userDomainMask @@ -51,7 +65,7 @@ let appSupport = FileManager.default.urls( let database = try await OliphauntDatabase.open( configuration: OliphauntConfiguration( storage: .directory(appSupport.appending(path: "main.oliphaunt")), - extensions: ["vector"] + extensions: [OliphauntExtensionVector.descriptor, OliphauntExtensions.hstore] ) ) @@ -166,8 +180,8 @@ wrapped in either composite error. ### Select extensions -Select exact SQL extension names in app configuration. The app bundle contains -selected extension artifacts plus required dependencies. `CREATE EXTENSION` +Install external SwiftPM products and select their descriptors alongside contrib +descriptors in database configuration. `CREATE EXTENSION` succeeds when the selected runtime resources contain that extension for the Apple target. diff --git a/src/docs/content/sdk/swift/index.mdx b/src/docs/content/sdk/swift/index.mdx index 5e8299f13..ba5a9b208 100644 --- a/src/docs/content/sdk/swift/index.mdx +++ b/src/docs/content/sdk/swift/index.mdx @@ -18,7 +18,19 @@ this SDK, so the Swift lifecycle model is the Apple behavior source. Add the Swift package from Xcode or `Package.swift`: ```swift -.package(url: "https://github.com/f0rr0/oliphaunt.git", from: "0.7.0") +// Package.swift +let package = Package( + name: "Example", + platforms: [.iOS(.v17), .macOS(.v14)], + dependencies: [ + .package(url: "https://github.com/f0rr0/oliphaunt.git", from: "0.7.0"), + .package(url: "https://github.com/f0rr0/oliphaunt-extension-vector.git", from: "0.2.0") + ], + targets: [.executableTarget(name: "Example", dependencies: [ + .product(name: "Oliphaunt", package: "oliphaunt"), + .product(name: "OliphauntExtensionVector", package: "oliphaunt-extension-vector") + ])] +) ``` The package carries the Swift API and the native runtime artifacts for supported @@ -30,6 +42,7 @@ Open a database from an async context: ```swift import Oliphaunt +import OliphauntExtensionVector let appSupport = FileManager.default.urls( for: .applicationSupportDirectory, @@ -39,7 +52,7 @@ let appSupport = FileManager.default.urls( let database = try await OliphauntDatabase.open( configuration: OliphauntConfiguration( storage: .directory(appSupport.appending(path: "main.oliphaunt")), - extensions: ["vector"] + extensions: [OliphauntExtensionVector.descriptor, OliphauntExtensions.hstore] ) ) @@ -60,8 +73,8 @@ against the resident backend and preserves transaction ordering. - Use `.directory(url)` for persistent user data. Omit it for SDK-owned temporary storage. -- Select exact SQL extension names so the app bundle contains only selected - extension artifacts and declared dependencies. +- Install external packages and explicitly select their descriptors alongside + contrib descriptors when opening each database. - Use backup and restore APIs for export, import, and user-visible data movement. diff --git a/src/docs/content/sdk/typescript/guide.mdx b/src/docs/content/sdk/typescript/guide.mdx index 344ac5a9e..abda95326 100644 --- a/src/docs/content/sdk/typescript/guide.mdx +++ b/src/docs/content/sdk/typescript/guide.mdx @@ -42,7 +42,7 @@ Install the npm package. Runtime assets and helper executables resolve through package configuration. ```sh -npm install @oliphaunt/ts +npm install @oliphaunt/ts @oliphaunt/extension-vector ``` @@ -56,11 +56,13 @@ database. ```ts -import { Oliphaunt } from '@oliphaunt/ts'; +import { extensions, Oliphaunt } from '@oliphaunt/ts'; +import { directory } from '@oliphaunt/ts/storage/node'; +import vector from '@oliphaunt/extension-vector'; const db = await Oliphaunt.open({ - storage: { kind: 'directory', path: './app-data/main.oliphaunt' }, - extensions: ['vector'], + storage: directory('./app-data/main.oliphaunt'), + extensions: [vector, extensions.hstore], }); const rows = await db.query('SELECT 1::text AS value'); @@ -181,8 +183,9 @@ rejecting. ### Select extensions -Select exact SQL extension names before open. Generated resources include only -selected extensions and mandatory dependencies. +Install external packages and select their descriptors alongside contrib descriptors +when opening a database. Base contrib ships with the SDK; external bytes come +from their independently versioned packages. diff --git a/src/docs/content/sdk/typescript/index.mdx b/src/docs/content/sdk/typescript/index.mdx index bbda367b5..b03a42cfc 100644 --- a/src/docs/content/sdk/typescript/index.mdx +++ b/src/docs/content/sdk/typescript/index.mdx @@ -24,7 +24,7 @@ delegates through Swift and Kotlin. Install the package from npm: ```sh -npm install @oliphaunt/ts +npm install @oliphaunt/ts @oliphaunt/extension-vector ``` npm is the native-runtime distribution for Node.js, Bun, and Deno. Deno imports @@ -39,11 +39,13 @@ delegates through Swift and Kotlin. Open a database from TypeScript: ```ts -import { Oliphaunt } from '@oliphaunt/ts'; +import { extensions, Oliphaunt } from '@oliphaunt/ts'; +import { directory } from '@oliphaunt/ts/storage/node'; +import vector from '@oliphaunt/extension-vector'; const db = await Oliphaunt.open({ - storage: { kind: 'directory', path: './app-data/main.oliphaunt' }, - extensions: ['vector'], + storage: directory('./app-data/main.oliphaunt'), + extensions: [vector, extensions.hstore], }); const rows = await db.query('SELECT 1::text AS value'); diff --git a/src/docs/content/sdk/wasix-rust/dump-restore.mdx b/src/docs/content/sdk/wasix-rust/dump-restore.mdx index 0e28b2cfd..871936e0d 100644 --- a/src/docs/content/sdk/wasix-rust/dump-restore.mdx +++ b/src/docs/content/sdk/wasix-rust/dump-restore.mdx @@ -112,13 +112,13 @@ oliphaunt-wasix-dump --directory ./.oliphaunt ``` Select a non-default database or user with `--database` and `--username`. If -the root uses installed extensions, repeat `--extension NAME` for every -extension that must be mounted before PostgreSQL starts: +the root uses contrib extensions, repeat `--extension NAME` for each required +contrib member. External descriptors are supplied through the Rust API: ```sh oliphaunt-wasix-dump --directory ./.oliphaunt \ --database app --username owner \ - --extension vector --extension pg_trgm + --extension hstore --extension pg_trgm ``` Pass through normal `pg_dump` shaping flags after `--`: diff --git a/src/docs/content/sdk/wasix-rust/guide.mdx b/src/docs/content/sdk/wasix-rust/guide.mdx index 3a3ccac74..026a511b9 100644 --- a/src/docs/content/sdk/wasix-rust/guide.mdx +++ b/src/docs/content/sdk/wasix-rust/guide.mdx @@ -178,25 +178,24 @@ another backend. ### Select extensions -Enable only the exact Cargo features the application uses, then select the -matching typed extension value on the builder. Selection makes the artifact -and required pre-start settings available; it does not run `CREATE EXTENSION`, -`LOAD`, or migration SQL. Optional extension payloads are not part of the core -runtime. The matching associated constant exists only when that leaf feature is -enabled. `Extension::ALL` and `Extension::by_sql_name` enumerate only the -enabled set, so an unavailable artifact cannot be selected and fail later at -runtime. +Contrib ships with the base SDK. Install external packages independently and +select descriptors for both. Selection makes resources available; migrations +still own `CREATE EXTENSION` and `LOAD`. ```toml [dependencies] -oliphaunt-wasix = { version = "0.1", features = ["extension-pgtap"] } +oliphaunt-wasix = "0.2" +oliphaunt-extension-vector-wasix = "0.2" ``` ```rust -use oliphaunt_wasix::{Extension, Oliphaunt}; +use oliphaunt_wasix::{extensions, Oliphaunt}; +use oliphaunt_extension_vector_wasix::VECTOR; -let mut database = Oliphaunt::builder().extension(Extension::PGTAP).open()?; -database.execute("CREATE EXTENSION pgtap")?; +let mut database = Oliphaunt::builder() + .extensions([VECTOR, extensions::HSTORE]) + .open()?; +database.execute("CREATE EXTENSION vector; CREATE EXTENSION hstore")?; ``` @@ -277,7 +276,7 @@ An independent pump/recovery failure is ## Troubleshooting Check whether the selected managed root is new or complete, the packaged -cluster seed and exact runtime/AOT asset pair, enabled extension features, +cluster seed and exact runtime/AOT asset pair, selected extension packages, directory ownership, and SQLSTATE-bearing PostgreSQL errors. Browser, Node, Bun, Deno, and Electron actor/direct/Worker behavior, the explicit `/worker` entry point, IndexedDB, and recovery behavior are documented diff --git a/src/docs/content/sdk/wasix-typescript/guide.mdx b/src/docs/content/sdk/wasix-typescript/guide.mdx index 39d784d7d..0b58224fe 100644 --- a/src/docs/content/sdk/wasix-typescript/guide.mdx +++ b/src/docs/content/sdk/wasix-typescript/guide.mdx @@ -176,19 +176,22 @@ database remains open and must be closed after the callback settles. ### Select extensions -Import exact WASIX extension descriptors and pass them to `open()`: +Install `@oliphaunt/extension-vector-wasix` separately. Contrib ships with the +base SDK; select both descriptors explicitly: ```ts -import pgtap from '@oliphaunt/extension-pgtap-wasix'; +import { Oliphaunt, extensions } from '@oliphaunt/wasix-ts'; +import vector from '@oliphaunt/extension-vector-wasix'; -const database = await Oliphaunt.open({ extensions: [pgtap] }); -await database.execute('CREATE EXTENSION pgtap'); +const database = await Oliphaunt.open({ extensions: [vector, extensions.hstore] }); +await database.execute('CREATE EXTENSION vector; CREATE EXTENSION hstore'); ``` In browsers, selection materializes only the verified carrier artifacts and required startup/preload configuration. On Node.js, Bun, Deno, and Electron, -the addon embeds the frozen qualified catalog and validates each selected -descriptor before resolving its SQL name to those exact compiled artifacts. +the addon receives the installed package payloads and validates their owner, +runtime compatibility, target, and hashes before loading them. External bytes +are owned by their packages; the base addon includes the runtime and contrib. Reopen persistent data with the same selection. Neither path silently runs `CREATE EXTENSION`, `LOAD`, schema, post-create, upgrade, or migration SQL. Applications and ORM migrations own that ordinary PostgreSQL lifecycle. diff --git a/src/docs/src/components/home-interactive.tsx b/src/docs/src/components/home-interactive.tsx index 1605d4c29..12e4f80aa 100644 --- a/src/docs/src/components/home-interactive.tsx +++ b/src/docs/src/components/home-interactive.tsx @@ -396,10 +396,11 @@ database.close()`, brand: 'react', packageName: '@oliphaunt/react-native', language: 'typescript', - code: `import { Oliphaunt } from '@oliphaunt/react-native'; + code: `import { Oliphaunt, directory } from '@oliphaunt/react-native'; +import { Paths } from 'expo-file-system'; const db = await Oliphaunt.open({ - storage: { kind: 'applicationData', name: 'main' }, + storage: directory(Paths.document.uri + 'main.oliphaunt'), extensions: [], }); diff --git a/src/docs/src/components/oliphaunt.tsx b/src/docs/src/components/oliphaunt.tsx index ba4378077..3004e8c56 100644 --- a/src/docs/src/components/oliphaunt.tsx +++ b/src/docs/src/components/oliphaunt.tsx @@ -270,7 +270,8 @@ const embeddedModelRows = [ }, { title: 'Exact extensions', - description: 'Apps select SQL extension names before packaging or opening the database.', + description: + 'Apps install external packages and explicitly select extension descriptors before opening.', icon: ShieldCheck, }, { @@ -702,8 +703,8 @@ const extensionFlow = [ title: 'Select exact extensions', description: ( <> - Use the native SDK's SQL-name form, Rust WASIX typed values, or imported WASIX TypeScript - descriptors. + Install external packages through your ecosystem package manager, then select external and + contrib descriptors explicitly for each database. ), }, @@ -1584,7 +1585,6 @@ const firstQueryExamples = [ packageName: '@oliphaunt/ts', code: `const db = await Oliphaunt.open({ storage: { kind: 'directory', path: 'main.oliphaunt' }, - extensions: ['vector'], }); const rows = await db.query('select 1 as ready'); @@ -1596,7 +1596,6 @@ await db.close();`, code: `let mut db = Oliphaunt::builder() .storage(DatabaseStorage::Directory("main.oliphaunt".into())) .direct() - .extension(Extension::VECTOR) .open()?; db.query("select 1 as ready")?; @@ -1607,8 +1606,7 @@ db.close()?;`, packageName: 'Oliphaunt', code: `let db = try await OliphauntDatabase.open( configuration: OliphauntConfiguration( - storage: .directory(appStorage.appending(path: "main.oliphaunt")), - extensions: ["vector"] + storage: .directory(appStorage.appending(path: "main.oliphaunt")) ) ) @@ -1624,11 +1622,11 @@ export function FirstQueryFlow() {

First query shape

- The same storage, extension, query, and lifecycle concepts in ecosystem-native syntax. + The same storage, query, and lifecycle concepts in ecosystem-native syntax.

    - {['Product', 'Storage', 'Extensions', 'Query', 'Close'].map((item, index) => ( + {['Product', 'Storage', 'Query', 'Close'].map((item, index) => (
  1. {String(index + 1).padStart(2, '0')} @@ -1861,9 +1859,9 @@ export function ExactExtensionRule() {

    Extension selection is exact and product-native.

    - Native SDKs select exact SQL names, Rust WASIX uses exact Cargo features and typed - values, and WASIX TypeScript imports exact -wasix descriptors. Every form - includes only the selected carrier closure. + Every SDK explicitly selects contrib and external descriptors. Contrib ships with the + base SDK; external packages own their bytes and versions. Package dependencies determine + what ships, and database configuration determines what is used.

    diff --git a/src/docs/tools/check-docs-product.mjs b/src/docs/tools/check-docs-product.mjs index 9036624a0..ba7327245 100644 --- a/src/docs/tools/check-docs-product.mjs +++ b/src/docs/tools/check-docs-product.mjs @@ -835,13 +835,11 @@ function assertReferencePageCoverage() { title: 'Extensions', components: ['ExactExtensionRule', 'ExtensionArtifactFlow'], headings: [ - 'Native selection', - 'Rust WASIX selection', - 'WASIX TypeScript selection', - 'Platform Behavior', - 'Dependencies', - 'External Extensions', - 'Verifying App Artifacts', + 'Native Rust', + 'Rust WASIX', + 'TypeScript', + 'Mobile packaging', + 'Dependencies and versions', ], }, { diff --git a/src/extensions/external/pg_hashids/release.toml b/src/extensions/external/pg_hashids/release.toml index 9343649a9..b8522373e 100644 --- a/src/extensions/external/pg_hashids/release.toml +++ b/src/extensions/external/pg_hashids/release.toml @@ -23,7 +23,12 @@ registry_packages = [ "npm:@oliphaunt/extension-pg-hashids-linux-x64-gnu", "npm:@oliphaunt/extension-pg-hashids-macos-arm64", "npm:@oliphaunt/extension-pg-hashids-wasix", + "npm:@oliphaunt/extension-pg-hashids-wasix-linux-arm64-gnu", + "npm:@oliphaunt/extension-pg-hashids-wasix-linux-x64-gnu", + "npm:@oliphaunt/extension-pg-hashids-wasix-macos-arm64", + "npm:@oliphaunt/extension-pg-hashids-wasix-windows-x64-msvc", "npm:@oliphaunt/extension-pg-hashids-windows-x64-msvc", + "maven:dev.oliphaunt.extensions:oliphaunt-extension-pg-hashids", "maven:dev.oliphaunt.extensions:oliphaunt-extension-pg-hashids-android-arm64-v8a", "maven:dev.oliphaunt.extensions:oliphaunt-extension-pg-hashids-android-x86_64", ] diff --git a/src/extensions/external/pg_ivm/release.toml b/src/extensions/external/pg_ivm/release.toml index f2bf46f0c..963859cff 100644 --- a/src/extensions/external/pg_ivm/release.toml +++ b/src/extensions/external/pg_ivm/release.toml @@ -23,7 +23,12 @@ registry_packages = [ "npm:@oliphaunt/extension-pg-ivm-linux-x64-gnu", "npm:@oliphaunt/extension-pg-ivm-macos-arm64", "npm:@oliphaunt/extension-pg-ivm-wasix", + "npm:@oliphaunt/extension-pg-ivm-wasix-linux-arm64-gnu", + "npm:@oliphaunt/extension-pg-ivm-wasix-linux-x64-gnu", + "npm:@oliphaunt/extension-pg-ivm-wasix-macos-arm64", + "npm:@oliphaunt/extension-pg-ivm-wasix-windows-x64-msvc", "npm:@oliphaunt/extension-pg-ivm-windows-x64-msvc", + "maven:dev.oliphaunt.extensions:oliphaunt-extension-pg-ivm", "maven:dev.oliphaunt.extensions:oliphaunt-extension-pg-ivm-android-arm64-v8a", "maven:dev.oliphaunt.extensions:oliphaunt-extension-pg-ivm-android-x86_64", ] diff --git a/src/extensions/external/pg_textsearch/release.toml b/src/extensions/external/pg_textsearch/release.toml index 53ebc0370..43b58cba4 100644 --- a/src/extensions/external/pg_textsearch/release.toml +++ b/src/extensions/external/pg_textsearch/release.toml @@ -23,7 +23,12 @@ registry_packages = [ "npm:@oliphaunt/extension-pg-textsearch-linux-x64-gnu", "npm:@oliphaunt/extension-pg-textsearch-macos-arm64", "npm:@oliphaunt/extension-pg-textsearch-wasix", + "npm:@oliphaunt/extension-pg-textsearch-wasix-linux-arm64-gnu", + "npm:@oliphaunt/extension-pg-textsearch-wasix-linux-x64-gnu", + "npm:@oliphaunt/extension-pg-textsearch-wasix-macos-arm64", + "npm:@oliphaunt/extension-pg-textsearch-wasix-windows-x64-msvc", "npm:@oliphaunt/extension-pg-textsearch-windows-x64-msvc", + "maven:dev.oliphaunt.extensions:oliphaunt-extension-pg-textsearch", "maven:dev.oliphaunt.extensions:oliphaunt-extension-pg-textsearch-android-arm64-v8a", "maven:dev.oliphaunt.extensions:oliphaunt-extension-pg-textsearch-android-x86_64", ] diff --git a/src/extensions/external/pg_uuidv7/release.toml b/src/extensions/external/pg_uuidv7/release.toml index 019f090ad..f7b94c221 100644 --- a/src/extensions/external/pg_uuidv7/release.toml +++ b/src/extensions/external/pg_uuidv7/release.toml @@ -23,7 +23,12 @@ registry_packages = [ "npm:@oliphaunt/extension-pg-uuidv7-linux-x64-gnu", "npm:@oliphaunt/extension-pg-uuidv7-macos-arm64", "npm:@oliphaunt/extension-pg-uuidv7-wasix", + "npm:@oliphaunt/extension-pg-uuidv7-wasix-linux-arm64-gnu", + "npm:@oliphaunt/extension-pg-uuidv7-wasix-linux-x64-gnu", + "npm:@oliphaunt/extension-pg-uuidv7-wasix-macos-arm64", + "npm:@oliphaunt/extension-pg-uuidv7-wasix-windows-x64-msvc", "npm:@oliphaunt/extension-pg-uuidv7-windows-x64-msvc", + "maven:dev.oliphaunt.extensions:oliphaunt-extension-pg-uuidv7", "maven:dev.oliphaunt.extensions:oliphaunt-extension-pg-uuidv7-android-arm64-v8a", "maven:dev.oliphaunt.extensions:oliphaunt-extension-pg-uuidv7-android-x86_64", ] diff --git a/src/extensions/external/pgtap/release.toml b/src/extensions/external/pgtap/release.toml index a8254531c..6467a09ab 100644 --- a/src/extensions/external/pgtap/release.toml +++ b/src/extensions/external/pgtap/release.toml @@ -20,6 +20,7 @@ registry_packages = [ "npm:@oliphaunt/extension-pgtap-macos-arm64", "npm:@oliphaunt/extension-pgtap-wasix", "npm:@oliphaunt/extension-pgtap-windows-x64-msvc", + "maven:dev.oliphaunt.extensions:oliphaunt-extension-pgtap", "maven:dev.oliphaunt.extensions:oliphaunt-extension-pgtap-android-arm64-v8a", "maven:dev.oliphaunt.extensions:oliphaunt-extension-pgtap-android-x86_64", ] diff --git a/src/extensions/external/postgis/release.toml b/src/extensions/external/postgis/release.toml index 8096c464a..1fc503998 100644 --- a/src/extensions/external/postgis/release.toml +++ b/src/extensions/external/postgis/release.toml @@ -23,7 +23,12 @@ registry_packages = [ "npm:@oliphaunt/extension-postgis-linux-x64-gnu", "npm:@oliphaunt/extension-postgis-macos-arm64", "npm:@oliphaunt/extension-postgis-wasix", + "npm:@oliphaunt/extension-postgis-wasix-linux-arm64-gnu", + "npm:@oliphaunt/extension-postgis-wasix-linux-x64-gnu", + "npm:@oliphaunt/extension-postgis-wasix-macos-arm64", + "npm:@oliphaunt/extension-postgis-wasix-windows-x64-msvc", "npm:@oliphaunt/extension-postgis-windows-x64-msvc", + "maven:dev.oliphaunt.extensions:oliphaunt-extension-postgis", "maven:dev.oliphaunt.extensions:oliphaunt-extension-postgis-android-arm64-v8a", "maven:dev.oliphaunt.extensions:oliphaunt-extension-postgis-android-x86_64", ] diff --git a/src/extensions/external/vector/release.toml b/src/extensions/external/vector/release.toml index 7c25de556..e21d48c1c 100644 --- a/src/extensions/external/vector/release.toml +++ b/src/extensions/external/vector/release.toml @@ -23,7 +23,12 @@ registry_packages = [ "npm:@oliphaunt/extension-vector-linux-x64-gnu", "npm:@oliphaunt/extension-vector-macos-arm64", "npm:@oliphaunt/extension-vector-wasix", + "npm:@oliphaunt/extension-vector-wasix-linux-arm64-gnu", + "npm:@oliphaunt/extension-vector-wasix-linux-x64-gnu", + "npm:@oliphaunt/extension-vector-wasix-macos-arm64", + "npm:@oliphaunt/extension-vector-wasix-windows-x64-msvc", "npm:@oliphaunt/extension-vector-windows-x64-msvc", + "maven:dev.oliphaunt.extensions:oliphaunt-extension-vector", "maven:dev.oliphaunt.extensions:oliphaunt-extension-vector-android-arm64-v8a", "maven:dev.oliphaunt.extensions:oliphaunt-extension-vector-android-x86_64", ] diff --git a/src/extensions/tools/check-extension-model.py b/src/extensions/tools/check-extension-model.py index 856edfb3d..6f23beb30 100755 --- a/src/extensions/tools/check-extension-model.py +++ b/src/extensions/tools/check-extension-model.py @@ -57,8 +57,10 @@ ROOT / "src/sdks/kotlin/oliphaunt/src/generated/extensions.json", ROOT / "src/sdks/react-native/src/generated/extensions.json", ) +GENERATED_RUST_CONTRIB_MODULE = ROOT / "src/sdks/rust/src/extensions.rs" GENERATED_RUST_SDK_MODULE = ROOT / "src/sdks/rust/src/generated/extensions.rs" GENERATED_TS_SDK_MODULE = ROOT / "src/sdks/js/src/generated/extensions.ts" +GENERATED_SWIFT_SDK_MODULE = ROOT / "src/sdks/swift/Sources/Oliphaunt/OliphauntExtensions.swift" GENERATED_KOTLIN_SDK_MODULE = ROOT / "src/sdks/kotlin/oliphaunt/src/commonMain/kotlin/dev/oliphaunt/GeneratedExtensions.kt" GENERATED_KOTLIN_GRADLE_PLUGIN_CATALOG = ( ROOT @@ -1328,6 +1330,13 @@ def generated_mobile_extension_smoke_module(metadata: dict) -> str: "sqlName": row["sql-name"], "createsExtension": row["creates-extension"], "selectedExtensionDependencies": row["selected-extension-dependencies"], + "descriptor": { + "schema": "oliphaunt-native-extension-v1", + "sqlName": row["sql-name"], + "product": row["artifact-product"], + "packageName": row["npm-package"], + **({} if row["runtime-bound"] else {"version": (ROOT / "src/extensions/external" / row["id"] / "VERSION").read_text().strip()}), + }, } for row in metadata.get("extensions", []) ] @@ -1342,11 +1351,14 @@ def generated_mobile_extension_smoke_module(metadata: dict) -> str: source = ( f"// This file is generated by {CHECK_EXTENSION_MODEL_PATH}.\n" "// Do not edit by hand. It belongs only to installed mobile qualification.\n\n" + "import type { NativeExtensionDescriptor } from '@oliphaunt/react-native';\n\n" "export type GeneratedMobileExtensionProof = {\n" " readonly sqlName: string;\n" + " readonly descriptor: NativeExtensionDescriptor;\n" " readonly createsExtension: boolean;\n" " readonly selectedExtensionDependencies: readonly string[];\n" "};\n\n" + f"export const GENERATED_MOBILE_ICU = {{ schema: 'oliphaunt-native-icu-v1', packageName: '@oliphaunt/icu', version: {json.dumps((ROOT / 'src/runtimes/liboliphaunt/native/VERSION').read_text().strip())} }} as const;\n\n" f"export const GENERATED_MOBILE_EXTENSION_METADATA_SHA256 = {json.dumps(metadata['extension-catalog-sha256'])} as const;\n\n" f"export const GENERATED_MOBILE_EXTENSION_PLAN = {json.dumps(extension_plan, indent=2)} as const satisfies readonly GeneratedMobileExtensionProof[];\n\n" f"export const GENERATED_MOBILE_EXTENSION_SMOKE = {json.dumps(recipes, indent=2, sort_keys=True)} as const satisfies Readonly>;\n" @@ -1354,6 +1366,50 @@ def generated_mobile_extension_smoke_module(metadata: dict) -> str: return format_typescript_source(source, GENERATED_MOBILE_SMOKE_MODULE) +def generated_wasix_contrib_shim(metadata: dict) -> str: + return "// Generated by check-extension-model.py.\n" + "\n".join( + "export declare const " + row["id"] + ": import('./types.js').WasixExtensionDescriptor;" + for row in sorted(metadata["extensions"], key=lambda row: row["id"]) if row["runtime-bound"] + ) + "\n" + + +def generated_rust_contrib_module(metadata: dict) -> str: + body = "\n".join( + "/// Select the `" + row["sql-name"] + "` contrib extension.\n" + + "pub const " + row["id"].upper() + ": crate::ExtensionDescriptor = crate::ExtensionDescriptor {\n" + + " sql_name: " + json.dumps(row["sql-name"]) + ",\n" + + " product: " + json.dumps(row["artifact-product"]) + ",\n" + + " version: None,\n runtime_version: env!(\"OLIPHAUNT_NATIVE_RUNTIME_VERSION\"),\n resources: &[],\n};\n" + for row in sorted(metadata["extensions"], key=lambda row: row["id"]) if row["runtime-bound"] + ) + return f"// This file is generated by {CHECK_EXTENSION_MODEL_PATH}.\n// Do not edit by hand.\n\n" + body + + +def generated_swift_extension_module(metadata: dict) -> str: + rows = sorted(metadata.get("extensions", []), key=lambda row: str(row["sql-name"])) + body = "\n".join( + " public static let " + row["id"].split("_")[0] + + "".join(part.title() for part in row["id"].split("_")[1:]) + + " = OliphauntExtension(sqlName: " + json.dumps(row["sql-name"]) + + ", product: " + json.dumps(row["artifact-product"]) + ")" + for row in rows if row["runtime-bound"] + ) + contracts = "\n".join( + " " + json.dumps(row["sql-name"]) + ": (dependencies: " + + json.dumps(row["selected-extension-dependencies"]) + ", sharedPreloads: " + + json.dumps(row["shared-preload-libraries"]) + ", module: " + + (json.dumps(row["native-module-stem"]) if row["native-module-stem"] is not None else "nil") + ")," + for row in rows + ) + return ( + f"// This file is generated by {CHECK_EXTENSION_MODEL_PATH}.\n" + "// Do not edit by hand.\n\n" + "public enum OliphauntExtensions {\n" + body + "\n}\n\n" + + "let oliphauntExtensionRuntimeContracts: [String: (dependencies: [String], sharedPreloads: [String], module: String?)] = [\n" + + contracts + "\n]\n" + ) + + def generated_kotlin_extension_module(metadata: dict) -> str: rows = sorted(metadata.get("extensions", []), key=lambda row: str(row["sql-name"])) body = "\n".join( @@ -1363,16 +1419,28 @@ def generated_kotlin_extension_module(metadata: dict) -> str: + f"createsExtension = {'true' if row['creates-extension'] else 'false'}, " + "nativeModuleStem = " + (json.dumps(str(row["native-module-stem"])) if row["native-module-stem"] is not None else "null") + + ", dependencies = listOf(" + ", ".join(json.dumps(name) for name in row["selected-extension-dependencies"]) + ")" + + ", sharedPreloads = listOf(" + ", ".join(json.dumps(name) for name in row["shared-preload-libraries"]) + ")" + ")," for row in rows ) + contrib = "\n\n".join( + " @JvmField public val " + row["id"].upper() + " = ExtensionDescriptor(" + + json.dumps(row["sql-name"]) + ", " + json.dumps(row["artifact-product"]) + ")" + for row in rows if row["runtime-bound"] + ) return ( f"// This file is generated by {CHECK_EXTENSION_MODEL_PATH}.\n" "// Do not edit by hand.\n\n" "package dev.oliphaunt\n\n" + "public object Extensions {\n" + f"{contrib}\n" + "}\n\n" "internal data class GeneratedExtensionRuntimeContract(\n" " val createsExtension: Boolean,\n" " val nativeModuleStem: String?,\n" + " val dependencies: List = emptyList(),\n" + " val sharedPreloads: List = emptyList(),\n" ")\n\n" "internal val generatedExtensionRuntimeContracts: Map = mapOf(\n" f"{body}\n" @@ -1944,6 +2012,10 @@ def validate_generated_file(path: Path, expected: dict, write: bool) -> None: def validate_generated_sdk_metadata(catalog: dict, write: bool) -> None: metadata = generated_sdk_metadata(catalog) + validate_generated_text_file( + ROOT / "src/bindings/wasix-ts/src/contrib-carrier-shim.d.ts", + generated_wasix_contrib_shim(metadata), write, + ) ios_static_dependencies = generated_ios_static_dependencies(catalog) validate_generated_file(GENERATED_SDK_METADATA, metadata, write) validate_generated_file( @@ -1981,6 +2053,16 @@ def validate_generated_sdk_metadata(catalog: dict, write: bool) -> None: generated_kotlin_extension_module(metadata), write, ) + validate_generated_text_file( + GENERATED_RUST_CONTRIB_MODULE, + generated_rust_contrib_module(metadata), + write, + ) + validate_generated_text_file( + GENERATED_SWIFT_SDK_MODULE, + generated_swift_extension_module(metadata), + write, + ) validate_generated_text_file( GENERATED_KOTLIN_GRADLE_PLUGIN_CATALOG, generated_kotlin_gradle_plugin_catalog(metadata), diff --git a/src/runtimes/liboliphaunt/icu/Cargo.toml b/src/runtimes/liboliphaunt/icu/Cargo.toml index f8d4c7f6a..cba1c30e7 100644 --- a/src/runtimes/liboliphaunt/icu/Cargo.toml +++ b/src/runtimes/liboliphaunt/icu/Cargo.toml @@ -8,7 +8,7 @@ readme = "README.md" repository = "https://github.com/f0rr0/oliphaunt" homepage = "https://oliphaunt.dev" documentation = "https://docs.rs/oliphaunt-icu" -license = "MIT AND Unicode-3.0" +license = "MIT AND PostgreSQL AND Unicode-3.0" links = "oliphaunt_artifact_oliphaunt_icu" build = "build.rs" include = [ @@ -21,11 +21,15 @@ include = [ "THIRD_PARTY_NOTICES.md", "THIRD_PARTY_NOTICES.oliphaunt-wasix.md", "THIRD_PARTY_LICENSES/ICU-LICENSE", + "THIRD_PARTY_LICENSES/PostgreSQL-COPYRIGHT", ] [lib] path = "src/lib.rs" +[dependencies] +oliphaunt-resources = { version = "0.2.0", path = "../../../sdks/rust/crates/oliphaunt-resources" } + [build-dependencies] sha2 = "0.10" tar = "0.4" diff --git a/src/runtimes/liboliphaunt/icu/README.md b/src/runtimes/liboliphaunt/icu/README.md index 3c2e9d1d0..f3d9b1918 100644 --- a/src/runtimes/liboliphaunt/icu/README.md +++ b/src/runtimes/liboliphaunt/icu/README.md @@ -1,6 +1,19 @@ # oliphaunt-icu -Optional ICU data files for Oliphaunt runtimes. +Optional ICU data and matching PostgreSQL catalog seeds for Oliphaunt. -Applications add this package only when they need PostgreSQL ICU collations. -Ordinary Oliphaunt runtime carriers do not include ICU data. +```toml +[dependencies] +oliphaunt = "0.2" +oliphaunt-icu = "0.2" +``` + +```rust +let db = oliphaunt::Oliphaunt::builder() + .icu(oliphaunt_icu::ICU) + .open()?; +``` + +The package embeds the current native target's seed and ICU data. WASIX +consumers use the package's WASIX descriptor with `oliphaunt-wasix` instead. +Base runtime carriers include only the standard seed. diff --git a/src/runtimes/liboliphaunt/icu/build.rs b/src/runtimes/liboliphaunt/icu/build.rs index 84b515488..82da64bf6 100644 --- a/src/runtimes/liboliphaunt/icu/build.rs +++ b/src/runtimes/liboliphaunt/icu/build.rs @@ -12,6 +12,20 @@ const ARTIFACT_TARGET: &str = "portable"; const PACKAGED_ICU_ARCHIVE: &str = "payload/icu-data.tar.zst"; fn main() { + let native_version_file = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()) + .join("payload/native-runtime-version"); + println!("cargo:rerun-if-changed={}", native_version_file.display()); + let native_version = fs::read_to_string(&native_version_file) + .unwrap_or_else(|_| env::var("CARGO_PKG_VERSION").unwrap()); + let native_version = native_version.trim(); + assert!( + !native_version.is_empty() + && native_version + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b".-+".contains(&byte)), + "invalid native runtime version" + ); + println!("cargo:rustc-env=OLIPHAUNT_ICU_NATIVE_RUNTIME_VERSION={native_version}"); println!("cargo:rerun-if-env-changed=OLIPHAUNT_ICU_DATA_DIR"); println!("cargo:rerun-if-env-changed=OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD"); @@ -33,6 +47,7 @@ fn main() { ); } write_generated_icu(&out, None); + fs::write(out_dir.join("native_icu.rs"), "&[]\n").expect("write empty native ICU index"); } } @@ -40,7 +55,82 @@ fn emit_icu_artifact(out: &Path, out_dir: &Path, archive: &Path, icu_root: &Path let archive_sha256 = sha256_file(archive).expect("digest ICU data archive"); let data_tree_sha256 = logical_tree_sha256(icu_root).expect("digest ICU logical data tree"); write_generated_icu(out, Some((archive, &archive_sha256, &data_tree_sha256))); - emit_artifact_manifest(out_dir, icu_root, &data_tree_sha256); + let receipt = out_dir.join("native-icu.properties"); + fs::write(&receipt, format!("schema=oliphaunt-icu-data-v1\nartifactRole=icu-data\nicuDataVersion=76.1\nicuDataForm=files-le\nicuDataTreeSha256={data_tree_sha256}\n")).expect("write native ICU receipt"); + emit_artifact_manifest(out_dir, icu_root, &receipt); + let mut native = String::from("&[\n"); + for file in collect_files(icu_root) + .expect("collect native ICU files") + .into_iter() + .chain([receipt.clone()]) + { + let relative = if file == receipt { + "manifest.properties".to_owned() + } else { + format!( + "share/icu/{}", + file.strip_prefix(icu_root) + .expect("ICU file path") + .to_string_lossy() + .replace('\\', "/") + ) + }; + let digest = sha256_file(&file).expect("hash native ICU file"); + native.push_str(&format!( + "({:?}, include_bytes!({:?}), {digest:?}, false),\n", + format!("icu-data/oliphaunt-icu/{relative}"), + file + )); + } + if let Some((target, seed_root)) = native_seed_root() { + println!("cargo:rerun-if-changed={}", seed_root.display()); + if env::var_os("OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD").is_some() { + assert!( + seed_root.join("manifest.properties").is_file(), + "native ICU seed missing for {target}" + ); + } + for directory in + collect_directories(&seed_root).expect("collect native ICU seed directories") + { + let relative = directory + .strip_prefix(&seed_root) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + let name = format!("icu-data/oliphaunt-icu/native-seeds/{target}/{relative}/"); + native.push_str(&format!("({name:?}, b\"\", \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\", false),\n")); + } + for file in collect_files(&seed_root).expect("collect native ICU seed files") { + let relative = file + .strip_prefix(&seed_root) + .expect("seed file path") + .to_string_lossy() + .replace('\\', "/"); + let digest = sha256_file(&file).expect("hash native ICU seed file"); + native.push_str(&format!( + "({:?}, include_bytes!({:?}), {digest:?}, false),\n", + format!("icu-data/oliphaunt-icu/native-seeds/{target}/{relative}"), + file + )); + } + } + native.push_str("]\n"); + fs::write(out_dir.join("native_icu.rs"), native).expect("write native ICU resource index"); +} + +fn native_seed_root() -> Option<(&'static str, PathBuf)> { + let target = match env::var("TARGET").unwrap_or_default().as_str() { + "x86_64-unknown-linux-gnu" => "linux-x64-gnu", + "aarch64-unknown-linux-gnu" => "linux-arm64-gnu", + "aarch64-apple-darwin" => "macos-arm64", + "x86_64-pc-windows-msvc" => "windows-x64-msvc", + _ => return None, + }; + let root = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("manifest dir")) + .join("payload/native-seeds") + .join(target); + Some((target, root)) } fn find_packaged_icu_archive() -> Option { @@ -220,24 +310,42 @@ fn write_generated_icu(out: &Path, archive: Option<(&Path, &str, &str)>) { "pub const HAS_ICU_DATA: bool = true;\n\ pub const ICU_DATA_ARCHIVE_SHA256: Option<&str> = Some({archive_sha256:?});\n\ pub const ICU_DATA_TREE_SHA256: Option<&str> = Some({data_tree_sha256:?});\n\ - pub fn icu_data_archive() -> Option<&'static [u8]> {{ Some(include_bytes!({archive:?})) }}\n", + pub const fn icu_data_archive() -> Option<&'static [u8]> {{ Some(include_bytes!({archive:?})) }}\n", archive = archive.to_string_lossy(), ), None => "pub const HAS_ICU_DATA: bool = false;\n\ pub const ICU_DATA_ARCHIVE_SHA256: Option<&str> = None;\n\ pub const ICU_DATA_TREE_SHA256: Option<&str> = None;\n\ - pub fn icu_data_archive() -> Option<&'static [u8]> { None }\n" + pub const fn icu_data_archive() -> Option<&'static [u8]> { None }\n" .to_owned(), }; + let mut text = text; + for (name, file) in [ + ("ICU_SEED_ARCHIVE", "icu.tar.zst"), + ("ICU_SEED_MANIFEST", "icu.json"), + ] { + let seed = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("manifest dir")) + .join("payload/cluster-seeds") + .join(file); + println!("cargo:rerun-if-changed={}", seed.display()); + let body = if seed.is_file() { + format!("Some(include_bytes!({seed:?}))") + } else { + "None".into() + }; + text.push_str(&format!( + "pub const {name}: Option<&'static [u8]> = {body};\n" + )); + } fs::write(out, text).expect("write generated ICU data module"); } -fn emit_artifact_manifest(out_dir: &Path, icu_root: &Path, data_tree_sha256: &str) { +fn emit_artifact_manifest(out_dir: &Path, icu_root: &Path, receipt: &Path) { let version = env::var("CARGO_PKG_VERSION").expect("CARGO_PKG_VERSION is set by Cargo"); let manifest_path = out_dir.join("oliphaunt-artifact.toml"); let files = collect_files(icu_root).expect("collect ICU data files for manifest"); let mut text = format!( - "schema = {ARTIFACT_SCHEMA:?}\nproduct = {ARTIFACT_PRODUCT:?}\nversion = {version:?}\nkind = {ARTIFACT_KIND:?}\ntarget = {ARTIFACT_TARGET:?}\ndata_tree_sha256 = {data_tree_sha256:?}\ndata_version = \"76.1\"\ndata_form = \"files-le\"\n" + "schema = {ARTIFACT_SCHEMA:?}\nproduct = {ARTIFACT_PRODUCT:?}\nversion = {version:?}\nkind = {ARTIFACT_KIND:?}\ntarget = {ARTIFACT_TARGET:?}\n" ); for file in files { let relative = file @@ -253,10 +361,67 @@ fn emit_artifact_manifest(out_dir: &Path, icu_root: &Path, data_tree_sha256: &st sha256, )); } + let receipt_sha256 = sha256_file(receipt).expect("hash ICU receipt"); + text.push_str(&format!( + "\n[[files]]\nsource = {:?}\nrelative = \"manifest.properties\"\nsha256 = {receipt_sha256:?}\nexecutable = false\n", + receipt.display().to_string() + )); + if let Some((target, root)) = native_seed_root() { + let directories = collect_directories(&root) + .expect("collect seed directories") + .iter() + .map(|directory| { + format!( + "native-seeds/{target}/{}", + directory + .strip_prefix(&root) + .unwrap() + .to_string_lossy() + .replace('\\', "/") + ) + }) + .collect::>(); + let insert_at = text.find("\n[[files]]").unwrap_or(text.len()); + text.insert_str(insert_at, &format!("\ndirectories = {directories:?}\n")); + for file in collect_files(&root).expect("collect native ICU seed manifest files") { + let relative = file + .strip_prefix(&root) + .expect("seed relative path") + .to_string_lossy() + .replace('\\', "/"); + let sha256 = sha256_file(&file).expect("hash native ICU seed"); + text.push_str(&format!( + "\n[[files]]\nsource = {:?}\nrelative = {:?}\nsha256 = {:?}\nexecutable = false\n", + file.display().to_string(), + format!("native-seeds/{target}/{relative}"), + sha256 + )); + } + } fs::write(&manifest_path, text).expect("write ICU Cargo artifact manifest"); println!("cargo::metadata=manifest={}", manifest_path.display()); } +fn collect_directories(root: &Path) -> io::Result> { + let mut directories = Vec::new(); + if !root.exists() { + return Ok(directories); + } + for entry in fs::read_dir(root)? { + let entry = entry?; + let kind = entry.file_type()?; + if kind.is_symlink() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "seed symlink")); + } + if kind.is_dir() { + directories.push(entry.path()); + directories.extend(collect_directories(&entry.path())?); + } + } + directories.sort(); + Ok(directories) +} + fn collect_files(root: &Path) -> io::Result> { let mut files = Vec::new(); collect_files_inner(root, &mut files)?; diff --git a/src/runtimes/liboliphaunt/icu/src/lib.rs b/src/runtimes/liboliphaunt/icu/src/lib.rs index 8ed595a6b..d9cc5e262 100644 --- a/src/runtimes/liboliphaunt/icu/src/lib.rs +++ b/src/runtimes/liboliphaunt/icu/src/lib.rs @@ -2,6 +2,19 @@ include!(concat!(env!("OUT_DIR"), "/generated_icu.rs")); +/// Select this optional ICU package for a native or WASIX database. +pub const ICU: oliphaunt_resources::IcuData = oliphaunt_resources::IcuData { + version: env!("CARGO_PKG_VERSION"), + native_runtime_version: env!("OLIPHAUNT_ICU_NATIVE_RUNTIME_VERSION"), + runtime_version: env!("CARGO_PKG_VERSION"), + resources: include!(concat!(env!("OUT_DIR"), "/native_icu.rs")), + wasix_archive: icu_data_archive(), + wasix_archive_sha256: ICU_DATA_ARCHIVE_SHA256, + wasix_data_tree_sha256: ICU_DATA_TREE_SHA256, + wasix_seed_archive: ICU_SEED_ARCHIVE, + wasix_seed_manifest: ICU_SEED_MANIFEST, +}; + #[cfg(test)] mod tests { use super::*; diff --git a/src/runtimes/liboliphaunt/native/icu-npm/OliphauntICU.podspec b/src/runtimes/liboliphaunt/native/icu-npm/OliphauntICU.podspec index 1dba9a47d..2ab2ffe94 100644 --- a/src/runtimes/liboliphaunt/native/icu-npm/OliphauntICU.podspec +++ b/src/runtimes/liboliphaunt/native/icu-npm/OliphauntICU.podspec @@ -3,7 +3,7 @@ Pod::Spec.new do |s| s.version = '0.2.0' # x-release-please-version s.summary = 'Portable ICU data files for Oliphaunt runtimes.' s.homepage = 'https://oliphaunt.dev' - s.license = { :type => 'MIT AND Unicode-3.0' } + s.license = { :type => 'MIT AND PostgreSQL AND Unicode-3.0' } s.author = { 'Oliphaunt Maintainers' => 'https://github.com/f0rr0' } s.source = { :path => '.' } s.platforms = { :ios => '17.0', :osx => '14.0' } diff --git a/src/runtimes/liboliphaunt/native/icu-npm/README.md b/src/runtimes/liboliphaunt/native/icu-npm/README.md index 1bc99ffa1..a09f9f5c8 100644 --- a/src/runtimes/liboliphaunt/native/icu-npm/README.md +++ b/src/runtimes/liboliphaunt/native/icu-npm/README.md @@ -1,22 +1,26 @@ # @oliphaunt/icu -Portable ICU data files for Oliphaunt runtimes. +Optional ICU data and matching PostgreSQL catalog seeds for native Node, Bun, +Deno, and React Native. Install this package when you need ICU collations: -Install this package only when an application needs PostgreSQL ICU collations. -Ordinary Oliphaunt runtime packages do not include ICU data. +```sh +npm install @oliphaunt/icu +``` -The published package stores the ICU tree once under -`OliphauntICU.bundle/share/icu`, with its exact data receipt at -`OliphauntICU.bundle/manifest.properties`. Target-specific native runtime -packages carry their own matching PostgreSQL ICU-catalog cluster seed. Node, Bun, and -Deno consumers should resolve the data directory from -`oliphaunt.dataRelativePath` rather than hard-coding its location. +```ts +import icu from '@oliphaunt/icu'; -On Apple platforms, `OliphauntICU.podspec` installs the prebuilt -`OliphauntICU.bundle` as one resource. Copying the bundle atomically preserves -the ICU subdirectories and prevents locale files with the same basename from -colliding during the Xcode build. +const db = await Oliphaunt.open({ icu }); +``` -The carrier deliberately disables React Native autolinking on iOS and Android. -The `@oliphaunt/react-native` config plugin stages selected ICU data through its -app-owned native payload instead. +Import `Oliphaunt` from your SDK. Installing the package makes its resources +available; passing the descriptor selects ICU for that database. + +The package owns `OliphauntICU.bundle/share/icu` and the matching seeds under +`OliphauntICU.bundle/native-seeds`. The SDK resolves the current platform and +validates the seed against the data. Base runtime packages carry the standard +seed only. + +The React Native Expo plugin discovers this installed dependency and stages +its resources into the app. No additional plugin option or separate pod is +needed. Native Node, Bun, and Deno resolve resources from the imported package. diff --git a/src/runtimes/liboliphaunt/native/icu-npm/index.d.ts b/src/runtimes/liboliphaunt/native/icu-npm/index.d.ts new file mode 100644 index 000000000..f7ac04df1 --- /dev/null +++ b/src/runtimes/liboliphaunt/native/icu-npm/index.d.ts @@ -0,0 +1,8 @@ +declare const icu: Readonly<{ + schema: 'oliphaunt-native-icu-v1'; + packageName: '@oliphaunt/icu'; + version: string; + packageJsonUrl?: string; +}>; + +export = icu; diff --git a/src/runtimes/liboliphaunt/native/icu-npm/index.js b/src/runtimes/liboliphaunt/native/icu-npm/index.js new file mode 100644 index 000000000..cf46fd35a --- /dev/null +++ b/src/runtimes/liboliphaunt/native/icu-npm/index.js @@ -0,0 +1,7 @@ +const { version } = require('./package.json'); + +module.exports = Object.freeze({ + schema: 'oliphaunt-native-icu-v1', + packageName: '@oliphaunt/icu', + version, +}); diff --git a/src/runtimes/liboliphaunt/native/icu-npm/node.js b/src/runtimes/liboliphaunt/native/icu-npm/node.js new file mode 100644 index 000000000..79650dc0c --- /dev/null +++ b/src/runtimes/liboliphaunt/native/icu-npm/node.js @@ -0,0 +1,7 @@ +const descriptor = require('./index.js'); +const { pathToFileURL } = require('node:url'); + +module.exports = Object.freeze({ + ...descriptor, + packageJsonUrl: pathToFileURL(require.resolve('./package.json')).href, +}); diff --git a/src/runtimes/liboliphaunt/native/icu-npm/package.json b/src/runtimes/liboliphaunt/native/icu-npm/package.json index 12ce65942..dd69a5632 100644 --- a/src/runtimes/liboliphaunt/native/icu-npm/package.json +++ b/src/runtimes/liboliphaunt/native/icu-npm/package.json @@ -2,7 +2,7 @@ "name": "@oliphaunt/icu", "version": "0.2.0", "description": "Portable ICU data files for Oliphaunt runtimes.", - "license": "MIT AND Unicode-3.0", + "license": "MIT AND PostgreSQL AND Unicode-3.0", "type": "commonjs", "repository": { "type": "git", @@ -33,9 +33,20 @@ "LICENSE", "THIRD_PARTY_NOTICES.md", "THIRD_PARTY_NOTICES.liboliphaunt-native.md", - "THIRD_PARTY_LICENSES/ICU-LICENSE" + "THIRD_PARTY_LICENSES/PostgreSQL-COPYRIGHT", + "THIRD_PARTY_LICENSES/ICU-LICENSE", + "index.js", + "node.js", + "index.d.ts" ], "exports": { - "./package.json": "./package.json" - } + "./package.json": "./package.json", + ".": { + "types": "./index.d.ts", + "react-native": "./index.js", + "node": "./node.js", + "default": "./index.js" + } + }, + "sideEffects": false } diff --git a/src/runtimes/liboliphaunt/native/include/oliphaunt.h b/src/runtimes/liboliphaunt/native/include/oliphaunt.h index d96facff7..7f55b34bd 100644 --- a/src/runtimes/liboliphaunt/native/include/oliphaunt.h +++ b/src/runtimes/liboliphaunt/native/include/oliphaunt.h @@ -222,9 +222,11 @@ OLIPHAUNT_API int32_t oliphaunt_close(OliphauntHandle *handle); * Registers statically linked PostgreSQL extension modules for the embedded * backend's normal LOAD path. * - * Call this before oliphaunt_init in processes that link extension code directly - * into the application or SDK library. The registry is process-wide and becomes - * immutable once backend startup begins. Each extension name is the module stem + * Call this before loading a module linked into the application or SDK library. + * The process-wide registry permits additions after backend startup; repeating an + * identical entry is harmless, while replacing an existing module is rejected. + * Registration copies descriptors and retains them for the process lifetime. + * Each extension name is the module stem * used by SQL, for example AS 'vector', and each symbol row exposes the C * symbols PostgreSQL would otherwise resolve with dlsym(). */ diff --git a/src/runtimes/liboliphaunt/native/moon.yml b/src/runtimes/liboliphaunt/native/moon.yml index 264550cb1..4c60fe54c 100644 --- a/src/runtimes/liboliphaunt/native/moon.yml +++ b/src/runtimes/liboliphaunt/native/moon.yml @@ -327,6 +327,7 @@ tasks: - "/tools/release/native-mobile-abi-contract.mjs" - "/tools/release/stage-native-cluster-seed.mjs" - "/tools/release/finalize-native-runtime-carrier.mjs" + - "/tools/release/native-icu-seeds.mjs" - "/tools/release/strip_native_release_binaries.mjs" - "/tools/release/platform-binary-contract.mjs" - "@group(release-archive-contract)" @@ -379,6 +380,7 @@ tasks: - "/tools/release/native-mobile-abi-contract.mjs" - "/tools/release/stage-native-cluster-seed.mjs" - "/tools/release/finalize-native-runtime-carrier.mjs" + - "/tools/release/native-icu-seeds.mjs" - "/tools/release/strip_native_release_binaries.mjs" - "/tools/release/platform-binary-contract.mjs" - "@group(release-archive-contract)" @@ -435,6 +437,7 @@ tasks: - "/tools/release/native-mobile-abi-contract.mjs" - "/tools/release/stage-native-cluster-seed.mjs" - "/tools/release/finalize-native-runtime-carrier.mjs" + - "/tools/release/native-icu-seeds.mjs" - "/tools/release/strip_native_release_binaries.mjs" - "/tools/release/platform-binary-contract.mjs" - "/tools/release/validate-ios-carrier-zips.mjs" diff --git a/src/runtimes/liboliphaunt/native/smoke/liboliphaunt_static_extension_registry.c b/src/runtimes/liboliphaunt/native/smoke/liboliphaunt_static_extension_registry.c index 8db09b567..ab4282575 100644 --- a/src/runtimes/liboliphaunt/native/smoke/liboliphaunt_static_extension_registry.c +++ b/src/runtimes/liboliphaunt/native/smoke/liboliphaunt_static_extension_registry.c @@ -96,6 +96,23 @@ int main(void) { fail("static-extension registry did not preserve the valid long linker symbol"); } - puts("liboliphaunt static-extension registry long-symbol contract passed"); + OliphauntStaticExtension later = valid_extension[0]; + later.name = "later_database_extension"; + if (oliphaunt_register_static_extensions(&later, 1) != 0 || + oliphaunt_register_static_extensions(valid_extension, 1) != 0 || + oliphaunt_static_extension_lookup("later_database_extension") == NULL || + oliphaunt_static_extension_lookup("long_symbol_fixture") != registered || + oliphaunt_static_extension_symbol(registered, long_linker_symbol) != &long_symbol_target) { + fail("adding a later database extension invalidated an existing registration"); + } + OliphauntStaticExtension conflict[] = {later, valid_extension[0]}; + conflict[0].name = "must_not_be_registered"; + conflict[1].symbol_count = 0; + if (oliphaunt_register_static_extensions(conflict, 2) == 0 || + oliphaunt_static_extension_lookup("must_not_be_registered") != NULL || + oliphaunt_static_extension_lookup("long_symbol_fixture") != registered) { + fail("conflicting registration was not rejected atomically"); + } + puts("liboliphaunt static-extension registry identity and lifetime contracts passed"); return 0; } diff --git a/src/runtimes/liboliphaunt/native/src/liboliphaunt_static_extensions.c b/src/runtimes/liboliphaunt/native/src/liboliphaunt_static_extensions.c index 02e5addbc..27edbe759 100644 --- a/src/runtimes/liboliphaunt/native/src/liboliphaunt_static_extensions.c +++ b/src/runtimes/liboliphaunt/native/src/liboliphaunt_static_extensions.c @@ -9,12 +9,11 @@ typedef struct OliphauntRegisteredStaticExtension { OliphauntStaticExtensionSymbol *symbols; char *name; char **symbol_names; + struct OliphauntRegisteredStaticExtension *next; } OliphauntRegisteredStaticExtension; static pthread_mutex_t static_registry_mutex = PTHREAD_MUTEX_INITIALIZER; static OliphauntRegisteredStaticExtension *static_registry = NULL; -static size_t static_registry_count = 0; -static bool static_registry_frozen = false; #ifdef _MSC_VER extern const OliphauntStaticExtension *liboliphaunt_builtin_static_extensions(size_t *count); @@ -51,9 +50,9 @@ static const OliphauntStaticExtension *lookup_registered_static_extension(const if (name == NULL) { return NULL; } - for (size_t i = 0; i < static_registry_count; i++) { - if (strcmp(static_registry[i].extension.name, name) == 0) { - return &static_registry[i].extension; + for (OliphauntRegisteredStaticExtension *entry = static_registry; entry != NULL; entry = entry->next) { + if (strcmp(entry->extension.name, name) == 0) { + return &entry->extension; } } return NULL; @@ -244,25 +243,19 @@ static int copy_static_extensions( return 0; } -static bool static_registry_matches(const OliphauntStaticExtension *extensions, size_t count) { - if (static_registry_count != count) { +static bool static_extension_matches( + const OliphauntStaticExtension *existing, + const OliphauntStaticExtension *incoming) { + if (existing->magic != incoming->magic || + existing->init != incoming->init || + existing->symbol_count != incoming->symbol_count) { return false; } - for (size_t i = 0; i < count; i++) { - const OliphauntStaticExtension *existing = &static_registry[i].extension; - const OliphauntStaticExtension *incoming = &extensions[i]; - if (strcmp(existing->name, incoming->name) != 0 || - existing->magic != incoming->magic || - existing->init != incoming->init || - existing->symbol_count != incoming->symbol_count) { + for (size_t j = 0; j < existing->symbol_count; j++) { + if (strcmp(existing->symbols[j].name, incoming->symbols[j].name) != 0 || + existing->symbols[j].address != incoming->symbols[j].address) { return false; } - for (size_t j = 0; j < existing->symbol_count; j++) { - if (strcmp(existing->symbols[j].name, incoming->symbols[j].name) != 0 || - existing->symbols[j].address != incoming->symbols[j].address) { - return false; - } - } } return true; } @@ -271,33 +264,43 @@ static int32_t oliphaunt_register_static_extensions_impl(const OliphauntStaticEx if (validate_static_extensions(extensions, count) != 0) { return -1; } + /* Entries live for the process lifetime: PostgreSQL retains descriptor pointers. + * Additions never move or replace an entry already visible to a backend. */ pthread_mutex_lock(&static_registry_mutex); - if (static_registry_frozen && static_registry_matches(extensions, count)) { - pthread_mutex_unlock(&static_registry_mutex); - return 0; + OliphauntRegisteredStaticExtension *pending = NULL; + for (size_t i = 0; i < count; i++) { + const OliphauntStaticExtension *existing = lookup_registered_static_extension(extensions[i].name); + if (existing != NULL) { + if (!static_extension_matches(existing, &extensions[i])) { + set_error(NULL, "conflicting static extension registration for an existing module"); + goto failure; + } + continue; + } + OliphauntRegisteredStaticExtension *entry = NULL; + if (copy_static_extensions(&extensions[i], 1, &entry) != 0) { + goto failure; + } + entry->next = pending; + pending = entry; + } + while (pending != NULL) { + OliphauntRegisteredStaticExtension *entry = pending; + pending = entry->next; + entry->next = static_registry; + static_registry = entry; } pthread_mutex_unlock(&static_registry_mutex); + return 0; - OliphauntRegisteredStaticExtension *new_entries = NULL; - if (copy_static_extensions(extensions, count, &new_entries) != 0) { - return -1; +failure: + while (pending != NULL) { + OliphauntRegisteredStaticExtension *entry = pending; + pending = entry->next; + free_static_registry_entries(entry, 1); } - - pthread_mutex_lock(&static_registry_mutex); - if (static_registry_frozen) { - pthread_mutex_unlock(&static_registry_mutex); - free_static_registry_entries(new_entries, count); - set_error(NULL, "static extension registry cannot be changed after backend startup"); - return -1; - } - OliphauntRegisteredStaticExtension *old_entries = static_registry; - size_t old_count = static_registry_count; - static_registry = new_entries; - static_registry_count = count; pthread_mutex_unlock(&static_registry_mutex); - - free_static_registry_entries(old_entries, old_count); - return 0; + return -1; } int32_t oliphaunt_register_static_extensions(const OliphauntStaticExtension *extensions, size_t count) { @@ -315,7 +318,6 @@ const OliphauntStaticExtension *oliphaunt_static_extension_lookup(const char *fi const OliphauntStaticExtension *builtins = builtin_static_extensions(&builtin_count); const OliphauntStaticExtension *builtin = lookup_static_extension(builtins, builtin_count, name); pthread_mutex_lock(&static_registry_mutex); - static_registry_frozen = true; const OliphauntStaticExtension *registered = lookup_registered_static_extension(name); pthread_mutex_unlock(&static_registry_mutex); if (builtin != NULL) { diff --git a/src/runtimes/liboliphaunt/wasix/crates/assets/build.rs b/src/runtimes/liboliphaunt/wasix/crates/assets/build.rs index 3953fc9b1..96e878e32 100644 --- a/src/runtimes/liboliphaunt/wasix/crates/assets/build.rs +++ b/src/runtimes/liboliphaunt/wasix/crates/assets/build.rs @@ -483,8 +483,6 @@ fn write_generated_assets(out: &Path, asset_dir: &Path, selected_extensions: &[S let runtime = asset_dir.join("oliphaunt.wasix.tar.zst"); let standard_seed_archive = asset_dir.join("cluster-seeds/standard.tar.zst"); let standard_seed_manifest = asset_dir.join("cluster-seeds/standard.json"); - let icu_seed_archive = asset_dir.join("cluster-seeds/icu.tar.zst"); - let icu_seed_manifest = asset_dir.join("cluster-seeds/icu.json"); let initdb = asset_dir.join("bin/initdb.wasix.wasm"); for required in [&manifest, &runtime, &initdb] { @@ -495,10 +493,9 @@ fn write_generated_assets(out: &Path, asset_dir: &Path, selected_extensions: &[S required.display() ); } - for (profile, archive, seed_manifest) in [ - ("standard", &standard_seed_archive, &standard_seed_manifest), - ("icu", &icu_seed_archive, &icu_seed_manifest), - ] { + for (profile, archive, seed_manifest) in + [("standard", &standard_seed_archive, &standard_seed_manifest)] + { assert!( archive.is_file() && seed_manifest.is_file(), "generated asset directory {} is missing the required {profile} cluster seed; expected both {} and {}", @@ -510,8 +507,6 @@ fn write_generated_assets(out: &Path, asset_dir: &Path, selected_extensions: &[S let standard_seed_archive_body = optional_include_bytes_body(&standard_seed_archive); let standard_seed_manifest_body = optional_include_bytes_body(&standard_seed_manifest); - let icu_seed_archive_body = optional_include_bytes_body(&icu_seed_archive); - let icu_seed_manifest_body = optional_include_bytes_body(&icu_seed_manifest); let extension_sql_names = selected_extension_sql_names_body(selected_extensions); let extension_aot_sql_names = selected_extension_aot_sql_names_body(selected_extensions); let extension_archive_body = extension_archive_body(selected_extensions); @@ -527,8 +522,6 @@ fn write_generated_assets(out: &Path, asset_dir: &Path, selected_extensions: &[S pub fn runtime_archive() -> Option<&'static [u8]> {{ Some(include_bytes!({runtime})) }}\n\ pub fn standard_cluster_seed_archive() -> Option<&'static [u8]> {{ {standard_seed_archive_body} }}\n\ pub fn standard_cluster_seed_manifest() -> Option<&'static [u8]> {{ {standard_seed_manifest_body} }}\n\ - pub fn icu_cluster_seed_archive() -> Option<&'static [u8]> {{ {icu_seed_archive_body} }}\n\ - pub fn icu_cluster_seed_manifest() -> Option<&'static [u8]> {{ {icu_seed_manifest_body} }}\n\ pub fn initdb_wasm() -> Option<&'static [u8]> {{ Some(include_bytes!({initdb})) }}\n\ pub fn extension_archive(name: &str) -> Option<&'static [u8]> {{\n{extension_archive_body} }}\n\ pub fn expected_extension_archive_sha256(name: &str) -> Option<&'static str> {{\n{extension_sha256_body} }}\n\ @@ -539,8 +532,6 @@ fn write_generated_assets(out: &Path, asset_dir: &Path, selected_extensions: &[S runtime = rust_string_literal(&runtime), standard_seed_archive_body = standard_seed_archive_body, standard_seed_manifest_body = standard_seed_manifest_body, - icu_seed_archive_body = icu_seed_archive_body, - icu_seed_manifest_body = icu_seed_manifest_body, initdb = rust_string_literal(&initdb), extension_sql_names = extension_sql_names, extension_aot_sql_names = extension_aot_sql_names, @@ -558,8 +549,6 @@ fn write_generated_assets(out: &Path, asset_dir: &Path, selected_extensions: &[S &runtime, &standard_seed_archive, &standard_seed_manifest, - &icu_seed_archive, - &icu_seed_manifest, &initdb, ], ); @@ -582,8 +571,6 @@ fn write_source_only_assets(out: &Path, selected_extensions: &[SelectedExtension pub fn runtime_archive() -> Option<&'static [u8]> { None } pub fn standard_cluster_seed_archive() -> Option<&'static [u8]> { None } pub fn standard_cluster_seed_manifest() -> Option<&'static [u8]> { None } -pub fn icu_cluster_seed_archive() -> Option<&'static [u8]> { None } -pub fn icu_cluster_seed_manifest() -> Option<&'static [u8]> { None } pub fn initdb_wasm() -> Option<&'static [u8]> { None } "##, ); diff --git a/src/runtimes/liboliphaunt/wasix/release.toml b/src/runtimes/liboliphaunt/wasix/release.toml index c0c5225b2..ff73aa9d1 100644 --- a/src/runtimes/liboliphaunt/wasix/release.toml +++ b/src/runtimes/liboliphaunt/wasix/release.toml @@ -16,6 +16,10 @@ registry_packages = [ "crates:oliphaunt-wasix-tools-aot-x86_64-unknown-linux-gnu", "npm:@oliphaunt/liboliphaunt-wasix", "npm:@oliphaunt/liboliphaunt-wasix-tools", + "npm:@oliphaunt/liboliphaunt-wasix-tools-linux-arm64-gnu", + "npm:@oliphaunt/liboliphaunt-wasix-tools-linux-x64-gnu", + "npm:@oliphaunt/liboliphaunt-wasix-tools-macos-arm64", + "npm:@oliphaunt/liboliphaunt-wasix-tools-windows-x64-msvc", "npm:@oliphaunt/wasix-icu", ] release_artifacts = [ diff --git a/src/runtimes/wasix-napi/Cargo.toml b/src/runtimes/wasix-napi/Cargo.toml index 63dcce0b9..97c503eae 100644 --- a/src/runtimes/wasix-napi/Cargo.toml +++ b/src/runtimes/wasix-napi/Cargo.toml @@ -16,98 +16,17 @@ crate-type = ["cdylib"] [features] default = [] extensions = ["oliphaunt-wasix/extensions"] -tools = ["oliphaunt-wasix/tools", "dep:oliphaunt-wasix-tools"] +tools = ["oliphaunt-wasix/__internal-tools"] test-noop = ["napi/noop"] -release = [ - "tools", - "extension-amcheck", - "extension-auto-explain", - "extension-bloom", - "extension-btree-gin", - "extension-btree-gist", - "extension-citext", - "extension-cube", - "extension-dict-int", - "extension-dict-xsyn", - "extension-earthdistance", - "extension-file-fdw", - "extension-fuzzystrmatch", - "extension-hstore", - "extension-intarray", - "extension-isn", - "extension-lo", - "extension-ltree", - "extension-pageinspect", - "extension-pg-buffercache", - "extension-pg-freespacemap", - "extension-pg-hashids", - "extension-pg-ivm", - "extension-pg-surgery", - "extension-pg-textsearch", - "extension-pg-trgm", - "extension-pg-uuidv7", - "extension-pg-visibility", - "extension-pg-walinspect", - "extension-pgcrypto", - "extension-pgtap", - "extension-postgis", - "extension-seg", - "extension-tablefunc", - "extension-tcn", - "extension-tsm-system-rows", - "extension-tsm-system-time", - "extension-unaccent", - "extension-uuid-ossp", - "extension-vector", -] -extension-amcheck = ["extensions", "oliphaunt-wasix/extension-amcheck"] -extension-auto-explain = ["extensions", "oliphaunt-wasix/extension-auto-explain"] -extension-bloom = ["extensions", "oliphaunt-wasix/extension-bloom"] -extension-btree-gin = ["extensions", "oliphaunt-wasix/extension-btree-gin"] -extension-btree-gist = ["extensions", "oliphaunt-wasix/extension-btree-gist"] -extension-citext = ["extensions", "oliphaunt-wasix/extension-citext"] -extension-cube = ["extensions", "oliphaunt-wasix/extension-cube"] -extension-dict-int = ["extensions", "oliphaunt-wasix/extension-dict-int"] -extension-dict-xsyn = ["extensions", "oliphaunt-wasix/extension-dict-xsyn"] -extension-earthdistance = ["extensions", "oliphaunt-wasix/extension-earthdistance"] -extension-file-fdw = ["extensions", "oliphaunt-wasix/extension-file-fdw"] -extension-fuzzystrmatch = ["extensions", "oliphaunt-wasix/extension-fuzzystrmatch"] -extension-hstore = ["extensions", "oliphaunt-wasix/extension-hstore"] -extension-intarray = ["extensions", "oliphaunt-wasix/extension-intarray"] -extension-isn = ["extensions", "oliphaunt-wasix/extension-isn"] -extension-lo = ["extensions", "oliphaunt-wasix/extension-lo"] -extension-ltree = ["extensions", "oliphaunt-wasix/extension-ltree"] -extension-pageinspect = ["extensions", "oliphaunt-wasix/extension-pageinspect"] -extension-pg-buffercache = ["extensions", "oliphaunt-wasix/extension-pg-buffercache"] -extension-pg-freespacemap = ["extensions", "oliphaunt-wasix/extension-pg-freespacemap"] -extension-pg-hashids = ["extensions", "oliphaunt-wasix/extension-pg-hashids"] -extension-pg-ivm = ["extensions", "oliphaunt-wasix/extension-pg-ivm"] -extension-pg-surgery = ["extensions", "oliphaunt-wasix/extension-pg-surgery"] -extension-pg-textsearch = ["extensions", "oliphaunt-wasix/extension-pg-textsearch"] -extension-pg-trgm = ["extensions", "oliphaunt-wasix/extension-pg-trgm"] -extension-pg-uuidv7 = ["extensions", "oliphaunt-wasix/extension-pg-uuidv7"] -extension-pg-visibility = ["extensions", "oliphaunt-wasix/extension-pg-visibility"] -extension-pg-walinspect = ["extensions", "oliphaunt-wasix/extension-pg-walinspect"] -extension-pgcrypto = ["extensions", "oliphaunt-wasix/extension-pgcrypto"] -extension-pgtap = ["extensions", "oliphaunt-wasix/extension-pgtap"] -extension-postgis = ["extensions", "oliphaunt-wasix/extension-postgis"] -extension-seg = ["extensions", "oliphaunt-wasix/extension-seg"] -extension-tablefunc = ["extensions", "oliphaunt-wasix/extension-tablefunc"] -extension-tcn = ["extensions", "oliphaunt-wasix/extension-tcn"] -extension-tsm-system-rows = ["extensions", "oliphaunt-wasix/extension-tsm-system-rows"] -extension-tsm-system-time = ["extensions", "oliphaunt-wasix/extension-tsm-system-time"] -extension-unaccent = ["extensions", "oliphaunt-wasix/extension-unaccent"] -extension-uuid-ossp = ["extensions", "oliphaunt-wasix/extension-uuid-ossp"] -extension-vector = ["extensions", "oliphaunt-wasix/extension-vector"] +release = ["extensions", "tools"] [dependencies] napi = { version = "=3.12.2", default-features = false, features = ["napi8"] } napi-derive = { version = "=3.6.3", default-features = false, features = ["strict", "type-def"] } liboliphaunt-wasix-portable = { version = "*", path = "../liboliphaunt/wasix/crates/assets" } -oliphaunt-icu = { version = "*", path = "../liboliphaunt/icu" } -oliphaunt-wasix = { version = "*", path = "../../bindings/wasix-rust/crates/oliphaunt-wasix", features = ["__internal-napi", "icu"] } -oliphaunt-wasix-tools = { version = "*", path = "../liboliphaunt/wasix/crates/tools", optional = true } +oliphaunt-wasix = { version = "*", path = "../../bindings/wasix-rust/crates/oliphaunt-wasix", features = ["__internal-napi"] } sha2 = "0.10" +serde_json = "1" [target.'cfg(unix)'.dependencies] rustix = { version = "=1.1.4", features = ["fs"] } diff --git a/src/runtimes/wasix-napi/build.rs b/src/runtimes/wasix-napi/build.rs index f8f669cc8..811075088 100644 --- a/src/runtimes/wasix-napi/build.rs +++ b/src/runtimes/wasix-napi/build.rs @@ -5,7 +5,6 @@ const RELEASE_INPUT_ENVS: &[&str] = &[ "OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR", "OLIPHAUNT_WASM_GENERATED_AOT_DIR", "OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT", - "OLIPHAUNT_ICU_DATA_DIR", "OLIPHAUNT_WASIX_NAPI_BUILD_INPUTS", ]; @@ -35,12 +34,8 @@ fn validate_release_inputs() { "manifest.json", "oliphaunt.wasix.tar.zst", "bin/initdb.wasix.wasm", - "bin/pg_dump.wasix.wasm", - "bin/psql.wasix.wasm", "cluster-seeds/standard.tar.zst", "cluster-seeds/standard.json", - "cluster-seeds/icu.tar.zst", - "cluster-seeds/icu.json", ] { required_file(&portable.join(relative), "portable WASIX release payload"); } @@ -54,7 +49,7 @@ fn validate_release_inputs() { }; required_file( &target_aot.join("manifest.json"), - "target WASIX core/tools AOT manifest", + "target WASIX core AOT manifest", ); let extension_root = required_directory("OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT"); @@ -66,24 +61,11 @@ fn validate_release_inputs() { "exact WASIX extension artifact root {} must not be empty", extension_root.display(), ); - let icu_root = required_directory("OLIPHAUNT_ICU_DATA_DIR"); - assert!( - std::fs::read_dir(&icu_root) - .expect("read ICU data root") - .next() - .is_some(), - "ICU data root {} must not be empty", - icu_root.display(), - ); - let inventory = required_path("OLIPHAUNT_WASIX_NAPI_BUILD_INPUTS"); required_file(&inventory, "validated WASIX N-API build-input inventory"); println!("cargo::rerun-if-changed={}", inventory.display()); - // `oliphaunt-wasix` relays the manifests emitted by the exact payload - // crates it compiled. Requiring all four proves Cargo selected embedded - // portable/core-AOT/tool/tool-AOT inputs, not only that similarly named - // files happened to exist in the workspace. + // Require the exact core payload manifests relayed by the SDK build. let target_suffix = match target.as_str() { "aarch64-apple-darwin" => "MACOS_ARM64", "aarch64-unknown-linux-gnu" => "LINUX_ARM64_GNU", @@ -93,22 +75,12 @@ fn validate_release_inputs() { }; for name in [ "DEP_OLIPHAUNT_ARTIFACT_WASIX_RELAY_LIBOLIPHAUNT_WASIX_RUNTIME_MANIFEST".to_owned(), - "DEP_OLIPHAUNT_ARTIFACT_WASIX_RELAY_OLIPHAUNT_WASIX_TOOLS_MANIFEST".to_owned(), format!( "DEP_OLIPHAUNT_ARTIFACT_WASIX_RELAY_LIBOLIPHAUNT_WASIX_AOT_{target_suffix}_MANIFEST" ), - format!( - "DEP_OLIPHAUNT_ARTIFACT_WASIX_RELAY_OLIPHAUNT_WASIX_TOOLS_AOT_{target_suffix}_MANIFEST" - ), ] { required_env_file(&name, "relayed WASIX Cargo artifact manifest"); } - if std::env::var_os("CARGO_FEATURE_ICU").is_some() { - required_env_file( - "DEP_OLIPHAUNT_ARTIFACT_WASIX_RELAY_OLIPHAUNT_ICU_MANIFEST", - "relayed ICU Cargo artifact manifest", - ); - } } fn required_path(name: &str) -> PathBuf { diff --git a/src/runtimes/wasix-napi/moon.yml b/src/runtimes/wasix-napi/moon.yml index 286be71bf..5da3feeb1 100644 --- a/src/runtimes/wasix-napi/moon.yml +++ b/src/runtimes/wasix-napi/moon.yml @@ -52,7 +52,7 @@ tasks: script: | set -e node --check tools/smoke-packaged-addon.mjs - cargo check --manifest-path Cargo.toml --locked --no-default-features + cargo check --manifest-path Cargo.toml --locked --no-default-features --features extensions,tools env: CARGO_TARGET_DIR: "../../../target/moon/oliphaunt-wasix-napi/compile" inputs: @@ -61,7 +61,6 @@ tasks: group: "code" - project: "liboliphaunt-wasix" group: "crates" - - "/src/runtimes/liboliphaunt/icu/**/*" - "Cargo.toml" - "build.rs" - "src/**/*" @@ -74,7 +73,7 @@ tasks: script: | set -e bun test tools/detect-linux-libc.test.mjs tools/portable-command.test.mjs - cargo test --manifest-path Cargo.toml --locked --no-default-features --features test-noop --lib + cargo test --manifest-path Cargo.toml --locked --no-default-features --features extensions,tools,test-noop --lib env: CARGO_TARGET_DIR: "../../../target/moon/oliphaunt-wasix-napi/unit" inputs: diff --git a/src/runtimes/wasix-napi/src/extension_package.rs b/src/runtimes/wasix-napi/src/extension_package.rs new file mode 100644 index 000000000..5f11f49d0 --- /dev/null +++ b/src/runtimes/wasix-napi/src/extension_package.rs @@ -0,0 +1,463 @@ +//! Loads installed extension dependencies. These packages are trusted native +//! dependencies, like the addon itself; this is not a loader for remote or +//! caller-supplied serialized code. Artifact hashes come from the installed +//! package manifests, never from the database's configuration or stored files. + +use std::collections::BTreeMap; +use std::fs; +use std::path::{Component, Path, PathBuf}; +use std::sync::{Mutex, OnceLock}; + +use napi::{Error, Result}; +use napi_derive::napi; +use oliphaunt_wasix::{Extension, ExtensionPackage}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +#[napi(object)] +pub struct NativeExtensionPackage { + pub sql_name: String, + pub product: String, + pub version: String, + pub package_json: String, + pub aot_package_json: Option, +} + +#[napi(object)] +pub struct NativeToolPackage { + pub package_json: String, + pub aot_package_json: String, +} + +#[cfg(feature = "tools")] +#[napi(js_name = "registerTools", catch_unwind)] +pub fn register_tools(selection: NativeToolPackage) -> Result<()> { + let (root, manifest) = package(&selection.package_json)?; + let metadata = &manifest["oliphaunt"]; + let version = string(&manifest, "version")?; + let name = "@oliphaunt/liboliphaunt-wasix-tools"; + if string(&manifest, "name")? != name + || version != super::RUNTIME_VERSION + || string(metadata, "kind")? != "wasix-tools" + || string(metadata, "runtimeVersion")? != super::RUNTIME_VERSION + { + return Err(fail("installed tools package does not match the runtime")); + } + let mut modules = Vec::new(); + for name in ["pg_dump", "psql"] { + let module = &metadata["tools"][name]; + let hash = string(module, "sha256")?; + let bytes = payload(&root, string(module, "path")?, hash)?; + if module["size"].as_u64() != Some(bytes.len() as u64) { + return Err(fail("installed tool module size mismatch")); + } + modules.push((name, bytes, hash.to_owned())); + } + let expected_name = format!("{name}-{}", target()); + let (aot_root, aot_package) = package(&selection.aot_package_json)?; + let aot_metadata = &aot_package["oliphaunt"]; + if string(&aot_package, "name")? != expected_name + || string(&aot_package, "version")? != version + || string(aot_metadata, "kind")? != "wasix-tools-aot" + || string(aot_metadata, "target")? != target() + || string(aot_metadata, "runtimeVersion")? != super::RUNTIME_VERSION + || manifest["optionalDependencies"][&expected_name].as_str() != Some(version) + { + return Err(fail( + "installed tools AOT package does not match its owner or host", + )); + } + let manifest_bytes = payload( + &aot_root, + "aot-manifest.json", + string(aot_metadata, "manifestSha256")?, + )?; + let aot = json(&manifest_bytes)?; + let mut artifacts = Vec::new(); + for artifact in aot["artifacts"] + .as_array() + .ok_or_else(|| fail("tools AOT artifacts missing"))? + { + let name = string(artifact, "name")?; + if !["tool:pg_dump", "tool:psql"].contains(&name) { + return Err(fail("tools AOT contains unrelated code")); + } + artifacts.push(( + name.to_owned(), + payload( + &aot_root, + string(artifact, "path")?, + string(artifact, "sha256")?, + )?, + )); + } + let key = format!( + "tools@{version}:{:x}:{}", + Sha256::digest(&manifest_bytes), + modules + .iter() + .map(|(_, _, hash)| hash.as_str()) + .collect::>() + .join(":") + ); + let mut packages = PACKAGES + .get_or_init(|| Mutex::new(BTreeMap::new())) + .lock() + .map_err(|_| fail("package cache poisoned"))?; + let package = if let Some(package) = packages.get(&key) { + **package + } else { + let modules = Box::leak( + modules + .into_iter() + .map(|(name, bytes, hash)| { + ( + name, + &*Box::leak(bytes.into_boxed_slice()), + &*Box::leak(hash.into_boxed_str()), + ) + }) + .collect::>() + .into_boxed_slice(), + ); + let artifacts = Box::leak( + artifacts + .into_iter() + .map(|(name, bytes)| { + ( + &*Box::leak(name.into_boxed_str()), + &*Box::leak(bytes.into_boxed_slice()), + ) + }) + .collect::>() + .into_boxed_slice(), + ); + // SAFETY: the host explicitly imports these installed native dependencies. + // Validate exact package ownership and all file identities before handing + // the trusted artifacts to the runtime's engine and module checks. + let package = unsafe { + ExtensionPackage::from_trusted_release( + "oliphaunt-wasix-tools", + Box::leak(version.to_owned().into_boxed_str()), + super::RUNTIME_VERSION, + modules, + Box::leak( + String::from_utf8(manifest_bytes) + .map_err(|_| fail("invalid tools AOT UTF-8"))? + .into_boxed_str(), + ), + artifacts, + ) + }; + packages.insert(key, Box::leak(Box::new(package))); + package + }; + oliphaunt_wasix::tools::register_installed_package(package) + .map_err(|error| fail(error.to_string())) +} + +static PACKAGES: OnceLock>> = OnceLock::new(); + +fn fail(message: impl Into) -> Error { + super::invalid_argument(message) +} + +fn string<'a>(value: &'a Value, key: &str) -> Result<&'a str> { + value + .get(key) + .and_then(Value::as_str) + .filter(|value| !value.is_empty() && !value.contains('\0')) + .ok_or_else(|| fail(format!("installed extension manifest is missing {key}"))) +} + +fn json(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes) + .map_err(|error| fail(format!("invalid installed extension manifest: {error}"))) +} + +fn read(path: &Path) -> Result> { + fs::read(path).map_err(|error| { + fail(format!( + "read installed extension {}: {error}", + path.display() + )) + }) +} + +fn package(path: &str) -> Result<(PathBuf, Value)> { + let path = Path::new(path); + if !path.is_absolute() || path.file_name().is_none_or(|name| name != "package.json") { + return Err(fail( + "extension package must identify an installed absolute package.json path", + )); + } + let path = fs::canonicalize(path) + .map_err(|error| fail(format!("resolve installed extension package: {error}")))?; + let root = path + .parent() + .ok_or_else(|| fail("extension package has no root"))? + .to_owned(); + Ok((root, json(&read(&path)?)?)) +} + +fn payload(root: &Path, relative: &str, expected: &str) -> Result> { + let path = Path::new(relative); + if relative.contains('\\') + || path + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err(fail("extension package contains an unsafe payload path")); + } + let path = fs::canonicalize(root.join(path)) + .map_err(|error| fail(format!("resolve extension payload: {error}")))?; + if !path.starts_with(root) || !path.is_file() { + return Err(fail("extension payload escapes its installed package")); + } + let bytes = read(&path)?; + if format!("{:x}", Sha256::digest(&bytes)) != expected { + return Err(fail(format!( + "installed extension payload hash mismatch: {relative}" + ))); + } + Ok(bytes) +} + +fn target() -> &'static str { + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + { + return "linux-x64-gnu"; + } + #[cfg(all(target_os = "linux", target_arch = "aarch64"))] + { + return "linux-arm64-gnu"; + } + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + { + return "macos-arm64"; + } + #[cfg(all(target_os = "windows", target_arch = "x86_64"))] + { + return "windows-x64-msvc"; + } + #[allow(unreachable_code)] + "unsupported" +} + +pub(super) fn load(selection: NativeExtensionPackage) -> Result { + let extension = Extension::by_sql_name(&selection.sql_name) + .ok_or_else(|| fail("unknown selected WASIX extension"))?; + let expected_product = format!( + "oliphaunt-extension-{}", + selection.sql_name.replace('_', "-") + ); + if selection.product != expected_product { + return Err(fail( + "external extension descriptor has the wrong release product", + )); + } + let expected_package = format!( + "@oliphaunt/{}-wasix", + &selection.product["oliphaunt-".len()..] + ); + let (root, manifest) = package(&selection.package_json)?; + let metadata = &manifest["oliphaunt"]; + if string(&manifest, "name")? != expected_package + || string(&manifest, "version")? != selection.version + || string(metadata, "product")? != selection.product + || string(metadata, "kind")? != "exact-extension-wasix" + || string(metadata, "runtime")? != "wasix" + || string(metadata, "wasixRuntimeProduct")? != "liboliphaunt-wasix" + || string(metadata, "wasixRuntimeVersion")? != super::RUNTIME_VERSION + { + return Err(fail( + "selected extension does not match its installed package or runtime", + )); + } + let carrier = &metadata["carriers"][&selection.sql_name]; + let archive_hash = string(carrier, "sha256")?; + let archive = payload(&root, string(carrier, "path")?, archive_hash)?; + if carrier["size"].as_u64() != Some(archive.len() as u64) { + return Err(fail("installed extension archive size mismatch")); + } + let needs_aot = carrier["requiresAot"] + .as_bool() + .ok_or_else(|| fail("extension carrier is missing requiresAot"))?; + let mut aot_manifest = String::new(); + let mut artifacts = Vec::new(); + if needs_aot { + let (aot_root, aot_package) = package( + selection + .aot_package_json + .as_deref() + .ok_or_else(|| fail("selected extension is missing its host AOT package"))?, + )?; + let expected_aot_package = format!("{expected_package}-{}", target()); + let aot_metadata = &aot_package["oliphaunt"]; + if string(&aot_package, "name")? != expected_aot_package + || string(&aot_package, "version")? != selection.version + || string(aot_metadata, "product")? != selection.product + || string(aot_metadata, "kind")? != "wasix-extension-aot" + || string(aot_metadata, "target")? != target() + || string(aot_metadata, "runtimeVersion")? != super::RUNTIME_VERSION + || manifest["optionalDependencies"][&expected_aot_package].as_str() + != Some(selection.version.as_str()) + { + return Err(fail( + "extension AOT package does not match its owner, version, runtime, or host", + )); + } + let bytes = payload( + &aot_root, + "aot-manifest.json", + string(aot_metadata, "manifestSha256")?, + )?; + let aot = json(&bytes)?; + for artifact in aot["artifacts"] + .as_array() + .ok_or_else(|| fail("extension AOT artifacts missing"))? + { + let name = string(artifact, "name")?; + let prefix = format!("extension:{}", selection.sql_name); + if name != prefix && !name.starts_with(&format!("{prefix}:")) { + return Err(fail( + "extension AOT package contains another extension's code", + )); + } + artifacts.push(( + name.to_owned(), + payload( + &aot_root, + string(artifact, "path")?, + string(artifact, "sha256")?, + )?, + )); + } + aot_manifest = + String::from_utf8(bytes).map_err(|_| fail("extension AOT manifest is not UTF-8"))?; + } else if selection.aot_package_json.is_some() { + return Err(fail("SQL-only extension must not supply an AOT package")); + } + let key = format!( + "{}@{}:{archive_hash}:{:x}", + selection.product, + selection.version, + Sha256::digest(aot_manifest.as_bytes()) + ); + let mut packages = PACKAGES + .get_or_init(|| Mutex::new(BTreeMap::new())) + .lock() + .map_err(|_| fail("extension package cache poisoned"))?; + if let Some(package) = packages.get(&key) { + return Ok(extension.with_package(package)); + } + // Match the lifetime of imported native modules. Only validated packages + // are retained, once per exact content identity, across worker environments. + let archives = Box::leak( + vec![( + &*Box::leak(selection.sql_name.into_boxed_str()), + &*Box::leak(archive.into_boxed_slice()), + &*Box::leak(archive_hash.to_owned().into_boxed_str()), + )] + .into_boxed_slice(), + ); + let artifacts = Box::leak( + artifacts + .into_iter() + .map(|(name, bytes)| { + ( + &*Box::leak(name.into_boxed_str()), + &*Box::leak(bytes.into_boxed_slice()), + ) + }) + .collect::>() + .into_boxed_slice(), + ); + // SAFETY: this FFI loader accepts installed native package dependencies from + // the host package resolver, under the same trust as importing their code. + // It never accepts remote assets or a caller's executable bytes/digest pair. + // Owner, exact version, host, runtime and all package-owned file identities + // have been checked above; the runtime additionally validates AOT engine, + // source fingerprint, raw bytes and WebAssembly identity before deserializing. + let package = unsafe { + ExtensionPackage::from_trusted_release( + Box::leak(selection.product.into_boxed_str()), + Box::leak(selection.version.into_boxed_str()), + super::RUNTIME_VERSION, + archives, + Box::leak(aot_manifest.into_boxed_str()), + artifacts, + ) + }; + let package = Box::leak(Box::new(package)); + packages.insert(key, package); + Ok(extension.with_package(package)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn installed_package_checks_owner_version_hash_and_containment() { + let root = std::env::temp_dir().join(format!( + "oliphaunt-installed-extension-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(root.join("extensions/pgtap")).unwrap(); + let hash = format!("{:x}", Sha256::digest(b"fixture")); + let mut manifest = serde_json::json!({ + "name": "@oliphaunt/extension-pgtap-wasix", "version": "1.3.4", + "oliphaunt": { "product": "oliphaunt-extension-pgtap", "kind": "exact-extension-wasix", + "runtime": "wasix", "wasixRuntimeProduct": "liboliphaunt-wasix", "wasixRuntimeVersion": super::super::RUNTIME_VERSION, + "carriers": { "pgtap": { "path": "extensions/pgtap/extension.tar.zst", "sha256": hash, "size": 7, "requiresAot": false } } } + }); + let package_file = root.join("package.json"); + let selection = || NativeExtensionPackage { + sql_name: "pgtap".into(), + product: "oliphaunt-extension-pgtap".into(), + version: "1.3.4".into(), + package_json: package_file.to_str().unwrap().into(), + aot_package_json: None, + }; + fs::write(root.join("extensions/pgtap/extension.tar.zst"), b"fixture").unwrap(); + fs::write(&package_file, manifest.to_string()).unwrap(); + assert_eq!(load(selection()).unwrap().sql_name(), "pgtap"); + #[cfg(feature = "tools")] + assert!( + register_tools(NativeToolPackage { + package_json: package_file.to_str().unwrap().into(), + aot_package_json: package_file.to_str().unwrap().into(), + }) + .unwrap_err() + .to_string() + .contains("does not match") + ); + fs::write(root.join("extensions/pgtap/extension.tar.zst"), b"corrupt").unwrap(); + assert!( + load(selection()) + .unwrap_err() + .to_string() + .contains("hash mismatch") + ); + manifest["version"] = "1.3.5".into(); + fs::write(&package_file, manifest.to_string()).unwrap(); + assert!( + load(selection()) + .unwrap_err() + .to_string() + .contains("does not match") + ); + assert!( + payload(&root, "../package.json", &hash) + .unwrap_err() + .to_string() + .contains("unsafe payload path") + ); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/src/runtimes/wasix-napi/src/icu_data.rs b/src/runtimes/wasix-napi/src/icu_data.rs new file mode 100644 index 000000000..0f31e84db --- /dev/null +++ b/src/runtimes/wasix-napi/src/icu_data.rs @@ -0,0 +1,87 @@ +use napi::{Result, bindgen_prelude::Buffer}; +use napi_derive::napi; +use oliphaunt_wasix::IcuData; +use sha2::{Digest, Sha256}; +use std::sync::OnceLock; + +#[napi(object)] +pub struct NativeIcuData { + pub version: String, + pub runtime_version: String, + pub archive: Buffer, + pub archive_sha256: String, + pub data_tree_sha256: String, + pub seed_archive: Buffer, + pub seed_archive_sha256: String, + pub seed_manifest: Buffer, + pub seed_manifest_sha256: String, +} + +static ICU: OnceLock = OnceLock::new(); + +pub(super) fn load(profile: &str, input: Option) -> Result> { + let Some(input) = input else { + if profile == "icu" { + return Err(super::invalid_argument( + "ICU profile requires the optional ICU package", + )); + } + return Ok(None); + }; + if profile != "icu" || input.runtime_version != super::RUNTIME_VERSION { + return Err(super::invalid_argument( + "ICU data does not match the selected runtime/profile", + )); + } + for (bytes, expected) in [ + (&input.archive, &input.archive_sha256), + (&input.seed_archive, &input.seed_archive_sha256), + (&input.seed_manifest, &input.seed_manifest_sha256), + ] { + if format!("{:x}", Sha256::digest(bytes.as_ref())) != *expected { + return Err(super::invalid_argument("ICU package payload hash mismatch")); + } + } + let seed: serde_json::Value = serde_json::from_slice(&input.seed_manifest) + .map_err(|_| super::invalid_argument("invalid ICU seed manifest"))?; + if seed["catalogProfile"] != "icu" + || seed["runtime"]["version"] != input.runtime_version + || seed["icu"]["dataTreeSha256"] != input.data_tree_sha256 + { + return Err(super::invalid_argument("ICU data and seed do not match")); + } + let runtime = liboliphaunt_wasix_portable::manifest() + .map_err(|error| super::invalid_argument(error.to_string()))?; + let expected = runtime + .cluster_seeds + .get("icu") + .ok_or_else(|| super::invalid_argument("runtime has no ICU seed identity"))?; + if expected.sha256 != input.seed_archive_sha256 + || expected.icu_data_tree_sha256.as_deref() != Some(&input.data_tree_sha256) + { + return Err(super::invalid_argument( + "ICU package does not match the runtime seed identity", + )); + } + let data = ICU.get_or_init(|| IcuData { + version: Box::leak(input.version.clone().into_boxed_str()), + runtime_version: super::RUNTIME_VERSION, + native_runtime_version: "unavailable", + resources: &[], + wasix_archive: Some(Box::leak(input.archive.to_vec().into_boxed_slice())), + wasix_archive_sha256: Some(Box::leak(input.archive_sha256.clone().into_boxed_str())), + wasix_data_tree_sha256: Some(Box::leak(input.data_tree_sha256.clone().into_boxed_str())), + wasix_seed_archive: Some(Box::leak(input.seed_archive.to_vec().into_boxed_slice())), + wasix_seed_manifest: Some(Box::leak(input.seed_manifest.to_vec().into_boxed_slice())), + }); + if data.version != input.version + || data.wasix_archive_sha256 != Some(&input.archive_sha256) + || data.wasix_seed_archive != Some(input.seed_archive.as_ref()) + || data.wasix_seed_manifest != Some(input.seed_manifest.as_ref()) + { + return Err(super::invalid_argument( + "conflicting ICU packages for one runtime", + )); + } + Ok(Some(*data)) +} diff --git a/src/runtimes/wasix-napi/src/lib.rs b/src/runtimes/wasix-napi/src/lib.rs index 634134823..a21167511 100644 --- a/src/runtimes/wasix-napi/src/lib.rs +++ b/src/runtimes/wasix-napi/src/lib.rs @@ -4,6 +4,11 @@ //! owners directly. Promise settlement is the only owner-to-JavaScript hop; //! no Tokio runtime or Node async-work queue participates in database work. +mod extension_package; +mod icu_data; +use extension_package::NativeExtensionPackage; +use icu_data::NativeIcuData; + use std::collections::BTreeMap; use std::mem; use std::panic::{AssertUnwindSafe, catch_unwind}; @@ -66,6 +71,8 @@ pub struct NativeOpenOptions { #[napi(js_name = "startupGucs")] pub startup_gucs: BTreeMap, pub extensions: Vec, + pub extension_packages: Option>, + pub icu: Option, } #[napi(object)] @@ -84,6 +91,8 @@ pub struct NativeServerOpenOptions { #[napi(js_name = "startupGucs")] pub startup_gucs: BTreeMap, pub extensions: Vec, + pub extension_packages: Option>, + pub icu: Option, pub listen: NativeListenOptions, } @@ -569,7 +578,6 @@ pub fn supported_profiles() -> Vec<&'static str> { #[napi(js_name = "payloadIdentity", catch_unwind)] pub fn payload_identity(component: String) -> Result { static STANDARD_SEED_MANIFEST: OnceLock = OnceLock::new(); - static ICU_SEED_MANIFEST: OnceLock = OnceLock::new(); let manifest = embedded_portable_manifest()?; match component.as_str() { "runtimeArchive" => embedded_identity( @@ -590,29 +598,6 @@ pub fn payload_identity(component: String) -> Result { liboliphaunt_wasix_portable::standard_cluster_seed_manifest(), &STANDARD_SEED_MANIFEST, ), - "icuDataArchive" => embedded_identity( - "ICU data archive", - oliphaunt_icu::icu_data_archive(), - oliphaunt_icu::ICU_DATA_ARCHIVE_SHA256.ok_or_else(|| { - Error::new( - Status::GenericFailure, - "WASIX ICU data archive has no embedded SHA-256 identity".to_owned(), - ) - })?, - ), - "icuSeedArchive" => { - let seed = embedded_seed(manifest, "icu")?; - embedded_identity( - "ICU cluster seed archive", - liboliphaunt_wasix_portable::icu_cluster_seed_archive(), - &seed.sha256, - ) - } - "icuSeedManifest" => hashed_embedded_identity( - "ICU cluster seed manifest", - liboliphaunt_wasix_portable::icu_cluster_seed_manifest(), - &ICU_SEED_MANIFEST, - ), _ => Err(invalid_argument(format!( "unsupported WASIX payload component {component:?}" ))), @@ -651,8 +636,11 @@ pub fn tool_identity(name: String) -> Result { static PG_DUMP: OnceLock = OnceLock::new(); static PSQL: OnceLock = OnceLock::new(); let (bytes, identity) = match name.as_str() { - "pg_dump" => (oliphaunt_wasix_tools::pg_dump_wasm(), &PG_DUMP), - "psql" => (oliphaunt_wasix_tools::psql_wasm(), &PSQL), + "pg_dump" => ( + oliphaunt_wasix::tools::installed_tool_wasm("pg_dump"), + &PG_DUMP, + ), + "psql" => (oliphaunt_wasix::tools::installed_tool_wasm("psql"), &PSQL), _ => { return Err(invalid_argument(format!( "unsupported WASIX tool {name:?}; expected \"pg_dump\" or \"psql\"" @@ -676,6 +664,8 @@ fn configure_direct_database(options: NativeOpenOptions) -> Result Result Result Result Result Result Result { } #[cfg(feature = "extensions")] -fn resolve_extensions(names: Vec) -> Result> { +fn resolve_extensions( + names: Vec, + packages: Vec, +) -> Result> { + let mut selected = BTreeMap::new(); + for package in packages { + if !names.contains(&package.sql_name) { + return Err(invalid_argument("unselected extension package supplied")); + } + let extension = extension_package::load(package)?; + if selected.insert(extension.sql_name(), extension).is_some() { + return Err(invalid_argument("duplicate external extension package")); + } + } names .into_iter() .map(|name| { - Extension::by_sql_name(&name).ok_or_else(|| { - invalid_argument(format!( - "WASIX extension {name:?} is unknown or unavailable in this runtime" - )) - }) + selected + .get(name.as_str()) + .copied() + .or_else(|| Extension::by_sql_name(&name)) + .ok_or_else(|| { + invalid_argument(format!( + "WASIX extension {name:?} is unknown or unavailable in this runtime" + )) + }) }) .collect() } @@ -821,14 +841,15 @@ fn resolve_extensions(names: Vec) -> Result> { fn apply_direct_extensions( builder: OliphauntBuilder, names: Vec, + packages: Vec, ) -> Result { #[cfg(feature = "extensions")] { - Ok(builder.extensions(resolve_extensions(names)?)) + Ok(builder.extensions(resolve_extensions(names, packages)?)) } #[cfg(not(feature = "extensions"))] { - if names.is_empty() { + if names.is_empty() && packages.is_empty() { Ok(builder) } else { Err(missing_release_feature("extensions", "open")) @@ -839,14 +860,15 @@ fn apply_direct_extensions( fn apply_async_extensions( builder: AsyncOliphauntBuilder, names: Vec, + packages: Vec, ) -> Result { #[cfg(feature = "extensions")] { - Ok(builder.extensions(resolve_extensions(names)?)) + Ok(builder.extensions(resolve_extensions(names, packages)?)) } #[cfg(not(feature = "extensions"))] { - if names.is_empty() { + if names.is_empty() && packages.is_empty() { Ok(builder) } else { Err(missing_release_feature("extensions", "open")) @@ -857,14 +879,15 @@ fn apply_async_extensions( fn apply_server_extensions( builder: AsyncOliphauntServerBuilder, names: Vec, + packages: Vec, ) -> Result { #[cfg(feature = "extensions")] { - Ok(builder.extensions(resolve_extensions(names)?)) + Ok(builder.extensions(resolve_extensions(names, packages)?)) } #[cfg(not(feature = "extensions"))] { - if names.is_empty() { + if names.is_empty() && packages.is_empty() { Ok(builder) } else { Err(missing_release_feature("extensions", "server open")) diff --git a/src/runtimes/wasix-napi/tools/build-native.sh b/src/runtimes/wasix-napi/tools/build-native.sh index dc0ac4644..03700c711 100755 --- a/src/runtimes/wasix-napi/tools/build-native.sh +++ b/src/runtimes/wasix-napi/tools/build-native.sh @@ -106,14 +106,13 @@ cargo_target_dir="$product_target_root/cargo-release" build_inputs_file="$product_target_root/build-inputs/$target_id.json" mkdir -p "$prebuild_dir" -# Release addons must consume the exact portable runtime, target AOT, exact -# extension, and ICU payloads staged by the same CI run. Export the canonical +# Release addons consume the exact core runtime, target AOT and contrib +# payloads staged by the same CI run. Export the canonical # dependency build-script variables explicitly so no source-only fallback can # be selected through a package-local or stale workspace probe. export OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR="${OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR:-$workspace_root/target/oliphaunt-wasix/assets}" export OLIPHAUNT_WASM_GENERATED_AOT_DIR="${OLIPHAUNT_WASM_GENERATED_AOT_DIR:-$workspace_root/target/oliphaunt-wasix/aot}" export OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT="${OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT:-$workspace_root/target/extension-artifacts}" -export OLIPHAUNT_ICU_DATA_DIR="${OLIPHAUNT_ICU_DATA_DIR:-$workspace_root/target/oliphaunt-wasix/wasix-build/work/icu-wasix/share/icu}" export OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD=1 export OLIPHAUNT_WASIX_NAPI_BUILD_INPUTS="$build_inputs_file" @@ -123,7 +122,6 @@ build_input_args=( --portable-root "$OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR" --aot-root "$OLIPHAUNT_WASM_GENERATED_AOT_DIR" --extension-root "$OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT" - --icu-root "$OLIPHAUNT_ICU_DATA_DIR" ) tools/dev/bun.sh src/runtimes/wasix-napi/tools/check-build-inputs.mjs \ "${build_input_args[@]}" \ @@ -245,19 +243,11 @@ function expectedIdentity(record, kind) { return `${record.sha256}:${size}`; } -const portableTools = buildInputs.inputs?.portableTools; -if ( - buildInputs.schema !== "oliphaunt-wasix-napi-build-inputs-v1" - || JSON.stringify(portableTools?.map(({ name }) => name)) !== JSON.stringify(["pg_dump", "psql"]) -) { - throw new Error(`${buildInputsPath} has an incompatible portable tool inventory`); -} -for (const tool of portableTools) { - const actual = addon.toolIdentity(tool.name); - const expected = expectedIdentity(tool, `${tool.name} tool`); - if (actual !== expected) { - throw new Error(`${addonPath} reports ${tool.name} tool identity ${actual}; expected ${expected}`); - } +if (typeof addon.registerTools !== "function") throw new Error("addon has no installed tools loader"); +for (const name of ["pg_dump", "psql"]) { + let missing = false; + try { addon.toolIdentity(name); } catch { missing = true; } + if (!missing) throw new Error(`base addon unexpectedly embeds ${name}`); } const portableExtensions = (buildInputs.inputs?.extensionArtifacts ?? []) @@ -283,9 +273,6 @@ for (const component of [ "runtimeArchive", "standardSeedArchive", "standardSeedManifest", - "icuDataArchive", - "icuSeedArchive", - "icuSeedManifest", ]) { const identity = addon.payloadIdentity(component); if (!/^[0-9a-f]{64}:[1-9][0-9]*$/.test(identity)) { diff --git a/src/runtimes/wasix-napi/tools/check-build-inputs.mjs b/src/runtimes/wasix-napi/tools/check-build-inputs.mjs index f8b117275..33aa99249 100755 --- a/src/runtimes/wasix-napi/tools/check-build-inputs.mjs +++ b/src/runtimes/wasix-napi/tools/check-build-inputs.mjs @@ -5,7 +5,6 @@ import { lstatSync, mkdirSync, readFileSync, - readdirSync, writeFileSync, } from "node:fs"; import path from "node:path"; @@ -13,7 +12,7 @@ import { fileURLToPath } from "node:url"; import { compareText, - exactExtensionProducts, + contribCarrierDescriptor, extensionArtifactProductRoot, extensionSqlNames, extensionWasixAotMemberSqlNames, @@ -44,7 +43,6 @@ function parseArguments(argv) { "portable-root", "aot-root", "extension-root", - "icu-root", ]) { if (!options[required]) fail(`--${required} is required`); } @@ -143,7 +141,7 @@ function portableInputs(portableRoot) { ); regularFile(path.join(portableRoot, "bin/initdb.wasix.wasm"), "portable WASIX initdb module"); - for (const profile of ["standard", "icu"]) { + for (const profile of ["standard"]) { const seed = manifest["cluster-seeds"]?.[profile]; if (!seed || typeof seed !== "object" || Array.isArray(seed)) { fail(`portable WASIX manifest is missing ${profile} cluster seed metadata`); @@ -158,13 +156,7 @@ function portableInputs(portableRoot) { regularFile(path.join(portableRoot, seedManifest), `${profile} cluster seed manifest`); } - const portableTools = [ - ["pg_dump", "bin/pg_dump.wasix.wasm"], - ["psql", "bin/psql.wasix.wasm"], - ].map(([name, relative]) => { - const file = path.join(portableRoot, relative); - regularFile(file, `portable WASIX ${name} module`); - return { name, path: repoPath(file, `portable WASIX ${name} module`), sha256: sha256(file) }; + return { name, path: repoPath(file, `portable WASIX ${name} module`), sha256: sha256(file) }; }); return { @@ -174,7 +166,6 @@ function portableInputs(portableRoot) { path: repoPath(manifestFile, "portable WASIX manifest"), sha256: sha256(manifestFile), }, - portableTools, }, }; } @@ -219,9 +210,6 @@ function runtimeAotInputs(aotRoot, targetTriple, sourceFingerprint) { "host WASIX AOT manifest", (name) => !name.startsWith("extension:"), ); - for (const tool of ["tool:pg_dump", "tool:psql"]) { - if (!names.has(tool)) fail(`host WASIX AOT manifest is missing ${tool}`); - } if (![...names].some((name) => !name.startsWith("tool:"))) { fail("host WASIX AOT manifest contains tools but no core runtime artifacts"); } @@ -242,7 +230,7 @@ function manifestMembers(manifest, product) { function extensionInputs(extensionRoot, target, targetTriple, sourceFingerprint) { directory(extensionRoot, "WASIX extension artifact root"); - return exactExtensionProducts(PREFIX).map((product) => { + return [contribCarrierDescriptor(PREFIX).artifactProduct].map((product) => { const productRoot = extensionArtifactProductRoot(product, "wasix", extensionRoot, PREFIX); const manifestFile = path.join(productRoot, "extension-artifacts.json"); const manifest = readJson(manifestFile, `${product} extension artifact manifest`); @@ -308,37 +296,10 @@ function extensionInputs(extensionRoot, target, targetTriple, sourceFingerprint) }).sort((left, right) => compareText(left.product, right.product)); } -function visitRegularFiles(root, files = []) { - for (const entry of readdirSync(root, { withFileTypes: true }).sort((left, right) => compareText(left.name, right.name))) { - const file = path.join(root, entry.name); - if (entry.isSymbolicLink()) fail(`ICU input contains a symlink: ${repoPath(file, "ICU input")}`); - if (entry.isDirectory()) visitRegularFiles(file, files); - else if (entry.isFile()) files.push(file); - else fail(`ICU input contains a non-regular entry: ${repoPath(file, "ICU input")}`); - } - return files; -} - -function icuInput(icuRoot) { - directory(icuRoot, "ICU data root"); - const files = visitRegularFiles(icuRoot); - if (files.length === 0) fail("ICU data root must not be empty"); - const records = files.map((file) => { - const relative = path.relative(icuRoot, file).split(path.sep).join("/"); - return `${sha256(file)} ${relative}\n`; - }); - return { - path: repoPath(icuRoot, "ICU data root"), - sha256: sha256Bytes(Buffer.from(records.join(""), "utf8")), - fileCount: files.length, - }; -} - function buildInventory(options) { const portableRoot = path.resolve(options["portable-root"]); const aotRoot = path.resolve(options["aot-root"]); const extensionRoot = path.resolve(options["extension-root"]); - const icuRoot = path.resolve(options["icu-root"]); const portable = portableInputs(portableRoot); return { schema: "oliphaunt-wasix-napi-build-inputs-v1", @@ -357,7 +318,6 @@ function buildInventory(options) { options["target-triple"], portable.manifest["source-fingerprint"], ), - icuData: icuInput(icuRoot), }, }; } diff --git a/src/runtimes/wasix-napi/tools/package-platform.mjs b/src/runtimes/wasix-napi/tools/package-platform.mjs index 1f2c12f94..b4cf6af00 100755 --- a/src/runtimes/wasix-napi/tools/package-platform.mjs +++ b/src/runtimes/wasix-napi/tools/package-platform.mjs @@ -106,7 +106,8 @@ function main() { || typeof buildInputs.targetTriple !== "string" || buildInputs.targetTriple.length === 0 || !Array.isArray(buildInputs.inputs?.extensionArtifacts) - || buildInputs.inputs.extensionArtifacts.length === 0 + || buildInputs.inputs.extensionArtifacts.length !== 1 + || buildInputs.inputs.extensionArtifacts[0].product !== "oliphaunt-extension-contrib-pg18" ) { throw new Error(`${path.basename(buildInputsFile)} has incompatible WASIX N-API build inputs`); } diff --git a/src/sdks/js/README.md b/src/sdks/js/README.md index b273ff7a9..6df132f76 100644 --- a/src/sdks/js/README.md +++ b/src/sdks/js/README.md @@ -8,9 +8,10 @@ use the separate WASIX TypeScript package. ```ts import Oliphaunt from '@oliphaunt/ts'; +import { directory } from '@oliphaunt/ts/storage/node'; const db = await Oliphaunt.open({ - storage: { kind: 'directory', path: '.oliphaunt' }, + storage: directory('.oliphaunt'), startupGUCs: { application_name: 'my-app' }, }); @@ -199,15 +200,36 @@ the SDK deliberately does not guess that an owner is stale. ## Runtime and extensions -Platform native runtime, Node addon, broker, and ICU packages are optional -dependencies selected for the installed host. Explicit library, runtime, addon, -broker, or server paths exist for packaging and development scenarios. Native -client-tool packages remain separate products and are not SDK dependencies. +The SDK installs PostgreSQL and the supported contrib distribution for the host. +External extensions and ICU data are separate dependencies: -Extensions are selected by exact PostgreSQL SQL name through `extensions`. -Runtime artifact discovery remains internal. The package intentionally does not -publish capability profiles, supported-mode introspection, package-size reports, -generic streams, protocol parsers, or backup format helpers. +```sh +npm install @oliphaunt/ts @oliphaunt/extension-vector +``` + +```ts +import Oliphaunt, { extensions } from '@oliphaunt/ts'; +import { directory } from '@oliphaunt/ts/storage/node'; +import vector from '@oliphaunt/extension-vector'; + +const db = await Oliphaunt.open({ + storage: directory('./postgres'), + extensions: [vector, extensions.hstore], +}); +try { + await db.execute('CREATE EXTENSION vector'); + await db.execute('CREATE EXTENSION hstore'); +} finally { + await db.close(); +} +``` + +`directory` accepts a filesystem path or local `file:` URL. Contrib needs no +additional application dependency, but its descriptor must be passed explicitly. +The imported external descriptor identifies the installed package and version; +resource resolution is internal. Selection never runs migration SQL. -The package has one public code entrypoint, `@oliphaunt/ts`, plus -`@oliphaunt/ts/package.json` for package metadata. +For ICU collations, install `@oliphaunt/icu`, import its default `icu` value, +and pass `icu` to `Oliphaunt.open`. The same extension and ICU options apply to +broker and local-server configuration. Explicit library and runtime paths remain +available for advanced packaging and development. diff --git a/src/sdks/js/package.json b/src/sdks/js/package.json index 33bc208fa..12f4e87ca 100644 --- a/src/sdks/js/package.json +++ b/src/sdks/js/package.json @@ -53,6 +53,10 @@ }, "./package.json": { "default": "./package.json" + }, + "./storage/node": { + "types": "./lib/storage/node.d.ts", + "default": "./lib/storage/node.js" } }, "main": "lib/index.js", diff --git a/src/sdks/js/src/__tests__/client.test.ts b/src/sdks/js/src/__tests__/client.test.ts index aec45e84b..9dbe8e834 100644 --- a/src/sdks/js/src/__tests__/client.test.ts +++ b/src/sdks/js/src/__tests__/client.test.ts @@ -43,6 +43,8 @@ test('exposes the minimal database lifecycle and byte backup contract', async () username: 'app', database: 'appdb', extensions: [], + extensionDescriptors: [], + icu: undefined, startupArgs: ['-c', 'work_mem=16MB'], }); assert.deepEqual(await db.execute('UPDATE things SET value = 1'), { @@ -125,6 +127,8 @@ test('snapshots open configuration before asynchronous storage work', async () = username: 'before', database: 'before', extensions: [], + extensionDescriptors: [], + icu: undefined, libraryPath: undefined, runtimeDirectory: undefined, brokerExecutable: undefined, @@ -229,6 +233,8 @@ test('snapshots server storage and nested configuration before asynchronous work username: 'postgres', database: 'postgres', extensions: [], + extensionDescriptors: [], + icu: undefined, libraryPath: undefined, runtimeDirectory: undefined, brokerExecutable: undefined, diff --git a/src/sdks/js/src/__tests__/config.test.ts b/src/sdks/js/src/__tests__/config.test.ts index 9ec221b13..03812fc9d 100644 --- a/src/sdks/js/src/__tests__/config.test.ts +++ b/src/sdks/js/src/__tests__/config.test.ts @@ -1,3 +1,4 @@ +import { extensions } from '../extensions.js'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { test } from 'vitest'; @@ -28,7 +29,16 @@ test('normalizes only the public database and server configuration', () => { startupGUCs: { work_mem: '16MB' }, username: 'app_user', database: 'app_db', - extensions: [' vector ', '', 'hstore'], + extensions: [ + { + schema: 'oliphaunt-native-extension-v1', + product: 'oliphaunt-extension-vector', + packageName: '@oliphaunt/extension-vector', + sqlName: 'vector', + version: '0.8.2', + }, + extensions.hstore, + ], }, { instanceDirectory: '/app/root', temporaryDirectory: false }, ); diff --git a/src/sdks/js/src/__tests__/native-bindings.test.ts b/src/sdks/js/src/__tests__/native-bindings.test.ts index 93efb21d9..78ef6128c 100644 --- a/src/sdks/js/src/__tests__/native-bindings.test.ts +++ b/src/sdks/js/src/__tests__/native-bindings.test.ts @@ -52,7 +52,7 @@ async function main(): Promise { await testNodeNativeBindingUsesExplicitAssetsAndAddon(); await testDenoAssetResolverHonorsExplicitPaths(); await testDenoPackageManagedResolverUsesStandardCarrierRuntime(); - await testDenoNativeBindingRejectsPackageManagedExtensions(); + await testDenoNativeBindingRequiresCompleteExplicitRuntime(); await testDenoNativeBindingUsesSeparateModuleDirectoryWithoutAmbientMutation(); } @@ -412,7 +412,7 @@ async function testDenoAssetResolverHonorsExplicitPaths(): Promise { } } -async function testDenoNativeBindingRejectsPackageManagedExtensions(): Promise { +async function testDenoNativeBindingRequiresCompleteExplicitRuntime(): Promise { const previousDeno = (globalThis as { Deno?: unknown }).Deno; const previousLibrary = process.env.LIBOLIPHAUNT_PATH; const previousRuntime = process.env.OLIPHAUNT_RUNTIME_DIR; @@ -574,7 +574,7 @@ async function testDenoNativeBindingRejectsPackageManagedExtensions(): Promise @@ -588,7 +588,7 @@ async function testDenoNativeBindingRejectsPackageManagedExtensions(): Promise { + expect(snapshotNativeExtensions([])).toEqual([]); + const selected = snapshotNativeExtensions([vector, extensions.hstore]); + expect(selected.map((item) => item.sqlName)).toEqual(['vector', 'hstore']); + expect(Object.isFrozen(selected[0])).toBe(true); + expect(selected[0]).not.toBe(vector); + expect(snapshotNativeIcu(undefined)).toBeUndefined(); +}); + +test('resource selection rejects raw names, wrong runtimes, and conflicting versions', () => { + expect(() => snapshotNativeExtensions(['vector'] as never)).toThrow(/descriptors/); + expect(() => + snapshotNativeExtensions([{ ...vector, schema: 'oliphaunt-wasix-extension-v1' }] as never), + ).toThrow(); + expect(() => snapshotNativeExtensions([vector, { ...vector, version: '0.8.3' }])).toThrow( + /conflicting/, + ); + expect(() => snapshotNativeExtensions([{ ...vector, version: undefined }])).toThrow(/version/); + expect(() => snapshotNativeIcu(true as never)).toThrow(/descriptor/); +}); + +test('Node directory follows filesystem string and URL conventions', () => { + expect(directory(new URL('file:///tmp/my%20db'))).toEqual(directory('/tmp/my db')); + expect(directory('file:///tmp/db')).toEqual({ kind: 'directory', path: 'file:///tmp/db' }); + expect(() => directory(new URL('https://example.com/db'))).toThrow(); + expect(() => directory('\0')).toThrow(); +}); diff --git a/src/sdks/js/src/client.ts b/src/sdks/js/src/client.ts index 00b64641f..35f0fac79 100644 --- a/src/sdks/js/src/client.ts +++ b/src/sdks/js/src/client.ts @@ -1,3 +1,4 @@ +import { snapshotNativeExtensions, snapshotNativeIcu } from '@oliphaunt/js-core/resources'; import { mkdir, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -1180,7 +1181,9 @@ function snapshotCommonOpenConfig(config: OpenConfig | ServerOpenConfig) { startupGUCs: config.startupGUCs === undefined ? undefined : { ...config.startupGUCs }, username: config.username, database: config.database, - extensions: config.extensions === undefined ? undefined : [...config.extensions], + extensions: + config.extensions === undefined ? undefined : snapshotNativeExtensions(config.extensions), + icu: snapshotNativeIcu(config.icu), runtimeDirectory: config.runtimeDirectory, }; } diff --git a/src/sdks/js/src/config.ts b/src/sdks/js/src/config.ts index 1cd1ed317..df1d1a45c 100644 --- a/src/sdks/js/src/config.ts +++ b/src/sdks/js/src/config.ts @@ -1,3 +1,9 @@ +import { + snapshotNativeExtensions, + snapshotNativeIcu, + type NativeExtensionDescriptor, + type NativeIcuDescriptor, +} from '@oliphaunt/js-core/resources'; import { join } from 'node:path'; import { @@ -25,6 +31,8 @@ export type NormalizedOpenConfig = { username: string; database: string; extensions: string[]; + extensionDescriptors: NativeExtensionDescriptor[]; + icu?: NativeIcuDescriptor; libraryPath?: string; runtimeDirectory?: string; brokerExecutable?: string; @@ -42,7 +50,11 @@ export function normalizeOpenConfig( validateDirectoryPath(resolvedStorage.instanceDirectory, 'database storage directory'); validateStartupIdentity(config.username ?? DEFAULT_USERNAME, 'username'); validateStartupIdentity(config.database ?? DEFAULT_DATABASE, 'database'); - const extensions = config.extensions ? validateExtensionIds(config.extensions) : []; + const extensionDescriptors = snapshotNativeExtensions(config.extensions ?? []); + const extensions = validateExtensionIds( + extensionDescriptors.map((extension) => extension.sqlName), + ); + const icu = snapshotNativeIcu(config.icu); const topology = config.topology === 'server' ? 'server' : normalizeDatabaseTopology(config.topology); validateNativeStartupGUCs(topology, config.startupGUCs ?? {}); @@ -77,6 +89,8 @@ export function normalizeOpenConfig( username: config.username ?? DEFAULT_USERNAME, database: config.database ?? DEFAULT_DATABASE, extensions, + extensionDescriptors, + icu, libraryPath, runtimeDirectory, brokerExecutable, diff --git a/src/sdks/js/src/extensions.ts b/src/sdks/js/src/extensions.ts new file mode 100644 index 000000000..b710ec10c --- /dev/null +++ b/src/sdks/js/src/extensions.ts @@ -0,0 +1,22 @@ +import type { NativeExtensionDescriptor } from '@oliphaunt/js-core/resources'; +import { GENERATED_EXTENSION_METADATA } from './generated/extensions.js'; + +type ContribId = Extract< + (typeof GENERATED_EXTENSION_METADATA)[number], + { readonly runtimeBound: true } +>['id']; + +/** Bundled contrib resources. Listing a value selects it; no SQL is executed. */ +export const extensions: Readonly> = Object.freeze( + Object.fromEntries( + GENERATED_EXTENSION_METADATA.filter((row) => row.runtimeBound).map((row) => [ + row.id, + Object.freeze({ + schema: 'oliphaunt-native-extension-v1' as const, + sqlName: row.sqlName, + product: row.artifactProduct, + packageName: row.npmPackage, + }), + ]), + ) as Record, +); diff --git a/src/sdks/js/src/index.ts b/src/sdks/js/src/index.ts index ae8698e85..5e13f2a06 100644 --- a/src/sdks/js/src/index.ts +++ b/src/sdks/js/src/index.ts @@ -53,3 +53,6 @@ import type { OliphauntClient } from './types.js'; export const Oliphaunt: OliphauntClient = createOliphauntClient(); export default Oliphaunt; + +export { extensions } from './extensions.js'; +export type { NativeExtensionDescriptor, NativeIcuDescriptor } from '@oliphaunt/js-core/resources'; diff --git a/src/sdks/js/src/native/assets-deno.ts b/src/sdks/js/src/native/assets-deno.ts index 45b1dce90..3100dafb7 100644 --- a/src/sdks/js/src/native/assets-deno.ts +++ b/src/sdks/js/src/native/assets-deno.ts @@ -1,3 +1,4 @@ +import type { NativeIcuDescriptor } from '@oliphaunt/js-core/resources'; import { createRequire } from 'node:module'; import { join } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -82,16 +83,23 @@ type ResolvedDenoIcuResources = { export async function resolveDenoNativeInstall( libraryPath?: string, + icuDescriptor?: NativeIcuDescriptor, ): Promise { const explicit = resolveExplicitLibraryPath(libraryPath); if (explicit !== undefined) { const deno = optionalDenoRuntime(); const versions = deno === undefined ? undefined : await packageVersions(deno); const icuDataDirectory = - deno === undefined || versions === undefined + deno === undefined || versions === undefined || icuDescriptor === undefined ? undefined - : (await resolveDenoIcuResources(deno, versions.icuVersion, versions.icuPackage)) - ?.dataDirectory; + : ( + await resolveDenoIcuResources( + deno, + versions.icuVersion, + versions.icuPackage, + icuDescriptor, + ) + )?.dataDirectory; return { libraryPath: explicit, runtimeDirectory: resolveExplicitRuntimeDirectory(), @@ -103,7 +111,15 @@ export async function resolveDenoNativeInstall( const deno = denoRuntime(); const versions = await packageVersions(deno); - const icu = await resolveDenoIcuResources(deno, versions.icuVersion, versions.icuPackage); + const icu = + icuDescriptor === undefined + ? undefined + : await resolveDenoIcuResources( + deno, + versions.icuVersion, + versions.icuPackage, + icuDescriptor, + ); const target = liboliphauntPackageTarget(deno.build.os, deno.build.arch); return resolvePackageNativeInstall(deno, target, versions.liboliphauntVersion, icu); } @@ -262,11 +278,15 @@ async function resolveDenoIcuResources( deno: DenoRuntime, expectedVersion: string, packageName: string, -): Promise { - const packageJsonUrl = optionalResolvePackageJsonUrl(packageName); - if (packageJsonUrl === undefined) { - return undefined; + descriptor: NativeIcuDescriptor, +): Promise { + if (descriptor.packageName !== packageName || descriptor.version !== expectedVersion) { + throw new Error(`Selected ICU package must be ${packageName}@${expectedVersion}`); } + const packageJsonUrl = + descriptor.packageJsonUrl === undefined + ? resolvePackageJsonUrl(packageName) + : new URL(descriptor.packageJsonUrl); const packageJson = JSON.parse(await deno.readTextFile(packageJsonUrl)) as IcuPackageMetadata; validateDenoIcuPackageMetadata(packageJson, packageName, expectedVersion); const metadata = packageJson.oliphaunt!; @@ -396,23 +416,6 @@ function resolvePackageJsonUrl(packageName: string): URL { } } -function optionalResolvePackageJsonUrl(packageName: string): URL | undefined { - const specifier = `${packageName}/package.json`; - const resolver = (import.meta as ImportMeta & { resolve?: (specifier: string) => string }) - .resolve; - if (resolver === undefined) { - return optionalResolvePackageJsonUrlWithRequire(specifier); - } - try { - return new URL(resolver(specifier)); - } catch (error) { - if (importMetaResolveUnsupported(error)) { - return optionalResolvePackageJsonUrlWithRequire(specifier); - } - return undefined; - } -} - function resolvePackageJsonUrlWithRequire(packageName: string, specifier: string): URL { const resolved = optionalResolvePackageJsonUrlWithRequire(specifier); if (resolved !== undefined) { diff --git a/src/sdks/js/src/native/assets-node.ts b/src/sdks/js/src/native/assets-node.ts index 32991de3c..abce92b0d 100644 --- a/src/sdks/js/src/native/assets-node.ts +++ b/src/sdks/js/src/native/assets-node.ts @@ -1,3 +1,5 @@ +import type { NativeExtensionDescriptor, NativeIcuDescriptor } from '@oliphaunt/js-core/resources'; +import { fileURLToPath } from 'node:url'; import { createHash, randomUUID } from 'node:crypto'; import { createReadStream } from 'node:fs'; import { cp, lstat, mkdir, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'; @@ -84,6 +86,7 @@ type IcuPackageMetadata = { }; type ResolvedNodeIcuResources = { + seedDirectory: string; dataDirectory: string; dataTreeSha256: string; }; @@ -171,14 +174,15 @@ const NPM_EXTENSION_CONTRACT_MEMBER_FIELDS = [ export async function resolveNodeNativeInstall( libraryPath?: string, + icuDescriptor?: NativeIcuDescriptor, ): Promise { const versions = await packageVersions(); const explicit = resolveExplicitLibraryPath(libraryPath); if (explicit !== undefined) { - const icuDataDirectory = await resolveNodeIcuDataDirectory( - versions.icuVersion, - versions.icuPackage, - ); + const icuDataDirectory = + icuDescriptor === undefined + ? undefined + : (await resolveSelectedIcu(icuDescriptor, versions.icuVersion)).dataDirectory; return { libraryPath: explicit, runtimeDirectory: resolveExplicitRuntimeDirectory(), @@ -188,7 +192,10 @@ export async function resolveNodeNativeInstall( }; } - const icu = await resolveNodeIcuResources(versions.icuVersion, versions.icuPackage); + const icu = + icuDescriptor === undefined + ? undefined + : await resolveSelectedIcu(icuDescriptor, versions.icuVersion); const target = liboliphauntPackageTarget(platform(), arch()); return resolvePackageNativeInstall(target, versions.liboliphauntVersion, icu); } @@ -196,12 +203,15 @@ export async function resolveNodeNativeInstall( export async function prepareNodeExtensionInstall( install: ResolvedNativeInstall, extensions: ReadonlyArray = [], - options: { explicitRuntimeDirectory?: boolean } = {}, + options: { + explicitRuntimeDirectory?: boolean; + descriptors?: readonly NativeExtensionDescriptor[]; + } = {}, ): Promise { if (options.explicitRuntimeDirectory === true && extensions.length > 0) { return validatePreparedNodeRuntimeExtensions(install, extensions); } - return materializeNodeExtensionInstall(install, extensions); + return materializeNodeExtensionInstall(install, extensions, options.descriptors); } export async function validatePreparedNodeRuntimeExtensions( @@ -226,6 +236,7 @@ export async function validatePreparedNodeRuntimeExtensions( export async function materializeNodeExtensionInstall( install: ResolvedNativeInstall, extensions: ReadonlyArray = [], + descriptors: readonly NativeExtensionDescriptor[] = [], ): Promise { const selected = selectedExtensionClosure(extensions); if (selected.length === 0) { @@ -242,7 +253,12 @@ export async function materializeNodeExtensionInstall( const target = liboliphauntPackageTarget(platform(), arch()); const packages = await Promise.all( selected.map((sqlName) => - resolveExtensionPackage(sqlName, target.id, versions.liboliphauntVersion), + resolveExtensionPackage( + sqlName, + target.id, + versions.liboliphauntVersion, + descriptors.find((value) => value.sqlName === sqlName), + ), ), ); const cacheKey = runtimeCacheKey({ @@ -315,6 +331,24 @@ export async function materializeNodeExtensionInstall( return { ...install, runtimeDirectory, moduleDirectory }; } +async function resolveSelectedIcu( + descriptor: NativeIcuDescriptor, + runtimeVersion: string, +): Promise { + if (descriptor.version !== runtimeVersion) { + throw new Error( + `ICU package ${descriptor.version} is incompatible with runtime ${runtimeVersion}`, + ); + } + const resources = await resolveNodeIcuResources( + descriptor.version, + descriptor.packageName, + descriptor.packageJsonUrl, + ); + if (resources === undefined) throw new Error('selected ICU package is not installed'); + return resources; +} + export async function resolveNodeIcuDataDirectory( expectedVersion?: string, packageName?: string, @@ -325,6 +359,7 @@ export async function resolveNodeIcuDataDirectory( async function resolveNodeIcuResources( expectedVersion?: string, packageName?: string, + packageJsonUrl?: string, ): Promise { const versions = expectedVersion === undefined || packageName === undefined @@ -332,7 +367,8 @@ async function resolveNodeIcuResources( : undefined; const expected = expectedVersion ?? versions?.icuVersion; const name = packageName ?? versions?.icuPackage ?? '@oliphaunt/icu'; - const packageJsonPath = optionalResolvePackageJson(name); + const packageJsonPath = + packageJsonUrl === undefined ? optionalResolvePackageJson(name) : fileURLToPath(packageJsonUrl); if (packageJsonPath === undefined) { return undefined; } @@ -383,7 +419,11 @@ async function resolveNodeIcuResources( if (receiptDigest !== dataTreeSha256) { throw new Error(`${name} ICU data receipt does not match package metadata`); } - return { dataDirectory, dataTreeSha256 }; + return { + dataDirectory, + dataTreeSha256, + seedDirectory: join(dirname(manifestPath), 'native-seeds'), + }; } async function packageVersions(): Promise<{ @@ -423,6 +463,7 @@ async function resolveExtensionPackage( sqlName: string, target: string, liboliphauntVersion: string, + descriptor?: NativeExtensionDescriptor, ): Promise { const extension = generatedExtensionBySqlName(sqlName); if (extension === undefined) { @@ -434,6 +475,7 @@ async function resolveExtensionPackage( extension, targetPackageName, target, + descriptor, ); const packageJsonPath = resolvedTarget.packageJsonPath; const packageRoot = dirname(packageJsonPath); @@ -1341,11 +1383,7 @@ async function resolvePackageNativeInstall( 'cluster-seed', `${target.packageName} clusterSeedRelativePath`, ); - const icuClusterSeedRelativePath = requireNativeClusterSeedPath( - packageJson.oliphaunt.icuClusterSeedRelativePath, - 'cluster-seed-icu', - `${target.packageName} icuClusterSeedRelativePath`, - ); + const carrierManifestPath = join(packageRoot, 'manifest.properties'); await requireFile(carrierManifestPath, `${target.packageName} runtime carrier receipt`); validateNativeRuntimeCarrierReceipt( @@ -1386,8 +1424,8 @@ async function resolvePackageNativeInstall( icu === undefined ? standardClusterSeedDirectory : resolvePackageRelativePath( - packageRoot, - icuClusterSeedRelativePath, + icu.seedDirectory, + clusterSeedTarget, `${target.packageName} ICU cluster seed metadata`, ); let icuDataTreeSha256: string | undefined; @@ -1525,12 +1563,24 @@ async function resolveExtensionTargetPackageJson( extension: GeneratedExtensionMetadata, targetPackageName: string, target: string, + descriptor?: NativeExtensionDescriptor, ): Promise<{ packageJsonPath: string; ownerVersion: string }> { const packageName = extension.npmPackage; const expectedMembers = extensionOwnerMembers(extension); const isBundle = expectedMembers.length > 1; - const packageJsonPath = optionalResolvePackageJson(packageName); + if ( + descriptor !== undefined && + (descriptor.product !== extension.artifactProduct || descriptor.packageName !== packageName) + ) { + throw new Error(`extension descriptor identity does not match ${extension.sqlName}`); + } + const packageJsonPath = + descriptor?.packageJsonUrl === undefined + ? optionalResolvePackageJson(packageName) + : fileURLToPath(descriptor.packageJsonUrl); if (packageJsonPath === undefined) { + if (descriptor !== undefined) + throw new Error(`${packageName} selected by an imported descriptor is not installed`); if (isBundle) { throw new Error( `${packageName} is not installed; add it to the application dependencies for CREATE EXTENSION support`, @@ -1568,6 +1618,11 @@ async function resolveExtensionTargetPackageJson( if (typeof packageJson.version !== 'string' || packageJson.version.length === 0) { throw new Error(`${packageName} package metadata is missing version`); } + if (descriptor?.version !== undefined && packageJson.version !== descriptor.version) { + throw new Error( + `${packageName} version ${packageJson.version} does not match imported descriptor ${descriptor.version}`, + ); + } const resolvedTargetPackageName = packageJson.oliphaunt.targetPackageNames?.[target] ?? targetPackageName; if (resolvedTargetPackageName !== targetPackageName) { diff --git a/src/sdks/js/src/native/deno.ts b/src/sdks/js/src/native/deno.ts index e5838d6a2..ae41bf39f 100644 --- a/src/sdks/js/src/native/deno.ts +++ b/src/sdks/js/src/native/deno.ts @@ -4,7 +4,8 @@ import { errorMessage, replaceNativeIcuDataEnvironment, } from './common.js'; -import { resolveDenoNativeInstall, validatePreparedDenoRuntimeExtensions } from './assets-deno.js'; +import { resolveDenoNativeInstall } from './assets-deno.js'; +import { prepareNodeExtensionInstall } from './assets-node.js'; import { dirname, join } from 'node:path'; import { copyNativeClusterSeed, @@ -144,57 +145,43 @@ export async function createDenoNativeBinding( return { async open(config: NativeOpenConfig): Promise { assertDenoDirectAdmissionOpen(); + const selectedInstall = + config.icu === undefined + ? install + : await resolveDenoNativeInstall(options.libraryPath, config.icu); const explicitRuntimeDirectory = config.runtimeDirectory !== undefined || install.packageManaged === false; - let openConfig = { - ...config, - runtimeDirectory: config.runtimeDirectory ?? install.runtimeDirectory, - }; - let moduleDirectory: string | undefined; - if ( - openConfig.extensions.length > 0 && - (openConfig.runtimeDirectory === undefined || - (install.packageManaged && openConfig.runtimeDirectory === install.runtimeDirectory)) - ) { - throw new Error( - `Deno direct execution does not automatically materialize extension packages; pass runtimeDirectory with the selected extension assets or use Node/Bun direct execution. Selected extensions: ${openConfig.extensions.join(', ')}`, - ); - } - if (openConfig.extensions.length > 0) { - const validated = await validatePreparedDenoRuntimeExtensions({ - deno, - runtimeDirectory: openConfig.runtimeDirectory, - extensions: openConfig.extensions, - source: 'Deno direct explicit runtimeDirectory', - }); - openConfig = { - ...openConfig, - runtimeDirectory: validated.runtimeDirectory, + let extensionInstall = await prepareNodeExtensionInstall( + { + ...selectedInstall, + runtimeDirectory: config.runtimeDirectory ?? selectedInstall.runtimeDirectory, + clusterSeedDirectory: + config.runtimeDirectory === undefined + ? selectedInstall.clusterSeedDirectory + : undefined, + }, + config.extensions, + { explicitRuntimeDirectory, descriptors: config.extensionDescriptors }, + ); + if (explicitRuntimeDirectory && extensionInstall.runtimeDirectory !== undefined) { + extensionInstall = { + ...extensionInstall, + ...(await resolveExactNativeRuntimeProfile(extensionInstall.runtimeDirectory)), + clusterSeedDirectory: undefined, }; - // Keep canonical lib/postgresql subprocess-owned during initdb. The - // separate lib/modules $libdir is carried in the native config. - moduleDirectory = validated.moduleDirectory; - applyNativeRuntimeLibraryEnvironment(validated.runtimeDirectory); - } - const runtimeProfile = - explicitRuntimeDirectory && openConfig.runtimeDirectory !== undefined - ? await resolveExactNativeRuntimeProfile(openConfig.runtimeDirectory) - : { - icuDataDirectory: install.icuDataDirectory, - catalogProfile: install.catalogProfile ?? ('standard' as const), - }; - if (explicitRuntimeDirectory) { - replaceNativeIcuDataEnvironment(runtimeProfile.icuDataDirectory); - applyNativeRuntimeLibraryEnvironment(openConfig.runtimeDirectory); } + replaceNativeIcuDataEnvironment(extensionInstall.icuDataDirectory); + applyNativeRuntimeLibraryEnvironment(extensionInstall.runtimeDirectory); + const openConfig = { ...config, runtimeDirectory: extensionInstall.runtimeDirectory }; + const moduleDirectory = extensionInstall.moduleDirectory; await prepareDenoPgdata( deno, openConfig.pgdata, openConfig.username, openConfig.runtimeDirectory, - config.runtimeDirectory === undefined ? install.clusterSeedDirectory : undefined, - runtimeProfile.icuDataDirectory, - runtimeProfile.catalogProfile, + extensionInstall.clusterSeedDirectory, + extensionInstall.icuDataDirectory, + extensionInstall.catalogProfile, ); const packed = packConfigPointers({ ...openConfig, moduleDirectory }, (value) => pointerOf(deno, value), diff --git a/src/sdks/js/src/native/node.ts b/src/sdks/js/src/native/node.ts index fefbfb578..6520774eb 100644 --- a/src/sdks/js/src/native/node.ts +++ b/src/sdks/js/src/native/node.ts @@ -47,18 +47,25 @@ export async function createNodeNativeBinding( return { async open(config: NativeOpenConfig): Promise { + const selectedInstall = + config.icu === undefined + ? install + : await resolveNodeNativeInstall(options.libraryPath, config.icu); const explicitRuntimeDirectory = config.runtimeDirectory !== undefined || install.packageManaged === false; let extensionInstall = await prepareNodeExtensionInstall( { - ...install, - runtimeDirectory: config.runtimeDirectory ?? install.runtimeDirectory, + ...selectedInstall, + runtimeDirectory: config.runtimeDirectory ?? selectedInstall.runtimeDirectory, clusterSeedDirectory: - config.runtimeDirectory === undefined ? install.clusterSeedDirectory : undefined, + config.runtimeDirectory === undefined + ? selectedInstall.clusterSeedDirectory + : undefined, }, config.extensions, { explicitRuntimeDirectory, + descriptors: config.extensionDescriptors, }, ); if (explicitRuntimeDirectory && extensionInstall.runtimeDirectory !== undefined) { @@ -69,6 +76,7 @@ export async function createNodeNativeBinding( }; replaceNativeIcuDataEnvironment(extensionInstall.icuDataDirectory); } + replaceNativeIcuDataEnvironment(extensionInstall.icuDataDirectory); applyNativeRuntimeLibraryEnvironment(extensionInstall.runtimeDirectory); await prepareNodePgdata( config.pgdata, diff --git a/src/sdks/js/src/native/types.ts b/src/sdks/js/src/native/types.ts index ff7ce0da7..88a158e95 100644 --- a/src/sdks/js/src/native/types.ts +++ b/src/sdks/js/src/native/types.ts @@ -1,3 +1,5 @@ +import type { NativeExtensionDescriptor, NativeIcuDescriptor } from '@oliphaunt/js-core/resources'; + export type NativeBindingOptions = { libraryPath?: string; nodeAddonPath?: string; @@ -9,6 +11,8 @@ export type NativeOpenConfig = { username: string; database: string; extensions: string[]; + extensionDescriptors?: NativeExtensionDescriptor[]; + icu?: NativeIcuDescriptor; startupArgs: string[]; }; diff --git a/src/sdks/js/src/runtime/broker.ts b/src/sdks/js/src/runtime/broker.ts index bbe3f2cab..f7e074a69 100644 --- a/src/sdks/js/src/runtime/broker.ts +++ b/src/sdks/js/src/runtime/broker.ts @@ -1,3 +1,4 @@ +import type { NativeExtensionDescriptor, NativeIcuDescriptor } from '@oliphaunt/js-core/resources'; import { createRequire } from 'node:module'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -5,7 +6,6 @@ import { arch, platform } from 'node:os'; import { readFile, stat } from 'node:fs/promises'; import type { NormalizedOpenConfig } from '../config.js'; -import type { DenoRuntime } from '../native/assets-deno.js'; import { ICU_DATA_ENV, envVar, @@ -524,59 +524,17 @@ async function resolveBrokerNativeInstall(config: { libraryPath?: string; runtimeDirectory?: string; extensions?: readonly string[]; + extensionDescriptors?: readonly NativeExtensionDescriptor[]; + icu?: NativeIcuDescriptor; }): Promise { const extensions = config.extensions ?? []; - if (runtimeName() === 'deno') { - if ( - extensions.length > 0 && - config.runtimeDirectory === undefined && - envVar(LIBOLIPHAUNT_RUNTIME_DIR_ENV) === undefined - ) { - throw new Error( - `Deno broker execution does not automatically materialize extension packages; pass runtimeDirectory with the selected extension assets or use Node/Bun broker execution. Selected extensions: ${extensions.join(', ')}`, - ); - } - const assets = await import('../native/assets-deno.js'); - const deno = (globalThis as { Deno?: unknown }).Deno; - const install = await assets.resolveDenoNativeInstall(config.libraryPath); - const runtimeDirectory = config.runtimeDirectory ?? install.runtimeDirectory; - if ( - extensions.length > 0 && - (runtimeDirectory === undefined || - (install.packageManaged && config.runtimeDirectory === undefined)) - ) { - throw new Error( - `Deno broker execution does not automatically materialize extension packages; pass runtimeDirectory with the selected extension assets or use Node/Bun broker execution. Selected extensions: ${extensions.join(', ')}`, - ); - } - const validated = - extensions.length === 0 - ? { runtimeDirectory, moduleDirectory: undefined } - : await assets.validatePreparedDenoRuntimeExtensions({ - deno: deno as DenoRuntime, - runtimeDirectory, - extensions, - source: 'Deno broker explicit runtimeDirectory', - }); - const explicitRuntimeDirectory = - config.runtimeDirectory !== undefined || install.packageManaged === false; - const profile = - explicitRuntimeDirectory && validated.runtimeDirectory !== undefined - ? await resolveExactNativeRuntimeProfile(validated.runtimeDirectory) - : { - icuDataDirectory: install.icuDataDirectory, - catalogProfile: install.catalogProfile ?? ('standard' as const), - }; - return { - libraryPath: install.libraryPath, - runtimeDirectory: validated.runtimeDirectory, - ...profile, - moduleDirectory: validated.moduleDirectory, - }; - } - const assets = await import('../native/assets-node.js'); - const install = await assets.resolveNodeNativeInstall(config.libraryPath); + const install = + runtimeName() === 'deno' + ? await import('../native/assets-deno.js').then((module) => + module.resolveDenoNativeInstall(config.libraryPath, config.icu), + ) + : await assets.resolveNodeNativeInstall(config.libraryPath, config.icu); const explicitRuntimeDirectory = config.runtimeDirectory !== undefined || install.packageManaged === false; const resolved = { @@ -587,6 +545,7 @@ async function resolveBrokerNativeInstall(config: { }; const prepared = await assets.prepareNodeExtensionInstall(resolved, extensions, { explicitRuntimeDirectory, + descriptors: config.extensionDescriptors, }); if (!explicitRuntimeDirectory || prepared.runtimeDirectory === undefined) { return { diff --git a/src/sdks/js/src/runtime/direct.ts b/src/sdks/js/src/runtime/direct.ts index 82fb242fd..39cfcbc22 100644 --- a/src/sdks/js/src/runtime/direct.ts +++ b/src/sdks/js/src/runtime/direct.ts @@ -17,6 +17,8 @@ export function directRuntimeBinding(binding: NativeBinding): RuntimeBinding { username: config.username, database: config.database, extensions: config.extensions, + extensionDescriptors: config.extensionDescriptors, + icu: config.icu, startupArgs: config.startupArgs, }); }, diff --git a/src/sdks/js/src/runtime/server.ts b/src/sdks/js/src/runtime/server.ts index be9d6a42a..5080c9f7d 100644 --- a/src/sdks/js/src/runtime/server.ts +++ b/src/sdks/js/src/runtime/server.ts @@ -1,3 +1,4 @@ +import type { NativeExtensionDescriptor, NativeIcuDescriptor } from '@oliphaunt/js-core/resources'; import { spawn } from 'node:child_process'; import { chmod, lstat, mkdir, mkdtemp, readdir, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -288,6 +289,8 @@ async function openServer(config: NormalizedOpenConfig): Promise { serverExecutable: config.serverExecutable, runtimeDirectory: config.runtimeDirectory, extensions: config.extensions, + extensionDescriptors: config.extensionDescriptors, + icu: config.icu, }); const executable = tools.executable; const toolDirectory = tools.toolDirectory; @@ -543,6 +546,8 @@ export async function resolveServerTools(options: { serverExecutable?: string; runtimeDirectory?: string; extensions?: readonly string[]; + extensionDescriptors?: readonly NativeExtensionDescriptor[]; + icu?: NativeIcuDescriptor; }): Promise { const candidates = [ options.serverExecutable, @@ -565,7 +570,11 @@ export async function resolveServerTools(options: { if (options.serverExecutable !== undefined || options.runtimeDirectory !== undefined) { throw new Error(`set serverExecutable, runtimeDirectory, or ${OLIPHAUNT_POSTGRES_ENV}`); } - const install = await resolvePackageManagedServerInstall(options.extensions ?? []); + const install = await resolvePackageManagedServerInstall( + options.extensions ?? [], + options.extensionDescriptors, + options.icu, + ); if (install.runtimeDirectory !== undefined) { const toolDirectory = join(install.runtimeDirectory, 'bin'); const executable = join(toolDirectory, executableName('postgres')); @@ -587,28 +596,22 @@ export async function resolveServerTools(options: { ); } -async function resolvePackageManagedServerInstall(extensions: readonly string[]): Promise<{ +async function resolvePackageManagedServerInstall( + extensions: readonly string[], + descriptors: readonly NativeExtensionDescriptor[] = [], + icu?: NativeIcuDescriptor, +): Promise<{ runtimeDirectory?: string; icuDataDirectory?: string; catalogProfile?: 'standard' | 'icu'; }> { - if (runtimeName() === 'deno') { - if (extensions.length > 0) { - throw new Error( - `Deno server execution does not automatically materialize extension packages; pass runtimeDirectory with the selected extension assets or use Node/Bun openServer(). Selected extensions: ${extensions.join(', ')}`, - ); - } - const install = await import('../native/assets-deno.js').then((module) => - module.resolveDenoNativeInstall(), - ); - return { - runtimeDirectory: install.runtimeDirectory, - icuDataDirectory: install.icuDataDirectory, - catalogProfile: install.catalogProfile, - }; - } - - return materializeNodeExtensionInstall(await resolveNodeNativeInstall(), extensions); + const install = + runtimeName() === 'deno' + ? await import('../native/assets-deno.js').then((module) => + module.resolveDenoNativeInstall(undefined, icu), + ) + : await resolveNodeNativeInstall(undefined, icu); + return materializeNodeExtensionInstall(install, extensions, descriptors); } async function optionalTool( diff --git a/src/sdks/js/src/storage/node.ts b/src/sdks/js/src/storage/node.ts new file mode 100644 index 000000000..4f5345db9 --- /dev/null +++ b/src/sdks/js/src/storage/node.ts @@ -0,0 +1,13 @@ +import { fileURLToPath } from 'node:url'; + +import { validateDirectoryPath } from '../config.js'; +import type { DatabaseStorage } from '../types.js'; + +/** Persist a database in a host directory. Oliphaunt creates it when opened. */ +export function directory(path: string | URL): DatabaseStorage { + const location = typeof path === 'string' ? path : fileURLToPath(path); + validateDirectoryPath(location, 'database storage directory'); + return Object.freeze({ kind: 'directory', path: location }); +} + +export default directory; diff --git a/src/sdks/js/src/types.ts b/src/sdks/js/src/types.ts index 4f4a1615f..20a918e87 100644 --- a/src/sdks/js/src/types.ts +++ b/src/sdks/js/src/types.ts @@ -1,3 +1,5 @@ +import type { NativeExtensionDescriptor, NativeIcuDescriptor } from '@oliphaunt/js-core/resources'; + export type DatabaseStorage = | { readonly kind: 'temporaryDirectory' } | { readonly kind: 'directory'; readonly path: string }; @@ -19,7 +21,8 @@ export type OpenConfig = { startupGUCs?: Readonly>; username?: string; database?: string; - extensions?: ReadonlyArray; + extensions?: ReadonlyArray; + icu?: NativeIcuDescriptor; libraryPath?: string; runtimeDirectory?: string; brokerExecutable?: string; diff --git a/src/sdks/kotlin/README.md b/src/sdks/kotlin/README.md index 213c0aefb..3f026838b 100644 --- a/src/sdks/kotlin/README.md +++ b/src/sdks/kotlin/README.md @@ -6,10 +6,8 @@ the only supported and published application facade. ## Android setup -Apply `dev.oliphaunt.android` and depend on -`dev.oliphaunt:oliphaunt-android`. The plugin packages the matching runtime and -only the selected extension artifacts; applications do not build PostgreSQL at -runtime. +Apply the Android plugin and install the SDK. Add vector as an independently +versioned dependency; contrib ships with the SDK. ```kotlin plugins { @@ -18,48 +16,68 @@ plugins { dependencies { implementation("dev.oliphaunt:oliphaunt-android:0.2.0") -} - -oliphaunt { - icu.set(true) // Omit unless PostgreSQL ICU collations are required. + implementation("dev.oliphaunt.extensions:oliphaunt-extension-vector:0.2.0") } ``` -The Gradle plugin resolves ICU data together with the matching Android cluster -seed. This is a build-time package choice, not a database-open mode. - -Open with the Android `Oliphaunt` object and an application `Context`. - -`username` selects an existing PostgreSQL role. A new root is bootstrapped with -the fixed `postgres` role, so create additional roles from `postgres` before -opening that root as them. +The plugin reads resolved dependencies for each Android variant and packages +the required native artifacts. No duplicate extension or version list is needed. +Select resources explicitly when opening each database: ```kotlin +import dev.oliphaunt.* +import dev.oliphaunt.extensions.vector.Vector + val db = Oliphaunt.open( context = applicationContext, config = OliphauntConfig( storage = DatabaseStorage.Directory(filesDir.resolve("database")), - startupGucs = listOf(PostgresStartupGuc("application_name", "my-app")), - extensions = listOf("vector"), + startupGucs = mapOf("application_name" to "my-app"), + extensions = listOf(Vector.descriptor, Extensions.HSTORE), ), ) +try { + db.execute("CREATE EXTENSION vector") + db.execute("CREATE EXTENSION hstore") + val rows = db.query("SELECT '[1,2,3]'::vector <-> '[1,2,4]'::vector AS distance") + println(rows.rows.first().value("distance", PostgresDecoders.double)) +} finally { + db.close() +} +``` -db.execute( - "INSERT INTO widgets(name) VALUES ($1)", - listOf(QueryParam.string("ready")), -) -val rows = db.query("SELECT name FROM widgets") -println(rows.rows.first().value("name", PostgresDecoders.string)) +For ICU collations, add `dev.oliphaunt.runtime:oliphaunt-icu` at the compatible +native runtime version, import `dev.oliphaunt.icu.ICU`, and pass `icu = ICU.data`. +Installing resources determines what the app ships; the configuration determines +what each database selects. Adding native dependencies requires rebuilding the app. -val bytes = db.backup() -db.close() -Oliphaunt.restore( - context = applicationContext, - destination = filesDir.resolve("restored-database").path, - bytes = bytes, -) +`username` selects an existing PostgreSQL role. New roots start with `postgres`; +create additional roles before reopening a root as them. + +## Java on Android + +Java uses the same dependencies and runtime. Call the blocking facade on an +application worker thread and use try-with-resources: + +```java +import dev.oliphaunt.*; +import dev.oliphaunt.extensions.vector.Vector; +import java.io.File; + +var config = OliphauntConfig.builder() + .storage(new DatabaseStorage.Directory(new File(context.getFilesDir(), "database"))) + .startupGuc("application_name", "my-app") + .extensions(Vector.descriptor, Extensions.HSTORE) + .build(); +try (var db = OliphauntJava.open(context, config)) { + db.execute("CREATE EXTENSION vector"); + db.execute("CREATE EXTENSION hstore"); +} ``` +Use `.icu(ICU.data)` for the optional ICU dependency. Kotlin retains its suspend +API; Java's facade owns the same native session and adapts the calling convention. + ## API contract `query` executes one statement and returns ordered nullable bytes, complete @@ -144,11 +162,12 @@ and physical format. Initialization validates PGDATA first and publishes the descriptor last. Nonempty descriptorless roots and symlink structural directories are rejected without mutation. -`PostgresStartupGuc` is the only tuning vocabulary. Values map directly to +`startupGucs` is a map of PostgreSQL setting names to string values. Values map directly to PostgreSQL `-c name=value` settings; the SDK has no durability, memory, runtime, or capability profiles. -`OliphauntConfig.extensions` accepts exact generated PostgreSQL SQL names. +`OliphauntConfig.extensions` accepts imported external descriptors and SDK contrib +values. Both are explicit selections; applications run `CREATE EXTENSION` in SQL. Packaging resolves dependencies and native registration; package manifests and size reports remain internal build concerns. diff --git a/src/sdks/kotlin/gradle/libs.versions.toml b/src/sdks/kotlin/gradle/libs.versions.toml index 0f198a399..f2f9ff180 100644 --- a/src/sdks/kotlin/gradle/libs.versions.toml +++ b/src/sdks/kotlin/gradle/libs.versions.toml @@ -10,6 +10,7 @@ dokka = "2.2.0" kover = "0.9.8" [libraries] +android-gradle-api = { module = "com.android.tools.build:gradle-api", version.ref = "android-gradle-plugin" } kotlinx-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlinx-coroutines" } kotlinx-coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "kotlinx-coroutines" } kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinx-serialization" } diff --git a/src/sdks/kotlin/oliphaunt-android-gradle-plugin/build.gradle.kts b/src/sdks/kotlin/oliphaunt-android-gradle-plugin/build.gradle.kts index d9723821b..67d7a6baa 100644 --- a/src/sdks/kotlin/oliphaunt-android-gradle-plugin/build.gradle.kts +++ b/src/sdks/kotlin/oliphaunt-android-gradle-plugin/build.gradle.kts @@ -6,6 +6,11 @@ plugins { alias(libs.plugins.maven.publish) } +dependencies { + compileOnly(libs.android.gradle.api) + testImplementation(libs.android.gradle.api) +} + group = providers.gradleProperty("GROUP").orElse("dev.oliphaunt").get() version = providers.gradleProperty("VERSION_NAME").orElse("0.0.0").get() diff --git a/src/sdks/kotlin/oliphaunt-android-gradle-plugin/src/main/java/dev/oliphaunt/android/OliphauntAndroidPlugin.java b/src/sdks/kotlin/oliphaunt-android-gradle-plugin/src/main/java/dev/oliphaunt/android/OliphauntAndroidPlugin.java index 99d4590ef..5456e93f6 100644 --- a/src/sdks/kotlin/oliphaunt-android-gradle-plugin/src/main/java/dev/oliphaunt/android/OliphauntAndroidPlugin.java +++ b/src/sdks/kotlin/oliphaunt-android-gradle-plugin/src/main/java/dev/oliphaunt/android/OliphauntAndroidPlugin.java @@ -1,6 +1,14 @@ package dev.oliphaunt.android; -import java.lang.reflect.Method; +import com.android.build.api.variant.AndroidComponentsExtension; +import com.android.build.api.variant.Variant; +import org.gradle.api.Action; +import org.gradle.api.artifacts.result.ResolvedComponentResult; +import org.gradle.api.artifacts.result.ResolvedDependencyResult; +import java.util.ArrayDeque; +import java.util.HashSet; +import java.util.TreeSet; +import java.util.TreeMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -10,7 +18,6 @@ import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.Directory; import org.gradle.api.provider.Provider; -import org.gradle.api.tasks.Sync; import org.gradle.api.tasks.TaskProvider; public final class OliphauntAndroidPlugin implements Plugin { @@ -65,22 +72,33 @@ public void apply(Project project) { .map(OliphauntAndroidPlugin::parseAndroidAbis) .orElse(List.of("arm64-v8a", "x86_64"))); - Provider assetRoot = - project.getLayout().getBuildDirectory().dir("generated/oliphaunt-android-assets"); - Provider jniRoot = - project.getLayout().getBuildDirectory().dir("generated/oliphaunt-android-jniLibs"); + for (String plugin : List.of("com.android.application", "com.android.library")) { + project.getPluginManager().withPlugin(plugin, ignored -> configureAndroid(project, extension)); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static void configureAndroid(Project project, OliphauntAndroidExtension extension) { + AndroidComponentsExtension components = project.getExtensions().getByType(AndroidComponentsExtension.class); + components.onVariants(components.selector().all(), (Action) variant -> configureVariant(project, extension, components, variant)); + } + + private static void configureVariant(Project project, OliphauntAndroidExtension extension, AndroidComponentsExtension components, Variant variant) { + String suffix = Character.toUpperCase(variant.getName().charAt(0)) + variant.getName().substring(1); + Provider selection = variant.getRuntimeConfiguration().getIncoming().getResolutionResult().getRootComponent() + .map(root -> resolvedSelection(root, extension.getSelectedExtensions().get(), extension.getExtensionVersions().get(), extension.getIcu().get())); Provider extensionJniRoot = project .getLayout() .getBuildDirectory() - .dir("generated/oliphaunt-android-extension-jniLibs"); + .dir("generated/oliphaunt-android-extension-jniLibs/" + variant.getName()); Provider resolvedRoot = - project.getLayout().getBuildDirectory().dir("oliphaunt/resolved-artifacts"); + project.getLayout().getBuildDirectory().dir("oliphaunt/resolved-artifacts/" + variant.getName()); Configuration runtimeArtifacts = project .getConfigurations() .create( - "oliphauntAndroidRuntimeArtifacts", + "oliphauntAndroidRuntimeArtifacts" + suffix, configuration -> { configuration.setCanBeConsumed(false); configuration.setCanBeResolved(true); @@ -90,7 +108,7 @@ public void apply(Project project) { project .getConfigurations() .create( - "oliphauntAndroidExtensionArtifacts", + "oliphauntAndroidExtensionArtifacts" + suffix, configuration -> { configuration.setCanBeConsumed(false); configuration.setCanBeResolved(true); @@ -100,33 +118,50 @@ public void apply(Project project) { project .getConfigurations() .create( - "oliphauntAndroidIcuArtifacts", + "oliphauntAndroidIcuArtifacts" + suffix, configuration -> { configuration.setCanBeConsumed(false); configuration.setCanBeResolved(true); configuration.setDescription("Optional Oliphaunt Android ICU data artifact resolved from Maven."); }); - project.afterEvaluate(ignored -> addDefaultArtifactDependencies(project, extension, runtimeArtifacts, extensionArtifacts, icuArtifacts)); + runtimeArtifacts.defaultDependencies(dependencies -> { + String version = extension.getLiboliphauntVersion().get(); + dependencies.add(project.getDependencies().create("dev.oliphaunt.runtime:liboliphaunt-runtime-resources-android-datum64:" + version + "@tar.gz")); + for (String abi : extension.getAndroidAbis().get()) { + dependencies.add(project.getDependencies().create("dev.oliphaunt.runtime:liboliphaunt-" + androidTarget(abi) + ":" + version + "@tar.gz")); + } + }); + extensionArtifacts.defaultDependencies(dependencies -> { + Selection selected = selection.get(); + for (OliphauntExtensionCatalog.Owner owner : OliphauntExtensionCatalog.resolveOwners(selected.extensions(), selected.versions(), extension.getLiboliphauntVersion().get())) { + for (String abi : extension.getAndroidAbis().get()) { + dependencies.add(project.getDependencies().create(owner.mavenGroup() + ":" + owner.mavenArtifact() + "-" + androidTarget(abi) + ":" + owner.version() + "@tar.gz")); + } + } + }); + icuArtifacts.defaultDependencies(dependencies -> { + if (selection.get().icu()) dependencies.add(project.getDependencies().create("dev.oliphaunt.runtime:oliphaunt-icu:" + extension.getLiboliphauntVersion().get() + "@tar.gz")); + }); TaskProvider resolve = project .getTasks() .register( - "resolveOliphauntAndroidAssets", + "resolveOliphauntAndroidAssets" + suffix, ResolveOliphauntAndroidAssetsTask.class, task -> { task.getVersion().set(extension.getLiboliphauntVersion()); - task.getSelectedExtensions().set(extension.getSelectedExtensions()); + task.getSelectedExtensions().set(selection.map(Selection::extensions)); task.getExtensionOwnerVersions() .set( project.provider( () -> OliphauntExtensionCatalog.ownerVersions( - extension.getSelectedExtensions().get(), - extension.getExtensionVersions().get(), + selection.map(Selection::extensions).get(), + selection.map(Selection::versions).get(), extension.getLiboliphauntVersion().get()))); - task.getIcu().set(extension.getIcu()); + task.getIcu().set(selection.map(Selection::icu)); task.getSelectedAbis().set(extension.getAndroidAbis()); task.getRuntimeArtifacts().from(runtimeArtifacts); task.getExtensionArtifacts().from(extensionArtifacts); @@ -140,7 +175,7 @@ public void apply(Project project) { project .getTasks() .register( - "linkOliphauntAndroidExtensions", + "linkOliphauntAndroidExtensions" + suffix, LinkOliphauntAndroidExtensionsTask.class, task -> { task.setDescription( @@ -156,169 +191,43 @@ public void apply(Project project) { task.getOutputDirectory().set(extensionJniRoot); }); - TaskProvider prepareAssets = - project - .getTasks() - .register( - "prepareOliphauntAndroidAssets", - Sync.class, - task -> { - task.dependsOn(resolve); - task.from(resolve.flatMap(ResolveOliphauntAndroidAssetsTask::getRuntimeResourcesDir)); - task.into(assetRoot); - }); - TaskProvider prepareJniLibs = - project - .getTasks() - .register( - "prepareOliphauntAndroidJniLibs", - Sync.class, - task -> { - task.dependsOn(resolve); - task.from(resolve.flatMap(ResolveOliphauntAndroidAssetsTask::getJniLibsDir)); - task.into(jniRoot); - }); - - project - .getPluginManager() - .withPlugin( - "com.android.application", - ignored -> - configureAndroid( - project, - assetRoot, - jniRoot, - extensionJniRoot, - prepareAssets, - prepareJniLibs, - linkExtensions)); - project - .getPluginManager() - .withPlugin( - "com.android.library", - ignored -> - configureAndroid( - project, - assetRoot, - jniRoot, - extensionJniRoot, - prepareAssets, - prepareJniLibs, - linkExtensions)); - } - private static void configureAndroid( - Project project, - Provider assetRoot, - Provider jniRoot, - Provider extensionJniRoot, - TaskProvider prepareAssets, - TaskProvider prepareJniLibs, - TaskProvider linkExtensions) { - Object android = project.getExtensions().findByName("android"); - if (android == null) { - throw new GradleException("dev.oliphaunt.android requires the Android application or library plugin"); + linkExtensions.configure(task -> task.getNdkDirectory().set(components.getSdkComponents().getNdkDirectory())); + if (variant.getSources().getAssets() == null || variant.getSources().getJniLibs() == null) { + throw new GradleException("Oliphaunt requires Android assets and JNI source directories"); } - Object sourceSets = invoke(android, "getSourceSets"); - Object main = invoke(sourceSets, "getByName", "main"); - invoke(invoke(main, "getAssets"), "srcDir", assetRoot.get().getAsFile()); - invoke(invoke(main, "getJniLibs"), "srcDir", jniRoot.get().getAsFile()); - invoke(invoke(main, "getJniLibs"), "srcDir", extensionJniRoot); - Object androidComponents = project.getExtensions().findByName("androidComponents"); - if (androidComponents == null) { - throw new GradleException( - "dev.oliphaunt.android requires an Android Gradle Plugin version exposing androidComponents"); - } - setNdkDirectoryProvider( - linkExtensions, invoke(invoke(androidComponents, "getSdkComponents"), "getNdkDirectory")); - project - .getTasks() - .matching(task -> task.getName().equals("preBuild")) - .configureEach( - task -> { - task.dependsOn(prepareAssets); - task.dependsOn(prepareJniLibs); - task.dependsOn(linkExtensions); - }); + variant.getSources().getAssets().addGeneratedSourceDirectory(resolve, ResolveOliphauntAndroidAssetsTask::getRuntimeResourcesDir); + variant.getSources().getJniLibs().addGeneratedSourceDirectory(resolve, ResolveOliphauntAndroidAssetsTask::getJniLibsDir); + variant.getSources().getJniLibs().addGeneratedSourceDirectory(linkExtensions, LinkOliphauntAndroidExtensionsTask::getOutputDirectory); } - @SuppressWarnings("unchecked") - private static void setNdkDirectoryProvider( - TaskProvider task, Object candidate) { - if (!(candidate instanceof Provider)) { - throw new GradleException( - "Android Gradle Plugin sdkComponents.ndkDirectory is not a Gradle Provider"); - } - Provider provider = (Provider) candidate; - task.configure(link -> link.getNdkDirectory().set(provider)); - } + private record Selection(List extensions, Map versions, boolean icu) implements java.io.Serializable {} - private static Object invoke(Object target, String method, Object... args) { - Method candidate = null; - for (Method methodCandidate : target.getClass().getMethods()) { - if (methodCandidate.getName().equals(method) && methodCandidate.getParameterCount() == args.length) { - candidate = methodCandidate; - break; + private static Selection resolvedSelection(ResolvedComponentResult root, List supplied, Map suppliedVersions, boolean suppliedIcu) { + TreeSet selected = new TreeSet<>(supplied); + TreeMap versions = new TreeMap<>(suppliedVersions); + selected.addAll(OliphauntExtensionCatalog.artifactProductMembers("oliphaunt-extension-contrib-pg18")); + boolean icu = suppliedIcu; + var queue = new ArrayDeque(); + var visited = new HashSet(); + queue.add(root); + while (!queue.isEmpty()) { + ResolvedComponentResult component = queue.remove(); + if (!visited.add(component.getId())) continue; + var module = component.getModuleVersion(); + if (module != null && module.getGroup().equals("dev.oliphaunt.extensions")) { + String product = module.getName(); + selected.addAll(OliphauntExtensionCatalog.artifactProductMembers(product)); + String owner = OliphauntExtensionCatalog.releaseProductForArtifactProduct(product); + String previous = versions.put(owner, module.getVersion()); + if (previous != null && !previous.equals(module.getVersion())) throw new GradleException("conflicting resolved versions for " + owner); } - } - if (candidate == null) { - throw new GradleException("Android Gradle Plugin API no longer exposes " + method + " on " + target.getClass()); - } - try { - return candidate.invoke(target, args); - } catch (ReflectiveOperationException error) { - throw new GradleException("failed to call Android Gradle Plugin API " + method, error); - } - } - - private static void addDefaultArtifactDependencies( - Project project, - OliphauntAndroidExtension extension, - Configuration runtimeArtifacts, - Configuration extensionArtifacts, - Configuration icuArtifacts) { - String runtimeVersion = extension.getLiboliphauntVersion().get(); - project - .getDependencies() - .add( - runtimeArtifacts.getName(), - "dev.oliphaunt.runtime:liboliphaunt-runtime-resources-android-datum64:" - + runtimeVersion - + "@tar.gz"); - for (String abi : extension.getAndroidAbis().get()) { - String artifact = switch (abi) { - case "arm64-v8a" -> "liboliphaunt-android-arm64-v8a"; - case "x86_64" -> "liboliphaunt-android-x86_64"; - default -> throw new GradleException("Oliphaunt Android runtime artifacts are published for arm64-v8a and x86_64, got " + abi); - }; - project.getDependencies().add(runtimeArtifacts.getName(), "dev.oliphaunt.runtime:" + artifact + ":" + runtimeVersion + "@tar.gz"); - } - if (extension.getIcu().get()) { - project - .getDependencies() - .add(icuArtifacts.getName(), "dev.oliphaunt.runtime:oliphaunt-icu:" + runtimeVersion + "@tar.gz"); - } - List extensionOwners = - OliphauntExtensionCatalog.resolveOwners( - extension.getSelectedExtensions().get(), - extension.getExtensionVersions().get(), - runtimeVersion); - for (OliphauntExtensionCatalog.Owner owner : extensionOwners) { - for (String abi : extension.getAndroidAbis().get()) { - project - .getDependencies() - .add( - extensionArtifacts.getName(), - owner.mavenGroup() - + ":" - + owner.mavenArtifact() - + "-" - + androidTarget(abi) - + ":" - + owner.version() - + "@tar.gz"); + if (module != null && module.getGroup().equals("dev.oliphaunt.runtime") && module.getName().equals("oliphaunt-icu")) icu = true; + for (var dependency : component.getDependencies()) { + if (dependency instanceof ResolvedDependencyResult resolved) queue.add(resolved.getSelected()); } } + return new Selection(List.copyOf(selected), Map.copyOf(versions), icu); } private static List parsePortableList(String raw) { diff --git a/src/sdks/kotlin/oliphaunt-android-gradle-plugin/src/main/java/dev/oliphaunt/android/ResolveOliphauntAndroidAssetsTask.java b/src/sdks/kotlin/oliphaunt-android-gradle-plugin/src/main/java/dev/oliphaunt/android/ResolveOliphauntAndroidAssetsTask.java index 54419f333..250e47a49 100644 --- a/src/sdks/kotlin/oliphaunt-android-gradle-plugin/src/main/java/dev/oliphaunt/android/ResolveOliphauntAndroidAssetsTask.java +++ b/src/sdks/kotlin/oliphaunt-android-gradle-plugin/src/main/java/dev/oliphaunt/android/ResolveOliphauntAndroidAssetsTask.java @@ -209,6 +209,17 @@ public void resolve() { unpackAndroidExtensionArchives(selectedExtensionFiles); refreshRuntimeCacheKey(resourceRoot); writeRuntimeResourceSizeReport(resourceRoot, selectedRuntimeArtifacts); + var selection = new java.util.TreeMap(); + selection.put("schema", "oliphaunt-sdk-resources-v1"); + selection.put("runtimeVersion", releaseVersion); + selection.put("icuVersion", includeIcu ? releaseVersion : ""); + for (String name : getSelectedExtensions().get()) { + var entry = OliphauntExtensionCatalog.require(name); + selection.put("extension." + name + ".product", entry.artifactProduct()); + selection.put("extension." + name + ".version", getExtensionOwnerVersions().get().getOrDefault(entry.releaseProduct(), releaseVersion)); + } + writeText(new File(resourceRoot, "sdk-resources.properties"), + selection.entrySet().stream().map(entry -> entry.getKey() + "=" + entry.getValue() + "\n").collect(java.util.stream.Collectors.joining())); } private List effectiveAbis() { @@ -2070,6 +2081,9 @@ private void mergeIcuDataArtifact(File archive) { fileSystemOperations.delete(spec -> spec.delete(destination)); copyTree(icuRoot.toPath(), destination.toPath()); File icuClusterSeed = new File(root, "cluster-seed-icu"); + File packagedSeed = new File(carrier, "native-seeds/" + ANDROID_CLUSTER_SEED_TARGET); + validateClusterSeed(packagedSeed, "icu", icuDigest); + copyTree(packagedSeed.toPath(), icuClusterSeed.toPath()); File icuClusterSeedManifest = new File(icuClusterSeed, "manifest.properties"); if (!new File(icuClusterSeed, "files/PG_VERSION").isFile() || !new File(icuClusterSeed, "files/global/pg_control").isFile() @@ -2422,6 +2436,7 @@ private static void validateAndroidRuntimeClosure(File root) { "liboliphaunt Android runtime resources have inconsistent mobileStaticRegistrySource"); } validateClusterSeed(new File(root, "cluster-seed"), "standard", ""); + if (!new File(root, "cluster-seed-icu").exists()) return; Properties icuSeed = validateClusterSeed(new File(root, "cluster-seed-icu"), "icu", null); String icuDigest = icuSeed.getProperty("icuDataTreeSha256", ""); diff --git a/src/sdks/kotlin/oliphaunt-maven-artifacts/build.gradle.kts b/src/sdks/kotlin/oliphaunt-maven-artifacts/build.gradle.kts index 8b0c36326..e1787914a 100644 --- a/src/sdks/kotlin/oliphaunt-maven-artifacts/build.gradle.kts +++ b/src/sdks/kotlin/oliphaunt-maven-artifacts/build.gradle.kts @@ -2,6 +2,7 @@ import groovy.json.JsonSlurper import org.gradle.api.GradleException import org.gradle.api.publish.maven.MavenPublication import org.gradle.api.tasks.bundling.Jar +import org.gradle.api.tasks.compile.JavaCompile import java.util.Locale plugins { @@ -147,6 +148,52 @@ publishing { publications { oliphauntArtifacts.forEach { artifact -> val publicationName = publicationName(artifact) + val icuDescriptor = artifact.groupId == "dev.oliphaunt.runtime" && artifact.artifactId == "oliphaunt-icu" + val descriptorSource = if (icuDescriptor) { + layout.buildDirectory.file("generated/oliphaunt-descriptors/$publicationName/src/ICU.java").get().asFile + } else artifact.file.takeIf { it.extension == "java" } + val generateIcuDescriptor = if (icuDescriptor) { + tasks.register("${publicationName}GenerateDescriptor") { + inputs.property("version", artifact.version) + outputs.file(descriptorSource!!) + doLast { + descriptorSource.parentFile.mkdirs() + descriptorSource.writeText(""" + package dev.oliphaunt.icu; + /** Optional ICU data supplied by this package. */ + public final class ICU { + private ICU() {} + public static final dev.oliphaunt.IcuData data = new dev.oliphaunt.IcuData("${artifact.version}"); + } + """.trimIndent() + "\n") + } + } + } else null + val descriptorClasses = layout.buildDirectory.dir("generated/oliphaunt-descriptors/$publicationName/classes") + val compileDescriptor = descriptorSource?.let { source -> + tasks.register("${publicationName}CompileDescriptor") { + val sdkJar = project(":oliphaunt").tasks.named("jvmJar") + dependsOn(sdkJar) + if (generateIcuDescriptor != null) dependsOn(generateIcuDescriptor) + source(source) + classpath = files(sdkJar) + destinationDirectory.set(descriptorClasses) + sourceCompatibility = "17" + targetCompatibility = "17" + } + } + val descriptorJar = compileDescriptor?.let { compile -> + tasks.register("${publicationName}DescriptorJar") { + dependsOn(compile) + archiveBaseName.set(artifact.artifactId) + archiveVersion.set(artifact.version) + destinationDirectory.set(layout.buildDirectory.dir("oliphaunt-maven-artifacts/$publicationName")) + from(descriptorClasses) + from(baseReleaseNoticeFiles) { into("META-INF") } + isPreserveFileTimestamps = false + isReproducibleFileOrder = true + } + } val placeholderRoot = layout.buildDirectory.dir("generated/oliphaunt-maven-artifacts/$publicationName") val placeholderSources = placeholderRoot.map { it.file("sources/README.md") } val placeholderJavadocs = placeholderRoot.map { it.file("javadoc/index.html") } @@ -161,7 +208,7 @@ publishing { } placeholderJavadocs.get().asFile.apply { parentFile.mkdirs() - writeText("$coordinate

    This binary carrier has no Java API.

    \n") + writeText("$coordinate

    ${if (descriptorSource != null) "Versioned resource descriptor for Kotlin and Java. See the sources archive." else "This binary carrier has no Java API."}

    \n") } } } @@ -174,7 +221,10 @@ publishing { destinationDirectory.set(layout.buildDirectory.dir("oliphaunt-maven-artifacts/$publicationName")) isPreserveFileTimestamps = false isReproducibleFileOrder = true - from(placeholderSources) + if (descriptorSource != null) { + if (generateIcuDescriptor != null) dependsOn(generateIcuDescriptor) + from(descriptorSource) + } else from(placeholderSources) from(baseReleaseNoticeFiles) { into("META-INF") filePermissions { @@ -203,9 +253,8 @@ publishing { groupId = artifact.groupId artifactId = artifact.artifactId version = artifact.version - artifact(artifact.file) { - extension = "tar.gz" - } + if (descriptorJar != null) artifact(descriptorJar) + if (descriptorJar == null || icuDescriptor) artifact(artifact.file) { extension = "tar.gz" } artifact(sourcesJar) artifact(javadocJar) pom { @@ -217,6 +266,17 @@ publishing { publicationProperties["oliphaunt.runtime.version"] = artifact.runtimeVersion } properties.set(publicationProperties) + if (descriptorSource != null) { + withXml { + val dependencies = asNode().appendNode("dependencies") + val dependency = dependencies.appendNode("dependency") + dependency.appendNode("groupId", "dev.oliphaunt") + dependency.appendNode("artifactId", "oliphaunt-android") + dependency.appendNode("version", project(":oliphaunt").version.toString()) + dependency.appendNode("type", "aar") + dependency.appendNode("scope", "compile") + } + } inceptionYear.set("2026") url.set("https://github.com/f0rr0/oliphaunt") licenses { @@ -272,8 +332,8 @@ tasks.register("validateOliphauntMavenArtifacts") { if (!artifact.file.isFile) { throw GradleException("Missing Maven artifact file for ${artifact.groupId}:${artifact.artifactId}: ${artifact.file}") } - if (!artifact.file.name.endsWith(".tar.gz")) { - throw GradleException("Oliphaunt Maven artifact ${artifact.file} must be a .tar.gz file") + if (!artifact.file.name.endsWith(".tar.gz") && artifact.file.extension != "java") { + throw GradleException("Oliphaunt Maven artifact ${artifact.file} must be a carrier .tar.gz or descriptor .java file") } if ((artifact.runtimeProduct == null) != (artifact.runtimeVersion == null)) { throw GradleException( diff --git a/src/sdks/kotlin/oliphaunt/src/androidMain/cpp/include/oliphaunt.h b/src/sdks/kotlin/oliphaunt/src/androidMain/cpp/include/oliphaunt.h index d96facff7..7f55b34bd 100644 --- a/src/sdks/kotlin/oliphaunt/src/androidMain/cpp/include/oliphaunt.h +++ b/src/sdks/kotlin/oliphaunt/src/androidMain/cpp/include/oliphaunt.h @@ -222,9 +222,11 @@ OLIPHAUNT_API int32_t oliphaunt_close(OliphauntHandle *handle); * Registers statically linked PostgreSQL extension modules for the embedded * backend's normal LOAD path. * - * Call this before oliphaunt_init in processes that link extension code directly - * into the application or SDK library. The registry is process-wide and becomes - * immutable once backend startup begins. Each extension name is the module stem + * Call this before loading a module linked into the application or SDK library. + * The process-wide registry permits additions after backend startup; repeating an + * identical entry is harmless, while replacing an existing module is rejected. + * Registration copies descriptors and retains them for the process lifetime. + * Each extension name is the module stem * used by SQL, for example AS 'vector', and each symbol row exposes the C * symbols PostgreSQL would otherwise resolve with dlsym(). */ diff --git a/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/AndroidNativeDirectEngine.kt b/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/AndroidNativeDirectEngine.kt index 580b11552..48bf19a99 100644 --- a/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/AndroidNativeDirectEngine.kt +++ b/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/AndroidNativeDirectEngine.kt @@ -47,6 +47,8 @@ internal class AndroidNativeDirectEngine( ?: env("OLIPHAUNT_INSTALL_DIR") ?: env("OLIPHAUNT_RUNTIME_DIR"), requestedExtensions = config.extensions, + descriptors = config.extensionDescriptors, + icu = config.icu, resourceRoot = resourceRoot, ) val storageDirectory = diff --git a/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntAndroid.kt b/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntAndroid.kt index f3a2d5c8f..89e161b83 100644 --- a/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntAndroid.kt +++ b/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntAndroid.kt @@ -11,11 +11,42 @@ public sealed interface DatabaseStorage { public data class OliphauntConfig( val storage: DatabaseStorage = DatabaseStorage.TemporaryDirectory, - val startupGucs: List = emptyList(), + val startupGucs: Map = emptyMap(), val username: String? = null, val database: String? = null, - val extensions: List = emptyList(), -) + val extensions: List = emptyList(), + val icu: IcuData? = null, +) { + public companion object { + @JvmStatic + public fun builder(): Builder = Builder() + } + + public class Builder internal constructor() { + private var storage: DatabaseStorage = DatabaseStorage.TemporaryDirectory + private var username: String? = null + private var database: String? = null + private var icu: IcuData? = null + private val gucs = linkedMapOf() + private val extensions = mutableListOf() + + public fun storage(value: DatabaseStorage): Builder = apply { storage = value } + public fun username(value: String): Builder = apply { username = value } + public fun database(value: String): Builder = apply { database = value } + public fun icu(value: IcuData): Builder = apply { icu = value } + public fun startupGuc(name: String, value: String): Builder = apply { gucs[name] = value } + public fun startupGucs(values: Map): Builder = apply { gucs.putAll(values) } + public fun extensions(vararg values: ExtensionDescriptor): Builder = apply { extensions.addAll(values) } + public fun build(): OliphauntConfig = OliphauntConfig( + storage, + gucs.toMap(), + username, + database, + selectedExtensionDescriptors(extensions), + icu, + ) + } +} public object Oliphaunt { public suspend fun open( @@ -51,8 +82,10 @@ internal fun OliphauntConfig.toEngineConfig(): EngineConfig = EngineConfig( DatabaseStorage.TemporaryDirectory -> EngineStorage.TemporaryDirectory is DatabaseStorage.Directory -> EngineStorage.Directory(selected.path.absolutePath) }, - startupGucs = startupGucs.toList(), + startupGucs = startupGucs.map { (name, value) -> PostgresStartupGuc(name, value) }, username = username, database = database, - extensions = extensions.toList(), + extensions = selectedExtensionDescriptors(extensions).map { it.sqlName }, + extensionDescriptors = selectedExtensionDescriptors(extensions), + icu = icu, ) diff --git a/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssets.kt b/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssets.kt index 9f3875549..34cf5148d 100644 --- a/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssets.kt +++ b/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssets.kt @@ -27,6 +27,7 @@ internal data class OliphauntAndroidAssetPackage( val target: String = "", val compatibilityKey: String = "", val icuDataTreeSha256: String = "", + val requestedExtensions: Set? = null, ) internal data class OliphauntPackageSizeReport( @@ -172,8 +173,10 @@ internal object OliphauntAndroidRuntimeAssets { explicitRuntimeDirectory: String?, requestedExtensions: Collection = emptyList(), resourceRoot: File? = null, + descriptors: List = emptyList(), + icu: IcuData? = null, ): OliphauntAndroidResolvedRuntime { - val requestedExtensionSet = validateExtensionIds(requestedExtensions) + val requestedExtensionSet = selectedExtensionClosure(validateExtensionIds(requestedExtensions)) val explicitRuntime = explicitRuntimeDirectory?.takeIf(String::isNotEmpty) if (explicitRuntime != null) { val sharedPreloadLibraries = @@ -182,7 +185,22 @@ internal object OliphauntAndroidRuntimeAssets { requestedExtensionSet, ) val runtimePackage = releaseShapedRuntimePackageForDirectory(explicitRuntime) - val clusterSeed = runtimePackage?.let(::matchingReleaseShapedClusterSeed) + if (descriptors.isNotEmpty() || icu != null) { + val root = runtimePackage?.resourceRoot + ?: throw OliphauntException("selected package descriptors require a release-shaped explicit runtime directory") + val receipt = File(root, "oliphaunt/sdk-resources.properties") + validateSelectedResourceReceipt(parseManifestText(receipt.readText(), receipt.path), descriptors, icu) + } + if (icu != null && runtimePackage?.runtimeFeatures?.contains("icu") != true) { + throw OliphauntException("selected ICU resources are absent from the explicit runtime directory") + } + val clusterSeed = runtimePackage?.let { + matchingReleaseShapedClusterSeed( + it.copy( + runtimeFeatures = if (icu == null) it.runtimeFeatures - "icu" else it.runtimeFeatures, + ), + ) + } return OliphauntAndroidResolvedRuntime( runtimeDirectory = explicitRuntime, clusterSeed = clusterSeed, @@ -220,15 +238,46 @@ internal object OliphauntAndroidRuntimeAssets { } else { filePackageManifestOrNull(resourceRoot, RUNTIME_ASSET_ROOT) } - val clusterSeed = matchingClusterSeed(packagedRuntime, standardClusterSeed, icuClusterSeed) - val runtimeDirectory = materializePackagedRuntime(context, requestedExtensionSet, packagedRuntime) + if (descriptors.isNotEmpty() || icu != null) { + val receiptPath = "oliphaunt/sdk-resources.properties" + val text = if (resourceRoot == null) { + context.assets.open(receiptPath).bufferedReader().use { it.readText() } + } else { + File(resourceRoot, receiptPath).readText() + } + validateSelectedResourceReceipt(parseManifestText(text, receiptPath), descriptors, icu) + } + if (icu != null && packagedRuntime?.runtimeFeatures?.contains("icu") != true) { + throw OliphauntException("selected ICU resources are not packaged in this application") + } + val preloads = requestedExtensionSet.flatMap { generatedExtensionRuntimeContract(it)?.sharedPreloads.orEmpty() }.toSet() + val selection = packagedRuntime?.copy( + runtimeFeatures = if (icu == null) packagedRuntime.runtimeFeatures - "icu" else packagedRuntime.runtimeFeatures, + sharedPreloadLibraries = preloads, + requestedExtensions = requestedExtensionSet, + ) + val clusterSeed = matchingClusterSeed(selection, standardClusterSeed, icuClusterSeed) + val runtimeDirectory = materializePackagedRuntime(context, requestedExtensionSet, selection) return OliphauntAndroidResolvedRuntime( runtimeDirectory = runtimeDirectory, clusterSeed = clusterSeed, - sharedPreloadLibraries = packagedRuntime?.sharedPreloadLibraries.orEmpty(), + sharedPreloadLibraries = preloads, ) } + internal fun validateSelectedResourceReceipt(values: Properties, descriptors: List, icu: IcuData?) { + if (values.getProperty("schema") != "oliphaunt-sdk-resources-v1") throw OliphauntException("unsupported SDK resource receipt") + for (descriptor in selectedExtensionDescriptors(descriptors)) { + val prefix = "extension.${descriptor.sqlName}" + if (values.getProperty("$prefix.product") != descriptor.product || + (descriptor.version != null && values.getProperty("$prefix.version") != descriptor.version) + ) { + throw OliphauntException("selected extension '${descriptor.sqlName}' does not match packaged product/version") + } + } + if (icu != null && values.getProperty("icuVersion") != icu.version) throw OliphauntException("selected ICU version does not match packaged resources") + } + internal fun validateExplicitRuntimeDirectory( runtimeDirectory: String, requestedExtensions: Collection, @@ -250,7 +299,7 @@ internal object OliphauntAndroidRuntimeAssets { requestedExtensions = requestedExtensionSet, runtimeFiles = File(runtimeDirectory), ) - return runtimePackage.sharedPreloadLibraries + return selectedExtensionClosure(requestedExtensionSet).flatMap { generatedExtensionRuntimeContract(it)?.sharedPreloads.orEmpty() }.toSet() } fun packageSizeReport(assetManager: AssetManager): OliphauntPackageSizeReport? = try { @@ -465,10 +514,13 @@ internal object OliphauntAndroidRuntimeAssets { "-PoliphauntRuntimeResourcesDir=.", ) requirePackagedExtensions(runtimePackage, requestedExtensions) + val selectionKey = java.security.MessageDigest.getInstance("SHA-256").digest( + (requestedExtensions.sorted().joinToString(",") + ":" + runtimePackage.runtimeFeatures.sorted().joinToString(",")).toByteArray(), + ).joinToString("") { "%02x".format(it) } val runtimeRoot = File( context.noBackupFilesDir, - "oliphaunt/runtime/${runtimePackage.cacheKey}", + "oliphaunt/runtime/${runtimePackage.cacheKey}/$selectionKey", ) materializeAssetPackage(context.assets, runtimePackage, runtimeRoot) requireExtensionInstallFiles(runtimePackage, requestedExtensions, runtimeRoot) @@ -1297,10 +1349,13 @@ internal object OliphauntAndroidRuntimeAssets { destination: File, ) { val resourceRoot = assetPackage.resourceRoot + val include: (String) -> Boolean = { relative -> + assetPackage.requestedExtensions?.let { includeSelectedRuntimeFile(relative, it, "icu" in assetPackage.runtimeFeatures) } ?: true + } if (resourceRoot == null) { - copyAssetTree(assetManager, "${assetPackage.assetRoot}/$FILES_DIR_NAME", destination) + copyAssetTree(assetManager, "${assetPackage.assetRoot}/$FILES_DIR_NAME", destination, include) } else { - copyFileTree(File(resourceRoot, "${assetPackage.assetRoot}/$FILES_DIR_NAME"), destination) + copyFileTree(File(resourceRoot, "${assetPackage.assetRoot}/$FILES_DIR_NAME"), destination, include) } } @@ -1315,7 +1370,10 @@ internal object OliphauntAndroidRuntimeAssets { assetManager: AssetManager, assetPath: String, destination: File, + include: (String) -> Boolean, + relative: String = "", ) { + if (!include(relative)) return val children = assetManager.list(assetPath) ?: throw OliphauntException("failed to list Android asset path $assetPath") @@ -1337,14 +1395,17 @@ internal object OliphauntAndroidRuntimeAssets { throw OliphauntException("failed to create directory ${destination.absolutePath}") } children.sorted().forEach { child -> - copyAssetTree(assetManager, "$assetPath/$child", File(destination, child)) + copyAssetTree(assetManager, "$assetPath/$child", File(destination, child), include, if (relative.isEmpty()) child else "$relative/$child") } } private fun copyFileTree( source: File, destination: File, + include: (String) -> Boolean, + relative: String = "", ) { + if (!include(relative)) return if (!source.exists()) { throw OliphauntException("missing Oliphaunt resource path ${source.absolutePath}") } @@ -1364,7 +1425,7 @@ internal object OliphauntAndroidRuntimeAssets { throw OliphauntException("failed to create directory ${destination.absolutePath}") } source.listFiles().orEmpty().sortedBy(File::getName).forEach { child -> - copyFileTree(child, File(destination, child.name)) + copyFileTree(child, File(destination, child.name), include, if (relative.isEmpty()) child.name else "$relative/${child.name}") } } diff --git a/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntJava.kt b/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntJava.kt new file mode 100644 index 000000000..2825f5fbd --- /dev/null +++ b/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntJava.kt @@ -0,0 +1,28 @@ +package dev.oliphaunt + +import android.content.Context +import kotlinx.coroutines.runBlocking + +/** Blocking Java access to the Kotlin SDK. Invoke on application worker threads. */ +public object OliphauntJava { + @JvmStatic + @JvmOverloads + public fun open(context: Context, config: OliphauntConfig = OliphauntConfig()): BlockingOliphauntDatabase = BlockingOliphauntDatabase(runBlocking { Oliphaunt.open(context, config) }) +} + +/** Owns the same native session as [OliphauntDatabase] and supports try-with-resources. */ +public class BlockingOliphauntDatabase internal constructor(private val database: OliphauntDatabase) : AutoCloseable { + @JvmOverloads + public fun execute(sql: String, parameters: List = emptyList()): CommandResult = runBlocking { database.execute(sql, parameters) } + + @JvmOverloads + public fun query(sql: String, parameters: List = emptyList()): QueryResult = runBlocking { database.query(sql, parameters) } + + public fun exec(sql: String): ExecResult = runBlocking { database.exec(sql) } + + public fun backup(): ByteArray = runBlocking { database.backup() } + + public fun cancel(): Unit = runBlocking { database.cancel() } + + override fun close(): Unit = runBlocking { database.close() } +} diff --git a/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssetsTest.kt b/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssetsTest.kt index 6810e8319..d386e0fb0 100644 --- a/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssetsTest.kt +++ b/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssetsTest.kt @@ -295,7 +295,7 @@ class OliphauntAndroidRuntimeAssetsTest { } @Test - fun validatesExplicitRuntimeDirectoryAgainstReleaseShapedResources() { + fun explicitRuntimeDirectoryPreloadsOnlySelectedExtensions() { val resourceRoot = Files.createTempDirectory("liboliphaunt-explicit-runtime").toFile() try { val runtimeFiles = @@ -311,7 +311,7 @@ class OliphauntAndroidRuntimeAssetsTest { listOf("vector"), ) - assertEquals(setOf("pg_search"), sharedPreloadLibraries) + assertEquals(emptySet(), sharedPreloadLibraries) } finally { resourceRoot.deleteRecursively() } diff --git a/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidTest.kt b/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidTest.kt index e6ca9b414..23c20df25 100644 --- a/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidTest.kt +++ b/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidTest.kt @@ -1,23 +1,55 @@ package dev.oliphaunt +import java.util.Properties import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue class OliphauntAndroidTest { @Test fun publicConfigurationConversionSnapshotsMutableCollections() { - val startupGucs = mutableListOf(PostgresStartupGuc("shared_buffers", "16MB")) - val extensions = mutableListOf("pgtap") + val startupGucs = mutableMapOf("shared_buffers" to "16MB") + val extensions = mutableListOf(ExtensionDescriptor("pgtap", "oliphaunt-extension-pgtap", "1.3.4")) val converted = OliphauntConfig( startupGucs = startupGucs, extensions = extensions, ).toEngineConfig() - startupGucs[0] = PostgresStartupGuc("work_mem", "64MB") - extensions[0] = "vector" + startupGucs["shared_buffers"] = "64MB" + extensions[0] = ExtensionDescriptor("vector", "oliphaunt-extension-vector", "0.8.2") assertEquals(listOf(PostgresStartupGuc("shared_buffers", "16MB")), converted.startupGucs) assertEquals(listOf("pgtap"), converted.extensions) } + + @Test + fun selectedResourcesKeepDependenciesAndRejectVersionSubstitution() { + val selected = selectedExtensionClosure(setOf("earthdistance", "vector")) + assertTrue("cube" in selected) + assertTrue(includeSelectedRuntimeFile("lib/postgresql/vector.so", selected, false)) + assertTrue(includeSelectedRuntimeFile("share/postgresql/extension/cube--1.5.sql", selected, false)) + assertFalse(includeSelectedRuntimeFile("share/postgresql/extension/hstore.control", selected, false)) + assertFalse(includeSelectedRuntimeFile("lib/postgresql/hstore.so", selected, false)) + assertFalse(includeSelectedRuntimeFile("share/icu/icudt.dat", selected, false)) + val receipt = Properties().apply { + setProperty("schema", "oliphaunt-sdk-resources-v1") + setProperty("extension.vector.product", "oliphaunt-extension-vector") + setProperty("extension.vector.version", "0.8.2") + } + OliphauntAndroidRuntimeAssets.validateSelectedResourceReceipt( + receipt, + listOf(ExtensionDescriptor("vector", "oliphaunt-extension-vector", "0.8.2")), + null, + ) + assertFailsWith { + OliphauntAndroidRuntimeAssets.validateSelectedResourceReceipt( + receipt, + listOf(ExtensionDescriptor("vector", "oliphaunt-extension-vector", "0.8.3")), + null, + ) + } + } } diff --git a/src/sdks/kotlin/oliphaunt/src/commonMain/kotlin/dev/oliphaunt/GeneratedExtensions.kt b/src/sdks/kotlin/oliphaunt/src/commonMain/kotlin/dev/oliphaunt/GeneratedExtensions.kt index f4823459c..f61467836 100644 --- a/src/sdks/kotlin/oliphaunt/src/commonMain/kotlin/dev/oliphaunt/GeneratedExtensions.kt +++ b/src/sdks/kotlin/oliphaunt/src/commonMain/kotlin/dev/oliphaunt/GeneratedExtensions.kt @@ -3,51 +3,119 @@ package dev.oliphaunt +public object Extensions { + @JvmField public val AMCHECK = ExtensionDescriptor("amcheck", "oliphaunt-extension-contrib-pg18") + + @JvmField public val AUTO_EXPLAIN = ExtensionDescriptor("auto_explain", "oliphaunt-extension-contrib-pg18") + + @JvmField public val BLOOM = ExtensionDescriptor("bloom", "oliphaunt-extension-contrib-pg18") + + @JvmField public val BTREE_GIN = ExtensionDescriptor("btree_gin", "oliphaunt-extension-contrib-pg18") + + @JvmField public val BTREE_GIST = ExtensionDescriptor("btree_gist", "oliphaunt-extension-contrib-pg18") + + @JvmField public val CITEXT = ExtensionDescriptor("citext", "oliphaunt-extension-contrib-pg18") + + @JvmField public val CUBE = ExtensionDescriptor("cube", "oliphaunt-extension-contrib-pg18") + + @JvmField public val DICT_INT = ExtensionDescriptor("dict_int", "oliphaunt-extension-contrib-pg18") + + @JvmField public val DICT_XSYN = ExtensionDescriptor("dict_xsyn", "oliphaunt-extension-contrib-pg18") + + @JvmField public val EARTHDISTANCE = ExtensionDescriptor("earthdistance", "oliphaunt-extension-contrib-pg18") + + @JvmField public val FILE_FDW = ExtensionDescriptor("file_fdw", "oliphaunt-extension-contrib-pg18") + + @JvmField public val FUZZYSTRMATCH = ExtensionDescriptor("fuzzystrmatch", "oliphaunt-extension-contrib-pg18") + + @JvmField public val HSTORE = ExtensionDescriptor("hstore", "oliphaunt-extension-contrib-pg18") + + @JvmField public val INTARRAY = ExtensionDescriptor("intarray", "oliphaunt-extension-contrib-pg18") + + @JvmField public val ISN = ExtensionDescriptor("isn", "oliphaunt-extension-contrib-pg18") + + @JvmField public val LO = ExtensionDescriptor("lo", "oliphaunt-extension-contrib-pg18") + + @JvmField public val LTREE = ExtensionDescriptor("ltree", "oliphaunt-extension-contrib-pg18") + + @JvmField public val PAGEINSPECT = ExtensionDescriptor("pageinspect", "oliphaunt-extension-contrib-pg18") + + @JvmField public val PG_BUFFERCACHE = ExtensionDescriptor("pg_buffercache", "oliphaunt-extension-contrib-pg18") + + @JvmField public val PG_FREESPACEMAP = ExtensionDescriptor("pg_freespacemap", "oliphaunt-extension-contrib-pg18") + + @JvmField public val PG_SURGERY = ExtensionDescriptor("pg_surgery", "oliphaunt-extension-contrib-pg18") + + @JvmField public val PG_TRGM = ExtensionDescriptor("pg_trgm", "oliphaunt-extension-contrib-pg18") + + @JvmField public val PG_VISIBILITY = ExtensionDescriptor("pg_visibility", "oliphaunt-extension-contrib-pg18") + + @JvmField public val PG_WALINSPECT = ExtensionDescriptor("pg_walinspect", "oliphaunt-extension-contrib-pg18") + + @JvmField public val PGCRYPTO = ExtensionDescriptor("pgcrypto", "oliphaunt-extension-contrib-pg18") + + @JvmField public val SEG = ExtensionDescriptor("seg", "oliphaunt-extension-contrib-pg18") + + @JvmField public val TABLEFUNC = ExtensionDescriptor("tablefunc", "oliphaunt-extension-contrib-pg18") + + @JvmField public val TCN = ExtensionDescriptor("tcn", "oliphaunt-extension-contrib-pg18") + + @JvmField public val TSM_SYSTEM_ROWS = ExtensionDescriptor("tsm_system_rows", "oliphaunt-extension-contrib-pg18") + + @JvmField public val TSM_SYSTEM_TIME = ExtensionDescriptor("tsm_system_time", "oliphaunt-extension-contrib-pg18") + + @JvmField public val UNACCENT = ExtensionDescriptor("unaccent", "oliphaunt-extension-contrib-pg18") + + @JvmField public val UUID_OSSP = ExtensionDescriptor("uuid-ossp", "oliphaunt-extension-contrib-pg18") +} + internal data class GeneratedExtensionRuntimeContract( val createsExtension: Boolean, val nativeModuleStem: String?, + val dependencies: List = emptyList(), + val sharedPreloads: List = emptyList(), ) internal val generatedExtensionRuntimeContracts: Map = mapOf( - "amcheck" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "amcheck"), - "auto_explain" to GeneratedExtensionRuntimeContract(createsExtension = false, nativeModuleStem = "auto_explain"), - "bloom" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "bloom"), - "btree_gin" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "btree_gin"), - "btree_gist" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "btree_gist"), - "citext" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "citext"), - "cube" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "cube"), - "dict_int" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "dict_int"), - "dict_xsyn" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "dict_xsyn"), - "earthdistance" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "earthdistance"), - "file_fdw" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "file_fdw"), - "fuzzystrmatch" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "fuzzystrmatch"), - "hstore" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "hstore"), - "intarray" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "_int"), - "isn" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "isn"), - "lo" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "lo"), - "ltree" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "ltree"), - "pageinspect" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pageinspect"), - "pg_buffercache" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_buffercache"), - "pg_freespacemap" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_freespacemap"), - "pg_hashids" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_hashids"), - "pg_ivm" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_ivm"), - "pg_surgery" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_surgery"), - "pg_textsearch" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_textsearch"), - "pg_trgm" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_trgm"), - "pg_uuidv7" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_uuidv7"), - "pg_visibility" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_visibility"), - "pg_walinspect" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_walinspect"), - "pgcrypto" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pgcrypto"), - "pgtap" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = null), - "postgis" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "postgis-3"), - "seg" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "seg"), - "tablefunc" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "tablefunc"), - "tcn" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "tcn"), - "tsm_system_rows" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "tsm_system_rows"), - "tsm_system_time" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "tsm_system_time"), - "unaccent" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "unaccent"), - "uuid-ossp" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "uuid-ossp"), - "vector" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "vector"), + "amcheck" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "amcheck", dependencies = listOf(), sharedPreloads = listOf()), + "auto_explain" to GeneratedExtensionRuntimeContract(createsExtension = false, nativeModuleStem = "auto_explain", dependencies = listOf(), sharedPreloads = listOf()), + "bloom" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "bloom", dependencies = listOf(), sharedPreloads = listOf()), + "btree_gin" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "btree_gin", dependencies = listOf(), sharedPreloads = listOf()), + "btree_gist" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "btree_gist", dependencies = listOf(), sharedPreloads = listOf()), + "citext" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "citext", dependencies = listOf(), sharedPreloads = listOf()), + "cube" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "cube", dependencies = listOf(), sharedPreloads = listOf()), + "dict_int" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "dict_int", dependencies = listOf(), sharedPreloads = listOf()), + "dict_xsyn" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "dict_xsyn", dependencies = listOf(), sharedPreloads = listOf()), + "earthdistance" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "earthdistance", dependencies = listOf("cube"), sharedPreloads = listOf()), + "file_fdw" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "file_fdw", dependencies = listOf(), sharedPreloads = listOf()), + "fuzzystrmatch" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "fuzzystrmatch", dependencies = listOf(), sharedPreloads = listOf()), + "hstore" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "hstore", dependencies = listOf(), sharedPreloads = listOf()), + "intarray" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "_int", dependencies = listOf(), sharedPreloads = listOf()), + "isn" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "isn", dependencies = listOf(), sharedPreloads = listOf()), + "lo" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "lo", dependencies = listOf(), sharedPreloads = listOf()), + "ltree" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "ltree", dependencies = listOf(), sharedPreloads = listOf()), + "pageinspect" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pageinspect", dependencies = listOf(), sharedPreloads = listOf()), + "pg_buffercache" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_buffercache", dependencies = listOf(), sharedPreloads = listOf()), + "pg_freespacemap" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_freespacemap", dependencies = listOf(), sharedPreloads = listOf()), + "pg_hashids" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_hashids", dependencies = listOf(), sharedPreloads = listOf()), + "pg_ivm" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_ivm", dependencies = listOf(), sharedPreloads = listOf()), + "pg_surgery" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_surgery", dependencies = listOf(), sharedPreloads = listOf()), + "pg_textsearch" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_textsearch", dependencies = listOf(), sharedPreloads = listOf("pg_textsearch")), + "pg_trgm" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_trgm", dependencies = listOf(), sharedPreloads = listOf()), + "pg_uuidv7" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_uuidv7", dependencies = listOf(), sharedPreloads = listOf()), + "pg_visibility" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_visibility", dependencies = listOf(), sharedPreloads = listOf()), + "pg_walinspect" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pg_walinspect", dependencies = listOf(), sharedPreloads = listOf()), + "pgcrypto" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "pgcrypto", dependencies = listOf(), sharedPreloads = listOf()), + "pgtap" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = null, dependencies = listOf(), sharedPreloads = listOf()), + "postgis" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "postgis-3", dependencies = listOf(), sharedPreloads = listOf()), + "seg" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "seg", dependencies = listOf(), sharedPreloads = listOf()), + "tablefunc" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "tablefunc", dependencies = listOf(), sharedPreloads = listOf()), + "tcn" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "tcn", dependencies = listOf(), sharedPreloads = listOf()), + "tsm_system_rows" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "tsm_system_rows", dependencies = listOf(), sharedPreloads = listOf()), + "tsm_system_time" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "tsm_system_time", dependencies = listOf(), sharedPreloads = listOf()), + "unaccent" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "unaccent", dependencies = listOf(), sharedPreloads = listOf()), + "uuid-ossp" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "uuid-ossp", dependencies = listOf(), sharedPreloads = listOf()), + "vector" to GeneratedExtensionRuntimeContract(createsExtension = true, nativeModuleStem = "vector", dependencies = listOf(), sharedPreloads = listOf()), ) internal val generatedExtensionSqlNames: Set = generatedExtensionRuntimeContracts.keys diff --git a/src/sdks/kotlin/oliphaunt/src/commonMain/kotlin/dev/oliphaunt/Oliphaunt.kt b/src/sdks/kotlin/oliphaunt/src/commonMain/kotlin/dev/oliphaunt/Oliphaunt.kt index 197f2357c..ebdd270e5 100644 --- a/src/sdks/kotlin/oliphaunt/src/commonMain/kotlin/dev/oliphaunt/Oliphaunt.kt +++ b/src/sdks/kotlin/oliphaunt/src/commonMain/kotlin/dev/oliphaunt/Oliphaunt.kt @@ -9,7 +9,7 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -public data class PostgresStartupGuc( +internal data class PostgresStartupGuc( val name: String, val value: String, ) @@ -28,6 +28,8 @@ internal data class EngineConfig( val username: String? = null, val database: String? = null, val extensions: List = emptyList(), + val extensionDescriptors: List = emptyList(), + val icu: IcuData? = null, ) internal fun validateStartupIdentity( @@ -1118,6 +1120,7 @@ public class OliphauntDatabase private constructor( config.copy( startupGucs = startupGucs, extensions = validateGeneratedExtensionIds(extensions), + extensionDescriptors = selectedExtensionDescriptors(config.extensionDescriptors), ) return OliphauntDatabase(engine.open(normalized)) } diff --git a/src/sdks/kotlin/oliphaunt/src/commonMain/kotlin/dev/oliphaunt/Resources.kt b/src/sdks/kotlin/oliphaunt/src/commonMain/kotlin/dev/oliphaunt/Resources.kt new file mode 100644 index 000000000..e055e3f05 --- /dev/null +++ b/src/sdks/kotlin/oliphaunt/src/commonMain/kotlin/dev/oliphaunt/Resources.kt @@ -0,0 +1,69 @@ +package dev.oliphaunt + +/** A resource selected from an extension package or the SDK's contrib values. */ +public data class ExtensionDescriptor( + val sqlName: String, + val product: String, + val version: String? = null, +) { + init { + require(Regex("[a-z0-9][a-z0-9_-]*").matches(sqlName)) { "invalid extension SQL name" } + require(Regex("oliphaunt-extension-[a-z0-9-]+").matches(product)) { "invalid extension product" } + require(version != null || product == "oliphaunt-extension-contrib-pg18") { + "external extension descriptors must declare their package version" + } + version?.let(::validateResourceVersion) + } +} + +/** Optional ICU data supplied by its separately installed package. */ +public data class IcuData(val version: String) { + init { + validateResourceVersion(version) + } +} + +private fun validateResourceVersion(version: String) { + require(Regex("[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?").matches(version)) { + "resource descriptor must declare a package version" + } +} + +internal fun selectedExtensionDescriptors(values: List): List { + val selected = linkedMapOf() + for (value in values) { + val previous = selected.put(value.sqlName, value) + require(previous == null || previous == value) { "conflicting extension descriptors for '${value.sqlName}'" } + } + return selected.values.toList() +} + +internal fun selectedExtensionClosure(names: Collection): Set { + val selected = linkedSetOf() + fun visit(name: String) { + if (!selected.add(name)) return + val contract = generatedExtensionRuntimeContract(name) + ?: throw OliphauntException("unknown extension '$name'") + contract.dependencies.forEach(::visit) + } + names.forEach(::visit) + return selected +} + +internal fun includeSelectedRuntimeFile(path: String, extensions: Set, icu: Boolean): Boolean { + if (!icu && (path == "share/icu" || path.startsWith("share/icu/"))) return false + val file = path.substringAfterLast('/') + if (path.startsWith("share/postgresql/extension/")) { + val owner = generatedExtensionSqlNames.firstOrNull { name -> + file == "$name.control" || file == "$name.sql" || file.startsWith("$name--") + } + if (owner != null && owner !in extensions) return false + } + if (path.startsWith("lib/postgresql/")) { + val owners = generatedExtensionRuntimeContracts.filterValues { contract -> + contract.nativeModuleStem?.let { file == "$it.so" || file == "$it.dylib" } == true + }.keys + if (owners.isNotEmpty() && owners.none { it in extensions }) return false + } + return true +} diff --git a/src/sdks/kotlin/tests/public-api-consumer/src/main/java/dev/oliphaunt/consumer/JavaPublicApiConsumer.java b/src/sdks/kotlin/tests/public-api-consumer/src/main/java/dev/oliphaunt/consumer/JavaPublicApiConsumer.java new file mode 100644 index 000000000..c1d91b9f3 --- /dev/null +++ b/src/sdks/kotlin/tests/public-api-consumer/src/main/java/dev/oliphaunt/consumer/JavaPublicApiConsumer.java @@ -0,0 +1,28 @@ +package dev.oliphaunt.consumer; + +import android.content.Context; +import dev.oliphaunt.DatabaseStorage; +import dev.oliphaunt.ExtensionDescriptor; +import dev.oliphaunt.Extensions; +import dev.oliphaunt.IcuData; +import dev.oliphaunt.OliphauntConfig; +import dev.oliphaunt.OliphauntJava; +import java.io.File; +import java.util.Map; + +/** Compile-only proof of the Java API against the packaged Android AAR. */ +public final class JavaPublicApiConsumer { + public static void useDatabase(Context context, File directory, ExtensionDescriptor vector, IcuData icu) { + var config = OliphauntConfig.builder() + .storage(new DatabaseStorage.Directory(directory)) + .startupGucs(Map.of("application_name", "java-consumer")) + .extensions(vector, Extensions.HSTORE) + .icu(icu) + .build(); + try (var database = OliphauntJava.open(context, config)) { + database.execute("CREATE EXTENSION vector"); + database.execute("CREATE EXTENSION hstore"); + database.query("SELECT '[1,2,3]'::vector <-> '[1,2,4]'::vector"); + } + } +} diff --git a/src/sdks/kotlin/tests/public-api-consumer/src/main/kotlin/dev/oliphaunt/consumer/PublicApiConsumer.kt b/src/sdks/kotlin/tests/public-api-consumer/src/main/kotlin/dev/oliphaunt/consumer/PublicApiConsumer.kt index 31497a795..70a506992 100644 --- a/src/sdks/kotlin/tests/public-api-consumer/src/main/kotlin/dev/oliphaunt/consumer/PublicApiConsumer.kt +++ b/src/sdks/kotlin/tests/public-api-consumer/src/main/kotlin/dev/oliphaunt/consumer/PublicApiConsumer.kt @@ -17,7 +17,6 @@ import dev.oliphaunt.PostgresErrorField import dev.oliphaunt.PostgresException import dev.oliphaunt.PostgresNotice import dev.oliphaunt.PostgresOid -import dev.oliphaunt.PostgresStartupGuc import dev.oliphaunt.QueryDescription import dev.oliphaunt.QueryParam import dev.oliphaunt.QueryResult @@ -48,10 +47,10 @@ internal suspend fun compileOliphauntPublicApi( val config = OliphauntConfig( storage = persistentStorage, - startupGucs = listOf(PostgresStartupGuc("application_name", "public-api-consumer")), + startupGucs = mapOf("application_name" to "public-api-consumer"), username = "postgres", database = "postgres", - extensions = listOf("vector"), + extensions = listOf(dev.oliphaunt.Extensions.HSTORE), ) val database: OliphauntDatabase = Oliphaunt.open( diff --git a/src/sdks/react-native/README.md b/src/sdks/react-native/README.md index ffd83023e..f7ad5b3a6 100644 --- a/src/sdks/react-native/README.md +++ b/src/sdks/react-native/README.md @@ -6,52 +6,56 @@ native SDKs while using platform-native storage and lifecycle adapters. ## Setup and use -Install the package, run CocoaPods for iOS, and apply the package's Expo config -plugin when using Expo prebuild. The supported platforms and packaged targets -are declared by the repository SDK manifest; the package does not advertise -future platform targets. +Install the SDK and any external extension dependencies, then rebuild the native app: -Enable PostgreSQL ICU collations through the Expo plugin only when the app needs -them: +```sh +npm install @oliphaunt/react-native @oliphaunt/extension-vector +``` + +React Native shares `@oliphaunt/extension-vector` with native Node. With Expo, +add the config plugin; it reads installed dependencies and versions: ```json { "expo": { - "plugins": [["@oliphaunt/react-native", { "icu": true }]] + "plugins": ["@oliphaunt/react-native"] } } ``` -The plugin packages ICU data with the matching platform cluster seed; this is a -build-time choice and does not add a database-open option. +Use the directory supplied by your platform filesystem API. For example, with +Expo FileSystem: ```typescript -import Oliphaunt from '@oliphaunt/react-native'; +import { Directory, Paths } from 'expo-file-system'; +import Oliphaunt, { directory, extensions } from '@oliphaunt/react-native'; +import vector from '@oliphaunt/extension-vector'; const db = await Oliphaunt.open({ - storage: { kind: 'applicationData', name: 'primary' }, + storage: directory(new Directory(Paths.document, 'postgres').uri), startupGUCs: { application_name: 'my-app' }, + extensions: [vector, extensions.hstore], }); - -await db.execute('CREATE TABLE events(value text)'); -await db.execute('INSERT INTO events(value) VALUES ($1)', ['ready']); -const result = await db.query('SELECT value FROM events'); -console.log(result.rows[0]?.value); - -const bytes = await db.backup(); -await db.close(); -await Oliphaunt.restore( - { kind: 'applicationData', name: 'restored' }, - bytes, -); +try { + await db.execute('CREATE EXTENSION vector'); + await db.execute('CREATE EXTENSION hstore'); +} finally { + await db.close(); +} ``` -`username` selects an existing PostgreSQL role. New roots are bootstrapped with -`postgres`; create other roles before reopening the root as them. +`directory` accepts a native path or local file URI. Omit storage for an SDK-owned +temporary directory. Restore accepts a persistent directory destination, such as +`Oliphaunt.restore(directory(restoredDirectoryUri), bytes)`. + +Contrib ships with the SDK but must be selected explicitly. For ICU collations, +install `@oliphaunt/icu`, import its default `icu` value, and pass `icu` when +opening. The plugin packages installed resources; each database selects its own +extensions and ICU option. There is no repeated plugin extension list or ICU flag. +Native dependency changes require a native rebuild, including when using Metro. -Storage is `temporaryDirectory`, an explicit `directory`, or an -`applicationData` name resolved by the native platform adapter. Restore accepts -only persistent directory/application-data destinations. +`username` selects an existing PostgreSQL role. New roots start with `postgres`; +create other roles before reopening the root as them. ## API contract diff --git a/src/sdks/react-native/android/src/main/cpp/OliphauntJsiBindings.cpp b/src/sdks/react-native/android/src/main/cpp/OliphauntJsiBindings.cpp index f4e5441b8..d5b6b541e 100644 --- a/src/sdks/react-native/android/src/main/cpp/OliphauntJsiBindings.cpp +++ b/src/sdks/react-native/android/src/main/cpp/OliphauntJsiBindings.cpp @@ -909,10 +909,6 @@ class OliphauntModuleJSIBindings runtime, destination.getProperty(runtime, "storagePath"), "restore storagePath"); - auto storageName = copyOptionalStringArgument( - runtime, - destination.getProperty(runtime, "storageName"), - "restore storageName"); std::vector artifact = copyBinaryArgument(runtime, args[1]); auto promiseConstructor = runtime.global().getPropertyAsFunction(runtime, "Promise"); auto executor = jsi::Function::createFromHostFunction( @@ -923,7 +919,6 @@ class OliphauntModuleJSIBindings callInvoker, storageKind = std::move(storageKind), storagePath = std::move(storagePath), - storageName = std::move(storageName), artifact = std::move(artifact)]( jsi::Runtime &runtime, const jsi::Value &, @@ -956,7 +951,6 @@ class OliphauntModuleJSIBindings try { auto storageKindString = jni::make_jstring(storageKind); auto storagePathString = jni::make_jstring(storagePath.value_or("")); - auto storageNameString = jni::make_jstring(storageName.value_or("")); auto artifactArray = makeByteArray(artifact); static const auto callbackConstructor = OliphauntJsiPromiseCallback::javaClassStatic() @@ -967,7 +961,6 @@ class OliphauntModuleJSIBindings static const auto restoreBytes = OliphauntModuleJSIBindings::javaClassStatic() ->getMethod File(validatePath(storagePath, "restore destination directory")) - "applicationData" -> File( - File(reactContext.filesDir, "Oliphaunt"), - validateApplicationDataName(storageName), - ) else -> throw IllegalArgumentException("unknown restore destination kind '$storageKind'") } Oliphaunt.restore( @@ -392,10 +387,6 @@ class OliphauntModule( "directory" -> DatabaseStorage.Directory( File(validatePath(config.string("storagePath"), "database storage directory")), ) - "applicationData" -> { - val name = validateApplicationDataName(config.string("storageName")) - DatabaseStorage.Directory(File(File(reactContext.filesDir, "Oliphaunt"), name)) - } else -> throw IllegalArgumentException("unknown database storage kind '$kind'") } val runtimeDirectory = reactNativeRuntimeDirectory(null)?.let(::File) @@ -405,10 +396,11 @@ class OliphauntModule( return ReactNativeAndroidOpenConfig( config = OliphauntConfig( storage = storage, - startupGucs = config.startupGucs("startupGUCs"), + startupGucs = config.startupGucs("startupGUCs").associate { it.name to it.value }, username = username, database = database, - extensions = config.stringList("extensions"), + extensions = config.extensionDescriptors(), + icu = config.string("icuVersion")?.let { dev.oliphaunt.IcuData(it) }, ), runtimeDirectory = runtimeDirectory, resourceRoot = null, @@ -454,6 +446,19 @@ class OliphauntModule( } } + private fun ReadableMap.extensionDescriptors(): List { + val values = array("extensions") ?: return emptyList() + return (0 until values.size()).map { index -> + require(values.getType(index) == ReadableType.Map) { "extensions must contain descriptors" } + val value = requireNotNull(values.getMap(index)) + dev.oliphaunt.ExtensionDescriptor( + sqlName = requireNotNull(value.string("sqlName")) { "extension sqlName is required" }, + product = requireNotNull(value.string("product")) { "extension product is required" }, + version = value.string("version"), + ) + } + } + private fun ReadableMap.startupIdentity(name: String): String? { val value = string(name) ?: return null if (value.isBlank()) { @@ -487,18 +492,6 @@ class OliphauntModule( return value } - private fun validateApplicationDataName(value: String?): String { - val name = value?.trim().orEmpty() - if (name == "." || name == ".." || !PORTABLE_STORAGE_NAME.matches(name)) { - throw IllegalArgumentException( - "applicationData storage name must contain 1 to 128 ASCII letters, digits, dot, underscore or hyphen", - ) - } - return name - } - - private val PORTABLE_STORAGE_NAME = Regex("[A-Za-z0-9._-]{1,128}") - private fun validatePathOverride(value: String?, name: String): String? { if (value == null) { return null diff --git a/src/sdks/react-native/app.plugin.js b/src/sdks/react-native/app.plugin.js index 0b12f5c6e..747c04a94 100644 --- a/src/sdks/react-native/app.plugin.js +++ b/src/sdks/react-native/app.plugin.js @@ -59,6 +59,60 @@ function normalizeOptions(options = {}) { }; } +// Follow the installed dependency graph, including aliases and nested package versions. +// Dependency declarations determine the app's shipping set; open() remains explicit. +function resolveInstalledResources(projectRoot) { + const { createRequire } = require('node:module'); + const queue = [path.join(projectRoot, 'package.json')]; + const visited = new Set(); + const resourcePackages = new Map(); + const selected = new Set(); + while (queue.length > 0) { + const manifestFile = fs.realpathSync(queue.pop()); + if (visited.has(manifestFile)) continue; + visited.add(manifestFile); + const manifest = readJsonObject(manifestFile, 'installed dependency manifest'); + const members = extensionMetadata.extensions.filter((row) => row['npm-package'] === manifest.name); + if (members.length > 0 || manifest.name === '@oliphaunt/icu') { + const previous = resourcePackages.get(manifest.name); + if (previous && readJsonObject(previous, 'resource package').version !== manifest.version) { + throw new Error(`app dependencies resolve conflicting versions of ${manifest.name}`); + } + resourcePackages.set(manifest.name, manifestFile); + for (const member of members) selected.add(member['sql-name']); + } + const resolve = createRequire(manifestFile).resolve; + const names = new Set([ + ...Object.keys(manifest.dependencies ?? {}), + ...Object.keys(manifest.optionalDependencies ?? {}), + ...Object.keys(manifest.peerDependencies ?? {}), + ]); + for (const name of names) { + try { + queue.push(resolve(`${name}/package.json`)); + } catch (error) { + if (error.code === 'ERR_PACKAGE_PATH_NOT_EXPORTED') { + let directory = path.dirname(resolve(name)); + while (!fs.existsSync(path.join(directory, 'package.json'))) { + const parent = path.dirname(directory); + if (parent === directory) throw error; + directory = parent; + } + queue.push(path.join(directory, 'package.json')); + } else if (error.code !== 'MODULE_NOT_FOUND' + || (manifest.dependencies?.[name] && !manifest.optionalDependencies?.[name])) { + throw error; + } + } + } + } + return { + extensions: [...selected].sort(), + icu: resourcePackages.has('@oliphaunt/icu'), + packageJsonResolver: (name, paths) => resourcePackages.get(name) ?? resolvePackageJson(name, paths), + }; +} + function optionalString(value) { if (value == null) { return undefined; @@ -1009,7 +1063,9 @@ function patchAndroidGradle(androidRoot, normalized) { function withOliphaunt(config, options = {}) { const plugin = require('expo/config-plugins'); - const normalized = normalizeOptions(options); + if (Object.hasOwn(options, 'extensions') || Object.hasOwn(options, 'icu')) { + throw new Error('Oliphaunt build resources come from installed dependencies; select extensions and ICU in open()'); + } // Expo's built-in iOS mods consume this synchronously and propagate it to // both the Xcode project and Podfile.properties.json during prebuild. config = ensureIosConfigDeploymentTarget(config); @@ -1018,11 +1074,13 @@ function withOliphaunt(config, options = {}) { 'android', (modConfig) => { const projectRoot = modConfig.modRequest.projectRoot; + const installedResources = resolveInstalledResources(projectRoot); + const normalized = normalizeOptions({ ...options, ...installedResources }); const androidRoot = path.join(projectRoot, 'android'); const installedExtensions = resolveInstalledExtensionOwners( projectRoot, normalized.extensions, - { liboliphauntVersion: normalized.liboliphauntVersion }, + { liboliphauntVersion: normalized.liboliphauntVersion, packageJsonResolver: installedResources.packageJsonResolver }, ); const androidOptions = { ...normalized, @@ -1048,8 +1106,10 @@ function withOliphaunt(config, options = {}) { 'ios', (modConfig) => { const projectRoot = modConfig.modRequest.projectRoot; + const installedResources = resolveInstalledResources(projectRoot); + const normalized = normalizeOptions({ ...options, ...installedResources }); const iosRoot = path.join(projectRoot, 'ios'); - stageIosAppPayload(projectRoot, iosRoot, normalized); + stageIosAppPayload(projectRoot, iosRoot, normalized, { packageJsonResolver: installedResources.packageJsonResolver }); writeJson(path.join(iosRoot, 'oliphaunt.json'), normalized); writeJson(path.join(iosRoot, 'OliphauntExtensions.json'), { extensions: normalized.extensions, @@ -1083,3 +1143,5 @@ module.exports.iosPodfileBlock = iosPodfileBlock; module.exports.ensureIosDeploymentTarget = ensureIosDeploymentTarget; module.exports.ensureIosConfigDeploymentTarget = ensureIosConfigDeploymentTarget; module.exports.insertAppGradlePlugin = insertAppGradlePlugin; + +module.exports.resolveInstalledResources = resolveInstalledResources; diff --git a/src/sdks/react-native/ios/Oliphaunt.mm b/src/sdks/react-native/ios/Oliphaunt.mm index 966d649d5..780340ace 100644 --- a/src/sdks/react-native/ios/Oliphaunt.mm +++ b/src/sdks/react-native/ios/Oliphaunt.mm @@ -335,11 +335,21 @@ static void OliphauntSetIfPresent(NSMutableDictionary *dictionary, NSString *key NSMutableDictionary *dictionary = [NSMutableDictionary new]; dictionary[@"storageKind"] = config.storageKind(); OliphauntSetIfPresent(dictionary, @"storagePath", config.storagePath()); - OliphauntSetIfPresent(dictionary, @"storageName", config.storageName()); OliphauntSetIfPresent(dictionary, @"startupGUCs", RCTConvertOptionalVecToArray(config.startupGUCs())); OliphauntSetIfPresent(dictionary, @"username", config.username()); OliphauntSetIfPresent(dictionary, @"database", config.database()); - OliphauntSetIfPresent(dictionary, @"extensions", RCTConvertOptionalVecToArray(config.extensions())); + if (auto extensions = config.extensions()) { + NSMutableArray *values = [NSMutableArray new]; + for (const auto &extension : *extensions) { + NSMutableDictionary *value = [NSMutableDictionary new]; + value[@"sqlName"] = extension.sqlName(); + value[@"product"] = extension.product(); + OliphauntSetIfPresent(value, @"version", extension.version()); + [values addObject:value]; + } + dictionary[@"extensions"] = values; + } + OliphauntSetIfPresent(dictionary, @"icuVersion", config.icuVersion()); return dictionary; } @@ -725,13 +735,11 @@ - (void)backupDataForJsi:(double)handle completion:(OliphauntDataCompletion)comp - (void)restoreDataForJsi:(NSString *)storageKind storagePath:(NSString *_Nullable)storagePath - storageName:(NSString *_Nullable)storageName backupData:(NSData *)backupData completion:(OliphauntVoidCompletion)completion { [OliphauntAdapterDatabase restoreWithStorageKind:storageKind storagePath:storagePath - storageName:storageName backupData:backupData completion:completion]; } @@ -1087,10 +1095,6 @@ - (void)installJSIBindingsWithRuntime:(facebook::jsi::Runtime &)runtime runtime, destination.getProperty(runtime, "storagePath"), "restore storagePath"); - NSString *storageName = OliphauntCopyOptionalNSStringArgument( - runtime, - destination.getProperty(runtime, "storageName"), - "restore storageName"); std::vector artifact = OliphauntCopyBinaryArgument(runtime, args[1]); auto artifactData = [NSData dataWithBytes:artifact.data() length:artifact.size()]; auto promiseConstructor = runtime.global().getPropertyAsFunction(runtime, "Promise"); @@ -1098,7 +1102,7 @@ - (void)installJSIBindingsWithRuntime:(facebook::jsi::Runtime &)runtime runtime, facebook::jsi::PropNameID::forAscii(runtime, "liboliphauntRestoreExecutor"), 2, - [weakSelf, callInvoker, storageKind, storagePath, storageName, artifactData]( + [weakSelf, callInvoker, storageKind, storagePath, artifactData]( facebook::jsi::Runtime &runtime, const facebook::jsi::Value &, const facebook::jsi::Value *promiseArgs, @@ -1129,7 +1133,6 @@ - (void)installJSIBindingsWithRuntime:(facebook::jsi::Runtime &)runtime [strongSelf restoreDataForJsi:storageKind storagePath:storagePath - storageName:storageName backupData:artifactData completion:^(NSError *_Nullable error) { if (error != nil) { diff --git a/src/sdks/react-native/ios/OliphauntAdapter.h b/src/sdks/react-native/ios/OliphauntAdapter.h index f397325cf..85bb64f43 100644 --- a/src/sdks/react-native/ios/OliphauntAdapter.h +++ b/src/sdks/react-native/ios/OliphauntAdapter.h @@ -16,7 +16,6 @@ typedef void (^OliphauntVoidCompletion)(NSError *_Nullable error); + (void)openWithConfig:(NSDictionary *)config completion:(OliphauntOpenCompletion)completion; + (void)restoreWithStorageKind:(NSString *)storageKind storagePath:(nullable NSString *)storagePath - storageName:(nullable NSString *)storageName backupData:(NSData *)backupData completion:(OliphauntVoidCompletion)completion; diff --git a/src/sdks/react-native/ios/OliphauntAdapter.swift b/src/sdks/react-native/ios/OliphauntAdapter.swift index f25a32a3d..c26174fc9 100644 --- a/src/sdks/react-native/ios/OliphauntAdapter.swift +++ b/src/sdks/react-native/ios/OliphauntAdapter.swift @@ -52,19 +52,17 @@ public final class OliphauntAdapterDatabase: NSObject, @unchecked Sendable { } } - @objc(restoreWithStorageKind:storagePath:storageName:backupData:completion:) + @objc(restoreWithStorageKind:storagePath:backupData:completion:) public static func restore( storageKind: String, storagePath: String?, - storageName: String?, backupData: Data, completion: @escaping (NSError?) -> Void ) { do { let destination = try restoreDestination( storageKind: storageKind, - storagePath: storagePath, - storageName: storageName + storagePath: storagePath ) let completionBox = CompletionBox(completion) Task(priority: .userInitiated) { @@ -82,8 +80,7 @@ public final class OliphauntAdapterDatabase: NSObject, @unchecked Sendable { private static func restoreDestination( storageKind: String, - storagePath: String?, - storageName: String? + storagePath: String? ) throws -> URL { switch storageKind { case "directory": @@ -93,17 +90,6 @@ public final class OliphauntAdapterDatabase: NSObject, @unchecked Sendable { throw adapterError("restore destination directory must not be empty or contain NUL bytes") } return URL(fileURLWithPath: storagePath, isDirectory: true) - case "applicationData": - let name = try applicationDataName(storageName) - guard let support = FileManager.default.urls( - for: .applicationSupportDirectory, - in: .userDomainMask - ).first else { - throw adapterError("failed to resolve application data restore directory") - } - return support - .appendingPathComponent("Oliphaunt", isDirectory: true) - .appendingPathComponent(name, isDirectory: true) default: throw adapterError("unknown restore destination kind '\(storageKind)'") } @@ -212,13 +198,25 @@ public final class OliphauntAdapterDatabase: NSObject, @unchecked Sendable { let storage = try parseDatabaseStorage(config) let username = try startupIdentity(config, "username") let database = try startupIdentity(config, "database") - let extensions = try stringArray(config, "extensions") + let rawExtensions = config["extensions"] ?? [] + guard let descriptorValues = rawExtensions as? [NSDictionary] else { + throw adapterError("extensions must contain descriptors") + } + let extensions = try descriptorValues.map { value in + guard let sqlName = try string(value, "sqlName"), + let product = try string(value, "product") else { + throw adapterError("extension sqlName and product are required") + } + return OliphauntExtension(sqlName: sqlName, product: product, version: try string(value, "version")) + } + let icu = try string(config, "icuVersion").map { OliphauntIcuData(version: $0) } let configuration = OliphauntConfiguration( storage: storage, - startupGUCs: try startupGUCs(config, "startupGUCs"), + startupGUCs: Dictionary(try startupGUCs(config, "startupGUCs").map { ($0.name, $0.value) }, uniquingKeysWith: { _, last in last }), username: username, database: database, - extensions: extensions + extensions: extensions, + icu: icu ) return ParsedOpenConfig(configuration: configuration) } @@ -276,58 +274,11 @@ public final class OliphauntAdapterDatabase: NSObject, @unchecked Sendable { throw adapterError("directory storage requires storagePath") } return .directory(URL(fileURLWithPath: path, isDirectory: true)) - case "applicationData": - guard let name = try nonBlankString( - config, - "storageName", - emptyMessage: "applicationData storage name must not be empty" - ) else { - throw adapterError("applicationData storage requires storageName") - } - guard isPortableStorageName(name) else { - throw adapterError( - "applicationData storage name must contain 1 to 128 ASCII letters, digits, dot, underscore or hyphen" - ) - } - guard let baseURL = FileManager.default.urls( - for: .applicationSupportDirectory, - in: .userDomainMask - ).first else { - throw adapterError("failed to resolve application data storage directory") - } - return .directory( - baseURL - .appendingPathComponent("Oliphaunt", isDirectory: true) - .appendingPathComponent(name, isDirectory: true) - ) case let kind: throw adapterError("unknown database storage kind '\(kind)'") } } - private static func isPortableStorageName(_ value: String) -> Bool { - let bytes = value.utf8 - guard !bytes.isEmpty, bytes.count <= 128, value != ".", value != ".." else { - return false - } - return bytes.allSatisfy { byte in - (byte >= 65 && byte <= 90) || - (byte >= 97 && byte <= 122) || - (byte >= 48 && byte <= 57) || - byte == 46 || byte == 95 || byte == 45 - } - } - - private static func applicationDataName(_ value: String?) throws -> String { - let name = value?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" - guard isPortableStorageName(name) else { - throw adapterError( - "applicationData storage name must contain 1 to 128 ASCII letters, digits, dot, underscore or hyphen" - ) - } - return name - } - private static func startupIdentity(_ dictionary: NSDictionary, _ key: String) throws -> String? { guard let value = try string(dictionary, key) else { return nil } if value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { diff --git a/src/sdks/react-native/package.json b/src/sdks/react-native/package.json index bfb6c0fd9..9678cf4e2 100644 --- a/src/sdks/react-native/package.json +++ b/src/sdks/react-native/package.json @@ -32,7 +32,8 @@ }, "oliphaunt": { "swiftSdkVersion": "0.7.0", - "kotlinSdkVersion": "0.2.0" + "kotlinSdkVersion": "0.2.0", + "liboliphauntVersion": "0.2.0" }, "files": [ "android", diff --git a/src/sdks/react-native/release.toml b/src/sdks/react-native/release.toml index 11583790e..4cc2c54d1 100644 --- a/src/sdks/react-native/release.toml +++ b/src/sdks/react-native/release.toml @@ -19,3 +19,8 @@ parser = "json:oliphaunt.swiftSdkVersion" source_product = "oliphaunt-kotlin" path = "src/sdks/react-native/package.json" parser = "json:oliphaunt.kotlinSdkVersion" + +[compatibility_versions.oliphaunt-react-native-liboliphaunt] +source_product = "liboliphaunt-native" +path = "src/sdks/react-native/package.json" +parser = "json:oliphaunt.liboliphauntVersion" diff --git a/src/sdks/react-native/src/__tests__/client.test.ts b/src/sdks/react-native/src/__tests__/client.test.ts index 6244e24cc..b41ef220b 100644 --- a/src/sdks/react-native/src/__tests__/client.test.ts +++ b/src/sdks/react-native/src/__tests__/client.test.ts @@ -1,3 +1,4 @@ +import { extensions } from '../extensions'; import assert from 'node:assert/strict'; import { test, vi } from 'vitest'; @@ -175,6 +176,8 @@ async function testPublicEntrypointIsMinimal(): Promise { 'array', 'binary', 'default', + 'directory', + 'extensions', 'json', 'postgresOids', 'text', @@ -190,21 +193,24 @@ async function testPublicEntrypointIsMinimal(): Promise { async function testOpenUsesNativeDirectDefaults(): Promise { const native = new MockNative(); const db = await createOliphauntClient(native).open({ - storage: { kind: 'applicationData', name: 'primary' }, + storage: { kind: 'directory', path: '/data/primary' }, startupGUCs: { search_path: 'public' }, username: 'postgres', database: 'app', - extensions: ['hstore'], + extensions: [extensions.hstore], }); assert.deepEqual(native.openCalls, [ { - storageKind: 'applicationData', - storageName: 'primary', + storageKind: 'directory', + storagePath: '/data/primary', startupGUCs: ['search_path=public'], username: 'postgres', database: 'app', - extensions: ['hstore'], + extensions: [ + { sqlName: 'hstore', product: 'oliphaunt-extension-contrib-pg18', version: undefined }, + ], + icuVersion: undefined, }, ]); await db.close(); @@ -1214,9 +1220,8 @@ class MockNative implements Spec { readonly forgottenClosedGenerations: number[] = []; readonly restoreCalls: Array<{ destination: { - storageKind: 'directory' | 'applicationData'; + storageKind: 'directory'; storagePath?: string; - storageName?: string; }; payload: string; }> = []; @@ -1405,9 +1410,8 @@ class MockNative implements Spec { async restoreJsi( destination: { - storageKind: 'directory' | 'applicationData'; + storageKind: 'directory'; storagePath?: string; - storageName?: string; }, artifact: Uint8Array, ): Promise { @@ -1453,9 +1457,8 @@ type GlobalWithJsi = typeof globalThis & { backup(handle: number): Promise; restore( destination: { - storageKind: 'directory' | 'applicationData'; + storageKind: 'directory'; storagePath?: string; - storageName?: string; }, artifact: Uint8Array, ): Promise; diff --git a/src/sdks/react-native/src/__tests__/config-plugin.test.ts b/src/sdks/react-native/src/__tests__/config-plugin.test.ts index 79bdc0fe1..60363fef2 100644 --- a/src/sdks/react-native/src/__tests__/config-plugin.test.ts +++ b/src/sdks/react-native/src/__tests__/config-plugin.test.ts @@ -18,6 +18,7 @@ const { readCarrierSummary, releaseOwnerForSqlName, resolveInstalledExtensionOwners, + resolveInstalledResources, resolveIosCarrierManifests, selectedExtensionClosure, serializeExtensionVersions, @@ -598,3 +599,38 @@ test('carrier discovery and staging fail closed', () => { fs.rmSync(root, { force: true, recursive: true }); } }); + +test('shipping resources follow resolved dependencies, aliases, and nested versions', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'oliphaunt-resource-graph-')); + try { + writeJson(path.join(root, 'package.json'), { + dependencies: { feature: '1.0.0' }, + optionalDependencies: { absent: '1.0.0' }, + }); + const feature = path.join(root, 'node_modules/feature'); + writeJson(path.join(feature, 'package.json'), { + name: 'feature', + dependencies: { vectors: 'npm:@oliphaunt/extension-vector@0.8.2', '@oliphaunt/icu': '0.2.0' }, + }); + const vector = path.join(feature, 'node_modules/vectors/package.json'); + writeJson(vector, { name: '@oliphaunt/extension-vector', version: '0.8.2' }); + writeJson(path.join(feature, 'node_modules/@oliphaunt/icu/package.json'), { + name: '@oliphaunt/icu', + version: '0.2.0', + }); + const resources = resolveInstalledResources(root); + assert.deepEqual(resources.extensions, ['vector']); + assert.equal(resources.icu, true); + assert.equal(resources.packageJsonResolver('@oliphaunt/extension-vector', [root]), vector); + writeJson(path.join(root, 'package.json'), { + dependencies: { feature: '1.0.0', '@oliphaunt/extension-vector': '0.9.0' }, + }); + writeJson(path.join(root, 'node_modules/@oliphaunt/extension-vector/package.json'), { + name: '@oliphaunt/extension-vector', + version: '0.9.0', + }); + assert.throws(() => resolveInstalledResources(root), /conflicting versions/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/src/sdks/react-native/src/__tests__/storage.test.ts b/src/sdks/react-native/src/__tests__/storage.test.ts new file mode 100644 index 000000000..37289d53a --- /dev/null +++ b/src/sdks/react-native/src/__tests__/storage.test.ts @@ -0,0 +1,31 @@ +import { expect, test } from 'vitest'; +import { directory } from '../storage'; + +test('directory accepts native paths and decodes local filesystem URIs once', () => { + expect(directory('/data/my db')).toEqual({ kind: 'directory', path: '/data/my db' }); + expect(directory('file:///data/my%20db')).toEqual(directory('/data/my db')); + expect(directory('file:///data/a%2520b').path).toBe('/data/a%20b'); + expect(Object.isFrozen(directory('/data/db'))).toBe(true); +}); + +test('directory rejects non-local resources and malformed file URIs', () => { + for (const input of [ + '', + ' ', + 'relative/db', + './db', + '../db', + '/data/\0db', + 'file:///data/%00db', + 'file://remote/db', + 'https://example.com/db', + 'content://files/db', + 'file:///db?q=1', + 'file:///db#fragment', + 'file:///data%2fdb', + 'file:///data%5cdb', + 'file:///bad%', + ]) { + expect(() => directory(input), input).toThrow(); + } +}); diff --git a/src/sdks/react-native/src/client.ts b/src/sdks/react-native/src/client.ts index 71175acbc..f8ba8b711 100644 --- a/src/sdks/react-native/src/client.ts +++ b/src/sdks/react-native/src/client.ts @@ -37,6 +37,12 @@ import { type RawQueryResult, type TransactionStatus, } from './query'; +import { + snapshotNativeExtensions, + snapshotNativeIcu, + type NativeExtensionDescriptor, + type NativeIcuDescriptor, +} from '@oliphaunt/js-core/resources'; import { generatedExtensionBySqlName } from './generated/extensions'; import type { NativeOpenConfig, Spec as NativeOliphauntModule } from './specs/NativeOliphaunt'; @@ -46,8 +52,7 @@ type ProtocolChunkCallback = (chunk: Uint8Array) => undefined; export type DatabaseStorage = | { readonly kind: 'temporaryDirectory' } - | { readonly kind: 'directory'; readonly path: string } - | { readonly kind: 'applicationData'; readonly name: string }; + | { readonly kind: 'directory'; readonly path: string }; type QueryReadOptions = Omit; @@ -58,7 +63,8 @@ export type OpenConfig = { startupGUCs?: Readonly>; username?: string; database?: string; - extensions?: ReadonlyArray; + extensions?: ReadonlyArray; + icu?: NativeIcuDescriptor; }; export type OliphauntClient = { @@ -1094,20 +1100,13 @@ export function createOliphauntClient( } function normalizeRestoreDestination(destination: RestoreDestination): { - storageKind: 'directory' | 'applicationData'; + storageKind: 'directory'; storagePath?: string; - storageName?: string; } { if (destination.kind === 'directory') { validatePath(destination.path, 'restore destination directory'); return { storageKind: 'directory', storagePath: destination.path }; } - if (destination.kind === 'applicationData') { - return { - storageKind: 'applicationData', - storageName: validateApplicationDataName(destination.name), - }; - } throw new Error( `unknown restore destination kind '${String((destination as { kind?: unknown }).kind)}'`, ); @@ -1118,12 +1117,16 @@ function normalizeOpenConfig(config: OpenConfig): NativeOpenConfig { validateStartupIdentity(config.database, 'database'); const startupGUCs = config.startupGUCs ? validateStartupGUCs(config.startupGUCs) : undefined; const storage = normalizeDatabaseStorage(config.storage); + const descriptors = snapshotNativeExtensions(config.extensions ?? []); + validateExtensionIds(descriptors.map((value) => value.sqlName)); + const icu = snapshotNativeIcu(config.icu); return { ...storage, startupGUCs, username: config.username, database: config.database, - extensions: config.extensions ? validateExtensionIds(config.extensions) : undefined, + extensions: descriptors.map(({ sqlName, product, version }) => ({ sqlName, product, version })), + icuVersion: icu?.version, }; } @@ -1138,7 +1141,7 @@ function validatePath(value: string, label: string): void { function normalizeDatabaseStorage( storage: DatabaseStorage | undefined, -): Pick { +): Pick { if (storage === undefined) { return { storageKind: 'temporaryDirectory' }; } @@ -1152,25 +1155,9 @@ function normalizeDatabaseStorage( validatePath(storage.path, 'database storage directory'); return { storageKind: 'directory', storagePath: storage.path }; } - if (storage.kind === 'applicationData') { - return { - storageKind: 'applicationData', - storageName: validateApplicationDataName(storage.name), - }; - } throw new Error(`unknown database storage kind ${String((storage as { kind?: unknown }).kind)}`); } -function validateApplicationDataName(value: string): string { - const name = value.trim(); - if (!/^[A-Za-z0-9._-]{1,128}$/.test(name) || name === '.' || name === '..') { - throw new Error( - 'applicationData storage name must contain 1 to 128 ASCII letters, digits, dot, underscore or hyphen', - ); - } - return name; -} - function validateStartupIdentity(value: string | undefined, label: string): void { if (value === undefined) { return; diff --git a/src/sdks/react-native/src/extensions.ts b/src/sdks/react-native/src/extensions.ts new file mode 100644 index 000000000..04bdc335b --- /dev/null +++ b/src/sdks/react-native/src/extensions.ts @@ -0,0 +1,22 @@ +import type { NativeExtensionDescriptor } from '@oliphaunt/js-core/resources'; +import { GENERATED_EXTENSION_METADATA } from './generated/extensions'; + +type ContribId = Extract< + (typeof GENERATED_EXTENSION_METADATA)[number], + { readonly runtimeBound: true } +>['id']; + +/** Bundled contrib resources. Listing a value selects it; no SQL is executed. */ +export const extensions: Readonly> = Object.freeze( + Object.fromEntries( + GENERATED_EXTENSION_METADATA.filter((row) => row.runtimeBound).map((row) => [ + row.id, + Object.freeze({ + schema: 'oliphaunt-native-extension-v1' as const, + sqlName: row.sqlName, + product: row.artifactProduct, + packageName: row.npmPackage, + }), + ]), + ) as Record, +); diff --git a/src/sdks/react-native/src/index.ts b/src/sdks/react-native/src/index.ts index c34ae433a..3edc1891a 100644 --- a/src/sdks/react-native/src/index.ts +++ b/src/sdks/react-native/src/index.ts @@ -51,3 +51,8 @@ export { export const Oliphaunt: import('./client').OliphauntClient = createOliphauntClient(NativeOliphaunt); export default Oliphaunt; + +export { extensions } from './extensions'; +export type { NativeExtensionDescriptor, NativeIcuDescriptor } from '@oliphaunt/js-core/resources'; + +export { directory } from './storage'; diff --git a/src/sdks/react-native/src/jsiTransport.ts b/src/sdks/react-native/src/jsiTransport.ts index eb01380c5..f7544b983 100644 --- a/src/sdks/react-native/src/jsiTransport.ts +++ b/src/sdks/react-native/src/jsiTransport.ts @@ -28,9 +28,8 @@ export type JsiRawProtocolTransport = { readonly backup: (handle: number) => Promise; readonly restore: ( destination: { - storageKind: 'directory' | 'applicationData'; + storageKind: 'directory'; storagePath?: string; - storageName?: string; }, artifact: Uint8Array, ) => Promise; @@ -135,9 +134,8 @@ export async function backupJsi( export async function restoreJsi( transport: JsiRawProtocolTransport, destination: { - storageKind: 'directory' | 'applicationData'; + storageKind: 'directory'; storagePath?: string; - storageName?: string; }, artifact: Uint8Array, ): Promise { diff --git a/src/sdks/react-native/src/specs/NativeOliphaunt.ts b/src/sdks/react-native/src/specs/NativeOliphaunt.ts index 4aa73cb01..25d1be9ae 100644 --- a/src/sdks/react-native/src/specs/NativeOliphaunt.ts +++ b/src/sdks/react-native/src/specs/NativeOliphaunt.ts @@ -4,11 +4,11 @@ import { TurboModuleRegistry } from 'react-native'; export type NativeOpenConfig = { storageKind: string; storagePath?: string; - storageName?: string; startupGUCs?: Array; username?: string; database?: string; - extensions?: Array; + extensions?: Array<{ sqlName: string; product: string; version?: string }>; + icuVersion?: string; }; export interface Spec extends TurboModule { diff --git a/src/sdks/react-native/src/storage.ts b/src/sdks/react-native/src/storage.ts new file mode 100644 index 000000000..e9152c39b --- /dev/null +++ b/src/sdks/react-native/src/storage.ts @@ -0,0 +1,31 @@ +import type { DatabaseStorage } from './client'; + +/** + * Persist a database in a native directory. Accepts a filesystem path or the + * local file URI returned by a mobile filesystem API; performs no file IO. + */ +export function directory(location: string): Extract { + let path = location; + if (/^file:/i.test(location)) { + const url = new URL(location); + if (url.hostname !== '' && url.hostname !== 'localhost') { + throw new TypeError('database storage directory must be a local file URI'); + } + if (url.search || url.hash || /%2f|%5c/i.test(url.pathname)) { + throw new TypeError( + 'database storage file URI must not contain a query, fragment, or encoded separator', + ); + } + path = decodeURIComponent(url.pathname); + } else if (/^[a-z][a-z0-9+.-]*:/i.test(location)) { + throw new TypeError('database storage directory must be a filesystem path or local file URI'); + } + if (!path.startsWith('/') || path.includes('\0')) { + throw new TypeError( + 'database storage directory must be an absolute native path without NUL bytes', + ); + } + return Object.freeze({ kind: 'directory', path }); +} + +export default directory; diff --git a/src/sdks/react-native/tools/expo-android-runner.sh b/src/sdks/react-native/tools/expo-android-runner.sh index 0a1f08945..54b795529 100755 --- a/src/sdks/react-native/tools/expo-android-runner.sh +++ b/src/sdks/react-native/tools/expo-android-runner.sh @@ -293,12 +293,7 @@ pack_react_native_sdk_if_needed() { fi patch_expo_example_react_native_dependency "file:$tarball" - if [ ! -d "$example_dir/node_modules/@oliphaunt/react-native" ] || - [ "$tarball" -nt "$example_dir/node_modules/@oliphaunt/react-native/package.json" ]; then - install_expo_example_dependencies - else - echo "Expo example dependencies are current" - fi + install_expo_example_dependencies } install_react_native_sdk_tarball() { @@ -772,6 +767,9 @@ main() { need_cmd diff fi prepare_expo_example_workspace + bash "$root/tools/dev/bun.sh" "$root/tools/release/stage-react-native-resource-packages.mjs" \ + android "$(normalize_mobile_extensions)" "$android_icu_enabled" \ + "$scratch_root/resource-packages" "$example_dir" "$scratch_root/pnpm-workspace.yaml" pack_react_native_sdk_if_needed ensure_android_project local runtime_resources jni_libs source_so static_registry_source diff --git a/src/sdks/react-native/tools/expo-ios-runner.sh b/src/sdks/react-native/tools/expo-ios-runner.sh index 5762518f0..189fa59e9 100755 --- a/src/sdks/react-native/tools/expo-ios-runner.sh +++ b/src/sdks/react-native/tools/expo-ios-runner.sh @@ -230,6 +230,13 @@ prepare_runtime_resources() { "$package_root" } +installed_mobile_extensions() { + node - "$root/src/sdks/react-native/app.plugin.js" "$example_dir" <<'NODE' +const plugin = require(process.argv[2]); +process.stdout.write(plugin.resolveInstalledResources(process.argv[3]).extensions.join(",")); +NODE +} + find_ios_library_artifact() { local artifact="${OLIPHAUNT_EXPO_IOS_OLIPHAUNT_XCFRAMEWORK:-}" [ -n "$artifact" ] || artifact="${OLIPHAUNT_EXPO_IOS_OLIPHAUNT_FRAMEWORK:-}" @@ -506,18 +513,16 @@ configure_ios_carrier_inputs() { local selected_extensions icu_enabled selected_extensions="$(normalize_mobile_extensions)" icu_enabled="${OLIPHAUNT_EXPO_IOS_ICU:-0}" - node - "$example_dir/app.json" "$selected_extensions" "$icu_enabled" <<'NODE' + bash "$root/tools/dev/bun.sh" "$root/tools/release/stage-react-native-resource-packages.mjs" \ + "$carrier_manifest" "$selected_extensions" "$icu_enabled" \ + "$scratch_root/resource-packages" "$example_dir" "$scratch_root/pnpm-workspace.yaml" + node - "$example_dir/app.json" <<'NODE' const fs = require("node:fs"); const file = process.argv[2]; -const extensions = process.argv[3].split(",").map((value) => value.trim()).filter(Boolean); -const icu = ["1", "true", "yes"].includes(process.argv[4].toLowerCase()); const value = JSON.parse(fs.readFileSync(file, "utf8")); -const plugins = Array.isArray(value.expo?.plugins) ? value.expo.plugins : []; -value.expo.plugins = plugins.filter((entry) => { - const name = Array.isArray(entry) ? entry[0] : entry; - return name !== "@oliphaunt/react-native"; -}); -value.expo.plugins.push(["@oliphaunt/react-native", { extensions, icu }]); +value.expo.plugins = (value.expo.plugins ?? []).filter(entry => + (Array.isArray(entry) ? entry[0] : entry) !== "@oliphaunt/react-native"); +value.expo.plugins.push("@oliphaunt/react-native"); fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`); NODE } @@ -835,7 +840,7 @@ build_ios_app() { echo "bundled: $resource_root/lib/liboliphaunt.dylib" >&2 fi local selected_extensions app_resource_files - selected_extensions="$(normalize_mobile_extensions)" + selected_extensions="$(installed_mobile_extensions)" app_resource_files="$scratch_root/ios-resource-files.txt" find "$resource_root" -type f -print >"$app_resource_files" oliphaunt_dev_assert_runtime_file_list "$selected_extensions" "iOS" <"$app_resource_files" @@ -980,7 +985,7 @@ main() { install_react_native_sdk_from_source_for_reuse if [ "$runner" = "crash" ]; then crash_storage="$crash_storage_override" - [ -n "$crash_storage" ] || crash_storage="app-data:oliphaunt-crash-recovery-$crash_storage_suffix" + [ -n "$crash_storage" ] || crash_storage="app-directory:oliphaunt-crash-recovery-$crash_storage_suffix" exercise_ios_device_crash_recovery "$device_id" "$crash_storage" return fi @@ -997,8 +1002,9 @@ main() { fi configure_iphoneos_signing local app - pack_react_native_sdk configure_ios_carrier_inputs + pack_react_native_sdk + install_expo_example_dependencies ensure_ios_project prepare_swift_sdk_artifact_git_repo_if_required patch_expo_modules_jsi_for_host_toolchain @@ -1010,7 +1016,7 @@ main() { "$app/OliphauntReactNativeResources.bundle/oliphaunt/runtime/manifest.properties" \ "iOS app" local selected_extensions - selected_extensions="$(normalize_mobile_extensions)" + selected_extensions="$(installed_mobile_extensions)" write_ios_build_artifact_report "$app" "$selected_extensions" if is_ios_build_only; then printf '\niOS build-only mobile artifact complete: %s\n' "$app" diff --git a/src/sdks/react-native/tools/expo-runner-ios-installed-app.sh b/src/sdks/react-native/tools/expo-runner-ios-installed-app.sh index e99671ba4..6987d6129 100644 --- a/src/sdks/react-native/tools/expo-runner-ios-installed-app.sh +++ b/src/sdks/react-native/tools/expo-runner-ios-installed-app.sh @@ -569,8 +569,8 @@ ios_runner_url() { fi if [ -n "$storage_arg" ]; then case "$storage_arg" in - app-data:*) - url="$url&liboliphauntApplicationData=$(urlencode "${storage_arg#app-data:}")" + app-directory:*) + url="$url&liboliphauntDirectoryName=$(urlencode "${storage_arg#app-directory:}")" ;; *) url="$url&liboliphauntStorageDirectory=$(urlencode "$storage_arg")" @@ -618,7 +618,7 @@ exercise_ios_crash_recovery() { if [ -z "$crash_storage_override" ]; then case "$crash_storage" in - app-data:*) + app-directory:*) ;; /*) rm -rf "$crash_storage" @@ -746,7 +746,7 @@ exercise_ios_device_crash_recovery() { if [ -z "$crash_storage_override" ]; then case "$crash_storage" in - app-data:*) + app-directory:*) ;; /*) rm -rf "$crash_storage" @@ -827,7 +827,7 @@ install_and_launch() { if [ "$runner" = "crash" ]; then local crash_storage="$crash_storage_override" - [ -n "$crash_storage" ] || crash_storage="app-data:oliphaunt-crash-recovery-$crash_storage_suffix" + [ -n "$crash_storage" ] || crash_storage="app-directory:oliphaunt-crash-recovery-$crash_storage_suffix" exercise_ios_device_crash_recovery "$device_id" "$crash_storage" return fi diff --git a/src/sdks/react-native/tools/expo-runner-reporting.sh b/src/sdks/react-native/tools/expo-runner-reporting.sh index 148b13802..cb43ee7ac 100644 --- a/src/sdks/react-native/tools/expo-runner-reporting.sh +++ b/src/sdks/react-native/tools/expo-runner-reporting.sh @@ -103,8 +103,10 @@ export_mobile_e2e_icu_expectation_from_android_apk() { extract_status=$? unzip -p "$apk" "assets/oliphaunt/cluster-seed/manifest.properties" >"$extracted/cluster-seed/manifest.properties" || extract_status=$? - unzip -p "$apk" "assets/oliphaunt/cluster-seed-icu/manifest.properties" >"$extracted/cluster-seed-icu/manifest.properties" || - extract_status=$? + if grep -Eq '^runtimeFeatures=([^,]*,)*icu(,|$)' "$manifest"; then + unzip -p "$apk" "assets/oliphaunt/cluster-seed-icu/manifest.properties" >"$extracted/cluster-seed-icu/manifest.properties" || + extract_status=$? + fi if [ "$extract_status" -ne 0 ]; then rm -rf "$extracted" echo "$label is missing its runtime or cluster-seed manifest: $apk" >&2 diff --git a/src/sdks/react-native/tools/expo-runner-runtime-resources.sh b/src/sdks/react-native/tools/expo-runner-runtime-resources.sh index 26aa95498..6d5ff9973 100644 --- a/src/sdks/react-native/tools/expo-runner-runtime-resources.sh +++ b/src/sdks/react-native/tools/expo-runner-runtime-resources.sh @@ -38,6 +38,7 @@ require_mobile_runtime_seed_closure() { fail "$configured_env does not contain the exact $target runtime-carrier receipt" local name profile role manifest for name in cluster-seed cluster-seed-icu; do + if [ "$name" = cluster-seed-icu ] && [ ! -d "$root/$name" ]; then continue; fi [ "$name" = cluster-seed ] && profile=standard || profile=icu role="cluster-seed-$profile" manifest="$root/$name/manifest.properties" @@ -80,7 +81,11 @@ install_mobile_runtime_seed_closure() { rm -rf "$package_root/oliphaunt/cluster-seed" "$package_root/oliphaunt/cluster-seed-icu" cp "$closure/manifest.properties" "$package_root/oliphaunt/manifest.properties" cp -R "$closure/cluster-seed" "$package_root/oliphaunt/cluster-seed" - cp -R "$closure/cluster-seed-icu" "$package_root/oliphaunt/cluster-seed-icu" + local include_icu="${OLIPHAUNT_EXPO_ANDROID_ICU:-${OLIPHAUNT_EXPO_IOS_ICU:-0}}" + if [ "$include_icu" = 1 ] || grep -Fxq runtimeFeatures=icu "$package_root/oliphaunt/runtime/manifest.properties" 2>/dev/null; then + [ -d "$closure/cluster-seed-icu" ] || fail "selected ICU qualification requires its optional seed in $closure/cluster-seed-icu" + cp -R "$closure/cluster-seed-icu" "$package_root/oliphaunt/cluster-seed-icu" + fi } bind_mobile_runtime_manifest_to_seed_closure() { @@ -89,8 +94,8 @@ bind_mobile_runtime_manifest_to_seed_closure() { target="$(sed -n 's/^clusterSeedTarget=//p' "$closure/manifest.properties")" features="$(sed -n 's/^runtimeFeatures=//p' "$manifest")" digest="$(sed -n 's/^icuDataTreeSha256=//p' "$manifest")" - seed_digest="$(sed -n 's/^icuDataTreeSha256=//p' "$closure/cluster-seed-icu/manifest.properties")" if [ "$features" = icu ]; then + seed_digest="$(sed -n 's/^icuDataTreeSha256=//p' "$package_root/oliphaunt/cluster-seed-icu/manifest.properties")" [ -n "$digest" ] && [ "$digest" = "$seed_digest" ] || fail "staged mobile ICU runtime does not match the canonical $target ICU cluster seed" else diff --git a/src/sdks/react-native/tools/native-resource-closure.mjs b/src/sdks/react-native/tools/native-resource-closure.mjs index e5888eb20..a642654c8 100644 --- a/src/sdks/react-native/tools/native-resource-closure.mjs +++ b/src/sdks/react-native/tools/native-resource-closure.mjs @@ -90,7 +90,7 @@ export async function logicalTreeSha256(root) { return digest.digest("hex"); } -async function validateClusterSeed(root, profile) { +export async function validateClusterSeed(root, profile) { const source = path.join(root, "manifest.properties"); await Promise.all([ fs.access(path.join(root, "files/PG_VERSION")), @@ -153,7 +153,9 @@ export async function validateNativeRuntimeClosure(root, { integrated } = {}) { : ""; requireProperty(runtime, "mobileStaticRegistrySource", expectedRegistrySource, source); await validateClusterSeed(path.join(root, "cluster-seed"), "standard"); - const icu = await validateClusterSeed(path.join(root, "cluster-seed-icu"), "icu"); + const seed = path.join(root, "cluster-seed-icu"); + const present = await fs.stat(seed).then(() => true, error => { if (error.code === "ENOENT") return false; throw error; }); + const icu = present ? await validateClusterSeed(seed, "icu") : undefined; const features = new Set((runtime.get("runtimeFeatures") ?? "").split(",").filter(Boolean)); integrated ??= features.has("icu"); if ([...features].some((feature) => feature !== "icu") || features.has("icu") !== integrated) { @@ -161,7 +163,7 @@ export async function validateNativeRuntimeClosure(root, { integrated } = {}) { } const runtimeDigest = runtime.get("icuDataTreeSha256") ?? ""; if (integrated) { - if (runtimeDigest !== icu.digest) { + if (icu === undefined || runtimeDigest !== icu.digest) { throw new Error(`${source} ICU identity does not match cluster-seed-icu`); } const data = path.join(root, "runtime/files/share/icu"); @@ -171,7 +173,7 @@ export async function validateNativeRuntimeClosure(root, { integrated } = {}) { } else if (runtimeDigest !== "") { throw new Error(`${source} selects ICU data without the ICU runtime feature`); } - return { icuDigest: icu.digest, runtime }; + return { icuDigest: icu?.digest, runtime }; } export async function validateIcuDataCarrier(root) { diff --git a/src/sdks/react-native/tools/stage-ios-app.mjs b/src/sdks/react-native/tools/stage-ios-app.mjs index 7ce5b4a0b..a49dfa274 100755 --- a/src/sdks/react-native/tools/stage-ios-app.mjs +++ b/src/sdks/react-native/tools/stage-ios-app.mjs @@ -19,6 +19,7 @@ import { parseProperties, requireProperty, validateIcuDataCarrier, + validateClusterSeed, validateNativeRuntimeClosure, } from "./native-resource-closure.mjs"; @@ -635,10 +636,11 @@ function validateBase(value, label, allowFileUrls) { exactKeys(base, ["assets", "product", "tag", "version"], label); if (base.product !== "liboliphaunt-native") fail(`${label}.product must be liboliphaunt-native`); const assets = validateAssetList(base.assets, `${label}.assets`, allowFileUrls); - noOtherRoles(assets, ["base-xcframework", "icu-data", "runtime-resources"], `${label}.assets`); + noOtherRoles(assets, ["base-xcframework", "icu-data", "runtime-resources", "icu-seed"], `${label}.assets`); const framework = exactlyOneRole(assets, "base-xcframework", `${label}.assets`); const runtime = exactlyOneRole(assets, "runtime-resources", `${label}.assets`); const icu = exactlyOneRole(assets, "icu-data", `${label}.assets`); + const icuSeed = exactlyOneRole(assets, "icu-seed", `${label}.assets`); const frameworkName = portable( path.posix.basename(framework.member), `${label} base-xcframework member basename`, @@ -655,7 +657,7 @@ function validateBase(value, label, allowFileUrls) { const expectedTag = `${base.product}-v${version}`; if (base.tag !== expectedTag) fail(`${label}.tag must be ${expectedTag}`); return { - assets: { framework, icu, runtime }, + assets: { framework, icu, icuSeed, runtime }, kind: "base", product: base.product, tag: base.tag, @@ -794,7 +796,7 @@ function validateLegalDocument(value, label, base, extensions) { if (!Array.isArray(legal.base)) fail(`${label}.base must be an array`); const baseGroups = legal.base.map((row, index) => validateLegalGroup(row, `${label}.base[${index}]`)); - const expectedBaseRoles = ["base-xcframework", "runtime-resources", "icu-data"]; + const expectedBaseRoles = ["base-xcframework", "runtime-resources", "icu-data", "icu-seed"]; if (JSON.stringify(baseGroups.map(({ assetRole }) => assetRole)) !== JSON.stringify(expectedBaseRoles)) { fail(`${label}.base asset roles must be exactly ${expectedBaseRoles.join(",")}`); } @@ -802,6 +804,7 @@ function validateLegalDocument(value, label, base, extensions) { [base.assets.framework.role, base.assets.framework], [base.assets.runtime.role, base.assets.runtime], [base.assets.icu.role, base.assets.icu], + [base.assets.icuSeed.role, base.assets.icuSeed], ]); for (const group of baseGroups) { if (!baseAssets.has(group.assetRole)) { @@ -1939,10 +1942,11 @@ async function stageSelectedLegalFiles({ [base.assets.framework.role, base.assets.framework], [base.assets.runtime.role, base.assets.runtime], [base.assets.icu.role, base.assets.icu], + [base.assets.icuSeed.role, base.assets.icuSeed], ]); const groups = []; for (const group of base.legal) { - if (group.assetRole === "icu-data" && !args.icu) continue; + if (["icu-data", "icu-seed"].includes(group.assetRole) && !args.icu) continue; const asset = baseAssets.get(group.assetRole); if (asset === undefined) fail(`base legal group references missing ${group.assetRole} asset`); groups.push({ @@ -2544,14 +2548,26 @@ async function stage(args, base, selected) { "oliphaunt", ); await copyTree(baseResources, resourceRoot); + const resourceSelection = { + schema: "oliphaunt-sdk-resources-v1", runtimeVersion: base.version, icuVersion: args.icu ? base.version : "", + }; + for (const carrier of selected) { + resourceSelection[`extension.${carrier.sqlName}.product`] = carrier.product; + resourceSelection[`extension.${carrier.sqlName}.version`] = carrier.version; + } + await fs.writeFile(path.join(resourceRoot, "sdk-resources.properties"), + Object.keys(resourceSelection).sort().map((key) => `${key}=${resourceSelection[key]}\n`).join("")); let icuDataTreeSha256 = ""; if (args.icu) { const icuClosure = await resolveAsset(base.assets.icu, args.cacheDir); await requirePayloadDirectory(icuClosure, "ICU closure carrier member"); const icu = await validateIcuDataCarrier(icuClosure); - if (icu.digest !== baseClosure.icuDigest) { + const seed = await resolveAsset(base.assets.icuSeed, args.cacheDir); + const seedIdentity = await validateClusterSeed(seed, "icu"); + if (icu.digest !== seedIdentity.digest) { fail("iOS ICU data does not match the target runtime's cluster-seed-icu"); } + await copyTree(seed, path.join(resourceRoot, "cluster-seed-icu")); icuDataTreeSha256 = icu.digest; await mergeTree(icu.data, path.join(resourceRoot, "runtime", "files", "share", "icu")); } diff --git a/src/sdks/react-native/tools/stage-ios-app.test.mjs b/src/sdks/react-native/tools/stage-ios-app.test.mjs index e84ba25fd..ad53e6bc5 100755 --- a/src/sdks/react-native/tools/stage-ios-app.test.mjs +++ b/src/sdks/react-native/tools/stage-ios-app.test.mjs @@ -577,6 +577,8 @@ async function baseAssets(root) { ], }, ]; + const seedLegal = { ...baseLegalSpecs[1], assetRole: "icu-seed", root: icuSeed }; + baseLegalSpecs.push(seedLegal); const legal = []; for (const spec of baseLegalSpecs) { legal.push(await legalGroup(spec.root, { @@ -597,6 +599,9 @@ async function baseAssets(root) { ); const frameworkArchive = path.join(archiveRoot, "liboliphaunt-1.0.0-apple-spm-xcframework.zip"); const icuArchive = path.join(archiveRoot, "liboliphaunt-1.0.0-icu-data.tar.gz"); + const icuSeedArchive = path.join(archiveRoot, "liboliphaunt-1.0.0-icu-seed-ios-datum64.tar.gz"); + await tarMembers(icuSeed, icuSeedArchive, ["manifest.properties", "files", ...seedLegal.files]); + await fs.rm(icuSeed, { recursive: true }); await tarMembers(path.dirname(runtime), runtimeArchive, [ path.basename(runtime), "LICENSE", @@ -621,6 +626,7 @@ async function baseAssets(root) { await asset("base-xcframework", frameworkArchive, "zip", "liboliphaunt.xcframework"), await asset("runtime-resources", runtimeArchive, "tar.gz", "oliphaunt"), await asset("icu-data", icuArchive, "tar.gz", "."), + await asset("icu-seed", icuSeedArchive, "tar.gz", "."), ], legal, }; diff --git a/src/sdks/react-native/tools/verify-ios-package.mjs b/src/sdks/react-native/tools/verify-ios-package.mjs index 1869c73f5..e32c5ea92 100755 --- a/src/sdks/react-native/tools/verify-ios-package.mjs +++ b/src/sdks/react-native/tools/verify-ios-package.mjs @@ -239,7 +239,7 @@ async function validateLegalSelection(payloadDir, selection, frozenSelected) { const baseRole = /^licenses\/base\/([^/]+)\//u.exec(destination)?.[1]; if (baseRole !== undefined) { requirePortableSelectionId(baseRole, `${label} base legal role`); - if (!new Set(["base-xcframework", "runtime-resources", "icu-data"]).has(baseRole)) { + if (!new Set(["base-xcframework", "runtime-resources", "icu-data", "icu-seed"]).has(baseRole)) { fail(`${label}.destination carries unknown base legal role ${baseRole}`); } if (row.source !== "base") fail(`${label}.source must be base for a base legal destination`); @@ -259,7 +259,7 @@ async function validateLegalSelection(payloadDir, selection, frozenSelected) { } extensionScopes.add(extension); } - if (destination.startsWith("licenses/base/icu-data/") && selection.icu !== true) { + if (["licenses/base/icu-data/", "licenses/base/icu-seed/"].some(prefix => destination.startsWith(prefix)) && selection.icu !== true) { fail(`${label}.destination carries unselected ICU legal material`); } if (!new Set(["license", "notice"]).has(row.kind) || !new Set(["base", "extension"]).has(row.source)) { @@ -285,7 +285,7 @@ async function validateLegalSelection(payloadDir, selection, frozenSelected) { } const expectedBaseScopes = [ "base-xcframework", - ...(selection.icu ? ["icu-data"] : []), + ...(selection.icu ? ["icu-data", "icu-seed"] : []), "runtime-resources", ].sort(compareText); requireExactDomain([...baseScopes].sort(compareText), expectedBaseScopes, `${payloadDir} base legal scopes`); @@ -537,10 +537,11 @@ async function validateStagedPackage(payloadDir, allowRuntimeDylib) { path.join(resourceRoot, "cluster-seed/files"), "iOS standard cluster seed", ); - await requirePayloadFiles( - path.join(resourceRoot, "cluster-seed-icu/files"), - "iOS ICU cluster seed", - ); + if (selection.icu) { + await requirePayloadFiles(path.join(resourceRoot, "cluster-seed-icu/files"), "iOS ICU cluster seed"); + } else if (await fs.stat(path.join(resourceRoot, "cluster-seed-icu")).then(() => true, error => { if (error.code === "ENOENT") return false; throw error; })) { + fail("unselected ICU seed must not be bundled"); + } await requireFile(path.join(resourceRoot, "package-size.tsv"), "iOS package-size report"); const baseFrameworks = await validateBaseLibrary(payloadDir, resourceRoot, allowRuntimeDylib); await validateNoBuildInputsInResources(resourceRoot); diff --git a/src/sdks/rust/Cargo.toml b/src/sdks/rust/Cargo.toml index b6e4258b9..66c6edc69 100644 --- a/src/sdks/rust/Cargo.toml +++ b/src/sdks/rust/Cargo.toml @@ -10,7 +10,7 @@ homepage.workspace = true license = "MIT" exclude = [ ".gitignore", - "crates/oliphaunt-build/**", + "crates/**", "moon.yml", "release.toml", "tools/**", @@ -33,6 +33,7 @@ broker-version = "0.2.0" native-version = "0.2.0" [dependencies] +oliphaunt-resources = { version = "0.2.0", path = "crates/oliphaunt-resources" } fs2 = "0.4" getrandom = "0.3" libloading = "0.8" @@ -40,5 +41,11 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[build-dependencies] +oliphaunt-build = { version = "0.2.0", path = "crates/oliphaunt-build" } + [dev-dependencies] tokio = { version = "1", features = ["rt", "time"] } diff --git a/src/sdks/rust/README.md b/src/sdks/rust/README.md index 0a9baa4f5..0be389c92 100644 --- a/src/sdks/rust/README.md +++ b/src/sdks/rust/README.md @@ -8,16 +8,17 @@ PostgreSQL driver or ORM through their connection string. ## Installation -Add `oliphaunt` and use `oliphaunt-build` from the build script so the matching -native runtime, tools, and selected extension artifacts are staged for the -target platform. - -```rust -fn main() { - oliphaunt_build::configure(); -} +```sh +cargo add oliphaunt ``` +The crate selects the matching native runtime and supported PostgreSQL contrib +artifacts. Ordinary applications need no build script, build dependency, or +`package.metadata.oliphaunt` block. Installed artifacts are embedded and prepared +in a validated reusable cache at first use. Preassembled application resources +remain available for deployments that require platform signing or prohibit +extracting executable files. + ## Execution placement and database topology Direct mode is the default. It runs the embedded backend in the application @@ -286,14 +287,33 @@ application error. ## Extensions and platform support -Choose extensions with `.extension(Extension::...)` or `.extensions(...)`. -Selection uses exact PostgreSQL SQL names and the generated PostgreSQL 18 -catalog. `Extension` is an opaque `Copy + Eq + Hash + Ord` selector with -uppercase associated constants, `ALL`, `by_sql_name`, and `sql_name`. Selection -makes artifacts and required pre-start configuration available but never runs -`CREATE EXTENSION`, `LOAD`, or migration SQL. Build and release tooling owns -artifact resolution; the runtime API does not expose package manifests, size -reports, capability profiles, or packaging internals. +Install an external extension as its own versioned dependency: + +```sh +cargo add oliphaunt-extension-vector +``` + +```rust,ignore +use oliphaunt::{extensions, Oliphaunt}; +use oliphaunt_extension_vector::VECTOR; + +let mut db = Oliphaunt::builder() + .extensions([VECTOR, extensions::HSTORE]) + .open()?; +db.execute("CREATE EXTENSION vector")?; +db.execute("CREATE EXTENSION hstore")?; +db.close()?; +``` + +Contrib descriptors come from `oliphaunt::extensions`; they require no additional +application dependency. External descriptors bind the selected package version +to its resources. Both must be selected explicitly for each database. Selection +prepares resources and startup requirements; applications run their own SQL +migrations, including `CREATE EXTENSION`. + +For ICU collations, add `oliphaunt-icu` and pass `.icu(oliphaunt_icu::ICU)` +to the builder. Omit it for standard initialization. Extension and ICU choices +also apply to the async and server builders. Supported native products and targets are declared by the repository SDK manifest and release packages. WASIX is a separate binding family and is not a diff --git a/src/sdks/rust/build.rs b/src/sdks/rust/build.rs index 15f0e873e..7bcb6abb8 100644 --- a/src/sdks/rust/build.rs +++ b/src/sdks/rust/build.rs @@ -12,6 +12,14 @@ const PACKAGED_QUERY_CORE: &str = "src/query_core.rs"; const CHECKOUT_QUERY_CORE: &str = "../../shared/rust-query-core/query_core.rs"; fn main() { + #[cfg(not(test))] + let embedded = oliphaunt_build::embed_resolved_artifacts() + .expect("validate and embed resolved Oliphaunt resource dependencies"); + #[cfg(not(test))] + println!( + "cargo::rustc-env=OLIPHAUNT_EMBEDDED_RESOURCES_RS={}", + embedded.display() + ); match build_instructions(env::vars()) { Ok(instructions) => { for instruction in instructions { diff --git a/src/sdks/rust/crates/oliphaunt-build/src/lib.rs b/src/sdks/rust/crates/oliphaunt-build/src/lib.rs index 5a0321dda..6373deafd 100644 --- a/src/sdks/rust/crates/oliphaunt-build/src/lib.rs +++ b/src/sdks/rust/crates/oliphaunt-build/src/lib.rs @@ -40,6 +40,110 @@ pub fn try_configure() -> Result { BuildContext::from_env()?.configure() } +/// Embed the artifact dependencies resolved for this crate, without application metadata. +/// The generated slice contains (resource path, bytes, SHA-256, executable) tuples. +pub fn embed_resolved_artifacts() -> Result { + let mut context = BuildContext::from_env()?; + context.artifact_manifest_paths = env::vars_os() + .filter_map(|(key, value)| { + let key = key.to_str()?; + let artifact = + key.starts_with(ARTIFACT_ENV_PREFIX) && key.ends_with(ARTIFACT_ENV_SUFFIX); + let sdk_relay = key.starts_with("DEP_OLIPHAUNT_ARTIFACT_RELAY_") + && !key.starts_with("DEP_OLIPHAUNT_ARTIFACT_RELAY_EXTENSION_"); + (artifact && !sdk_relay && !value.is_empty()).then(|| PathBuf::from(value)) + }) + .collect(); + let artifacts = context.read_artifact_manifests()?; + for artifact in &artifacts { + if artifact.target != context.target && artifact.target != "portable" { + return Err(Error::new(format!( + "{} targets {}, but Cargo is building {}", + artifact.label(), + artifact.target, + context.target + ))); + } + } + let versions = artifacts + .iter() + .filter_map(|artifact| { + if artifact.kind == ArtifactKind::NativeRuntime { + Some(artifact.version.as_str()) + } else if artifact.runtime_product.as_deref() == Some("liboliphaunt-native") { + artifact.runtime_version.as_deref() + } else { + None + } + }) + .collect::>(); + if versions.len() > 1 { + return Err(Error::new( + "resolved artifacts require conflicting native runtime versions", + )); + } + let manifest_path = context.manifest_dir.join("Cargo.toml"); + let manifest: toml::Value = toml::from_str( + &fs::read_to_string(&manifest_path) + .map_err(|error| Error::io("read package manifest", &manifest_path, error))?, + ) + .map_err(|error| Error::parse(&manifest_path, error))?; + let fallback = manifest + .get("package") + .and_then(|package| package.get("metadata")) + .and_then(|metadata| metadata.get("oliphaunt")) + .and_then(|metadata| metadata.get("native-version")) + .and_then(toml::Value::as_str) + .unwrap_or("unavailable"); + println!( + "cargo::rustc-env=OLIPHAUNT_NATIVE_RUNTIME_VERSION={}", + versions.first().copied().unwrap_or(fallback) + ); + let resources = context.out_dir.join("embedded-resources"); + let staged = stage_artifacts(&artifacts, &resources)?; + let mut files = BTreeMap::new(); + for artifact in staged { + for file in artifact.files { + files.insert(file.path.clone(), file); + } + } + let mut source = String::from("&[\n"); + for artifact in &artifacts { + for directory in &artifact.directories { + let relative = format!( + "{}/{}/{}/", + artifact.kind.as_str(), + artifact.product, + directory + ); + source.push_str(&format!( + "({relative:?}, b\"\", {:?}, false),\n", + sha256_hex(b"") + )); + } + } + for (relative, file) in files { + let path = resources.join(&relative); + source.push_str(&format!( + "({relative:?}, include_bytes!({:?}), {:?}, {}),\n", + path, file.sha256, file.executable + )); + } + source.push_str("]\n"); + let output = context.out_dir.join("embedded_resources.rs"); + fs::write(&output, source) + .map_err(|error| Error::io("write embedded resource index", &output, error))?; + for manifest in &context.artifact_manifest_paths { + println!("cargo::rerun-if-changed={}", manifest.display()); + } + for artifact in artifacts { + for file in artifact.files { + println!("cargo::rerun-if-changed={}", file.source.display()); + } + } + Ok(output) +} + /// Successful build-script output. #[derive(Debug, Clone, PartialEq, Eq)] pub struct BuildOutput { @@ -532,6 +636,12 @@ fn stage_artifacts( let artifact_dir = resources_dir .join(artifact.kind.as_str()) .join(&artifact.product); + for directory in &artifact.directories { + let relative = checked_relative_path(directory)?; + let dest = artifact_dir.join(relative); + fs::create_dir_all(&dest) + .map_err(|source| Error::io("create artifact directory", &dest, source))?; + } let mut locked_files = Vec::new(); for file in &artifact.files { let relative = checked_relative_path(&file.relative)?; @@ -589,6 +699,7 @@ fn stage_artifacts( runtime_product: artifact.runtime_product.clone(), runtime_version: artifact.runtime_version.clone(), files: locked_files, + directories: artifact.directories.clone(), }); } Ok(staged) @@ -818,6 +929,8 @@ struct ArtifactManifestDocument { #[serde(default)] files: Vec, #[serde(default)] + directories: Vec, + #[serde(default)] extensions: Vec, } @@ -858,6 +971,7 @@ impl ArtifactManifestDocument { extension: self.extension, dependencies: self.dependencies, files: self.files, + directories: self.directories, bundle_member: false, source_manifest: None, }]); @@ -872,6 +986,7 @@ impl ArtifactManifestDocument { || self.extension.is_some() || !self.dependencies.is_empty() || !self.files.is_empty() + || !self.directories.is_empty() || self.extensions.len() < 2 { return Err(Error::new(format!( @@ -899,6 +1014,7 @@ impl ArtifactManifestDocument { extension: Some(member.extension), dependencies: member.dependencies, files: member.files, + directories: Vec::new(), bundle_member: true, source_manifest: None, }); @@ -928,6 +1044,7 @@ struct ArtifactManifest { extension: Option, dependencies: Vec, files: Vec, + directories: Vec, bundle_member: bool, source_manifest: Option, } @@ -1061,9 +1178,6 @@ impl ArtifactManifest { "cluster-seed/manifest.properties", "cluster-seed/files/PG_VERSION", "cluster-seed/files/global/pg_control", - "cluster-seed-icu/manifest.properties", - "cluster-seed-icu/files/PG_VERSION", - "cluster-seed-icu/files/global/pg_control", ], )?; self.reject_files( @@ -1089,8 +1203,6 @@ impl ArtifactManifest { "bin/initdb.wasix.wasm", "cluster-seeds/standard.tar.zst", "cluster-seeds/standard.json", - "cluster-seeds/icu.tar.zst", - "cluster-seeds/icu.json", ], )?; self.reject_files( @@ -1277,6 +1389,8 @@ struct LockedArtifact { #[serde(skip_serializing_if = "Option::is_none")] runtime_version: Option, files: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + directories: Vec, } #[derive(Debug, Clone, Serialize)] diff --git a/src/sdks/rust/crates/oliphaunt-resources/Cargo.toml b/src/sdks/rust/crates/oliphaunt-resources/Cargo.toml new file mode 100644 index 000000000..ab151f2bd --- /dev/null +++ b/src/sdks/rust/crates/oliphaunt-resources/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "oliphaunt-resources" +version = "0.2.0" +edition = "2024" +rust-version = "1.93" +description = "Data-only package descriptors shared by Oliphaunt runtimes and optional resource crates." +readme = "README.md" +repository.workspace = true +homepage.workspace = true +license = "MIT" + +[lib] +path = "src/lib.rs" diff --git a/src/sdks/rust/crates/oliphaunt-resources/README.md b/src/sdks/rust/crates/oliphaunt-resources/README.md new file mode 100644 index 000000000..223337f40 --- /dev/null +++ b/src/sdks/rust/crates/oliphaunt-resources/README.md @@ -0,0 +1,3 @@ +# Oliphaunt resource descriptors + +Shared data types for SDKs and optional resource packages. Applications use the descriptors exported by their chosen extension or ICU package; no direct dependency on this crate is needed. This crate contains no runtime binaries or resource payloads. diff --git a/src/sdks/rust/crates/oliphaunt-resources/src/lib.rs b/src/sdks/rust/crates/oliphaunt-resources/src/lib.rs new file mode 100644 index 000000000..fd48b8e88 --- /dev/null +++ b/src/sdks/rust/crates/oliphaunt-resources/src/lib.rs @@ -0,0 +1,121 @@ +#![deny(unsafe_code)] +#![no_std] +//! Resource identities and immutable bytes, without runtime or filesystem dependencies. + +/// A package-owned resource: relative path, bytes, SHA-256, and executable bit. +/// A trailing slash declares a directory, with empty bytes and executable=false. +pub type EmbeddedResource = (&'static str, &'static [u8], &'static str, bool); + +/// A versioned native extension and its package-owned resources. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ExtensionDescriptor { + /// PostgreSQL extension name. + pub sql_name: &'static str, + /// Independently released artifact product. + pub product: &'static str, + /// Extension package release version; contrib follows the SDK runtime. + pub version: Option<&'static str>, + /// Native runtime release expected by the artifact. + pub runtime_version: &'static str, + /// Verified release resources embedded by the extension package. + pub resources: &'static [EmbeddedResource], +} + +/// Explicitly selected ICU data from the optional ICU crate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IcuData { + /// Native runtime release expected by this carrier. + pub native_runtime_version: &'static str, + /// ICU carrier package version. + pub version: &'static str, + /// WASIX runtime release expected by this carrier. + pub runtime_version: &'static str, + /// Package-owned ICU files and receipt. + pub resources: &'static [EmbeddedResource], + /// Portable WASIX ICU archive. + pub wasix_archive: Option<&'static [u8]>, + /// SHA-256 of the portable archive. + pub wasix_archive_sha256: Option<&'static str>, + /// SHA-256 of the installed logical ICU data tree. + pub wasix_data_tree_sha256: Option<&'static str>, + /// Matching ICU catalog seed, owned by this optional package. + pub wasix_seed_archive: Option<&'static [u8]>, + /// Matching ICU catalog seed manifest. + pub wasix_seed_manifest: Option<&'static [u8]>, +} + +/// Immutable resources owned by an independently released WASIX package. +/// +/// Applications use the descriptor exported by their extension crate. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct WasixPackage { + product: &'static str, + version: &'static str, + runtime_version: &'static str, + archives: &'static [(&'static str, &'static [u8], &'static str)], + aot_manifest: &'static str, + aot_artifacts: &'static [(&'static str, &'static [u8])], +} + +impl WasixPackage { + /// Package-owned product. + pub const fn product(self) -> &'static str { + self.product + } + /// Package-owned version. + pub const fn version(self) -> &'static str { + self.version + } + /// Package-owned runtime version. + pub const fn runtime_version(self) -> &'static str { + self.runtime_version + } + /// Package-owned archives. + pub const fn archives(self) -> &'static [(&'static str, &'static [u8], &'static str)] { + self.archives + } + /// Package-owned aot manifest. + pub const fn aot_manifest(self) -> &'static str { + self.aot_manifest + } + /// Package-owned aot artifacts. + pub const fn aot_artifacts(self) -> &'static [(&'static str, &'static [u8])] { + self.aot_artifacts + } + + /// Construct a descriptor in a generated extension package. + /// + /// # Safety + /// Every AOT artifact must have been produced by the trusted Oliphaunt build + /// for the declared engine and target. The manifest and archive identities + /// must belong to that same release. A caller-provided hash alone does not + /// establish this trust: arbitrary serialized native code is not safe input. + #[doc(hidden)] + #[allow(unsafe_code)] + pub const unsafe fn from_trusted_release( + product: &'static str, + version: &'static str, + runtime_version: &'static str, + archives: &'static [(&'static str, &'static [u8], &'static str)], + aot_manifest: &'static str, + aot_artifacts: &'static [(&'static str, &'static [u8])], + ) -> Self { + Self { + product, + version, + runtime_version, + archives, + aot_manifest, + aot_artifacts, + } + } +} + +/// A selected WASIX extension; package resources are omitted for bundled contrib. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WasixExtensionDescriptor { + /// PostgreSQL extension name. + pub sql_name: &'static str, + /// Resources from the independently versioned extension crate. + pub package: Option<&'static WasixPackage>, +} diff --git a/src/sdks/rust/moon.yml b/src/sdks/rust/moon.yml index 5a4935a39..478e6e1ae 100644 --- a/src/sdks/rust/moon.yml +++ b/src/sdks/rust/moon.yml @@ -109,13 +109,15 @@ tasks: script: | set -e node src/sdks/rust/tools/package-source.mjs - cargo package --manifest-path target/liboliphaunt-sdk-check/oliphaunt-rust/package-source/Cargo.toml --allow-dirty --no-verify cargo package --manifest-path target/liboliphaunt-sdk-check/oliphaunt-rust/package-source/Cargo.toml --allow-dirty --list > target/liboliphaunt-sdk-check/rust-cargo-package-list.txt - cargo package -p oliphaunt-build --locked --allow-dirty --no-verify + tools/dev/bun.sh -e 'import { mkdirSync } from "node:fs"; import { stageArtifacts } from "./tools/release/sdk-artifacts/rust.mjs"; const output = process.cwd() + "/target/liboliphaunt-sdk-check/oliphaunt-rust/package"; mkdirSync(output, { recursive: true }); stageArtifacts(output, output + "/work");' env: CARGO_TARGET_DIR: "target/moon/oliphaunt-rust/package" inputs: - "@group(legal-files)" + - "@group(release-archive-contract)" + - "/tools/release/sdk-artifacts/rust.mjs" + - "/tools/release/prepare-rust-release-source.mjs" - "@group(cargo-workspace)" - project: "shared-test-fixtures" group: "fixtures" @@ -125,7 +127,7 @@ tasks: outputs: - "/target/liboliphaunt-sdk-check/rust-cargo-package-list.txt" - "/target/liboliphaunt-sdk-check/oliphaunt-rust/package-source/**/*" - - "/target/moon/oliphaunt-rust/package/package/**/*" + - "/target/liboliphaunt-sdk-check/oliphaunt-rust/package/**/*" options: cache: true runFromWorkspaceRoot: true diff --git a/src/sdks/rust/release.toml b/src/sdks/rust/release.toml index 9e8771017..9499b082c 100644 --- a/src/sdks/rust/release.toml +++ b/src/sdks/rust/release.toml @@ -2,7 +2,7 @@ id = "oliphaunt-rust" owner = "@oliphaunt/sdk-rust" kind = "sdk" publish_targets = ["crates-io"] -registry_packages = ["crates:oliphaunt", "crates:oliphaunt-build"] +registry_packages = ["crates:oliphaunt", "crates:oliphaunt-build", "crates:oliphaunt-resources"] release_artifacts = ["cargo-crate", "runtime-resource-cli"] [compatibility_versions.native_runtime] diff --git a/src/sdks/rust/src/broker.rs b/src/sdks/rust/src/broker.rs index f047b8a1d..6b1a19590 100644 --- a/src/sdks/rust/src/broker.rs +++ b/src/sdks/rust/src/broker.rs @@ -564,6 +564,17 @@ fn spawn_broker( .stdout(Stdio::piped()) .stderr(Stdio::inherit()) .env(ENV_BROKER_AUTH_TOKEN, auth_token.as_str()); + command.env("OLIPHAUNT_SELECTED_ICU", if config.icu { "1" } else { "0" }); + if let Some(resources) = &config.resource_directory { + command.env("OLIPHAUNT_SELECTED_RESOURCES_DIR", resources); + } else { + command.env_remove("OLIPHAUNT_SELECTED_RESOURCES_DIR"); + } + if let Some(resources) = crate::build_resources::registered_build_resources_dir() + .or_else(crate::build_resources::embedded_base_resources_dir) + { + command.env("OLIPHAUNT_RESOURCES_DIR", resources); + } command.spawn().map_err(|err| { Error::Engine(format!( "spawn native broker {}: {err}", @@ -760,7 +771,15 @@ fn resolve_broker_executable() -> Option { if let Some(path) = resolve_broker_executable_next_to_current_exe() { return Some(path); } - resolve_broker_executable_from_asset_dir() + resolve_broker_executable_from_asset_dir().or_else(|| { + let resources = crate::build_resources::registered_build_resources_dir() + .or_else(crate::build_resources::embedded_base_resources_dir)?; + let target = current_broker_release_target()?; + target + .unpacked_executable_candidates(&resources.join("broker-helper/oliphaunt-broker")) + .into_iter() + .find(|path| path.is_file()) + }) } fn resolve_broker_executable_next_to_current_exe() -> Option { diff --git a/src/sdks/rust/src/broker_support.rs b/src/sdks/rust/src/broker_support.rs index 154171d8c..8365d4f17 100644 --- a/src/sdks/rust/src/broker_support.rs +++ b/src/sdks/rust/src/broker_support.rs @@ -52,6 +52,8 @@ pub fn open( extensions: Vec, ) -> Result { let config = OpenConfig { + resource_directory: std::env::var_os("OLIPHAUNT_SELECTED_RESOURCES_DIR").map(PathBuf::from), + icu: std::env::var("OLIPHAUNT_SELECTED_ICU").as_deref() == Ok("1"), mode: EngineMode::Direct, storage: DatabaseStorage::Directory(root), broker: NativeBrokerConfig::default(), diff --git a/src/sdks/rust/src/build_resources.rs b/src/sdks/rust/src/build_resources.rs index 4339f8d67..01c9943f6 100644 --- a/src/sdks/rust/src/build_resources.rs +++ b/src/sdks/rust/src/build_resources.rs @@ -5,10 +5,227 @@ use crate::error::{Error, Result}; static BUILD_RESOURCES_DIR: OnceLock>> = OnceLock::new(); +pub use oliphaunt_resources::EmbeddedResource; + +const BASE_RESOURCES: &[EmbeddedResource] = include!(env!("OLIPHAUNT_EMBEDDED_RESOURCES_RS")); +static EMBEDDED_BASE: OnceLock, String>> = OnceLock::new(); + +pub(crate) fn prepare_base_resources() -> Result> { + EMBEDDED_BASE + .get_or_init(|| { + if BASE_RESOURCES.is_empty() { + return Ok(None); + } + materialize_embedded_resources(BASE_RESOURCES) + .map(Some) + .map_err(|error| error.to_string()) + }) + .clone() + .map_err(Error::Engine) +} + +pub(crate) fn embedded_base_resources_dir() -> Option { + EMBEDDED_BASE + .get() + .and_then(|result| result.as_ref().ok()) + .cloned() + .flatten() +} + +pub(crate) fn materialize_embedded_resources(files: &[EmbeddedResource]) -> Result { + use fs2::FileExt; + use sha2::{Digest, Sha256}; + use std::fs::{self, OpenOptions}; + use std::path::{Component, Path}; + let mut selected = std::collections::BTreeMap::new(); + for &(relative, bytes, digest, executable) in files { + if (relative.ends_with('/') && (!bytes.is_empty() || executable)) + || relative.is_empty() + || relative.contains('\\') + || relative.contains(':') + || Path::new(relative) + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err(Error::InvalidConfig(format!( + "invalid embedded resource path {relative:?}" + ))); + } + if format!("{:x}", Sha256::digest(bytes)) != digest { + return Err(Error::InvalidConfig(format!( + "embedded resource checksum mismatch: {relative}" + ))); + } + if let Some(previous) = selected.insert(relative, (bytes, digest, executable)) + && previous != (bytes, digest, executable) + { + return Err(Error::InvalidConfig(format!( + "conflicting embedded resource: {relative}" + ))); + } + } + let mut hash = Sha256::new(); + for (relative, (_, digest, executable)) in &selected { + hash.update(relative.as_bytes()); + hash.update([0]); + hash.update(digest.as_bytes()); + hash.update([u8::from(*executable)]); + } + let key = format!("{:x}", hash.finalize()); + let cache = std::env::var_os("OLIPHAUNT_RUNTIME_CACHE_DIR") + .map(PathBuf::from) + .or_else(|| std::env::var_os("LOCALAPPDATA").map(PathBuf::from)) + .or_else(|| std::env::var_os("XDG_CACHE_HOME").map(PathBuf::from)) + .or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".cache"))) + .ok_or_else(|| { + Error::Engine("no cache directory available for native resources".to_owned()) + })? + .join("oliphaunt-embedded"); + let io = |error: std::io::Error| { + Error::Engine(format!("prepare embedded native resources: {error}")) + }; + fs::create_dir_all(&cache).map_err(io)?; + if fs::symlink_metadata(&cache) + .map_err(io)? + .file_type() + .is_symlink() + { + return Err(Error::Engine( + "embedded resource cache must not be a symbolic link".to_owned(), + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&cache, fs::Permissions::from_mode(0o700)).map_err(io)?; + } + let lock_path = cache.join(format!("{key}.lock")); + if fs::symlink_metadata(&lock_path).is_ok_and(|metadata| metadata.file_type().is_symlink()) { + return Err(Error::Engine( + "embedded resource lock must not be a symbolic link".to_owned(), + )); + } + let mut lock_options = OpenOptions::new(); + lock_options + .create(true) + .truncate(false) + .read(true) + .write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + lock_options.custom_flags(libc::O_NOFOLLOW); + } + let lock = lock_options.open(lock_path).map_err(io)?; + lock.lock_exclusive().map_err(io)?; + let directory = cache.join(&key); + let expected_paths = selected + .keys() + .flat_map(|relative| { + Path::new(relative) + .ancestors() + .filter(|path| !path.as_os_str().is_empty()) + .map(|path| directory.join(path)) + .collect::>() + }) + .collect::>(); + let mut pending = vec![directory.clone()]; + let mut inventory_valid = true; + while let Some(parent) = pending.pop() { + let Ok(entries) = fs::read_dir(&parent) else { + inventory_valid = false; + break; + }; + for entry in entries { + let Ok(entry) = entry else { + inventory_valid = false; + break; + }; + let path = entry.path(); + let Ok(kind) = entry.file_type() else { + inventory_valid = false; + break; + }; + if !expected_paths.contains(&path) + || kind.is_symlink() + || (!kind.is_file() && !kind.is_dir()) + { + inventory_valid = false; + break; + } + if kind.is_dir() { + pending.push(path); + } + } + if !inventory_valid { + break; + } + } + let valid = inventory_valid + && fs::symlink_metadata(&directory) + .is_ok_and(|metadata| metadata.is_dir() && !metadata.file_type().is_symlink()) + && selected.iter().all(|(relative, (_, digest, executable))| { + let path = directory.join(relative); + let Ok(metadata) = fs::symlink_metadata(&path) else { + return false; + }; + if relative.ends_with('/') { + return metadata.is_dir() && !metadata.file_type().is_symlink(); + } + if !metadata.is_file() || metadata.file_type().is_symlink() { + return false; + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if (metadata.permissions().mode() & 0o111 != 0) != *executable { + return false; + } + } + fs::read(path).is_ok_and(|bytes| format!("{:x}", Sha256::digest(bytes)) == *digest) + }); + if !valid { + let staging = cache.join(format!(".{key}-{}", std::process::id())); + if staging.exists() { + fs::remove_dir_all(&staging).map_err(io)?; + } + fs::create_dir(&staging).map_err(io)?; + for (relative, (bytes, _, executable)) in selected { + let path = staging.join(relative); + if relative.ends_with('/') { + fs::create_dir_all(&path).map_err(io)?; + continue; + } + fs::create_dir_all(path.parent().expect("resource parent")).map_err(io)?; + fs::write(&path, bytes).map_err(io)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions( + path, + fs::Permissions::from_mode(if executable { 0o700 } else { 0o600 }), + ) + .map_err(io)?; + } + } + if let Ok(metadata) = fs::symlink_metadata(&directory) { + if metadata.file_type().is_symlink() || !metadata.is_dir() { + fs::remove_file(&directory).map_err(io)?; + } else { + fs::remove_dir_all(&directory).map_err(io)?; + } + } + fs::rename(staging, &directory).map_err(io)?; + } + FileExt::unlock(&lock).map_err(io)?; + Ok(directory) +} + /// Register the Oliphaunt resource directory staged by `oliphaunt-build`. /// -/// Applications usually call [`crate::register_build_resources!`] once during startup -/// after their `build.rs` has called `oliphaunt_build::configure()`. The native +/// Advanced preassembled deployments call [`crate::register_build_resources!`] +/// after their `build.rs` has called `oliphaunt_build::configure()`. Ordinary +/// applications use automatically embedded resources. The native /// runtime locator uses this directory before falling back to explicit /// environment variables and source-tree build layouts. pub fn register_build_resources_dir(path: impl Into) -> Result<()> { @@ -65,6 +282,76 @@ macro_rules! register_build_resources { mod tests { use super::*; + #[test] + fn embedded_resources_reject_unsafe_paths_and_repair_modified_cache_files() { + use sha2::{Digest, Sha256}; + let digest: &'static str = + Box::leak(format!("{:x}", Sha256::digest(b"resource cache test")).into_boxed_str()); + for relative in ["../escape", "/absolute", "C:/escape", "a/../escape"] { + assert!( + materialize_embedded_resources(&[( + relative, + b"resource cache test", + digest, + false + )]) + .is_err() + ); + } + assert!(materialize_embedded_resources(&[("file", b"wrong", digest, false)]).is_err()); + let files = [( + "extension/test/payload", + &b"resource cache test"[..], + digest, + false, + )]; + let root = materialize_embedded_resources(&files).unwrap(); + assert_eq!(root, materialize_embedded_resources(&files).unwrap()); + std::fs::write(root.join("extension/test/payload"), b"modified").unwrap(); + let repaired = materialize_embedded_resources(&files).unwrap(); + assert_eq!( + std::fs::read(repaired.join("extension/test/payload")).unwrap(), + b"resource cache test" + ); + std::fs::write(root.join("unexpected.so"), b"unselected code").unwrap(); + materialize_embedded_resources(&files).unwrap(); + assert!(!root.join("unexpected.so").exists()); + #[cfg(unix)] + { + let outside = root.with_extension("symlink-test"); + std::fs::create_dir_all(outside.join("test")).unwrap(); + std::fs::write(outside.join("test/payload"), b"resource cache test").unwrap(); + std::fs::remove_dir_all(root.join("extension")).unwrap(); + std::os::unix::fs::symlink(&outside, root.join("extension")).unwrap(); + materialize_embedded_resources(&files).unwrap(); + assert!( + !std::fs::symlink_metadata(root.join("extension")) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read(outside.join("test/payload")).unwrap(), + b"resource cache test" + ); + std::fs::remove_dir_all(outside).unwrap(); + } + std::fs::remove_dir_all(root).unwrap(); + let directory_resources = [( + "cluster-seed/files/pg_wal/", + &b""[..], + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + false, + )]; + let directory_root = materialize_embedded_resources(&directory_resources).unwrap(); + assert!(directory_root.join("cluster-seed/files/pg_wal").is_dir()); + assert_eq!( + directory_root, + materialize_embedded_resources(&directory_resources).unwrap() + ); + std::fs::remove_dir_all(directory_root).unwrap(); + } + #[test] fn registration_and_macro_contract_is_process_wide_and_immutable() { assert_eq!(registered_build_resources_dir(), None); diff --git a/src/sdks/rust/src/builder.rs b/src/sdks/rust/src/builder.rs index 3fb7269cc..47232150d 100644 --- a/src/sdks/rust/src/builder.rs +++ b/src/sdks/rust/src/builder.rs @@ -9,7 +9,7 @@ use crate::database::{AsyncOliphaunt, AsyncOliphauntServer}; use crate::engine::{EngineSession, NativeRuntime}; use crate::error::{Error, Result}; use crate::executor::EngineExecutor; -use crate::extension::Extension; +use crate::extension::{ExtensionDescriptor, IcuData, prepare_selected_resources}; use crate::liboliphaunt::OliphauntRuntime; use crate::server::NativeServerRuntime; use crate::storage::DatabaseStorage; @@ -35,7 +35,8 @@ struct CommonOpenOptions { startup_gucs: Vec, username: String, database: String, - extensions: Vec, + extensions: Vec, + icu: Option, } impl Default for CommonOpenOptions { @@ -46,6 +47,7 @@ impl Default for CommonOpenOptions { username: DEFAULT_USERNAME.to_owned(), database: DEFAULT_DATABASE.to_owned(), extensions: Vec::new(), + icu: None, } } } @@ -57,6 +59,8 @@ impl CommonOpenOptions { broker: NativeBrokerConfig, server: NativeServerConfig, ) -> Result { + let (extensions, resource_directory) = + prepare_selected_resources(&self.extensions, self.icu)?; let config = OpenConfig { mode, storage: self.storage.clone(), @@ -65,7 +69,9 @@ impl CommonOpenOptions { startup_gucs: self.startup_gucs.clone(), username: self.username.clone(), database: self.database.clone(), - extensions: self.extensions.clone(), + extensions, + resource_directory, + icu: self.icu.is_some(), }; config.validate()?; Ok(config) @@ -145,17 +151,28 @@ impl AsyncOliphauntBuilder { self } - /// Make one bundled PostgreSQL extension artifact available to the database. + /// Select the optional package-owned ICU data. + pub fn icu(mut self, data: IcuData) -> Self { + self.common.icu = Some(data); + self + } + + /// Make one explicitly selected PostgreSQL extension artifact available to the database. /// Database-local installation remains the application's migration concern. - pub fn extension(mut self, extension: Extension) -> Self { - self.common.extensions.push(extension); + pub fn extension(mut self, extension: impl Into) -> Self { + self.common.extensions.push(extension.into()); self } - /// Make bundled PostgreSQL extension artifacts available to the database. + /// Make explicitly selected PostgreSQL extension artifacts available to the database. /// Database-local installation remains the application's migration concern. - pub fn extensions(mut self, extensions: impl IntoIterator) -> Self { - self.common.extensions.extend(extensions); + pub fn extensions( + mut self, + extensions: impl IntoIterator>, + ) -> Self { + self.common + .extensions + .extend(extensions.into_iter().map(Into::into)); self } @@ -240,17 +257,28 @@ impl AsyncOliphauntServerBuilder { self } - /// Make one bundled PostgreSQL extension artifact available to clients. + /// Select the optional package-owned ICU data. + pub fn icu(mut self, data: IcuData) -> Self { + self.common.icu = Some(data); + self + } + + /// Make one explicitly selected PostgreSQL extension artifact available to clients. /// Database-local installation remains the application's migration concern. - pub fn extension(mut self, extension: Extension) -> Self { - self.common.extensions.push(extension); + pub fn extension(mut self, extension: impl Into) -> Self { + self.common.extensions.push(extension.into()); self } - /// Make bundled PostgreSQL extension artifacts available to clients. + /// Make explicitly selected PostgreSQL extension artifacts available to clients. /// Database-local installation remains the application's migration concern. - pub fn extensions(mut self, extensions: impl IntoIterator) -> Self { - self.common.extensions.extend(extensions); + pub fn extensions( + mut self, + extensions: impl IntoIterator>, + ) -> Self { + self.common + .extensions + .extend(extensions.into_iter().map(Into::into)); self } @@ -278,6 +306,7 @@ impl AsyncOliphauntServerBuilder { } pub(crate) fn open_embedded_session(config: OpenConfig) -> Result> { + crate::build_resources::prepare_base_resources()?; match config.mode { EngineMode::Direct => OliphauntRuntime::from_env().open(config), EngineMode::Broker => NativeBrokerRuntime::from_config(&config.broker).open(config), @@ -286,6 +315,7 @@ pub(crate) fn open_embedded_session(config: OpenConfig) -> Result Result<(Box, String)> { + crate::build_resources::prepare_base_resources()?; let session = NativeServerRuntime::from_config(&config.server).open(config)?; let connection_string = session.connection_string().ok_or_else(|| { Error::Engine("native server did not expose its connection string".to_owned()) diff --git a/src/sdks/rust/src/config.rs b/src/sdks/rust/src/config.rs index dc57218c6..864c4ca60 100644 --- a/src/sdks/rust/src/config.rs +++ b/src/sdks/rust/src/config.rs @@ -125,6 +125,8 @@ pub(crate) struct OpenConfig { pub(crate) username: String, pub(crate) database: String, pub(crate) extensions: Vec, + pub(crate) resource_directory: Option, + pub(crate) icu: bool, } impl OpenConfig { @@ -139,6 +141,8 @@ impl OpenConfig { username: DEFAULT_USERNAME.to_owned(), database: DEFAULT_DATABASE.to_owned(), extensions: Vec::new(), + resource_directory: None, + icu: false, } } diff --git a/src/sdks/rust/src/direct.rs b/src/sdks/rust/src/direct.rs index 293c6e61d..f4260e2a3 100644 --- a/src/sdks/rust/src/direct.rs +++ b/src/sdks/rust/src/direct.rs @@ -13,7 +13,7 @@ use crate::error::{ Error, RawStreamCallbackOutput, RawStreamError, RawStreamResult, Result, SESSION_STATE_UNKNOWN, TransactionError, TransactionResult, }; -use crate::extension::Extension; +use crate::extension::{ExtensionDescriptor, IcuData}; use crate::liboliphaunt::OliphauntRuntime; use crate::protocol::{ProtocolRequest, ProtocolResponse}; use crate::query::{ @@ -112,16 +112,25 @@ impl OliphauntBuilder { self } + /// Select the optional package-owned ICU data. + pub fn icu(mut self, data: IcuData) -> Self { + self.inner = self.inner.icu(data); + self + } + /// Make one bundled PostgreSQL extension artifact available to the database. /// Database-local installation remains the application's migration concern. - pub fn extension(mut self, extension: Extension) -> Self { + pub fn extension(mut self, extension: impl Into) -> Self { self.inner = self.inner.extension(extension); self } /// Make bundled PostgreSQL extension artifacts available to the database. /// Database-local installation remains the application's migration concern. - pub fn extensions(mut self, extensions: impl IntoIterator) -> Self { + pub fn extensions( + mut self, + extensions: impl IntoIterator>, + ) -> Self { self.inner = self.inner.extensions(extensions); self } @@ -191,16 +200,25 @@ impl OliphauntServerBuilder { self } + /// Select the optional package-owned ICU data. + pub fn icu(mut self, data: IcuData) -> Self { + self.inner = self.inner.icu(data); + self + } + /// Make one bundled PostgreSQL extension artifact available to clients. /// Database-local installation remains the application's migration concern. - pub fn extension(mut self, extension: Extension) -> Self { + pub fn extension(mut self, extension: impl Into) -> Self { self.inner = self.inner.extension(extension); self } /// Make bundled PostgreSQL extension artifacts available to clients. /// Database-local installation remains the application's migration concern. - pub fn extensions(mut self, extensions: impl IntoIterator) -> Self { + pub fn extensions( + mut self, + extensions: impl IntoIterator>, + ) -> Self { self.inner = self.inner.extensions(extensions); self } diff --git a/src/sdks/rust/src/extension.rs b/src/sdks/rust/src/extension.rs index 67a95b1a9..59a7480dc 100644 --- a/src/sdks/rust/src/extension.rs +++ b/src/sdks/rust/src/extension.rs @@ -6,6 +6,66 @@ use crate::error::{Error, Result}; mod generated_extensions; pub use generated_extensions::Extension; +pub use oliphaunt_resources::{ExtensionDescriptor, IcuData}; + +impl From for ExtensionDescriptor { + fn from(extension: Extension) -> Self { + Self { + sql_name: extension.sql_name(), + product: "", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], + } + } +} + +pub(crate) fn prepare_selected_resources( + extensions: &[ExtensionDescriptor], + icu: Option, +) -> Result<(Vec, Option)> { + let mut selected = std::collections::BTreeMap::new(); + let mut files = Vec::new(); + for value in extensions { + let extension = Extension::by_sql_name(value.sql_name).ok_or_else(|| { + Error::InvalidConfig(format!("unknown extension '{}'", value.sql_name)) + })?; + if value.runtime_version != env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION") { + return Err(Error::InvalidConfig(format!( + "extension '{}' requires native runtime {}", + value.sql_name, value.runtime_version + ))); + } + if let Some(previous) = selected.insert(extension, *value) + && previous != *value + { + return Err(Error::InvalidConfig(format!( + "conflicting extension descriptors for '{}'", + value.sql_name + ))); + } + files.extend_from_slice(value.resources); + } + if let Some(icu) = icu { + if icu.native_runtime_version != env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION") + || icu.resources.is_empty() + { + return Err(Error::InvalidConfig( + "ICU descriptor has no compatible runtime resources".to_owned(), + )); + } + files.extend_from_slice(icu.resources); + } + let resources = if files.is_empty() { + None + } else { + Some(crate::build_resources::materialize_embedded_resources( + &files, + )?) + }; + Ok((selected.into_keys().collect(), resources)) +} + impl Extension { /// SQL extension name used by `CREATE EXTENSION`. pub const fn sql_name(self) -> &'static str { diff --git a/src/sdks/rust/src/extensions.rs b/src/sdks/rust/src/extensions.rs new file mode 100644 index 000000000..0b62576c9 --- /dev/null +++ b/src/sdks/rust/src/extensions.rs @@ -0,0 +1,290 @@ +// This file is generated by src/extensions/tools/check-extension-model.mjs. +// Do not edit by hand. + +/// Select the `amcheck` contrib extension. +pub const AMCHECK: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "amcheck", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `auto_explain` contrib extension. +pub const AUTO_EXPLAIN: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "auto_explain", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `bloom` contrib extension. +pub const BLOOM: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "bloom", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `btree_gin` contrib extension. +pub const BTREE_GIN: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "btree_gin", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `btree_gist` contrib extension. +pub const BTREE_GIST: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "btree_gist", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `citext` contrib extension. +pub const CITEXT: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "citext", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `cube` contrib extension. +pub const CUBE: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "cube", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `dict_int` contrib extension. +pub const DICT_INT: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "dict_int", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `dict_xsyn` contrib extension. +pub const DICT_XSYN: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "dict_xsyn", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `earthdistance` contrib extension. +pub const EARTHDISTANCE: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "earthdistance", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `file_fdw` contrib extension. +pub const FILE_FDW: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "file_fdw", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `fuzzystrmatch` contrib extension. +pub const FUZZYSTRMATCH: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "fuzzystrmatch", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `hstore` contrib extension. +pub const HSTORE: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "hstore", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `intarray` contrib extension. +pub const INTARRAY: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "intarray", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `isn` contrib extension. +pub const ISN: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "isn", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `lo` contrib extension. +pub const LO: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "lo", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `ltree` contrib extension. +pub const LTREE: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "ltree", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `pageinspect` contrib extension. +pub const PAGEINSPECT: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pageinspect", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `pg_buffercache` contrib extension. +pub const PG_BUFFERCACHE: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pg_buffercache", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `pg_freespacemap` contrib extension. +pub const PG_FREESPACEMAP: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pg_freespacemap", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `pg_surgery` contrib extension. +pub const PG_SURGERY: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pg_surgery", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `pg_trgm` contrib extension. +pub const PG_TRGM: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pg_trgm", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `pg_visibility` contrib extension. +pub const PG_VISIBILITY: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pg_visibility", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `pg_walinspect` contrib extension. +pub const PG_WALINSPECT: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pg_walinspect", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `pgcrypto` contrib extension. +pub const PGCRYPTO: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "pgcrypto", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `seg` contrib extension. +pub const SEG: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "seg", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `tablefunc` contrib extension. +pub const TABLEFUNC: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "tablefunc", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `tcn` contrib extension. +pub const TCN: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "tcn", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `tsm_system_rows` contrib extension. +pub const TSM_SYSTEM_ROWS: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "tsm_system_rows", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `tsm_system_time` contrib extension. +pub const TSM_SYSTEM_TIME: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "tsm_system_time", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `unaccent` contrib extension. +pub const UNACCENT: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "unaccent", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; + +/// Select the `uuid-ossp` contrib extension. +pub const UUID_OSSP: crate::ExtensionDescriptor = crate::ExtensionDescriptor { + sql_name: "uuid-ossp", + product: "oliphaunt-extension-contrib-pg18", + version: None, + runtime_version: env!("OLIPHAUNT_NATIVE_RUNTIME_VERSION"), + resources: &[], +}; diff --git a/src/sdks/rust/src/lib.rs b/src/sdks/rust/src/lib.rs index ec60036e9..feb0c7837 100644 --- a/src/sdks/rust/src/lib.rs +++ b/src/sdks/rust/src/lib.rs @@ -61,7 +61,7 @@ mod session; mod storage; #[cfg(test)] mod test_fixtures; -pub use build_resources::register_build_resources_dir; +pub use build_resources::{EmbeddedResource, register_build_resources_dir}; pub use builder::{AsyncOliphauntBuilder, AsyncOliphauntServerBuilder}; pub use config::ServerListen; pub use database::{AsyncOliphaunt, AsyncOliphauntServer, AsyncSql, AsyncTransaction}; @@ -73,7 +73,9 @@ pub use error::{ Error, ErrorKind, PostgresError, PostgresErrorField, RawStreamCallbackOutput, RawStreamError, RawStreamResult, Result, TransactionError, TransactionResult, }; -pub use extension::Extension; +pub use extension::{Extension, ExtensionDescriptor, IcuData}; +/// Contrib extensions supplied by the base SDK. +pub mod extensions; pub use query::{ CommandResult, DecodeError, ExecResult, FromSql, IntoParameter, Parameter, PostgresNotice, QueryField, QueryFormat, QueryResult, QueryRow, RowIndex, StatementDescription, diff --git a/src/sdks/rust/src/liboliphaunt/ffi.rs b/src/sdks/rust/src/liboliphaunt/ffi.rs index 44794f5f3..94ee3e727 100644 --- a/src/sdks/rust/src/liboliphaunt/ffi.rs +++ b/src/sdks/rust/src/liboliphaunt/ffi.rs @@ -211,7 +211,26 @@ fn resolve_library_path() -> Result { } pub(super) fn resolve_library_path_candidates() -> Vec { - env_path_candidates([ENV_OLIPHAUNT]) + let mut candidates = env_path_candidates([ENV_OLIPHAUNT]); + if let Some(root) = crate::build_resources::registered_build_resources_dir() + .or_else(|| std::env::var_os("OLIPHAUNT_RESOURCES_DIR").map(PathBuf::from)) + .or_else(crate::build_resources::embedded_base_resources_dir) + { + let payload = root.join("native-runtime/liboliphaunt-native"); + let path = if cfg!(windows) { + payload.join("bin/oliphaunt.dll") + } else { + payload.join(format!( + "lib/{}oliphaunt{}", + std::env::consts::DLL_PREFIX, + std::env::consts::DLL_SUFFIX + )) + }; + if path.is_file() { + candidates.push(path); + } + } + candidates } pub(super) fn env_path_candidates(names: [&str; N]) -> Vec { diff --git a/src/sdks/rust/src/liboliphaunt/root.rs b/src/sdks/rust/src/liboliphaunt/root.rs index 4b8e3b2b9..f05e707f8 100644 --- a/src/sdks/rust/src/liboliphaunt/root.rs +++ b/src/sdks/rust/src/liboliphaunt/root.rs @@ -111,10 +111,12 @@ impl PreparedNativeRoot { ))); } let pgdata = root.join("pgdata"); - let runtime_closure = runtime::resolve_runtime_closure( + let runtime_closure = runtime::resolve_runtime_closure_with_resources( NativeRuntimeProfile::for_mode(config.mode), extensions, None, + config.resource_directory.as_deref(), + Some(config.icu), )?; let runtime_dir = runtime_closure.runtime_dir; let mut pgdata_cleanup = CreatedPgdataCleanup::new(); diff --git a/src/sdks/rust/src/liboliphaunt/root/runtime.rs b/src/sdks/rust/src/liboliphaunt/root/runtime.rs index 73b2af3c1..8983be6a7 100644 --- a/src/sdks/rust/src/liboliphaunt/root/runtime.rs +++ b/src/sdks/rust/src/liboliphaunt/root/runtime.rs @@ -35,14 +35,35 @@ pub(super) struct ResolvedRuntimeClosure { pub(super) cluster_seed_dir: Option, } +#[cfg(feature = "internal-native-packaging")] pub(super) fn resolve_runtime_closure( profile: NativeRuntimeProfile, extensions: &[Extension], requested_catalog_profile: Option, +) -> Result { + resolve_runtime_closure_with_resources( + profile, + extensions, + requested_catalog_profile, + None, + None, + ) +} + +pub(super) fn resolve_runtime_closure_with_resources( + profile: NativeRuntimeProfile, + extensions: &[Extension], + requested_catalog_profile: Option, + selected_resources: Option<&Path>, + selected_icu: Option, ) -> Result { let install_dir = locate_native_install_dir()?; let package_resources_root = package_resources_root_for_install(&install_dir); - let available_icu_data = locate_native_icu_data()?; + let available_icu_data = if selected_icu == Some(false) { + None + } else { + locate_native_icu_data(selected_resources)? + }; let catalog_profile = requested_catalog_profile.unwrap_or_else(|| { if available_icu_data.is_some() { NativeCatalogProfile::Icu @@ -63,20 +84,15 @@ pub(super) fn resolve_runtime_closure( let icu_tree_sha256 = icu_data .as_ref() .and_then(|data| data.tree_sha256.as_deref()); - let runtime_dir = materialize_runtime( + let runtime_dir = materialize_runtime_with_resources( profile, &install_dir, extensions, icu_directory, icu_tree_sha256, + selected_resources, )?; - let package_closure_root = package_resources_root.filter(|resources_root| match &icu_data { - None => catalog_profile == NativeCatalogProfile::Standard, - Some(icu) => { - catalog_profile == NativeCatalogProfile::Icu - && icu.package_resources_root.as_ref() == Some(resources_root) - } - }); + let package_closure_root = package_resources_root; // Packaging materializes the seeds after resolving the runtime closure. Only an // ordinary SDK open consumes a seed that already belongs to a released carrier. let cluster_seed = if requested_catalog_profile.is_none() { @@ -113,14 +129,22 @@ pub(super) fn resolve_runtime_closure( }) } -pub(super) fn materialize_runtime( +fn materialize_runtime_with_resources( profile: NativeRuntimeProfile, install_dir: &Path, extensions: &[Extension], icu_data: Option<&Path>, icu_data_tree_sha256: Option<&str>, + selected_resources: Option<&Path>, ) -> Result { - let extension_artifact_dirs = locate_native_extension_artifact_dirs(); + let mut extension_artifact_dirs = Vec::new(); + if let Some(resources) = selected_resources { + for entry in sorted_read_dir(&resources.join("extension")).unwrap_or_default() { + extension_artifact_dirs.push(entry.path()); + } + } + extension_artifact_dirs.extend(locate_native_extension_artifact_dirs()); + let embedded_modules = if profile.needs_embedded_modules() { Some(locate_native_embedded_modules_dir(install_dir)?) } else { diff --git a/src/sdks/rust/src/liboliphaunt/root/runtime/locate.rs b/src/sdks/rust/src/liboliphaunt/root/runtime/locate.rs index e55f79bb3..da7e7bef4 100644 --- a/src/sdks/rust/src/liboliphaunt/root/runtime/locate.rs +++ b/src/sdks/rust/src/liboliphaunt/root/runtime/locate.rs @@ -144,11 +144,20 @@ pub(super) fn resources_dir_candidates() -> Vec { if let Some(path) = std::env::var_os(ENV_RESOURCES_DIR) { candidates.push(PathBuf::from(path)); } + if let Some(path) = crate::build_resources::embedded_base_resources_dir() { + candidates.push(path); + } candidates } -pub(super) fn locate_native_icu_data() -> Result> { - for resources_dir in resources_dir_candidates() { +pub(super) fn locate_native_icu_data( + selected_resources: Option<&Path>, +) -> Result> { + for resources_dir in selected_resources + .map(Path::to_path_buf) + .into_iter() + .chain(resources_dir_candidates()) + { let directory = resources_dir.join("icu-data/oliphaunt-icu/share/icu"); if !icu_data_dir_is_valid(&directory) { continue; @@ -180,11 +189,12 @@ pub(super) fn locate_native_cluster_seed( ) -> Result> { let payload = resources_dir.join("native-runtime/liboliphaunt-native"); let carrier_target = read_native_runtime_carrier(&payload.join("manifest.properties"))?; - let relative = match profile { - super::super::NativeCatalogProfile::Standard => "cluster-seed", - super::super::NativeCatalogProfile::Icu => "cluster-seed-icu", + let directory = match profile { + super::super::NativeCatalogProfile::Standard => payload.join("cluster-seed"), + super::super::NativeCatalogProfile::Icu => resources_dir + .join("icu-data/oliphaunt-icu/native-seeds") + .join(&carrier_target), }; - let directory = payload.join(relative); if !directory.is_dir() { return Ok(None); } diff --git a/src/sdks/rust/tools/package-source.mjs b/src/sdks/rust/tools/package-source.mjs index c87768edd..7f8129158 100644 --- a/src/sdks/rust/tools/package-source.mjs +++ b/src/sdks/rust/tools/package-source.mjs @@ -28,7 +28,7 @@ export function stageRustPackageSource(outputDir) { recursive: true, filter: (source) => path.basename(source) !== "target", }); - rmSync(path.join(destination, "crates/oliphaunt-build"), { recursive: true, force: true }); + rmSync(path.join(destination, "crates"), { recursive: true, force: true }); cpSync(path.join(ROOT, "src/shared/fixtures"), path.join(destination, "testdata"), { recursive: true, filter: (source) => path.basename(source) !== "moon.yml", @@ -43,7 +43,9 @@ export function stageRustPackageSource(outputDir) { const manifest = path.join(destination, "Cargo.toml"); let text = readFileSync(manifest, "utf8") .replace("repository.workspace = true", 'repository = "https://github.com/f0rr0/oliphaunt"') - .replace("homepage.workspace = true", 'homepage = "https://oliphaunt.dev"'); + .replace("homepage.workspace = true", 'homepage = "https://oliphaunt.dev"') + .replace(', path = "crates/oliphaunt-build"', "") + .replace(', path = "crates/oliphaunt-resources"', ""); if (!text.includes("[workspace]")) text = `${text.trimEnd()}\n\n[workspace]\n`; writeFileSync(manifest, text, "utf8"); return manifest; diff --git a/src/sdks/swift/README.md b/src/sdks/swift/README.md index 75ac0718e..b72f6aa7e 100644 --- a/src/sdks/swift/README.md +++ b/src/sdks/swift/README.md @@ -2,162 +2,55 @@ ## Install -Add Oliphaunt from Swift Package Manager: +Add the base package and any external extension packages through SwiftPM. The +base `Oliphaunt` product includes PostgreSQL, supported contrib extensions, and +the standard cluster seed. External extensions and ICU are separate packages. -```text -dependencies: [ - .package(url: "https://github.com/f0rr0/oliphaunt.git", exact: "0.7.0") -] +```swift +// Package.swift +let package = Package( + name: "Example", + platforms: [.iOS(.v17), .macOS(.v14)], + dependencies: [ + .package(url: "https://github.com/f0rr0/oliphaunt.git", exact: "0.7.0"), + .package(url: "https://github.com/f0rr0/oliphaunt-extension-vector.git", from: "0.2.0") + ], + targets: [.executableTarget(name: "Example", dependencies: [ + .product(name: "Oliphaunt", package: "oliphaunt"), + .product(name: "OliphauntExtensionVector", package: "oliphaunt-extension-vector") + ])] +) ``` -Then add the `Oliphaunt` product to the iOS or macOS app target. Release tags -are source tags for the Swift API and are paired with compatible -`liboliphaunt-native-v` GitHub release assets, for example -`liboliphaunt-native-v0.1.1`. Those assets contain the base Apple XCFramework, -portable runtime resources, and checksum manifest. -CocoaPods trunk is not a release path for Oliphaunt. The SwiftPM release tag -resolves a generated manifest with a checksum-pinned `liboliphaunt` binary -target; the SDK auto-discovers the bundled runtime resources from that framework -for ordinary native-direct opens. -Normal iOS and macOS app consumers do not install Rust, run Cargo, build -PostgreSQL, or copy local Oliphaunt artifacts. SwiftPM resolves the Swift API -and checksum-pinned binary/runtime assets for the selected release. - -Base Apple packages do not include full ICU data. Applications that need -PostgreSQL ICU collations add the `OliphauntICU` SwiftPM product to the same app -target as `Oliphaunt`. The generated release manifest exposes `OliphauntICU` as -a resource-only product containing the canonical ICU data. The target runtime -resources carry the matching platform-qualified cluster seed, and `Oliphaunt` -resolves the pair as one checked closure. Do not add `OliphauntICU` for -applications that do not use ICU collations. - -Optional PostgreSQL extensions are exact-extension artifacts. PostgreSQL 18 -contrib members share the logical `oliphaunt-extension-contrib-pg18` artifact; -its native carrier uses the `liboliphaunt-native` release and version. External -projects such as `vector` keep independent `oliphaunt-extension-vector` -releases. The base -Swift package does not publish hidden extension products or bundle unselected -extension files. Swift and React Native iOS integrations resolve selected SQL -names and their dependency closure from the checksum-bound JSON carrier -manifest. Each exact-extension release also carries a strict -`manifest.properties` metadata index for auditing and non-Swift tooling; it is -not a second remote-asset resolver. - -The SDK-owned SwiftPM integration generator starts from the selection-neutral -carrier embedded in the Swift source tag and composes independently versioned -extension release carriers. Given explicit SQL extension names, it resolves the -mandatory dependency closure, downloads into a content-addressed cache, verifies -byte sizes and SHA-256 checksums, safely extracts only runtime resources, and -emits a standalone consumer-owned local package with checksum-pinned binary -targets, C descriptors that strongly reference built symbols, -dependency-ordered Swift wrappers, and sanitized resource targets. Download one -carrier from every release owner that supplies a selected or mandatory dependency -extension, then pass each carrier explicitly: +Select both the external extension and contrib member explicitly when opening +each database: -```bash -CONTRIB_CARRIER=/path/to/oliphaunt-extension-contrib-pg18-X.Y.Z-swift-extension-carrier.json -POSTGIS_CARRIER=/path/to/oliphaunt-extension-postgis-X.Y.Z-swift-extension-carrier.json -PGTAP_CARRIER=/path/to/oliphaunt-extension-pgtap-X.Y.Z-swift-extension-carrier.json - -node src/sdks/swift/tools/render-extension-products.mjs \ - --extension-carrier "$CONTRIB_CARRIER" \ - --extension-carrier "$POSTGIS_CARRIER" \ - --extension-carrier "$PGTAP_CARRIER" \ - --extensions cube,postgis,pgtap \ - --output-dir /path/to/package/generated/swiftpm/extensions +```swift +import Foundation +import Oliphaunt +import OliphauntExtensionVector + +let directory = URL.applicationSupportDirectory.appending(path: "postgres") +let db = try await OliphauntDatabase.open(configuration: OliphauntConfiguration( + storage: .directory(directory), + extensions: [OliphauntExtensionVector.descriptor, OliphauntExtensions.hstore] +)) +try await db.exec("CREATE EXTENSION IF NOT EXISTS vector; CREATE EXTENSION IF NOT EXISTS hstore;") +try await db.close() ``` -The output path is a create-only completion transaction. The generator builds a -private sibling staging tree, verifies every materialized resource and local -XCFramework against a no-symlink file-and-directory inventory with SHA-256 for -every file, then -claims the still-absent output with an exclusive directory creation. It moves -`Package.swift` and the completion marker only after the package contents, so a -claimed directory without that marker is incomplete. It never replaces or -deletes an existing output, even an empty one or one with an earlier generator -marker; callers that intentionally regenerate must remove their known -destination before invoking it again. A generation failure retains private -staging for explicit caller cleanup, and a process crash during the final -multi-entry publication can retain a claimed incomplete output. The generator -never recursively removes either path. - -The input, staging parent, and output must be in a caller-controlled workspace -that is not concurrently mutated by another process running as the same OS -user. The command rejects symlink outputs, outputs that would contain its -checkout or carrier inputs, and any overlap in either direction with its cache, -base package, local XCFrameworks, or extension-resource roots. - -Every generated SwiftPM release source tag contains a schema-valid, -selection-neutral carrier at -`src/sdks/swift/Carriers/oliphaunt-react-native-ios-carriers.json`. It pins only -the compatible `liboliphaunt-native` base assets; its `carriers` and -`extensions` arrays are empty. The generator uses that checksum-locked, -Git-tree-addressed base carrier by default, so a pure Swift consumer does not -install the React Native npm package. The Swift source tag never snapshots or -changes the version of an optional extension. A local `--carrier -/path/to/...json` base override is available for release validation and advanced -tooling, but it is not how consumers select extensions. - -Each exact-extension artifact publishes a checksum-covered -`--swift-extension-carrier.json` asset on its release -owner's `-v` GitHub tag. This applies whether the -carrier predates or follows the Swift source tag. Compose the asset with the embedded base using the repeatable -`--extension-carrier` option: +The descriptor prepares its package resources and registers native symbols. +Applications do not need a build script, resource generator, or manual registry +call. Installation makes a package available to the application; `extensions` +selects resources for one database, and SQL migrations create the extensions. -```bash -node src/sdks/swift/tools/render-extension-products.mjs \ - --extension-carrier /path/to/oliphaunt-extension-vector-0.2.0-swift-extension-carrier.json \ - --extensions vector \ - --output-dir /path/to/package/generated/swiftpm/vector -``` +For ICU collations, also add the `oliphaunt-icu` Swift package from +`https://github.com/f0rr0/oliphaunt-icu.git` and its `OliphauntICU` product. Import +`OliphauntICU` and pass `icu: OliphauntICU.descriptor` in the configuration. This +package owns the ICU data and matching Apple cluster seeds. Omit the dependency +and configuration option when ICU is unnecessary. -One contrib bundle carrier supplies all 32 contrib rows; only the requested SQL -names and their mandatory dependencies enter the generated package. Multiple -external carrier files may be composed in the same command. Every carrier pins -its owning release, compatible `liboliphaunt-native` release, member-specific -asset checksums, and direct extension dependency releases. A mismatched base, -missing or version-skewed dependency, duplicate SQL row, unused carrier file, -or conflicting native dependency asset stops generation before output. - -Normal use accepts HTTPS assets only. `--offline` requires a complete verified -cache; `--allow-file-urls` exists only for local CI fixtures. Carrier resolution -pins `nativeRuntime.product` and `nativeRuntime.version`; generation requires -every extracted resource manifest to carry the same stable -`liboliphaunt-native` identity. Native carrier rows -include an extension XCFramework plus exact dependency XCFramework roles and -build-derived registration symbols. SQL-only rows such as `pgtap` carry runtime -resources without a fake binary target. - -The generator copies only `files/share/postgresql` from each resource artifact; -native libraries and build archives cannot enter a Swift resource bundle. Each -generated product registers its `Bundle.module` fragment before database open. -Oliphaunt's internal resource loader resolves mandatory dependencies and -atomically composes the extension-free base with exactly those registered fragments into a -deterministic cache entry. It regenerates runtime, static-registry, and size -metadata, rejects conflicting paths, and supports multiple independent native -extensions plus SQL-only extensions. Consumers add the generated product and -call its `register()` method before opening a database that requests that SQL -extension. The application adds the generated directory as a local Swift -package and depends on the generated products it selected. The published base -Oliphaunt package remains extension-free; exact-extension releases remain -checksum-covered GitHub carrier assets rather than pretending -to be separately published Swift packages. `extension-products.json` records -the complete frozen selection. Missing dependency rows, resources, symbols, -base products, or carrier assets fail generation. - -React Native iOS uses this Swift SDK through the npm package and its config -plugin. It does not carry a second native database runtime. - -## Compatibility - -| SDK | Native core | Apple distribution | -| --- | --- | --- | -| `Oliphaunt` `0.6.1` | `liboliphaunt` `0.1.1` | SwiftPM source tag plus checksum-covered GitHub release assets | - -Exact extensions are selected by PostgreSQL SQL extension name and released as -separate exact-extension artifacts. Selecting `vector` must only fetch/link -`vector` artifacts and mandatory manifest dependencies; unselected extension -XCFrameworks and runtime files must not enter the app bundle. +React Native iOS uses this Swift SDK through its npm package and config plugin. ## Quickstart @@ -165,9 +58,7 @@ XCFrameworks and runtime files must not enter the app bundle. let db = try await OliphauntDatabase.open( configuration: OliphauntConfiguration( storage: .directory(applicationDatabaseURL), - startupGUCs: [ - OliphauntStartupGUC("shared_buffers", "32MB") - ], + startupGUCs: ["shared_buffers": "32MB"], username: "postgres", database: "postgres" ) @@ -333,87 +224,23 @@ The native-direct env-backed test opens temporary storage, executes `SELECT 1` through PostgreSQL protocol bytes, cancels an active `pg_sleep`, creates a same-version physical backup through the C ABI, restores it into a new destination, and -closes the runtime. Exact extensions are accepted when the app links their -generated SwiftPM products and calls each product's `register()` method before -opening the database. Extension names are validated before loading native code. - -For iOS and app-bundled macOS builds, generated products package resources using -this layout; the SDK discovers them automatically: - -```text -oliphaunt/ - manifest.properties - runtime/ - manifest.properties - files/ - cluster-seed/ - manifest.properties - files/ - PG_VERSION - global/pg_control - cluster-seed-icu/ - manifest.properties - files/ - PG_VERSION - global/pg_control -``` - -The macOS XCFramework slice embeds the `macos-arm64` closure. Both iOS slices -embed the `ios-datum64` closure. SwiftPM links exactly one platform slice, so an -application receives one target-qualified closure. React Native uses the separate -`liboliphaunt--runtime-resources-ios-datum64.tar.gz` carrier when it -composes its app-owned resource bundle; there is no generic or multi-target -runtime-resource archive. - -The root receipt binds the closure to one seed target and the two sibling seed -paths. Both seed manifests use the exact native cluster-seed contract; extension -selection and static-registry metadata belong only to the runtime manifest. -`runtime/manifest.properties` must include -`schema=oliphaunt-runtime-resources-v1`, -`layout=postgres-runtime-files-v1`, `mode=native-direct`, -`cacheKey=`, and -two distinct extension domains. `selectedExtensions` is the complete, -dependency-closed set of packaged SQL identities, including module-only -products such as `auto_explain`. `extensions` is exactly the subset whose -catalog rows support `CREATE EXTENSION`; it must be a subset of -`selectedExtensions`. Runtime availability and requested-extension checks use -`selectedExtensions`, while control/install-SQL checks apply only to requested -members of `extensions`. Producers must always write both fields. The SDK -rejects a missing `selectedExtensions`; an explicitly empty value means that -no extensions were selected. -The runtime manifest uses an exact field set. When -`mobileStaticRegistryState=complete`, `mobileStaticRegistrySource` is exactly -`static-registry/oliphaunt_static_registry.c` for packaged generator output or -`swiftpm-linked-products` for SwiftPM product composition; it is empty for all -other registry states. - -The Swift SDK rejects unknown package layouts, materializes runtime files into -Application Support using the cache key, and hydrates a new standard or ICU -PGDATA root from its matching target-qualified cluster seed. iOS-family targets -reject selected extensions while the runtime static-registry state is -`pending`. -Apple mobile platforms require either a packaged cluster seed or existing -storage whose `pgdata` child contains `PG_VERSION`; they do not rely on executing `initdb` from app storage. -When a selected extension contains native modules, the Swift package must -link those modules with the generated static-registry source. Complete Rust -runtime-resource generator output includes -`static-registry/oliphaunt_static_registry.c`; the Swift C bridge discovers -`liboliphaunt_selected_static_extensions` and registers the returned rows -through `oliphaunt_register_static_extensions` before the first database open. -The manifest state is a release gate, not a loader substitute. -For release builds with exact prebuilt mobile archives, use -`render-extension-products.mjs` and its checksum-bound carrier inputs described -above. It is the single resolver for contrib bundles and independently versioned -external extensions: it selects the SQL dependency closure, runtime resources, -primary XCFrameworks, and identity-qualified dependency XCFrameworks before it -writes the consumer-owned SwiftPM package. The release `.properties` files are -strict metadata indexes for auditing and non-Swift tooling; the runtime SDK does -not maintain a second, weaker remote-asset resolver. Optional extension and -dependency XCFrameworks therefore never enter the generated package unless the -app selected their exact PostgreSQL extension name. -The generated registry source strongly references selected extension magic and -SQL symbols. If an app selects `vector` but omits the matching prebuilt -`liboliphaunt_extension_vector.xcframework`, the build should fail rather than -shipping an app that fails later at `CREATE EXTENSION vector`. -The generated resource root also includes `package-size.tsv` for release and -bundle-size auditing. +closes the runtime. The package tests also check explicit extension selection, +resource composition, cluster seed compatibility, and native registration. + +## Runtime resources + +SwiftPM selects the Apple XCFramework slice for the target. The base framework +contains the runtime and standard seed. Selected extension descriptors register +resource fragments; the SDK resolves required dependencies and composes the +selected resources into a cache. ICU data and its matching target seed come +from the explicitly selected ICU package. The SDK checks the seed target and +ICU data identity before hydrating a new database directory. + +On iOS, database initialization uses a packaged seed. Existing database storage +can be reopened without running a separate `initdb` executable. + +Release tooling generates base contrib targets and independent external package +trees from checksum-verified carrier manifests. `render-extension-products.mjs` +is also available for advanced assembly of prebuilt artifacts; it is not an +application setup step. Native libraries remain binary targets, and SQL/data +resources remain SwiftPM resources. diff --git a/src/sdks/swift/Sources/COliphaunt/bridge.c b/src/sdks/swift/Sources/COliphaunt/bridge.c index ee463af1b..da22d56f5 100644 --- a/src/sdks/swift/Sources/COliphaunt/bridge.c +++ b/src/sdks/swift/Sources/COliphaunt/bridge.c @@ -224,6 +224,15 @@ static int load_symbols(const char *library_path, OliphauntSymbols *symbols) { return 0; } +int32_t oliphaunt_swift_register_static_extensions(const OliphauntStaticExtension *extensions, size_t count) { + OliphauntSymbols symbols; + if (load_symbols(NULL, &symbols) != 0) return -1; + int32_t status = symbols.register_static_extensions(extensions, count); + if (status != 0) set_global_native_error(&symbols, NULL, "static extension registration failed"); + unload_symbols(&symbols); + return status; +} + static int register_selected_static_extensions(OliphauntSymbols *symbols) { dlerror(); OliphauntSelectedStaticExtensionsFn selected = NULL; diff --git a/src/sdks/swift/Sources/COliphaunt/include/COliphaunt.h b/src/sdks/swift/Sources/COliphaunt/include/COliphaunt.h index 6ee0734b8..651cd1a29 100644 --- a/src/sdks/swift/Sources/COliphaunt/include/COliphaunt.h +++ b/src/sdks/swift/Sources/COliphaunt/include/COliphaunt.h @@ -5,6 +5,8 @@ typedef struct OliphauntSession OliphauntSession; +int32_t oliphaunt_swift_register_static_extensions(const OliphauntStaticExtension *extensions, size_t count); + int32_t oliphaunt_swift_open( const char *library_path, const OliphauntConfig *config, diff --git a/src/sdks/swift/Sources/COliphaunt/include/oliphaunt.h b/src/sdks/swift/Sources/COliphaunt/include/oliphaunt.h index d96facff7..7f55b34bd 100644 --- a/src/sdks/swift/Sources/COliphaunt/include/oliphaunt.h +++ b/src/sdks/swift/Sources/COliphaunt/include/oliphaunt.h @@ -222,9 +222,11 @@ OLIPHAUNT_API int32_t oliphaunt_close(OliphauntHandle *handle); * Registers statically linked PostgreSQL extension modules for the embedded * backend's normal LOAD path. * - * Call this before oliphaunt_init in processes that link extension code directly - * into the application or SDK library. The registry is process-wide and becomes - * immutable once backend startup begins. Each extension name is the module stem + * Call this before loading a module linked into the application or SDK library. + * The process-wide registry permits additions after backend startup; repeating an + * identical entry is harmless, while replacing an existing module is rejected. + * Registration copies descriptors and retains them for the process lifetime. + * Each extension name is the module stem * used by SQL, for example AS 'vector', and each symbol row exposes the C * symbols PostgreSQL would otherwise resolve with dlsym(). */ diff --git a/src/sdks/swift/Sources/Oliphaunt/Oliphaunt.swift b/src/sdks/swift/Sources/Oliphaunt/Oliphaunt.swift index 55e894268..245559f77 100644 --- a/src/sdks/swift/Sources/Oliphaunt/Oliphaunt.swift +++ b/src/sdks/swift/Sources/Oliphaunt/Oliphaunt.swift @@ -5,16 +5,6 @@ enum OliphauntNativeCatalogProfile: String, Sendable { case icu } -public struct OliphauntStartupGUC: Equatable, Sendable { - public var name: String - public var value: String - - public init(_ name: String, _ value: String) { - self.name = name - self.value = value - } -} - public enum OliphauntDatabaseStorage: Equatable, Sendable { case temporaryDirectory case directory(URL) @@ -22,23 +12,26 @@ public enum OliphauntDatabaseStorage: Equatable, Sendable { public struct OliphauntConfiguration: Equatable, Sendable { public var storage: OliphauntDatabaseStorage - public var startupGUCs: [OliphauntStartupGUC] + public var startupGUCs: [String: String] public var username: String? public var database: String? - public var extensions: [String] + public var extensions: [OliphauntExtension] + public var icu: OliphauntIcuData? public init( storage: OliphauntDatabaseStorage = .temporaryDirectory, - startupGUCs: [OliphauntStartupGUC] = [], + startupGUCs: [String: String] = [:], username: String? = nil, database: String? = nil, - extensions: [String] = [] + extensions: [OliphauntExtension] = [], + icu: OliphauntIcuData? = nil ) { self.storage = storage self.startupGUCs = startupGUCs self.username = username self.database = database self.extensions = extensions + self.icu = icu } } @@ -63,18 +56,18 @@ func requireOliphauntFreshRootRole(_ username: String) throws { } } -func validateOliphauntStartupGUCs(_ gucs: [OliphauntStartupGUC]) throws { - for guc in gucs { - let name = guc.name.trimmingCharacters(in: .whitespacesAndNewlines) +func validateOliphauntStartupGUCs(_ gucs: [String: String]) throws { + for (key, value) in gucs { + let name = key.trimmingCharacters(in: .whitespacesAndNewlines) if name.isEmpty { throw OliphauntError.engine("PostgreSQL startup GUC name must not be empty") } - if name.utf8.contains(0) || guc.value.utf8.contains(0) { + if name.utf8.contains(0) || value.utf8.contains(0) { throw OliphauntError.engine("PostgreSQL startup GUC must not contain NUL bytes") } if !isPortablePostgresGUCName(name) { throw OliphauntError.engine( - "PostgreSQL startup GUC name '\(guc.name)': each dot-separated component must start " + + "PostgreSQL startup GUC name '\(key)': each dot-separated component must start " + "with an ASCII letter or '_', followed by ASCII letters, digits, '_', or '$'" ) } @@ -372,11 +365,8 @@ public actor OliphauntDatabase { try validateOliphauntStartupIdentity(configuration.username, label: "username") try validateOliphauntStartupIdentity(configuration.database, label: "database") try validateOliphauntStartupGUCs(configuration.startupGUCs) - var normalized = configuration - normalized.extensions = try OliphauntRuntimeResources.normalizedExtensionIds( - configuration.extensions - ) - return OliphauntDatabase(session: try await engine.open(configuration: normalized)) + try configuration.prepareExtensionResources() + return OliphauntDatabase(session: try await engine.open(configuration: configuration)) } public static func restore(destination: URL, bytes: Data) async throws { @@ -1240,6 +1230,7 @@ public struct OliphauntTransaction: Sendable { extension OliphauntConfiguration { func postgresStartupArgs(sharedPreloadLibraries: [String] = []) -> [String] { + let startupGUCs = startupGUCs.sorted { $0.key < $1.key }.map { (name: $0.key, value: $0.value) } let requiredPreloads = Set(sharedPreloadLibraries).sorted() if requiredPreloads.isEmpty { return startupGUCs.flatMap { guc in diff --git a/src/sdks/swift/Sources/Oliphaunt/OliphauntExtensionResources.swift b/src/sdks/swift/Sources/Oliphaunt/OliphauntExtensionResources.swift index 0b0eba2d2..4c00b513e 100644 --- a/src/sdks/swift/Sources/Oliphaunt/OliphauntExtensionResources.swift +++ b/src/sdks/swift/Sources/Oliphaunt/OliphauntExtensionResources.swift @@ -27,6 +27,40 @@ private enum OliphauntPackagedExtensionRegistry { } extension OliphauntRuntimeResources { + func validateSelectedResources(_ configuration: OliphauntConfiguration) throws { + let receiptURL = resourceRoot.appendingPathComponent("sdk-resources.properties") + let receipt = FileManager.default.fileExists(atPath: receiptURL.path) + ? try packagedExtensionProperties(at: receiptURL) : nil + if let receipt, receipt["schema"] != "oliphaunt-sdk-resources-v1" { + throw OliphauntError.engine("unsupported SDK resource receipt") + } + OliphauntPackagedExtensionRegistry.lock.lock() + let registered = OliphauntPackagedExtensionRegistry.resources + OliphauntPackagedExtensionRegistry.lock.unlock() + for descriptor in configuration.extensions { + if let receipt { + let prefix = "extension.\(descriptor.sqlName)" + guard receipt["\(prefix).product"] == descriptor.product, + descriptor.version == nil || receipt["\(prefix).version"] == descriptor.version else { + throw OliphauntError.engine("selected extension '\(descriptor.sqlName)' does not match packaged product/version") + } + } else if let version = descriptor.version { + guard let resource = registered[descriptor.sqlName], resource.product == descriptor.product, resource.version == version else { + throw OliphauntError.engine("selected extension '\(descriptor.sqlName)' has no matching registered package") + } + } + } + if let icu = configuration.icu { + var icuReceipt = receipt + if let directory = icu.resourceDirectory { + icuReceipt = try packagedExtensionProperties(at: directory.appendingPathComponent("sdk-resources.properties")) + } + guard icuReceipt?["icuVersion"] == icu.version else { + throw OliphauntError.engine("selected ICU version does not match packaged resources") + } + } + } + /// Registers a generated SwiftPM exact-extension resource fragment. /// Applications normally call the generated `OliphauntExtension*.register()` /// wrapper rather than invoking this packaging API directly. @@ -584,7 +618,7 @@ private func composedCacheKey( ) } -private func packagedExtensionFingerprint(_ values: [String]) -> String { +func packagedExtensionFingerprint(_ values: [String]) -> String { var hash: UInt64 = 14_695_981_039_346_656_037 for byte in values.joined(separator: "\u{1f}").utf8 { hash ^= UInt64(byte) diff --git a/src/sdks/swift/Sources/Oliphaunt/OliphauntExtensions.swift b/src/sdks/swift/Sources/Oliphaunt/OliphauntExtensions.swift new file mode 100644 index 000000000..f8e8c41b6 --- /dev/null +++ b/src/sdks/swift/Sources/Oliphaunt/OliphauntExtensions.swift @@ -0,0 +1,79 @@ +// This file is generated by src/extensions/tools/check-extension-model.mjs. +// Do not edit by hand. + +public enum OliphauntExtensions { + public static let amcheck = OliphauntExtension(sqlName: "amcheck", product: "oliphaunt-extension-contrib-pg18") + public static let autoExplain = OliphauntExtension(sqlName: "auto_explain", product: "oliphaunt-extension-contrib-pg18") + public static let bloom = OliphauntExtension(sqlName: "bloom", product: "oliphaunt-extension-contrib-pg18") + public static let btreeGin = OliphauntExtension(sqlName: "btree_gin", product: "oliphaunt-extension-contrib-pg18") + public static let btreeGist = OliphauntExtension(sqlName: "btree_gist", product: "oliphaunt-extension-contrib-pg18") + public static let citext = OliphauntExtension(sqlName: "citext", product: "oliphaunt-extension-contrib-pg18") + public static let cube = OliphauntExtension(sqlName: "cube", product: "oliphaunt-extension-contrib-pg18") + public static let dictInt = OliphauntExtension(sqlName: "dict_int", product: "oliphaunt-extension-contrib-pg18") + public static let dictXsyn = OliphauntExtension(sqlName: "dict_xsyn", product: "oliphaunt-extension-contrib-pg18") + public static let earthdistance = OliphauntExtension(sqlName: "earthdistance", product: "oliphaunt-extension-contrib-pg18") + public static let fileFdw = OliphauntExtension(sqlName: "file_fdw", product: "oliphaunt-extension-contrib-pg18") + public static let fuzzystrmatch = OliphauntExtension(sqlName: "fuzzystrmatch", product: "oliphaunt-extension-contrib-pg18") + public static let hstore = OliphauntExtension(sqlName: "hstore", product: "oliphaunt-extension-contrib-pg18") + public static let intarray = OliphauntExtension(sqlName: "intarray", product: "oliphaunt-extension-contrib-pg18") + public static let isn = OliphauntExtension(sqlName: "isn", product: "oliphaunt-extension-contrib-pg18") + public static let lo = OliphauntExtension(sqlName: "lo", product: "oliphaunt-extension-contrib-pg18") + public static let ltree = OliphauntExtension(sqlName: "ltree", product: "oliphaunt-extension-contrib-pg18") + public static let pageinspect = OliphauntExtension(sqlName: "pageinspect", product: "oliphaunt-extension-contrib-pg18") + public static let pgBuffercache = OliphauntExtension(sqlName: "pg_buffercache", product: "oliphaunt-extension-contrib-pg18") + public static let pgFreespacemap = OliphauntExtension(sqlName: "pg_freespacemap", product: "oliphaunt-extension-contrib-pg18") + public static let pgSurgery = OliphauntExtension(sqlName: "pg_surgery", product: "oliphaunt-extension-contrib-pg18") + public static let pgTrgm = OliphauntExtension(sqlName: "pg_trgm", product: "oliphaunt-extension-contrib-pg18") + public static let pgVisibility = OliphauntExtension(sqlName: "pg_visibility", product: "oliphaunt-extension-contrib-pg18") + public static let pgWalinspect = OliphauntExtension(sqlName: "pg_walinspect", product: "oliphaunt-extension-contrib-pg18") + public static let pgcrypto = OliphauntExtension(sqlName: "pgcrypto", product: "oliphaunt-extension-contrib-pg18") + public static let seg = OliphauntExtension(sqlName: "seg", product: "oliphaunt-extension-contrib-pg18") + public static let tablefunc = OliphauntExtension(sqlName: "tablefunc", product: "oliphaunt-extension-contrib-pg18") + public static let tcn = OliphauntExtension(sqlName: "tcn", product: "oliphaunt-extension-contrib-pg18") + public static let tsmSystemRows = OliphauntExtension(sqlName: "tsm_system_rows", product: "oliphaunt-extension-contrib-pg18") + public static let tsmSystemTime = OliphauntExtension(sqlName: "tsm_system_time", product: "oliphaunt-extension-contrib-pg18") + public static let unaccent = OliphauntExtension(sqlName: "unaccent", product: "oliphaunt-extension-contrib-pg18") + public static let uuidOssp = OliphauntExtension(sqlName: "uuid-ossp", product: "oliphaunt-extension-contrib-pg18") +} + +let oliphauntExtensionRuntimeContracts: [String: (dependencies: [String], sharedPreloads: [String], module: String?)] = [ + "amcheck": (dependencies: [], sharedPreloads: [], module: "amcheck"), + "auto_explain": (dependencies: [], sharedPreloads: [], module: "auto_explain"), + "bloom": (dependencies: [], sharedPreloads: [], module: "bloom"), + "btree_gin": (dependencies: [], sharedPreloads: [], module: "btree_gin"), + "btree_gist": (dependencies: [], sharedPreloads: [], module: "btree_gist"), + "citext": (dependencies: [], sharedPreloads: [], module: "citext"), + "cube": (dependencies: [], sharedPreloads: [], module: "cube"), + "dict_int": (dependencies: [], sharedPreloads: [], module: "dict_int"), + "dict_xsyn": (dependencies: [], sharedPreloads: [], module: "dict_xsyn"), + "earthdistance": (dependencies: ["cube"], sharedPreloads: [], module: "earthdistance"), + "file_fdw": (dependencies: [], sharedPreloads: [], module: "file_fdw"), + "fuzzystrmatch": (dependencies: [], sharedPreloads: [], module: "fuzzystrmatch"), + "hstore": (dependencies: [], sharedPreloads: [], module: "hstore"), + "intarray": (dependencies: [], sharedPreloads: [], module: "_int"), + "isn": (dependencies: [], sharedPreloads: [], module: "isn"), + "lo": (dependencies: [], sharedPreloads: [], module: "lo"), + "ltree": (dependencies: [], sharedPreloads: [], module: "ltree"), + "pageinspect": (dependencies: [], sharedPreloads: [], module: "pageinspect"), + "pg_buffercache": (dependencies: [], sharedPreloads: [], module: "pg_buffercache"), + "pg_freespacemap": (dependencies: [], sharedPreloads: [], module: "pg_freespacemap"), + "pg_hashids": (dependencies: [], sharedPreloads: [], module: "pg_hashids"), + "pg_ivm": (dependencies: [], sharedPreloads: [], module: "pg_ivm"), + "pg_surgery": (dependencies: [], sharedPreloads: [], module: "pg_surgery"), + "pg_textsearch": (dependencies: [], sharedPreloads: ["pg_textsearch"], module: "pg_textsearch"), + "pg_trgm": (dependencies: [], sharedPreloads: [], module: "pg_trgm"), + "pg_uuidv7": (dependencies: [], sharedPreloads: [], module: "pg_uuidv7"), + "pg_visibility": (dependencies: [], sharedPreloads: [], module: "pg_visibility"), + "pg_walinspect": (dependencies: [], sharedPreloads: [], module: "pg_walinspect"), + "pgcrypto": (dependencies: [], sharedPreloads: [], module: "pgcrypto"), + "pgtap": (dependencies: [], sharedPreloads: [], module: nil), + "postgis": (dependencies: [], sharedPreloads: [], module: "postgis-3"), + "seg": (dependencies: [], sharedPreloads: [], module: "seg"), + "tablefunc": (dependencies: [], sharedPreloads: [], module: "tablefunc"), + "tcn": (dependencies: [], sharedPreloads: [], module: "tcn"), + "tsm_system_rows": (dependencies: [], sharedPreloads: [], module: "tsm_system_rows"), + "tsm_system_time": (dependencies: [], sharedPreloads: [], module: "tsm_system_time"), + "unaccent": (dependencies: [], sharedPreloads: [], module: "unaccent"), + "uuid-ossp": (dependencies: [], sharedPreloads: [], module: "uuid-ossp"), + "vector": (dependencies: [], sharedPreloads: [], module: "vector"), +] diff --git a/src/sdks/swift/Sources/Oliphaunt/OliphauntNativeDirect.swift b/src/sdks/swift/Sources/Oliphaunt/OliphauntNativeDirect.swift index f5390387d..612e3cf0a 100644 --- a/src/sdks/swift/Sources/Oliphaunt/OliphauntNativeDirect.swift +++ b/src/sdks/swift/Sources/Oliphaunt/OliphauntNativeDirect.swift @@ -54,12 +54,22 @@ struct OliphauntNativeDirectEngine: OliphauntEngine { try validateOliphauntStartupIdentity(configuration.username, label: "username") try validateOliphauntStartupIdentity(configuration.database, label: "database") try validateOliphauntStartupGUCs(configuration.startupGUCs) - _ = try OliphauntRuntimeResources.validateExtensionIds(configuration.extensions) - let packagedRuntimeResources = try runtimeResources ?? OliphauntRuntimeResources.bundled( - containing: configuration.extensions + _ = try OliphauntRuntimeResources.validateExtensionIds(configuration.extensionSqlNames) + var packagedRuntimeResources = try runtimeResources ?? OliphauntRuntimeResources.bundled( + containing: configuration.extensionSqlNames ) + try packagedRuntimeResources?.validateSelectedResources(configuration) + packagedRuntimeResources?.icuSelected = configuration.icu != nil + if let icu = configuration.icu { + if let directory = icu.resourceDirectory { + try validateOliphauntDirectory(directory, label: "ICU resource directory") + packagedRuntimeResources?.icuResourceDirectories = [directory] + } + } else { + packagedRuntimeResources?.icuResourceDirectories = [] + } let resolvedRuntime = try resolveRuntime( - extensions: configuration.extensions, + extensions: configuration.extensionSqlNames, runtimeResources: packagedRuntimeResources ) let username = configuration.username ?? "postgres" diff --git a/src/sdks/swift/Sources/Oliphaunt/OliphauntResources.swift b/src/sdks/swift/Sources/Oliphaunt/OliphauntResources.swift new file mode 100644 index 000000000..8e775ae71 --- /dev/null +++ b/src/sdks/swift/Sources/Oliphaunt/OliphauntResources.swift @@ -0,0 +1,95 @@ +import Foundation + +/// An explicitly selected extension supplied by a package or the SDK's contrib distribution. +public struct OliphauntExtension: Equatable, Sendable { + public let sqlName: String + public let product: String + public let version: String? + private let prepareResources: @Sendable () throws -> Void + + public init( + sqlName: String, + product: String, + version: String? = nil, + prepare: @escaping @Sendable () throws -> Void = {} + ) { + self.sqlName = sqlName + self.product = product + self.version = version + self.prepareResources = prepare + } + + public static func == (lhs: Self, rhs: Self) -> Bool { + lhs.sqlName == rhs.sqlName && lhs.product == rhs.product && lhs.version == rhs.version + } + + func prepare() throws { + _ = try OliphauntRuntimeResources.validateExtensionIds([sqlName]) + guard product.hasPrefix("oliphaunt-extension-"), + product == "oliphaunt-extension-contrib-pg18" || version?.isEmpty == false else { + throw OliphauntError.engine("extension descriptor must identify its release product and version") + } + #if OLIPHAUNT_PACKAGED_CONTRIB + if product == "oliphaunt-extension-contrib-pg18" { try prepareBundledContrib(sqlName) } + #endif + try prepareResources() + } +} + +/// Optional ICU resources; the package owns their version and bundle location. +public struct OliphauntIcuData: Equatable, Sendable { + public let version: String + public let resourceDirectory: URL? + + public init(version: String, resourceDirectory: URL? = nil) { + self.version = version + self.resourceDirectory = resourceDirectory + } +} + +extension OliphauntConfiguration { + var extensionSqlNames: [String] { extensions.map(\.sqlName) } + + func prepareExtensionResources() throws { + var selected: [String: OliphauntExtension] = [:] + for value in extensions { + if let existing = selected[value.sqlName], existing != value { + throw OliphauntError.engine("conflicting extension descriptors for '\(value.sqlName)'") + } + selected[value.sqlName] = value + } + for name in selected.keys.sorted() { + try selected[name]?.prepare() + } + } +} + +func selectedOliphauntExtensions(_ names: [String]) throws -> Set { + var selected = Set() + func visit(_ name: String) throws { + guard selected.insert(name).inserted else { return } + guard let contract = oliphauntExtensionRuntimeContracts[name] else { + throw OliphauntError.engine("unknown extension '\(name)'") + } + for dependency in contract.dependencies { try visit(dependency) } + } + for name in names { try visit(name) } + return selected +} + +func includeSelectedOliphauntRuntimeFile(_ path: String, extensions: Set, icu: Bool) -> Bool { + if !icu && (path == "share/icu" || path.hasPrefix("share/icu/")) { return false } + let file = path.split(separator: "/").last.map(String.init) ?? "" + if path.hasPrefix("share/postgresql/extension/") { + if let owner = oliphauntExtensionRuntimeContracts.keys.first(where: { + file == "\($0).control" || file == "\($0).sql" || file.hasPrefix("\($0)--") + }), !extensions.contains(owner) { return false } + } + if path.hasPrefix("lib/postgresql/") { + let owners = oliphauntExtensionRuntimeContracts.filter { _, contract in + contract.module.map { file == "\($0).so" || file == "\($0).dylib" } ?? false + }.keys + if !owners.isEmpty && !owners.contains(where: { extensions.contains($0) }) { return false } + } + return true +} diff --git a/src/sdks/swift/Sources/Oliphaunt/OliphauntRuntimeResources.swift b/src/sdks/swift/Sources/Oliphaunt/OliphauntRuntimeResources.swift index 108e55e3b..f99eb0744 100644 --- a/src/sdks/swift/Sources/Oliphaunt/OliphauntRuntimeResources.swift +++ b/src/sdks/swift/Sources/Oliphaunt/OliphauntRuntimeResources.swift @@ -85,6 +85,7 @@ struct OliphauntExtensionSizeReport: Equatable, Sendable { var resourceRoot: URL var cacheRoot: URL var icuResourceDirectories: [URL]? + var icuSelected: Bool? = nil init( resourceRoot: URL, @@ -193,23 +194,26 @@ struct OliphauntExtensionSizeReport: Equatable, Sendable { func resolveRuntime( requestedExtensions: [String] = [] ) throws -> ResolvedOliphauntRuntimeResources { - let requested = try Self.validateExtensionIds(requestedExtensions) + let requested = try selectedOliphauntExtensions(Array(Self.validateExtensionIds(requestedExtensions))) let runtime = try assetPackage(kind: .runtime) try require(runtime: runtime, contains: requested) - let integratedIcu = runtime.runtimeFeatures.contains("icu") - let externalIcu = integratedIcu ? nil : try Self.icuDataCarrier( + let integratedIcu = runtime.runtimeFeatures.contains("icu") && icuSelected != false + let externalIcu = integratedIcu || icuSelected == false ? nil : try Self.icuDataCarrier( inResourceDirectories: icuResourceDirectories ?? defaultBundleResourceURLs() ) + if icuSelected == true && !integratedIcu && externalIcu == nil { + throw OliphauntError.engine("selected ICU resources are not packaged in this application") + } let profile: OliphauntNativeCatalogProfile = integratedIcu || externalIcu != nil ? .icu : .standard let seed = try matchingClusterSeed( profile: profile, runtime: runtime, icuDataTreeSha256: externalIcu?.treeSha256 ) - let target = try materialize(runtime, seed: seed, profile: profile, externalIcu: externalIcu) + let target = try materialize(runtime, seed: seed, profile: profile, externalIcu: externalIcu, selectedExtensions: requested) return ResolvedOliphauntRuntimeResources( directory: target, - sharedPreloadLibraries: runtime.sharedPreloadLibraries.sorted(), + sharedPreloadLibraries: Set(requested.flatMap { oliphauntExtensionRuntimeContracts[$0]?.sharedPreloads ?? [] }).sorted(), catalogProfile: profile, owner: self ) @@ -227,11 +231,11 @@ struct OliphauntExtensionSizeReport: Equatable, Sendable { ) } try require(runtime: runtime, contains: requested) - let profile: OliphauntNativeCatalogProfile = runtime.runtimeFeatures.contains("icu") ? .icu : .standard + let profile: OliphauntNativeCatalogProfile = runtime.runtimeFeatures.contains("icu") && icuSelected != false ? .icu : .standard _ = try matchingClusterSeed(profile: profile, runtime: runtime, icuDataTreeSha256: nil) return ResolvedOliphauntRuntimeResources( directory: runtimeDirectory, - sharedPreloadLibraries: runtime.sharedPreloadLibraries.sorted(), + sharedPreloadLibraries: Set(requested.flatMap { oliphauntExtensionRuntimeContracts[$0]?.sharedPreloads ?? [] }).sorted(), catalogProfile: profile, owner: self ) @@ -384,17 +388,20 @@ struct OliphauntExtensionSizeReport: Equatable, Sendable { _ runtime: AssetPackage, seed: AssetPackage, profile: OliphauntNativeCatalogProfile, - externalIcu: OliphauntIcuDataCarrier? + externalIcu: OliphauntIcuDataCarrier?, + selectedExtensions: Set ) throws -> URL { let digest = profile == .icu ? seed.icuDataTreeSha256 : "none" let target = cacheRoot .appendingPathComponent("runtime", isDirectory: true) .appendingPathComponent(runtime.cacheKey, isDirectory: true) + .appendingPathComponent(packagedExtensionFingerprint(selectedExtensions.sorted()), isDirectory: true) .appendingPathComponent(profile.rawValue, isDirectory: true) .appendingPathComponent(seed.cacheKey, isDirectory: true) .appendingPathComponent(digest, isDirectory: true) let identity = [ "runtime=\(runtime.cacheKey)", + "extensions=\(selectedExtensions.sorted().joined(separator: ","))", "target=\(oliphauntSwiftClusterSeedTarget)", "profile=\(profile.rawValue)", "seed=\(seed.cacheKey)", @@ -418,7 +425,9 @@ struct OliphauntExtensionSizeReport: Equatable, Sendable { isDirectory: true ) let result: Result = Result { - try copyTree(from: runtime.filesURL, to: temp) + try copyTree(from: runtime.filesURL, to: temp, include: { + includeSelectedOliphauntRuntimeFile($0, extensions: selectedExtensions, icu: profile == .icu) + }) if let externalIcu { let destination = temp .appendingPathComponent("share", isDirectory: true) @@ -558,7 +567,13 @@ struct OliphauntExtensionSizeReport: Equatable, Sendable { if case .runtime = kind { try validateRuntimeCarrierReceipt() } - let rootURL = kind.root(in: resourceRoot) + var rootURL = kind.root(in: resourceRoot) + if case .clusterSeed(.icu) = kind, + !FileManager.default.fileExists(atPath: rootURL.path), + let carrier = try Self.icuDataCarrier(inResourceDirectories: icuResourceDirectories ?? defaultBundleResourceURLs()) { + rootURL = carrier.dataURL.deletingLastPathComponent().deletingLastPathComponent() + .appendingPathComponent("native-seeds/\(oliphauntSwiftClusterSeedTarget)", isDirectory: true) + } let manifestURL = rootURL.appendingPathComponent("manifest.properties") guard FileManager.default.fileExists(atPath: manifestURL.path) else { return nil @@ -800,7 +815,9 @@ struct OliphauntExtensionSizeReport: Equatable, Sendable { ) } _ = try assetPackage(kind: .clusterSeed(.standard)) - _ = try assetPackage(kind: .clusterSeed(.icu)) + if FileManager.default.fileExists(atPath: AssetPackageKind.clusterSeed(.icu).root(in: resourceRoot).path) { + _ = try assetPackage(kind: .clusterSeed(.icu)) + } } private func readManifest(_ url: URL) throws -> [String: String] { @@ -1368,7 +1385,8 @@ private struct AssetPackage { var icuDataTreeSha256: String } -private func copyTree(from source: URL, to destination: URL) throws { +private func copyTree(from source: URL, to destination: URL, include: (String) -> Bool = { _ in true }, relative: String = "") throws { + guard include(relative) else { return } let values = try source.resourceValues(forKeys: [.isDirectoryKey, .isSymbolicLinkKey]) if values.isSymbolicLink == true { throw OliphauntError.engine("refusing to copy symbolic link in Oliphaunt resources: \(source.path)") @@ -1381,7 +1399,7 @@ private func copyTree(from source: URL, to destination: URL) throws { options: [] ) for child in children { - try copyTree(from: child, to: destination.appendingPathComponent(child.lastPathComponent)) + try copyTree(from: child, to: destination.appendingPathComponent(child.lastPathComponent), include: include, relative: relative.isEmpty ? child.lastPathComponent : "\(relative)/\(child.lastPathComponent)") } } else { try FileManager.default.createDirectory( diff --git a/src/sdks/swift/Sources/Oliphaunt/OliphauntStaticExtensionRegistry.swift b/src/sdks/swift/Sources/Oliphaunt/OliphauntStaticExtensionRegistry.swift new file mode 100644 index 000000000..436e256a7 --- /dev/null +++ b/src/sdks/swift/Sources/Oliphaunt/OliphauntStaticExtensionRegistry.swift @@ -0,0 +1,114 @@ +import COliphaunt +import Foundation + +/// Registers independently packaged static PostgreSQL extensions as one +/// process-wide liboliphaunt registry before a database loads them. +@_spi(ExtensionSupport) public enum OliphauntStaticExtensionRegistry { + private static let lock = NSLock() + nonisolated(unsafe) private static var descriptors: [String: UnsafePointer] = [:] + + /// Adds an exact-extension descriptor and republishes the complete selected + /// set. Existing modules cannot be replaced by a different implementation. + public static func register( + product: String, + sqlName: String, + version: String, + dependencies: [String], + nativeDependencies: [String], + sharedPreloadLibraries: [String], + nativeModuleStem: String?, + resourceRoot: URL, + descriptor: UnsafePointer? + ) throws { + lock.lock() + defer { lock.unlock() } + + switch (nativeModuleStem, descriptor) { + case (nil, nil): + break + case (let expectedStem?, let descriptor?): + guard let descriptorName = descriptor.pointee.name, + String(cString: descriptorName) == expectedStem + else { + throw OliphauntError.engine( + "static-extension descriptor name does not match \(sqlName) native module stem \(expectedStem)" + ) + } + default: + throw OliphauntError.engine( + "SwiftPM exact-extension \(sqlName) must provide both a native module stem and descriptor, or neither" + ) + } + + let insertedResource = try OliphauntRuntimeResources.registerPackagedExtensionResource( + product: product, + version: version, + sqlName: sqlName, + dependencies: dependencies, + nativeDependencies: nativeDependencies, + nativeModuleStem: nativeModuleStem, + sharedPreloadLibraries: sharedPreloadLibraries, + resourceRoot: resourceRoot + ) + guard let descriptor else { + return + } + + let previous = descriptors[sqlName] + do { + if let previous, previous != descriptor { + throw OliphauntError.engine( + "conflicting static-extension descriptors were linked for \(sqlName)" + ) + } + descriptors[sqlName] = descriptor + + let rows = descriptors.keys.sorted().compactMap { descriptors[$0]?.pointee } + let status = rows.withUnsafeBufferPointer { buffer in + oliphaunt_swift_register_static_extensions(buffer.baseAddress, buffer.count) + } + guard status == 0 else { + let nativeMessage = copyNativeLastError() + throw OliphauntError.engine( + "could not register selected static extensions while adding \(sqlName) " + + "into the native registry: \(nativeMessage)" + ) + } + } catch { + if let previous { + descriptors[sqlName] = previous + } else { + descriptors.removeValue(forKey: sqlName) + } + if insertedResource { + OliphauntRuntimeResources.unregisterPackagedExtensionResource( + sqlName: sqlName, + resourceRoot: resourceRoot + ) + } + throw error + } + } + + private static func copyNativeLastError() -> String { + let fallback = "unknown liboliphaunt static-extension registration error" + let required = oliphaunt_swift_copy_last_error(nil, nil, 0) + guard required > 0, required < Int.max else { + return fallback + } + var bytes = [CChar](repeating: 0, count: required + 1) + let currentRequired = bytes.withUnsafeMutableBufferPointer { buffer in + oliphaunt_swift_copy_last_error(nil, buffer.baseAddress, buffer.count) + } + if currentRequired >= bytes.count { + bytes = [CChar](repeating: 0, count: currentRequired + 1) + bytes.withUnsafeMutableBufferPointer { buffer in + _ = oliphaunt_swift_copy_last_error(nil, buffer.baseAddress, buffer.count) + } + } + let message = bytes.withUnsafeBufferPointer { buffer in + String(cString: buffer.baseAddress!) + } + return message.isEmpty ? fallback : message + } +} diff --git a/src/sdks/swift/Sources/OliphauntExtensionSupport/OliphauntExtensionSupport.swift b/src/sdks/swift/Sources/OliphauntExtensionSupport/OliphauntExtensionSupport.swift index bc0f66447..38a905f8d 100644 --- a/src/sdks/swift/Sources/OliphauntExtensionSupport/OliphauntExtensionSupport.swift +++ b/src/sdks/swift/Sources/OliphauntExtensionSupport/OliphauntExtensionSupport.swift @@ -2,14 +2,8 @@ import COliphaunt import Foundation @_spi(ExtensionSupport) import Oliphaunt -/// Registers independently packaged static PostgreSQL extensions as one -/// deterministic liboliphaunt registry before the first database starts. +/// Runtime support used by independently released extension packages. public enum OliphauntExtensionSupport { - private static let lock = NSLock() - nonisolated(unsafe) private static var descriptors: [String: UnsafePointer] = [:] - - /// Adds an exact-extension descriptor and republishes the complete selected - /// set. Calling this after a database has started fails in liboliphaunt. public static func register( product: String, sqlName: String, @@ -21,95 +15,11 @@ public enum OliphauntExtensionSupport { resourceRoot: URL, descriptor: UnsafePointer? ) throws { - lock.lock() - defer { lock.unlock() } - - switch (nativeModuleStem, descriptor) { - case (nil, nil): - break - case (let expectedStem?, let descriptor?): - guard let descriptorName = descriptor.pointee.name, - String(cString: descriptorName) == expectedStem - else { - throw OliphauntError.engine( - "static-extension descriptor name does not match \(sqlName) native module stem \(expectedStem)" - ) - } - default: - throw OliphauntError.engine( - "SwiftPM exact-extension \(sqlName) must provide both a native module stem and descriptor, or neither" - ) - } - - let insertedResource = try OliphauntRuntimeResources.registerPackagedExtensionResource( - product: product, - version: version, - sqlName: sqlName, - dependencies: dependencies, - nativeDependencies: nativeDependencies, - nativeModuleStem: nativeModuleStem, - sharedPreloadLibraries: sharedPreloadLibraries, - resourceRoot: resourceRoot + try OliphauntStaticExtensionRegistry.register( + product: product, sqlName: sqlName, version: version, + dependencies: dependencies, nativeDependencies: nativeDependencies, + sharedPreloadLibraries: sharedPreloadLibraries, nativeModuleStem: nativeModuleStem, + resourceRoot: resourceRoot, descriptor: descriptor ) - guard let descriptor else { - return - } - - let previous = descriptors[sqlName] - do { - if let previous, previous != descriptor { - throw OliphauntError.engine( - "conflicting static-extension descriptors were linked for \(sqlName)" - ) - } - descriptors[sqlName] = descriptor - - let rows = descriptors.keys.sorted().compactMap { descriptors[$0]?.pointee } - let status = rows.withUnsafeBufferPointer { buffer in - oliphaunt_register_static_extensions(buffer.baseAddress, buffer.count) - } - guard status == 0 else { - let nativeMessage = copyNativeLastError() - throw OliphauntError.engine( - "could not register selected static extensions while adding \(sqlName) " + - "before backend startup: \(nativeMessage)" - ) - } - } catch { - if let previous { - descriptors[sqlName] = previous - } else { - descriptors.removeValue(forKey: sqlName) - } - if insertedResource { - OliphauntRuntimeResources.unregisterPackagedExtensionResource( - sqlName: sqlName, - resourceRoot: resourceRoot - ) - } - throw error - } - } - - private static func copyNativeLastError() -> String { - let fallback = "unknown liboliphaunt static-extension registration error" - let required = oliphaunt_copy_last_error(nil, nil, 0) - guard required > 0, required < Int.max else { - return fallback - } - var bytes = [CChar](repeating: 0, count: required + 1) - let currentRequired = bytes.withUnsafeMutableBufferPointer { buffer in - oliphaunt_copy_last_error(nil, buffer.baseAddress, buffer.count) - } - if currentRequired >= bytes.count { - bytes = [CChar](repeating: 0, count: currentRequired + 1) - bytes.withUnsafeMutableBufferPointer { buffer in - _ = oliphaunt_copy_last_error(nil, buffer.baseAddress, buffer.count) - } - } - let message = bytes.withUnsafeBufferPointer { buffer in - String(cString: buffer.baseAddress!) - } - return message.isEmpty ? fallback : message } } diff --git a/src/sdks/swift/Tests/OliphauntTests/ExtensionResourceCompositionTests.swift b/src/sdks/swift/Tests/OliphauntTests/ExtensionResourceCompositionTests.swift index def642f26..bbc98889c 100644 --- a/src/sdks/swift/Tests/OliphauntTests/ExtensionResourceCompositionTests.swift +++ b/src/sdks/swift/Tests/OliphauntTests/ExtensionResourceCompositionTests.swift @@ -676,3 +676,19 @@ private func extensionCompositionProperties(_ url: URL) throws -> [String: Strin } return values } + +@Test +func explicitResourceSelectionKeepsDependenciesAndRejectsConflictingVersions() throws { + let selected = try selectedOliphauntExtensions(["earthdistance", "vector"]) + #expect(selected.contains("cube")) + #expect(includeSelectedOliphauntRuntimeFile("lib/postgresql/vector.so", extensions: selected, icu: false)) + #expect(includeSelectedOliphauntRuntimeFile("share/postgresql/extension/cube--1.5.sql", extensions: selected, icu: false)) + #expect(!includeSelectedOliphauntRuntimeFile("share/postgresql/extension/hstore.control", extensions: selected, icu: false)) + #expect(!includeSelectedOliphauntRuntimeFile("lib/postgresql/hstore.so", extensions: selected, icu: false)) + #expect(!includeSelectedOliphauntRuntimeFile("share/icu/icudt.dat", extensions: selected, icu: false)) + let configuration = OliphauntConfiguration(extensions: [ + OliphauntExtension(sqlName: "vector", product: "oliphaunt-extension-vector", version: "0.8.2"), + OliphauntExtension(sqlName: "vector", product: "oliphaunt-extension-vector", version: "0.8.3"), + ]) + #expect(throws: OliphauntError.self) { try configuration.prepareExtensionResources() } +} diff --git a/src/sdks/swift/Tests/OliphauntTests/OliphauntTests.swift b/src/sdks/swift/Tests/OliphauntTests/OliphauntTests.swift index a3c7c3637..e3fd14e69 100644 --- a/src/sdks/swift/Tests/OliphauntTests/OliphauntTests.swift +++ b/src/sdks/swift/Tests/OliphauntTests/OliphauntTests.swift @@ -1563,7 +1563,7 @@ func configurationForwardsOnlyExplicitPostgresSettings() async throws { let engine = TestEngine(session: session) _ = try await OliphauntDatabase.open( configuration: OliphauntConfiguration( - startupGUCs: [.init("shared_buffers", "16MB")], + startupGUCs: ["shared_buffers": "16MB"], username: "alice", database: "app" ), @@ -1586,26 +1586,26 @@ func freshRootAcceptsOnlyFixedBootstrapRole() throws { @Test func startupGUCNamesUsePortablePostgresGrammar() async throws { try validateOliphauntStartupGUCs([ - .init("_name", ""), - .init("ext.name$1", "on"), + "_name": "", + "ext.name$1": "on", ]) for name in ["1name", ".foo", "a..b", "a.1b", "ext.$name"] { #expect(throws: OliphauntError.self) { - try validateOliphauntStartupGUCs([.init(name, "1")]) + try validateOliphauntStartupGUCs([name: "1"]) } } #expect(throws: OliphauntError.self) { - try validateOliphauntStartupGUCs([.init("good", "bad\0value")]) + try validateOliphauntStartupGUCs(["good": "bad\0value"]) } for name in ["CONFIG_FILE", "data_directory"] { #expect(throws: OliphauntError.self) { - try validateOliphauntStartupGUCs([.init(name, "/tmp/other")]) + try validateOliphauntStartupGUCs([name: "/tmp/other"]) } } let config = OliphauntConfiguration(startupGUCs: [ - .init("work_mem", "16MB"), - .init("SHARED_PRELOAD_LIBRARIES", "auto_explain, pg_textsearch"), + "work_mem": "16MB", + "SHARED_PRELOAD_LIBRARIES": "auto_explain, pg_textsearch", ]) #expect(config.postgresStartupArgs(sharedPreloadLibraries: ["pg_textsearch", "z"]) == [ "-c", "work_mem=16MB", diff --git a/src/sdks/swift/Tests/PublicApiConsumer/Sources/OliphauntPublicApiConsumer/main.swift b/src/sdks/swift/Tests/PublicApiConsumer/Sources/OliphauntPublicApiConsumer/main.swift index 7b1e1fe84..95ec94f99 100644 --- a/src/sdks/swift/Tests/PublicApiConsumer/Sources/OliphauntPublicApiConsumer/main.swift +++ b/src/sdks/swift/Tests/PublicApiConsumer/Sources/OliphauntPublicApiConsumer/main.swift @@ -19,7 +19,7 @@ func compileTransactionFailureAPI( func compileOliphauntPublicAPI(restoreDestination: URL) async throws { let configuration = OliphauntConfiguration( storage: .temporaryDirectory, - startupGUCs: [.init("application_name", "public-api-consumer")], + startupGUCs: ["application_name": "public-api-consumer"], username: "postgres", database: "postgres" ) diff --git a/src/sdks/swift/tools/render-extension-products.mjs b/src/sdks/swift/tools/render-extension-products.mjs index f11f6342b..8c6482aa5 100755 --- a/src/sdks/swift/tools/render-extension-products.mjs +++ b/src/sdks/swift/tools/render-extension-products.mjs @@ -60,7 +60,7 @@ function parseArgs(argv) { else args.offline = true; continue; } - if (!["--carrier", "--extension-carrier", "--extensions", "--cache-dir", "--output-dir", "--base-package-path", "--base-package-url", "--base-package-version"].includes(arg)) { + if (!["--carrier", "--extension-carrier", "--extensions", "--cache-dir", "--output-dir", "--base-package-path", "--base-package-url", "--base-package-version", "--release-product"].includes(arg)) { usage(); fail(`unknown argument ${arg}`); } @@ -77,6 +77,7 @@ function parseArgs(argv) { if (arg === "--base-package-path") args.basePackagePath = path.resolve(value); if (arg === "--base-package-url") args.basePackageUrl = value; if (arg === "--base-package-version") args.basePackageVersion = value; + if (arg === "--release-product") args.releaseProduct = value; } if (!args.outputDir || !args.extensions?.length) { usage(); @@ -755,6 +756,7 @@ function renderSwift(extension, bySqlName) { ` public static let dependencies: [String] = [${extension.dependencies.map(swiftString).join(", ")}]\n\n` + ` public static let nativeDependencies: [String] = [${extension.nativeDependencies.map(({ name }) => swiftString(name)).join(", ")}]\n\n` + ` public static let sharedPreloadLibraries: [String] = [${extension.sharedPreloadLibraries.map(swiftString).join(", ")}]\n\n` + + ` public static let descriptor = OliphauntExtension(sqlName: sqlName, product: product, version: version, prepare: register)\n\n` + ` public static func register() throws {\n` + `${dependencyRegistrations ? `${dependencyRegistrations}\n` : ""}` + ` guard let resourceRoot = Bundle.module.url(forResource: "extension-artifact", withExtension: nil) else {\n` + @@ -843,6 +845,61 @@ function renderTargetDependency(dependency) { return `.product(name: ${swiftString(dependency.product)}, package: ${swiftString(dependency.package)})`; } +/** Generate the runtime-owned contrib distribution inside the base SDK target. */ +export async function writeBundledContrib(selection, outputDir) { + if (selection.extensions.some(extension => extension.product !== "oliphaunt-extension-contrib-pg18")) { + fail("the base Swift SDK can bundle only runtime-owned contrib extensions"); + } + if (selection.nativeDependencies.length > 0) { + fail("bundled contrib must not introduce separately owned native dependency products"); + } + const prefix = "generated/swiftpm/contrib"; + const generated = path.join(outputDir, prefix); + const swiftRoot = path.join(outputDir, "src/sdks/swift/Sources/Oliphaunt"); + const targets = []; + const dependencies = []; + const imports = ["import Foundation", "import COliphaunt"]; + const cases = []; + for (const extension of selection.extensions) { + if (extension.cTarget) { + await copyLocalBinaryArtifact(extension.asset, extension.binaryTarget, generated); + const cRoot = path.join(generated, "Sources", extension.cTarget); + await fs.mkdir(path.join(cRoot, "include"), { recursive: true }); + await fs.writeFile(path.join(cRoot, "include", `${extension.cTarget}.h`), renderHeader(extension)); + await fs.writeFile(path.join(cRoot, "registration.c"), renderC(extension)); + imports.push(`import ${extension.cTarget}`); + dependencies.push(extension.cTarget); + targets.push(...targetIR(extension, selection.bySqlName, true) + .filter(target => target.name !== extension.swiftTarget) + .map(target => ({ ...target, path: `${prefix}/${target.path}`, + ...(target.dependencies ? { dependencies: target.dependencies.map(dependency => typeof dependency === "string" ? dependency : dependency.product) } : {}) }))); + } + await copyResourceArtifact(extension, path.join(swiftRoot, "ContribResources", extension.sqlName)); + cases.push(` case ${swiftString(extension.sqlName)}:\n` + + extension.dependencies.map(name => ` try prepareBundledContrib(${swiftString(name)})\n`).join("") + + ` guard let root = Bundle.module.url(forResource: "extension-artifact", withExtension: nil, subdirectory: ${swiftString(`ContribResources/${extension.sqlName}/Resources`)}) else {\n` + + ` throw OliphauntError.engine("missing bundled contrib resources for ${extension.sqlName}")\n }\n` + + ` try OliphauntStaticExtensionRegistry.register(\n` + + ` product: ${swiftString(extension.product)}, sqlName: ${swiftString(extension.sqlName)}, version: ${swiftString(extension.version)},\n` + + ` dependencies: [${extension.dependencies.map(swiftString).join(", ")}], nativeDependencies: [],\n` + + ` sharedPreloadLibraries: [${extension.sharedPreloadLibraries.map(swiftString).join(", ")}],\n` + + ` nativeModuleStem: ${extension.nativeModuleStem === null ? "nil" : swiftString(extension.nativeModuleStem)},\n` + + ` resourceRoot: root, descriptor: ${extension.cFunction ? `${extension.cFunction}()` : "nil"}\n )`); + } + await fs.writeFile(path.join(swiftRoot, "OliphauntBundledContrib.swift"), + `${imports.join("\n")}\n\nfunc prepareBundledContrib(_ name: String) throws {\n switch name {\n${cases.join("\n")}\n default: throw OliphauntError.engine("unknown bundled contrib extension: \\(name)")\n }\n}\n`); + return { targets, dependencies }; +} + +export function renderSwiftTargets(targets) { + return targets.map(target => { + if (target.kind === "binaryTarget") { + return `.binaryTarget(name: ${swiftString(target.name)}, path: ${swiftString(target.path)})`; + } + return `.target(name: ${swiftString(target.name)}, dependencies: [${target.dependencies.map(renderTargetDependency).join(", ")}], path: ${swiftString(target.path)}, publicHeadersPath: "include")`; + }).join(",\n "); +} + function renderPackage(manifest, basePackagePath) { const products = manifest.products .map( @@ -880,13 +937,12 @@ function renderPackage(manifest, basePackagePath) { .join(",\n"); const baseDependency = basePackagePath ? `.package(name: "oliphaunt", path: ${swiftString(basePackagePath)})` - : `.package(\n url: ${swiftString(manifest.basePackage.url)},\n exact: ${swiftString(manifest.basePackage.version)}\n )`; + : `.package(\n url: ${swiftString(manifest.basePackage.url)},\n ${manifest.consumerOwned ? "exact" : "from"}: ${swiftString(manifest.basePackage.version)}\n )`; return `// swift-tools-version: 6.0\n\n` + `import PackageDescription\n\n` + - `// Generated by ${PREFIX}. Do not edit. This local package belongs to the\n` + - `// consuming application; exact-extension assets remain separately released.\n` + + `// Generated by ${PREFIX}. Do not edit.\n` + `let package = Package(\n` + - ` name: "OliphauntSelectedExtensions",\n` + + ` name: ${swiftString(manifest.consumerOwned ? "OliphauntSelectedExtensions" : manifest.products[0].name)},\n` + ` platforms: [.iOS(.v17), .macOS(.v14)],\n` + ` products: [\n${products}\n ],\n` + ` dependencies: [\n ${baseDependency}\n ],\n` + @@ -923,7 +979,10 @@ async function copyLocalBinaryArtifact(asset, targetName, outputDir) { }); } -async function writeGeneratedTree(selection, outputDir, basePackagePath, localBinaryTargets) { +async function writeGeneratedTree(selection, outputDir, basePackagePath, localBinaryTargets, releaseProduct) { + if (releaseProduct !== undefined && (selection.extensions.length !== 1 || selection.extensions[0].product !== releaseProduct)) { + fail("an independent SwiftPM release package must contain exactly its external extension product"); + } const products = []; const targets = []; const selected = []; @@ -956,7 +1015,9 @@ async function writeGeneratedTree(selection, outputDir, basePackagePath, localBi products.push({ name: extension.swiftTarget, targets: [extension.swiftTarget], type: "library" }); targets.push(...targetIR(extension, selection.bySqlName, localBinaryTargets)); selected.push({ - asset: extension.asset, + asset: releaseProduct === undefined || extension.asset === undefined + ? extension.asset + : { name: extension.asset.name, checksum: extension.asset.checksum }, createsExtension: extension.resources.createsExtension, dependencies: extension.dependencies, nativeDependencies: extension.nativeDependencies, @@ -974,7 +1035,7 @@ async function writeGeneratedTree(selection, outputDir, basePackagePath, localBi } const manifest = { basePackage: selection.basePackage, - consumerOwned: true, + consumerOwned: releaseProduct === undefined, nativeRuntime: selection.nativeRuntime, products, requiredBaseProducts: ["COliphaunt", "Oliphaunt", "OliphauntExtensionSupport"], @@ -1122,6 +1183,7 @@ export async function writeGenerated( basePackagePath, localBinaryTargets, protectedPaths, + releaseProduct, ) { const resolvedOutput = await safeGeneratedOutput(outputDir, protectedPaths); const parent = path.dirname(resolvedOutput); @@ -1138,7 +1200,7 @@ export async function writeGenerated( let outputComplete = false; let operationError; try { - await writeGeneratedTree(selection, staging, basePackagePath, localBinaryTargets); + await writeGeneratedTree(selection, staging, basePackagePath, localBinaryTargets, releaseProduct); const stagingEntries = await validatedStagingEntries(staging); const publishOutput = await safeGeneratedOutput(outputDir, protectedPaths); if (publishOutput !== resolvedOutput) { @@ -1229,6 +1291,7 @@ async function main() { })), ]), ].filter(({ path: protectedPath }) => protectedPath !== undefined), + args.releaseProduct, ); console.log( `${PREFIX}: generated ${selection.extensions.length} selected extension product(s) in ${path.resolve(args.outputDir)}`, diff --git a/src/sdks/swift/tools/swift-carrier-resolver.mjs b/src/sdks/swift/tools/swift-carrier-resolver.mjs index 38af3d069..c6fadc772 100644 --- a/src/sdks/swift/tools/swift-carrier-resolver.mjs +++ b/src/sdks/swift/tools/swift-carrier-resolver.mjs @@ -541,12 +541,13 @@ function validateBase(value, label, allowFileUrls) { const version = stableVersion(row.version, `${label}.version`); if (row.tag !== `${row.product}-v${version}`) fail(`${label}.tag must be ${row.product}-v${version}`); const rows = assets(row.assets, `${label}.assets`, allowFileUrls); - const allowed = new Set(["base-xcframework", "icu-data", "runtime-resources"]); + const allowed = new Set(["base-xcframework", "icu-data", "runtime-resources", "icu-seed"]); const unsupported = [...new Set(rows.filter(({ role }) => !allowed.has(role)).map(({ role }) => role))].sort(compareText); if (unsupported.length) fail(`${label}.assets has unsupported roles: ${unsupported.join(",")}`); const framework = oneRole(rows, "base-xcframework", `${label}.assets`); const runtime = oneRole(rows, "runtime-resources", `${label}.assets`); oneRole(rows, "icu-data", `${label}.assets`); + oneRole(rows, "icu-seed", `${label}.assets`); const expectedRuntimeName = `liboliphaunt-${version}-runtime-resources-ios-datum64.tar.gz`; if (runtime.name !== expectedRuntimeName) { diff --git a/src/sdks/swift/tools/swift-carrier-resolver.test.mjs b/src/sdks/swift/tools/swift-carrier-resolver.test.mjs index 9d01501aa..9aebef11f 100755 --- a/src/sdks/swift/tools/swift-carrier-resolver.test.mjs +++ b/src/sdks/swift/tools/swift-carrier-resolver.test.mjs @@ -13,8 +13,11 @@ import { resolveSwiftCarrierSelection, } from "./swift-carrier-resolver.mjs"; +import { validateSelection, writeBundledContrib, renderSwiftTargets } from "./render-extension-products.mjs"; +import { loadSwiftExtensionInventoryCatalog, validateSwiftExtensionResourceArtifact } from "./extension-resource-inventory.mjs"; + const sdk = path.resolve(import.meta.dirname, ".."); -const root = path.resolve(process.argv[2] ?? path.join(sdk, ".build", "carrier-test")); +const root = path.resolve(process.argv[2] ?? path.join(sdk, "../../../target/swift-carrier-test")); const generator = path.join(import.meta.dirname, "render-extension-products.mjs"); const schema = "oliphaunt-react-native-ios-carrier-v1"; const extensionCarrierSchema = "oliphaunt-swift-extension-carrier-v1"; @@ -265,11 +268,14 @@ async function base() { ); const icu = path.join(archives, "liboliphaunt-0.1.0-icu-data.tar.gz"); run("tar", ["--no-xattrs", "-czf", icu, "-C", path.join(root, "base", "icu"), "."]); + const icuSeed = path.join(archives, "liboliphaunt-0.1.0-icu-seed-ios-datum64.tar.gz"); + await fs.copyFile(icu, icuSeed); return { assets: [ await asset("base-xcframework", framework, "zip", "liboliphaunt.xcframework"), await asset("runtime-resources", runtime, "tar.gz", "oliphaunt"), await asset("icu-data", icu, "tar.gz", "."), + await asset("icu-seed", icuSeed, "tar.gz", "."), ], product: "liboliphaunt-native", tag: "liboliphaunt-native-v0.1.0", @@ -668,6 +674,31 @@ async function main() { ); } + const bundledOutput = path.join(root, "bundled-contrib"); + const contribInput = await resolveSwiftCarrierSelection({ carrierFile: carrier, cacheDir: cache, + allowFileUrls: true, localBinaryTargets: true, basePackageVersion: "0.1.0", extensions: ["earthdistance"] }); + const contrib = validateSelection(contribInput, root, { allowFileUrls: true, localBinaryTargets: true }); + const inventoryCatalog = await loadSwiftExtensionInventoryCatalog(); + for (const extension of contrib.extensions) { + extension.resources = await validateSwiftExtensionResourceArtifact({ extension, canonical: inventoryCatalog.get(extension.sqlName), + nativeRuntime: contrib.nativeRuntime, label: "bundled contrib fixture", allowMobileCarrierArchives: true }); + } + const bundled = await writeBundledContrib(contrib, bundledOutput); + const bundledSource = await fs.readFile(path.join(bundledOutput, "src/sdks/swift/Sources/Oliphaunt/OliphauntBundledContrib.swift"), "utf8"); + assert.match(bundledSource, /try prepareBundledContrib\("cube"\)/u); + assert.doesNotMatch(bundledSource, /postgis|pgtap/u); + assert.equal((await fs.stat(path.join(bundledOutput, "src/sdks/swift/Sources/Oliphaunt/ContribResources/cube/Resources/extension-artifact/manifest.properties"))).isFile(), true); + assert.match(renderSwiftTargets(bundled.targets), /generated\/swiftpm\/contrib\/Artifacts/u); + + const standalone = path.join(root, "standalone-pgtap"); + run(process.execPath, [generator, "--carrier", carrier, "--extension-carrier", pgtapCarrier, + "--extensions", "pgtap", "--release-product", "oliphaunt-extension-pgtap", + "--cache-dir", cache, "--allow-file-urls", "--base-package-version", "0.1.0", "--output-dir", standalone]); + const standaloneManifest = await fs.readFile(path.join(standalone, "Package.swift"), "utf8"); + assert.match(standaloneManifest, /name: "OliphauntExtensionPgtap"/u); + assert.match(standaloneManifest, /from: "0.1.0"/u); + assert.doesNotMatch(standaloneManifest, /OliphauntSelectedExtensions/u); + const pgtapRuntime = manifest.extensions.find(({ sqlName }) => sqlName === "pgtap").assets[0]; const cachedPgtap = path.join(cache, "extracted", pgtapRuntime.sha256); await fs.writeFile(path.join(cachedPgtap, "manifest.properties"), "tampered-cache-entry\n"); diff --git a/src/shared/js-core/package.json b/src/shared/js-core/package.json index b9d051afe..8f57e71d7 100644 --- a/src/shared/js-core/package.json +++ b/src/shared/js-core/package.json @@ -4,16 +4,43 @@ "private": true, "exports": { "./protocol": { - "import": { "types": "./dist/module/protocol.d.ts", "default": "./dist/module/protocol.js" }, - "require": { "types": "./dist/commonjs/protocol.d.ts", "default": "./dist/commonjs/protocol.js" } + "import": { + "types": "./dist/module/protocol.d.ts", + "default": "./dist/module/protocol.js" + }, + "require": { + "types": "./dist/commonjs/protocol.d.ts", + "default": "./dist/commonjs/protocol.js" + } }, "./query": { - "import": { "types": "./dist/module/query.d.ts", "default": "./dist/module/query.js" }, - "require": { "types": "./dist/commonjs/query.d.ts", "default": "./dist/commonjs/query.js" } + "import": { + "types": "./dist/module/query.d.ts", + "default": "./dist/module/query.js" + }, + "require": { + "types": "./dist/commonjs/query.d.ts", + "default": "./dist/commonjs/query.js" + } + }, + "./resources": { + "import": { + "types": "./dist/module/resources.d.ts", + "default": "./dist/module/resources.js" + }, + "require": { + "types": "./dist/commonjs/resources.d.ts", + "default": "./dist/commonjs/resources.js" + } } }, - "files": ["dist/module", "dist/commonjs"], - "scripts": { "typecheck": "tsc --noEmit" }, + "files": [ + "dist/module", + "dist/commonjs" + ], + "scripts": { + "typecheck": "tsc --noEmit" + }, "devDependencies": { "@types/node": "^24.10.1", "typescript": "catalog:" diff --git a/src/shared/js-core/src/resources.ts b/src/shared/js-core/src/resources.ts new file mode 100644 index 000000000..0e7136ad6 --- /dev/null +++ b/src/shared/js-core/src/resources.ts @@ -0,0 +1,105 @@ +/** An explicitly selected native extension, supplied by its package or the SDK. */ +export type NativeExtensionDescriptor = Readonly<{ + schema: 'oliphaunt-native-extension-v1'; + sqlName: string; + product: string; + packageName: string; + /** Contrib follows the SDK runtime version; external packages own their version. */ + version?: string; + /** Node entry points bind resolution to the imported package, including npm aliases. */ + packageJsonUrl?: string; +}>; + +export type NativeIcuDescriptor = Readonly<{ + schema: 'oliphaunt-native-icu-v1'; + packageName: '@oliphaunt/icu'; + version: string; + packageJsonUrl?: string; +}>; + +const versionPattern = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/; + +export function snapshotNativeExtensions( + values: readonly NativeExtensionDescriptor[], +): NativeExtensionDescriptor[] { + if (!Array.isArray(values)) { + throw new TypeError('extensions must be an array of imported extension descriptors'); + } + const selected = new Map(); + for (const value of values) { + if ( + value === null || + typeof value !== 'object' || + value.schema !== 'oliphaunt-native-extension-v1' || + typeof value.sqlName !== 'string' || + !/^[a-z0-9][a-z0-9_-]*$/.test(value.sqlName) || + typeof value.product !== 'string' || + !/^oliphaunt-extension-[a-z0-9-]+$/.test(value.product) || + value.packageName !== `@oliphaunt/${value.product.slice('oliphaunt-'.length)}` + ) { + throw new TypeError('extensions must contain imported native extension descriptors'); + } + if (value.product !== 'oliphaunt-extension-contrib-pg18' || value.version !== undefined) { + requireVersion(value.version); + } + requirePackageUrl(value.packageJsonUrl); + const copy = Object.freeze({ + schema: value.schema, + sqlName: value.sqlName, + product: value.product, + packageName: value.packageName, + ...(value.version === undefined ? {} : { version: value.version }), + ...(value.packageJsonUrl === undefined ? {} : { packageJsonUrl: value.packageJsonUrl }), + }); + const previous = selected.get(copy.sqlName); + if (previous !== undefined && JSON.stringify(previous) !== JSON.stringify(copy)) { + throw new TypeError(`conflicting extension descriptors for '${copy.sqlName}'`); + } + selected.set(copy.sqlName, copy); + } + return [...selected.values()]; +} + +export function snapshotNativeIcu( + value: NativeIcuDescriptor | undefined, +): NativeIcuDescriptor | undefined { + if (value === undefined) return undefined; + if ( + value === null || + typeof value !== 'object' || + value.schema !== 'oliphaunt-native-icu-v1' || + value.packageName !== '@oliphaunt/icu' + ) { + throw new TypeError('icu must be an imported native ICU descriptor'); + } + requireVersion(value.version); + requirePackageUrl(value.packageJsonUrl); + return Object.freeze({ + schema: value.schema, + packageName: value.packageName, + version: value.version, + ...(value.packageJsonUrl === undefined ? {} : { packageJsonUrl: value.packageJsonUrl }), + }); +} + +function requireVersion(value: unknown): void { + if (typeof value !== 'string' || !versionPattern.test(value)) { + throw new TypeError('resource descriptor must declare its package version'); + } +} + +function requirePackageUrl(value: unknown): void { + if (value === undefined) return; + if (typeof value !== 'string' || !value.startsWith('file:') || value.includes('\0')) { + throw new TypeError('resource package location must be a local file URL'); + } + const url = new URL(value); + if ( + url.protocol !== 'file:' || + url.search || + url.hash || + !url.pathname.endsWith('/package.json') + ) { + throw new TypeError('resource package location must identify a package.json file'); + } +} diff --git a/src/shared/js-core/tools/stage-package.mjs b/src/shared/js-core/tools/stage-package.mjs index 5b043e0dd..3be171d5e 100644 --- a/src/shared/js-core/tools/stage-package.mjs +++ b/src/shared/js-core/tools/stage-package.mjs @@ -11,11 +11,15 @@ export const JS_CORE_BUNDLE_FILES = Object.freeze([ "dist/commonjs/protocol.js", "dist/commonjs/query.d.ts", "dist/commonjs/query.js", + "dist/commonjs/resources.d.ts", + "dist/commonjs/resources.js", "dist/module/package.json", "dist/module/protocol.d.ts", "dist/module/protocol.js", "dist/module/query.d.ts", "dist/module/query.js", + "dist/module/resources.d.ts", + "dist/module/resources.js", "package.json", ]); diff --git a/tools/policy/generate-sdk-api-surface.mjs b/tools/policy/generate-sdk-api-surface.mjs index fb40e9c0f..9349cbbea 100755 --- a/tools/policy/generate-sdk-api-surface.mjs +++ b/tools/policy/generate-sdk-api-surface.mjs @@ -1021,6 +1021,7 @@ function rejectRustRootSymbols(symbols, crateName, names) { } function addFeatureSymbol(featureSymbols, featureGate, symbol) { + if (featureGate === '__internal-tools') featureGate = 'tools'; if (!featureGate) { return; } @@ -1141,15 +1142,6 @@ function requireWasixRustFeatureSurface(surface) { } } -function markdownFeatureList(featureSymbols) { - if (featureSymbols.length === 0) { - return '- none\n'; - } - return `${featureSymbols - .map(([feature, symbol]) => `- \`${feature}\`: \`${symbol}\``) - .join('\n')}\n`; -} - function requireExtractorFixture(label, symbols, required, forbidden) { for (const symbol of required) { if (!symbols.includes(symbol)) { @@ -1287,7 +1279,7 @@ function render() { 'Kotlin SDK', kotlinCommon, [ - 'PostgresStartupGuc.name', + 'ExtensionDescriptor.sqlName', 'QueryField.name', 'StatementResult.Command.result', ], @@ -1494,16 +1486,6 @@ function render() { output += `\n### \`${feature}\` feature\n\n`; output += markdownList(wasixRustFeatureSurface.featureSymbols.get(feature) ?? []); } - output += `\n### Individual \`extension-*\` features\n\n`; - output += - `Each leaf feature also enables \`extensions\`; the constant below additionally ` + - `requires the feature shown.\n\n`; - output += markdownFeatureList( - Array.from(wasixRustFeatureSurface.featureSymbols.entries()) - .filter(([feature]) => feature.startsWith('extension-')) - .flatMap(([feature, symbols]) => symbols.map(symbol => [feature, symbol])) - .sort(([leftFeature], [rightFeature]) => leftFeature.localeCompare(rightFeature)), - ); output += `\n## Native C ABI: liboliphaunt\n\n`; output += `### Types\n\n`; output += markdownList(nativeC.types); diff --git a/tools/release/bootstrap-publication-capsule.test.mjs b/tools/release/bootstrap-publication-capsule.test.mjs index b3b06dfba..1e89c79d8 100644 --- a/tools/release/bootstrap-publication-capsule.test.mjs +++ b/tools/release/bootstrap-publication-capsule.test.mjs @@ -116,6 +116,7 @@ function fixture() { const cargo = [ cargoFixture(stage, artifacts, "oliphaunt", versions.get("oliphaunt-rust")), cargoFixture(stage, artifacts, "oliphaunt-build", versions.get("oliphaunt-rust")), + cargoFixture(stage, artifacts, "oliphaunt-resources", versions.get("oliphaunt-rust")), ]; const npm = npmFixture(stage, artifacts, "@oliphaunt/ts", versions.get("oliphaunt-js")); const lock = freezePublicationCandidate(buildPublicationCandidate({ @@ -231,7 +232,7 @@ test("packs every locked publication file deterministically and atomically insta assert.equal(sha256(first), sha256(second), "capsule must be byte-for-byte deterministic"); assert.equal(manifest.approval.releaseRunId, APPROVAL.approvalRunId); assert.equal(manifest.approval.qualificationRunId, APPROVAL.qualificationRunId); - assert.equal(manifest.files.length, 3); + assert.equal(manifest.files.length, 4); const installed = verifyExtractBootstrapCapsule({ transport: first, @@ -340,7 +341,7 @@ test("the real bootstrap command checkpoints a mixed inventory before a bounded const decision = JSON.parse(readFileSync(resultFile, "utf8")); assert.equal(decision.decision, "deferred"); assert.equal(decision.newlyCompletedIds.length, 0); - assert.equal(decision.remainingIds.length, 2); + assert.equal(decision.remainingIds.length, 3); } finally { rmSync(value.root, { recursive: true, force: true }); } }); diff --git a/tools/release/build-cargo-sdk-ci-artifacts.mjs b/tools/release/build-cargo-sdk-ci-artifacts.mjs index a58f293b2..7d3b1ca44 100644 --- a/tools/release/build-cargo-sdk-ci-artifacts.mjs +++ b/tools/release/build-cargo-sdk-ci-artifacts.mjs @@ -23,7 +23,9 @@ export function cargoSdkPackageClosure(product) { return { cratePath: path.join(artifactRoot, `oliphaunt-${version}.crate`), allFeatures: true, - stubDependencyPrefixes: ["liboliphaunt-native-", "oliphaunt-broker-"], + dependencyCrates: ["oliphaunt-build", "oliphaunt-resources"].map(name => + path.join(artifactRoot, `${name}-${version}.crate`)), + stubDependencyPrefixes: ["liboliphaunt-native-", "oliphaunt-broker-", "oliphaunt-extension-contrib-pg18-"], }; } if (product === "oliphaunt-wasix-rust") { @@ -33,7 +35,7 @@ export function cargoSdkPackageClosure(product) { path.join(ROOT, "src/bindings/wasix-rust/crates/oliphaunt-wasix/Cargo.toml"), ], noDefaultFeatures: true, - features: ["extensions", "tools", "icu"], + features: ["extensions", "tools"], }; } throw new Error(`${TOOL}: missing Cargo SDK package-closure configuration: ${product}`); diff --git a/tools/release/build-cargo-sdk-ci-artifacts.test.mjs b/tools/release/build-cargo-sdk-ci-artifacts.test.mjs index 2125ce90b..dfd7955a2 100644 --- a/tools/release/build-cargo-sdk-ci-artifacts.test.mjs +++ b/tools/release/build-cargo-sdk-ci-artifacts.test.mjs @@ -18,8 +18,12 @@ test("selects exact final-crate closure checks only for the two Cargo SDKs", () assert.deepEqual(rust.stubDependencyPrefixes, [ "liboliphaunt-native-", "oliphaunt-broker-", + "oliphaunt-extension-contrib-pg18-", ]); + assert.deepEqual(rust.dependencyCrates, ["oliphaunt-build", "oliphaunt-resources"].map(name => + path.join(ROOT, `target/sdk-artifacts/oliphaunt-rust/${name}-${rustVersion}.crate`))); + const wasixVersion = currentProductVersionSync( "oliphaunt-wasix-rust", "cargo-sdk-wrapper.test", @@ -33,7 +37,7 @@ test("selects exact final-crate closure checks only for the two Cargo SDKs", () ), ); assert.equal(wasix.noDefaultFeatures, true); - assert.deepEqual(wasix.features, ["extensions", "tools", "icu"]); + assert.deepEqual(wasix.features, ["extensions", "tools"]); assert.deepEqual(wasix.pathDependencyManifests, [ path.join(ROOT, "src/bindings/wasix-rust/crates/oliphaunt-wasix/Cargo.toml"), ]); diff --git a/tools/release/build-linux-wasix-napi-baseline.sh b/tools/release/build-linux-wasix-napi-baseline.sh index b289592ef..f2c6093f7 100755 --- a/tools/release/build-linux-wasix-napi-baseline.sh +++ b/tools/release/build-linux-wasix-napi-baseline.sh @@ -78,7 +78,6 @@ workspace_path() { generated_assets="$(workspace_path OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR)" generated_aot="$(workspace_path OLIPHAUNT_WASM_GENERATED_AOT_DIR)" extension_artifacts="$(workspace_path OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT)" -icu_data="$(workspace_path OLIPHAUNT_ICU_DATA_DIR)" build_inputs="$(workspace_path OLIPHAUNT_WASIX_NAPI_BUILD_INPUTS)" cargo_root="${CARGO_HOME:-$HOME/.cargo}" @@ -144,7 +143,6 @@ docker_cargo() { --env "OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR=$generated_assets" \ --env "OLIPHAUNT_WASM_GENERATED_AOT_DIR=$generated_aot" \ --env "OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT=$extension_artifacts" \ - --env "OLIPHAUNT_ICU_DATA_DIR=$icu_data" \ --env "OLIPHAUNT_WASIX_NAPI_BUILD_INPUTS=$build_inputs" \ --env "EXPECTED_BUILDER_GLIBC=$expected_builder_glibc" \ --env "EXPECTED_RUST_RELEASE=$rust_release" \ diff --git a/tools/release/build_maven_artifact_manifest.mjs b/tools/release/build_maven_artifact_manifest.mjs index 9ccad88c1..582520a95 100644 --- a/tools/release/build_maven_artifact_manifest.mjs +++ b/tools/release/build_maven_artifact_manifest.mjs @@ -1,5 +1,6 @@ #!/usr/bin/env bun import fs from "node:fs/promises"; +import { stageNativeIcuArchive } from "./native-icu-seeds.mjs"; import path from "node:path"; import { currentVersion } from "./product-version.mjs"; @@ -221,7 +222,12 @@ async function runtimeRows(assetRoot) { if (artifact === undefined) { fail(`liboliphaunt-native Maven artifact ${coordinate} has no release asset mapping`); } - const file = await requireFile(path.join(assetRoot, artifact.filename), artifactId); + let file = await requireFile(path.join(assetRoot, artifact.filename), artifactId); + if (artifactId === "oliphaunt-icu") { + file = path.join(ROOT, "target/release/maven-descriptor-sources", `oliphaunt-icu-${version}.tar.gz`); + await fs.mkdir(path.dirname(file), { recursive: true }); + await stageNativeIcuArchive(assetRoot, version, file, ["android-datum64"]); + } assertMavenPayloadLegal(file, artifact.licenseProfile); rows.push( tsvRow({ @@ -262,6 +268,29 @@ async function extensionRows(extensionRoot, selectedProducts) { if (runtimeVersion !== currentRuntimeVersion) { fail(`${product} native runtime compatibility ${runtimeVersion} does not match ${runtimeProduct}@${currentRuntimeVersion}`); } + if (product !== "oliphaunt-extension-contrib-pg18") { + const id = product.slice("oliphaunt-extension-".length); + const className = id.split("-").map((word) => word[0].toUpperCase() + word.slice(1)).join(""); + const packageName = `dev.oliphaunt.extensions.${id.replaceAll("-", "")}`; + const file = path.join(ROOT, "target/release/maven-descriptor-sources", product, `${className}.java`); + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, `package ${packageName}; + +/** Native extension resources supplied by ${product}. */ +public final class ${className} { + private ${className}() {} + public static final dev.oliphaunt.ExtensionDescriptor descriptor = + new dev.oliphaunt.ExtensionDescriptor(${JSON.stringify(sqlNames[0])}, ${JSON.stringify(product)}, ${JSON.stringify(version)}); +} +`); + rows.push(tsvRow({ + groupId: "dev.oliphaunt.extensions", artifactId: product, version, file, + name: `Oliphaunt ${sqlNames[0]} extension`, description: "Versioned extension descriptor for Kotlin and Java Android applications.", + runtimeProduct, runtimeVersion, + licenseSpdx: releaseProfilePackageLicense("code-facade").spdx, + licenses: releaseProfileMavenLicenses("code-facade", { product, version }), + })); + } const productRoot = path.join( extensionArtifactProductRoot(product, "native", extensionRoot, PREFIX), "release-assets", diff --git a/tools/release/carrier-license-contract.test.mjs b/tools/release/carrier-license-contract.test.mjs index e9d67dfe6..d238c990f 100644 --- a/tools/release/carrier-license-contract.test.mjs +++ b/tools/release/carrier-license-contract.test.mjs @@ -13,7 +13,7 @@ const ROOT = path.resolve(import.meta.dir, "../.."); const CODE_LICENSE = "MIT"; const NATIVE_RUNTIME_LICENSE = "MIT AND PostgreSQL AND Unicode-3.0"; const NATIVE_TOOLS_LICENSE = "MIT AND PostgreSQL"; -const ICU_CARRIER_LICENSE = "MIT AND Unicode-3.0"; +const ICU_CARRIER_LICENSE = "MIT AND PostgreSQL AND Unicode-3.0"; function readCargoLicense(relative) { const file = path.join(ROOT, relative); @@ -79,7 +79,7 @@ test("native source facades and payload carriers declare their exact role licens ); }); -test("portable ICU carrier declares its exact data-only license closure", () => { +test("optional ICU carrier declares its data and PostgreSQL seed license closure", () => { assertNpmLicenses([ "src/runtimes/liboliphaunt/native/icu-npm/package.json", ], ICU_CARRIER_LICENSE); diff --git a/tools/release/check-cargo-package-test-closure.mjs b/tools/release/check-cargo-package-test-closure.mjs index cb81d3093..df6008e3e 100644 --- a/tools/release/check-cargo-package-test-closure.mjs +++ b/tools/release/check-cargo-package-test-closure.mjs @@ -16,6 +16,7 @@ import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { packagedCargoManifestText } from "./cargo-source-package.mjs"; import { readPortableArchiveEntries } from "../../src/shared/artifact-packaging/portable-archive.mjs"; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); @@ -186,7 +187,7 @@ function pathDependencyPatches(manifests, scratch, packagedManifest) { const packagedVersions = new Set( packagedDependencies .filter(({ name }) => name === dependency.name) - .map(({ version }) => exactVersion(version, dependency.name)), + .map(({ version }) => version?.replace(/^[=^]/u, "")), ); if (packagedVersions.size !== 1 || !packagedVersions.has(local.version)) { throw error( @@ -202,7 +203,11 @@ function pathDependencyPatches(manifests, scratch, packagedManifest) { } sourceDirectories.set(dependency.name, realSource); const staged = path.join(scratch, "path-dependency-sources", dependency.name); - if (!patches.has(dependency.name)) copyCleanDependencySource(directory, staged); + if (!patches.has(dependency.name)) { + copyCleanDependencySource(directory, staged); + const stagedManifest = path.join(staged, "Cargo.toml"); + writeFileSync(stagedManifest, packagedCargoManifestText(readFileSync(stagedManifest, "utf8"))); + } addPatch(patches, dependency.name, staged, resolvedManifest); } } @@ -335,6 +340,7 @@ export function verifyPackagedCargoTestClosure({ cratePath, targetDir = process.env.CARGO_TARGET_DIR ?? path.join(ROOT, "target/cargo-package-test-closure"), pathDependencyManifests = [], + dependencyCrates = [], stubDependencies = [], stubDependencyPrefixes = [], allFeatures = false, @@ -356,6 +362,10 @@ export function verifyPackagedCargoTestClosure({ scratch, extracted.manifest, ); + for (const dependencyCrate of dependencyCrates) { + const dependency = extractCrate(path.resolve(dependencyCrate), scratch); + addPatch(patches, dependency.identity.name, dependency.packageRoot, "packaged dependency"); + } const stubs = createStubPatches({ manifest: extracted.manifest, scratch, diff --git a/tools/release/check-cargo-package-test-closure.test.mjs b/tools/release/check-cargo-package-test-closure.test.mjs index 1c4dfeac1..a3f2c48c4 100644 --- a/tools/release/check-cargo-package-test-closure.test.mjs +++ b/tools/release/check-cargo-package-test-closure.test.mjs @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { + appendFileSync, mkdtempSync, mkdirSync, rmSync, @@ -90,6 +91,20 @@ test("compiles an unpacked crate offline with locked weak-feature carrier stubs" ); }); +test("compiles with a dependency extracted from its release crate", (t) => { + const root = fixture(t, "cargo-closure-packed-dependency"); + const cratePath = closureCrate(root); + const source = path.join(root, "carrier"); + writePackage(source, "carrier"); + appendFileSync(path.join(source, "Cargo.toml"), '\n[features]\nneeded = []\n'); + const dependencyCrate = manualCargoPackageSource(path.join(source, "Cargo.toml"), path.join(root, "dependency-crate"), { + root, rel: String, fail: message => { throw new Error(message); }, + }); + assert.deepEqual(verifyPackagedCargoTestClosure({ + cratePath, dependencyCrates: [dependencyCrate], targetDir: path.join(root, "target"), allFeatures: true, lib: true, + }), { name: "closure-fixture", version: "0.1.0" }); +}); + test("rejects conflicting path-patch sources for the same package identity", (t) => { const root = fixture(t, "cargo-closure-conflict"); const cratePath = closureCrate(root); diff --git a/tools/release/check-liboliphaunt-release-assets.mjs b/tools/release/check-liboliphaunt-release-assets.mjs index 4e7d5a60e..6ad9a0136 100644 --- a/tools/release/check-liboliphaunt-release-assets.mjs +++ b/tools/release/check-liboliphaunt-release-assets.mjs @@ -324,15 +324,15 @@ function validateNativeRuntimeCarrierEntries( "cluster-seed/manifest.properties", "cluster-seed/files/PG_VERSION", "cluster-seed/files/global/pg_control", - "cluster-seed-icu/manifest.properties", - "cluster-seed-icu/files/PG_VERSION", - "cluster-seed-icu/files/global/pg_control", ]) { if (!entries.get(member(required))?.isFile) { fail(`${file} is missing native runtime closure member ${member(required)}`); } } - for (const profile of ["cluster-seed", "cluster-seed-icu"]) { + if ([...entries.keys()].some(name => name.startsWith(member("cluster-seed-icu/")))) { + fail(`${file} base runtime must not bundle the optional ICU seed`); + } + for (const profile of ["cluster-seed"]) { const filesPrefix = member(`${profile}/files/`); const pgVersion = entries.get(`${filesPrefix}PG_VERSION`); const control = entries.get(`${filesPrefix}global/pg_control`); @@ -376,15 +376,7 @@ function validateNativeRuntimeCarrierEntries( "standard", { label: `${file} ${member("cluster-seed/manifest.properties")}`, target }, ); - validateNativeClusterSeedManifest( - Buffer.from(archiveText(entries, file, member("cluster-seed-icu/manifest.properties"))), - "icu", - { - label: `${file} ${member("cluster-seed-icu/manifest.properties")}`, - target, - icuDataTreeSha256, - }, - ); + } catch (error) { fail(error instanceof Error ? error.message : String(error)); } @@ -495,9 +487,6 @@ function validateRuntimeResourceArtifactContents( "oliphaunt/cluster-seed/manifest.properties", "oliphaunt/cluster-seed/files/PG_VERSION", "oliphaunt/cluster-seed/files/global/pg_control", - "oliphaunt/cluster-seed-icu/manifest.properties", - "oliphaunt/cluster-seed-icu/files/PG_VERSION", - "oliphaunt/cluster-seed-icu/files/global/pg_control", ]) { if (!names.has(requiredMember)) { fail(`${file} must contain ${requiredMember}`); @@ -632,6 +621,7 @@ const RELEASE_NOTICE_OPTIONS_BY_KIND = new Map([ ], ["runtime-resources", Object.freeze({ profile: "native-runtime-resources" })], ["icu-data", Object.freeze({ profile: "native-icu-data" })], + ["icu-seed", Object.freeze({ profile: "native-runtime-resources" })], ]); export function assertLiboliphauntArtifactReleaseNotices(file, kind) { @@ -692,6 +682,21 @@ async function validate(assetDir) { const icuDataTreeSha256 = validateIcuDataArtifactContents( path.join(assetDir, `liboliphaunt-${version}-icu-data.tar.gz`), ); + for (const row of allArtifactTargets({ product: PRODUCT, surface: "github-release" }).filter(row => row.kind === "icu-seed")) { + const file = path.join(assetDir, assetName(row, version)); + const entries = readArchiveEntries(file); + validateNativeClusterSeedManifest(Buffer.from(archiveText(entries, file, "manifest.properties")), "icu", { + target: row.target, icuDataTreeSha256, label: file, + }); + for (const required of ["files/PG_VERSION", "files/global/pg_control"]) { + if (!entries.get(required)?.isFile || entries.get(required).size === 0) fail(`${file} is missing ${required}`); + } + if (archiveText(entries, file, "files/PG_VERSION").trim() !== "18") fail(`${file} has wrong PostgreSQL version`); + if (!(entries.get("files/pg_wal") ?? entries.get("files/pg_wal/"))?.isDirectory) fail(`${file} is missing files/pg_wal`); + for (const transient of ["postmaster.pid", "postmaster.opts"]) { + if (entries.has(`files/${transient}`)) fail(`${file} includes transient ${transient}`); + } + } for (const target of ["ios-datum64", "android-datum64"]) { validateRuntimeResourceArtifactContents( path.join(assetDir, `liboliphaunt-${version}-runtime-resources-${target}.tar.gz`), diff --git a/tools/release/check-release-metadata.mjs b/tools/release/check-release-metadata.mjs index b7646700d..75613d276 100755 --- a/tools/release/check-release-metadata.mjs +++ b/tools/release/check-release-metadata.mjs @@ -511,7 +511,7 @@ function validateWasixContract(graph, catalog) { const dependencies = object(sdk.dependencies, "oliphaunt-wasix dependencies"); workspaceDependency(dependencies, RUNTIME_PACKAGE); workspaceDependency(dependencies, TOOLS_PACKAGE, { optional: true }); - workspaceDependency(dependencies, ICU_PACKAGE, { optional: true }); + assert(!(ICU_PACKAGE in dependencies), "WASIX SDK must not depend on optional ICU bytes"); const targetTables = object(sdk.target, "oliphaunt-wasix target dependencies"); for (const [cfg, name] of Object.entries(publicAotCargoDependencies())) { workspaceDependency(object(targetTables[cfg], `oliphaunt-wasix target ${cfg}`).dependencies, name); @@ -519,13 +519,13 @@ function validateWasixContract(graph, catalog) { for (const [cfg, name] of Object.entries(publicToolsAotCargoDependencies())) { workspaceDependency(object(targetTables[cfg], `oliphaunt-wasix target ${cfg}`).dependencies, name, { optional: true }); } - assert(sameStrings(sdk.features?.tools ?? [], publicToolsFeatureDependencies()), "oliphaunt-wasix tools feature must select exactly the split tool carriers"); + assert(sameStrings(sdk.features?.tools ?? [], ["__internal-tools", ...publicToolsFeatureDependencies()]), "oliphaunt-wasix tools feature must select exactly the split tool carriers"); assert(!("bundled" in object(sdk.features, "oliphaunt-wasix features")), "oliphaunt-wasix must not expose an inert bundled feature"); const extensionFeatures = exactExtensionProducts(TOOL) .flatMap((product) => extensionSqlNames(product, TOOL)) .map((sqlName) => `extension-${sqlName.replaceAll("_", "-")}`); const sdkExtensionFeatures = Object.keys(sdk.features).filter((feature) => feature.startsWith("extension-")); - assert(sameStrings(extensionFeatures, sdkExtensionFeatures), "oliphaunt-wasix extension features must exactly match modeled extensions"); + assert(sdkExtensionFeatures.length === 0, "WASIX SDK extensions must use independent packages, not SDK features"); const runtimeFeatures = Object.keys(readToml("src/runtimes/liboliphaunt/wasix/crates/assets/Cargo.toml").features ?? {}); assert(sameStrings(extensionFeatures, runtimeFeatures), "portable WASIX runtime features must exactly match modeled extensions"); const dump = (sdk.bin ?? []).find((entry) => entry.name === "oliphaunt-wasix-dump"); diff --git a/tools/release/check-staged-artifacts.mjs b/tools/release/check-staged-artifacts.mjs index 85ef79a03..c6a8e994c 100644 --- a/tools/release/check-staged-artifacts.mjs +++ b/tools/release/check-staged-artifacts.mjs @@ -633,6 +633,7 @@ function requireRegistryTargetDependency(crate, dependencies, cfg, name, version || Array.isArray(dependency) || typeof dependency !== "object" || dependency.version !== `=${version}` + || dependency.optional === true || ["path", "git", "registry"].some((key) => key in dependency) ) { fail( @@ -649,7 +650,7 @@ async function validateRustSdkCrate(crate) { fail(`${rel(crate)} must declare a Cargo package`); } const packageName = packageConfig.name; - if (!["oliphaunt", "oliphaunt-build"].includes(packageName)) { + if (!["oliphaunt", "oliphaunt-build", "oliphaunt-resources"].includes(packageName)) { fail(`${rel(crate)} contains unexpected oliphaunt-rust package ${JSON.stringify(packageName)}`); } const sdkVersion = await currentProductVersion("oliphaunt-rust", PREFIX); @@ -680,7 +681,7 @@ async function validateRustSdkCrate(crate) { } catch (error) { fail(error instanceof Error ? error.message : String(error)); } - if (packageName === "oliphaunt-build") { + if (packageName !== "oliphaunt") { return packageName; } @@ -702,7 +703,8 @@ async function validateRustSdkCrate(crate) { fail(`${rel(crate)} oliphaunt package must declare target-specific native release dependencies`); } const expectedCfgs = targetIds.map((target) => `cfg(${rustNativeTargetCfg(target)})`); - exactSortedStrings(`${rel(crate)} native target tables`, Object.keys(targetTables), expectedCfgs); + exactSortedStrings(`${rel(crate)} native target tables`, Object.keys(targetTables).filter(cfg => cfg !== "cfg(unix)"), expectedCfgs); + exactSortedStrings(`${rel(crate)} Unix system dependencies`, Object.keys(targetTables["cfg(unix)"]?.dependencies ?? {}), ["libc"]); const nativeVersion = productCompatibilityVersion("oliphaunt-rust", "liboliphaunt-native", PREFIX); const brokerVersion = productCompatibilityVersion("oliphaunt-rust", "oliphaunt-broker", PREFIX); @@ -718,6 +720,7 @@ async function validateRustSdkCrate(crate) { const expectedDependencies = [ `liboliphaunt-native-${target.target}`, `oliphaunt-broker-${target.target}`, + `oliphaunt-extension-contrib-pg18-${target.target}`, ]; exactSortedStrings( `${rel(crate)} target dependencies for ${cfg}`, @@ -726,6 +729,7 @@ async function validateRustSdkCrate(crate) { ); requireRegistryTargetDependency(crate, dependencies, cfg, expectedDependencies[0], nativeVersion); requireRegistryTargetDependency(crate, dependencies, cfg, expectedDependencies[1], brokerVersion); + requireRegistryTargetDependency(crate, dependencies, cfg, expectedDependencies[2], nativeVersion); } const sourceMembers = archiveTarNames(crate).filter((name) => name.endsWith("/src/lib.rs")); @@ -776,7 +780,8 @@ async function validateWasixSdkCrate(crate) { if (dependencies === null || Array.isArray(dependencies) || typeof dependencies !== "object") { fail(`${rel(crate)} must declare Cargo dependencies`); } - for (const name of [WASIX_RUNTIME_PACKAGE, WASIX_TOOLS_PACKAGE, ICU_PACKAGE].sort(compareText)) { + if (manifest.dependencies?.[ICU_PACKAGE]) fail(`${rel(crate)} base SDK must not depend on optional ICU bytes`); + for (const name of [WASIX_RUNTIME_PACKAGE, WASIX_TOOLS_PACKAGE].sort(compareText)) { const dependency = dependencies[name]; if (dependency === null || Array.isArray(dependency) || typeof dependency !== "object" || dependency.version !== `=${runtimeVersion}` || "path" in dependency) { fail(`${rel(crate)} dependency ${name} must use registry version =${runtimeVersion} without a path`); @@ -1288,7 +1293,7 @@ async function checkSdkProduct(product, { require }) { exactSortedStrings( `${product} staged Cargo packages`, packageNames, - ["oliphaunt", "oliphaunt-build"], + ["oliphaunt", "oliphaunt-build", "oliphaunt-resources"], ); const version = await currentProductVersion("oliphaunt-rust", PREFIX); requireCrateMatchesCargoListing( diff --git a/tools/release/check-staged-artifacts.test.mjs b/tools/release/check-staged-artifacts.test.mjs index 73819118f..c67f0f1e7 100644 --- a/tools/release/check-staged-artifacts.test.mjs +++ b/tools/release/check-staged-artifacts.test.mjs @@ -261,6 +261,7 @@ function selectionNeutralCarrier(version = "1.2.3") { ["base-xcframework", `liboliphaunt-${version}-apple-spm-xcframework.zip`, "zip", "liboliphaunt.xcframework", "a"], ["runtime-resources", `liboliphaunt-${version}-runtime-resources-ios-datum64.tar.gz`, "tar.gz", "oliphaunt", "b"], ["icu-data", `liboliphaunt-${version}-icu-data.tar.gz`, "tar.gz", ".", "c"], + ["icu-seed", `liboliphaunt-${version}-icu-seed-ios-datum64.tar.gz`, "tar.gz", ".", "d"], ].map(([role, name, format, member, digit], index) => ({ bytes: index + 1, format, diff --git a/tools/release/check_artifact_targets.mjs b/tools/release/check_artifact_targets.mjs index 5502438c8..340b28657 100644 --- a/tools/release/check_artifact_targets.mjs +++ b/tools/release/check_artifact_targets.mjs @@ -354,6 +354,12 @@ export function expectedArtifactTargetContract() { surfaces: [GITHUB, "maven", "react-native-android", "react-native-ios", "rust-native-direct", "swiftpm", "typescript-native-direct"], npm: "@oliphaunt/icu", }), + targetRow({ product: "liboliphaunt-native", id: "icu-seed-android-datum64", kind: "icu-seed", target: "android-datum64", asset: "liboliphaunt-{version}-icu-seed-android-datum64.tar.gz", surfaces: [GITHUB] }), + targetRow({ product: "liboliphaunt-native", id: "icu-seed-ios-datum64", kind: "icu-seed", target: "ios-datum64", asset: "liboliphaunt-{version}-icu-seed-ios-datum64.tar.gz", surfaces: [GITHUB] }), + targetRow({ product: "liboliphaunt-native", id: "icu-seed-linux-arm64-gnu", kind: "icu-seed", target: "linux-arm64-gnu", asset: "liboliphaunt-{version}-icu-seed-linux-arm64-gnu.tar.gz", surfaces: [GITHUB] }), + targetRow({ product: "liboliphaunt-native", id: "icu-seed-linux-x64-gnu", kind: "icu-seed", target: "linux-x64-gnu", asset: "liboliphaunt-{version}-icu-seed-linux-x64-gnu.tar.gz", surfaces: [GITHUB] }), + targetRow({ product: "liboliphaunt-native", id: "icu-seed-macos-arm64", kind: "icu-seed", target: "macos-arm64", asset: "liboliphaunt-{version}-icu-seed-macos-arm64.tar.gz", surfaces: [GITHUB] }), + targetRow({ product: "liboliphaunt-native", id: "icu-seed-windows-x64-msvc", kind: "icu-seed", target: "windows-x64-msvc", asset: "liboliphaunt-{version}-icu-seed-windows-x64-msvc.tar.gz", surfaces: [GITHUB] }), portableRow("liboliphaunt-native", "checksums", "checksums", "liboliphaunt-{version}-release-assets.sha256"), portableRow("liboliphaunt-wasix", "runtime-portable", "wasix-runtime", "liboliphaunt-wasix-{version}-runtime-portable.tar.zst"), portableRow("liboliphaunt-wasix", "icu-data", "icu-data", "liboliphaunt-wasix-{version}-icu-data.tar.zst"), diff --git a/tools/release/example-cargo-policy.mjs b/tools/release/example-cargo-policy.mjs index 75fce23ac..c3941b434 100644 --- a/tools/release/example-cargo-policy.mjs +++ b/tools/release/example-cargo-policy.mjs @@ -56,18 +56,7 @@ export const EXAMPLE_CARGO_POLICIES = Object.freeze([ crateDir: "examples/tauri/src-tauri", ignoredLock: "examples/tauri/src-tauri/Cargo.lock", wasixToolchain: false, - dependencyBindings: [ - dependencyBinding("oliphaunt-build", ["build-dependencies"]), - dependencyBinding("oliphaunt", ["dependencies"]), - dependencyBinding("liboliphaunt-native-linux-x64-gnu", ["target", LINUX_X64_GNU_TARGET, "dependencies"]), - dependencyBinding("oliphaunt-broker-linux-x64-gnu", ["target", LINUX_X64_GNU_TARGET, "dependencies"]), - dependencyBinding("oliphaunt-extension-contrib-pg18-linux-x64-gnu", ["target", LINUX_X64_GNU_TARGET, "dependencies"]), - ], - runtime: { - product: "liboliphaunt-native", - productParts: Object.freeze(["package", "metadata", "oliphaunt", "runtime"]), - versionParts: Object.freeze(["package", "metadata", "oliphaunt", "runtime-version"]), - }, + dependencyBindings: [dependencyBinding("oliphaunt", ["dependencies"])], requiredPackages: Object.freeze([ "oliphaunt", "oliphaunt-build", diff --git a/tools/release/example-cargo-policy.test.mjs b/tools/release/example-cargo-policy.test.mjs index 93ec79a77..037324ff5 100644 --- a/tools/release/example-cargo-policy.test.mjs +++ b/tools/release/example-cargo-policy.test.mjs @@ -55,19 +55,14 @@ describe("ephemeral example Cargo policy", () => { } }); - test("rejects stale runtime metadata, duplicate dependencies, and dependency scope drift", () => { + test("rejects duplicate dependencies and dependency scope drift", () => { const policy = EXAMPLE_CARGO_POLICIES.find(({ id }) => id === "native-tauri"); const bindings = exampleCargoReleaseVersionBindings().filter(({ policyId }) => policyId === policy.id); - const runtimeBinding = bindings.find(({ kind }) => kind === "runtime"); const manifest = Bun.TOML.parse(readFileSync(path.join(ROOT, policy.crateDir, "Cargo.toml"), "utf8")); - manifest.package.metadata.oliphaunt["runtime-version"] = "9.9.9"; manifest["dev-dependencies"] = { oliphaunt: bindings.find(({ name }) => name === "oliphaunt").expected }; delete manifest.dependencies.oliphaunt; const failures = validateExampleManifestPolicy(policy, manifest, bindings); - expect(failures).toContain( - `${policy.crateDir}/Cargo.toml runtime-version uses "9.9.9"; expected ${runtimeBinding.expected}`, - ); expect(failures).toContain( `${policy.crateDir}/Cargo.toml oliphaunt must remain at TOML path dependencies.oliphaunt`, ); diff --git a/tools/release/extension-registry-packages.mjs b/tools/release/extension-registry-packages.mjs index 4ddfca60e..14be276c8 100644 --- a/tools/release/extension-registry-packages.mjs +++ b/tools/release/extension-registry-packages.mjs @@ -1,4 +1,5 @@ import { + AOT_TARGET_TRIPLES, expectedExtensionAotTargets, wasixExtensionAotPackageName, wasixExtensionPackageName, @@ -46,6 +47,13 @@ export function extensionNpmWasixPackageForProduct(product) { return `${extensionNpmPackageForProduct(product)}-wasix`; } +export function extensionNpmWasixAotTargets(product, aotTargets = expectedExtensionAotTargets()) { + if (product === "oliphaunt-extension-contrib-pg18") return []; + return Object.entries(AOT_TARGET_TRIPLES) + .filter(([, triple]) => aotTargets.includes(triple)) + .map(([target]) => target).sort(compareText); +} + export function extensionNpmTargetPackageForProduct(product, target) { return `${extensionNpmPackageForProduct(product)}-${target}`; } @@ -121,9 +129,11 @@ export function extensionWasixCargoPackageNames( } export function extensionMavenPackageNames(product, androidTargets) { - return stringTargetList(androidTargets, "extension Android Maven targets") - .map((target) => `dev.oliphaunt.extensions:${product}-${target}`) - .sort(compareText); + const targets = stringTargetList(androidTargets, "extension Android Maven targets"); + return [ + ...(targets.length > 0 && product !== "oliphaunt-extension-contrib-pg18" ? [`dev.oliphaunt.extensions:${product}`] : []), + ...targets.map((target) => `dev.oliphaunt.extensions:${product}-${target}`), + ].sort(compareText); } export function extensionRegistryPackageEntries({ @@ -184,7 +194,12 @@ export function extensionWasixRegistryPackageEntries({ ...extensionWasixCargoPackageNames(product, { includeAot, aotTargets }) .map((name) => ({ kind: "crates", name })), ...(includeNpm - ? [{ kind: "npm", name: extensionNpmWasixPackageForProduct(product) }] + ? [ + { kind: "npm", name: extensionNpmWasixPackageForProduct(product) }, + ...(includeAot ? extensionNpmWasixAotTargets(product, aotTargets).map(target => ({ + kind: "npm", name: `${extensionNpmWasixPackageForProduct(product)}-${target}`, + })) : []), + ] : []), ]; } diff --git a/tools/release/extension-wasix-npm-packages.test.mjs b/tools/release/extension-wasix-npm-packages.test.mjs index bc618216a..d95616447 100644 --- a/tools/release/extension-wasix-npm-packages.test.mjs +++ b/tools/release/extension-wasix-npm-packages.test.mjs @@ -17,6 +17,7 @@ import { createDeterministicTar } from "./cargo-source-package.mjs"; import { stageExtensionNpmPackagesForTargets, stageExtensionWasixNpmPackages, + writeWasixExtensionAotNpmPackage, } from "./package-extension-release-carriers.mjs"; import { extensionNpmPackageForProduct, @@ -24,6 +25,7 @@ import { extensionRegistryPackageEntries, } from "./extension-registry-packages.mjs"; import { canonicalGzipSync } from "../../src/shared/artifact-packaging/portable-archive.mjs"; +import { canonicalWasixAotMetadata } from "./wasix-aot-manifest.mjs"; import { currentProductVersionSync, extensionReleaseVersion, @@ -47,6 +49,35 @@ function sha256Bytes(bytes) { return createHash("sha256").update(bytes).digest("hex"); } +test("packages exact host AOT bytes separately and rejects corrupted code before packing", () => { + const root = temporaryRoot("oliphaunt-extension-aot-"); + const sourceDir = path.join(root, "input"); + const output = path.join(root, "package"); + mkdirSync(sourceDir); + const raw = Buffer.from("trusted-build-fixture"); + const bytes = zstdCompressSync(raw); + writeFileSync(path.join(sourceDir, "vector.bin.zst"), bytes); + const canonical = canonicalWasixAotMetadata(); + const manifest = { + "format-version": 1, "source-lane": canonical.sourceLane, engine: canonical.engine, + "wasmer-version": canonical.wasmerVersion, "wasmer-wasix-version": canonical.wasmerWasixVersion, + "target-triple": "x86_64-unknown-linux-gnu", "source-fingerprint": "fixture", "postgres-version": "18.3", + artifacts: [{ name: "extension:vector", path: "vector.bin.zst", compressed: true, + sha256: sha256Bytes(bytes), "raw-sha256": sha256Bytes(raw), "raw-size": raw.length, + "module-sha256": "a".repeat(64) }], + }; + writeFileSync(path.join(sourceDir, "manifest.json"), JSON.stringify(manifest)); + const options = { product: "oliphaunt-extension-vector", version: "0.8.2", runtimeVersion: "0.2.0", + sqlName: "vector", target: "linux-x64-gnu", sourceDir }; + writeWasixExtensionAotNpmPackage(output, options); + const metadata = JSON.parse(readFileSync(path.join(output, "package.json"))); + expect(metadata.name).toBe("@oliphaunt/extension-vector-wasix-linux-x64-gnu"); + expect(metadata.oliphaunt.manifestSha256).toBe(sha256Bytes(readFileSync(path.join(output, "aot-manifest.json")))); + expect(readFileSync(path.join(output, "vector.bin.zst"))).toEqual(bytes); + writeFileSync(path.join(sourceDir, "vector.bin.zst"), "corrupt"); + expect(() => writeWasixExtensionAotNpmPackage(output, options)).toThrow("SHA-256"); +}); + function deterministicTar(stage, archiveRoot) { return createDeterministicTar(stage, archiveRoot, { fail(message) { @@ -379,8 +410,8 @@ test("contrib subpath imports carry their exact transitive dependency closure", const cube = inspectDescriptorWithNode(path.join(unpacked, "descriptors", "cube.js")).descriptor; expect(cube.carriers.map(({ sqlName }) => sqlName)).toEqual(["cube"]); const packageJson = JSON.parse(readFileSync(path.join(unpacked, "package.json"), "utf8")); - expect(packageJson.exports["."]).toBeUndefined(); - expect(Object.keys(packageJson.exports).sort()).toEqual(["./cube", "./earthdistance", "./package.json"]); + expect(packageJson.exports["."].import).toBe("./index.js"); + expect(Object.keys(packageJson.exports).sort()).toEqual([".", "./cube", "./earthdistance", "./package.json"]); expect(packageJson.oliphaunt.memberExports).toEqual({ cube: "./cube", earthdistance: "./earthdistance", diff --git a/tools/release/ios-carrier-manifest.mjs b/tools/release/ios-carrier-manifest.mjs index 1849bdc68..8c1adb47d 100644 --- a/tools/release/ios-carrier-manifest.mjs +++ b/tools/release/ios-carrier-manifest.mjs @@ -71,6 +71,7 @@ const BASE_LEGAL_PROFILES = Object.freeze([ memberPrefix: "", profile: "native-icu-data", }), + Object.freeze({ assetRole: "icu-seed", memberPrefix: "", profile: "native-runtime-resources" }), ]); function error(message) { @@ -495,6 +496,7 @@ function baseCarrier({ baseAssetDir, repository, localUrls, verifyMembers, archi name: `liboliphaunt-${version}-icu-data.tar.gz`, member: ".", }, + { role: "icu-seed", name: `liboliphaunt-${version}-icu-seed-ios-datum64.tar.gz`, member: "." }, ]; const assets = rows.map((row) => asset({ ...row, @@ -537,7 +539,7 @@ function frozenBaseCarrier(file) { if (base?.product !== product || base.version !== version || base.tag !== tag || !Array.isArray(base.assets)) { throw error(`${file} does not freeze the current ${product} base carrier`); } - const expectedRoles = ["base-xcframework", "runtime-resources", "icu-data"]; + const expectedRoles = ["base-xcframework", "runtime-resources", "icu-data", "icu-seed"]; if (JSON.stringify(base.assets.map(({ role }) => role)) !== JSON.stringify(expectedRoles)) { throw error(`${file} base carrier roles must be exactly ${expectedRoles.join(", ")}`); } diff --git a/tools/release/ios-carrier-manifest.test.mjs b/tools/release/ios-carrier-manifest.test.mjs index 7bdea6632..76b0a2e06 100644 --- a/tools/release/ios-carrier-manifest.test.mjs +++ b/tools/release/ios-carrier-manifest.test.mjs @@ -176,6 +176,7 @@ test("produces exact local and GitHub carrier envelopes without consulting trunc insideMember: false, profile: "native-runtime-resources", }); + archive(base, `liboliphaunt-${version}-icu-seed-ios-datum64.tar.gz`, "files", "tar.gz", { insideMember: false, profile: "native-runtime-resources" }); archive(base, `liboliphaunt-${version}-icu-data.tar.gz`, "share/icu", "tar.gz", { insideMember: false, profile: "native-icu-data", @@ -227,12 +228,13 @@ test("produces exact local and GitHub carrier envelopes without consulting trunc extensionManifests: [postgis, pgtap], localUrls: true, })); - assert.deepEqual(local.base.assets.map(({ role }) => role), ["base-xcframework", "runtime-resources", "icu-data"]); - assert.deepEqual(local.legal.base.map(({ assetRole }) => assetRole), ["base-xcframework", "runtime-resources", "icu-data"]); + assert.deepEqual(local.base.assets.map(({ role }) => role), ["base-xcframework", "runtime-resources", "icu-data", "icu-seed"]); + assert.deepEqual(local.legal.base.map(({ assetRole }) => assetRole), ["base-xcframework", "runtime-resources", "icu-data", "icu-seed"]); assert.deepEqual(local.legal.base.map(({ spdx }) => spdx), [ "MIT AND PostgreSQL AND Unicode-3.0", "MIT AND PostgreSQL", - "MIT AND Unicode-3.0", + "MIT AND PostgreSQL AND Unicode-3.0", + "MIT AND PostgreSQL", ]); assert.deepEqual(local.extensions.map(({ sqlName }) => sqlName), ["pgtap", "postgis"]); assert.deepEqual(local.legal.extensions.map(({ sqlName }) => sqlName), ["pgtap", "postgis"]); diff --git a/tools/release/maven-artifact-manifest-publication-lock.test.mjs b/tools/release/maven-artifact-manifest-publication-lock.test.mjs index 2639a80f7..25bc93329 100644 --- a/tools/release/maven-artifact-manifest-publication-lock.test.mjs +++ b/tools/release/maven-artifact-manifest-publication-lock.test.mjs @@ -235,7 +235,7 @@ test("the Maven manifest builder validates singleton upstream licenses in the ru "--extension-artifact-root", root, ]); - expect(readFileSync(manifest, "utf8").trimEnd().split("\n")).toHaveLength(2); + expect(readFileSync(manifest, "utf8").trimEnd().split("\n")).toHaveLength(3); for (const [label, options, pattern] of [ ["missing files namespace", { upstreamRoot: null }, /packed upstream license members differ/u], diff --git a/tools/release/maven-artifact-staging.mjs b/tools/release/maven-artifact-staging.mjs index b5df3b5c0..8e1d49bf1 100644 --- a/tools/release/maven-artifact-staging.mjs +++ b/tools/release/maven-artifact-staging.mjs @@ -13,6 +13,8 @@ import { writeFileSync, } from "node:fs"; import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { currentProductVersionSync } from "./release-artifact-targets.mjs"; import { createDeterministicZip } from "../../src/shared/artifact-packaging/archive-directory.mjs"; import { @@ -145,7 +147,7 @@ export function parseMavenArtifactManifest(file) { if (coordinates.has(coordinate)) throw error(`${label} repeats Maven coordinate ${coordinate}`); coordinates.add(coordinate); requiredText(rawArtifact, `${label} artifact path`); - if (!rawArtifact.endsWith(".tar.gz")) throw error(`${label} artifact must be a .tar.gz payload`); + if (!rawArtifact.endsWith(".tar.gz") && !rawArtifact.endsWith(".java")) throw error(`${label} artifact must be a .tar.gz payload or descriptor .java source`); const artifact = path.isAbsolute(rawArtifact) ? rawArtifact : path.resolve(ROOT, rawArtifact); requireArtifact(artifact, `${label} artifact ${relative(artifact)}`); requiredText(name, `${label} name`); @@ -174,6 +176,14 @@ export function parseMavenArtifactManifest(file) { }); } +function isIcuDescriptor(row) { + return row.groupId === "dev.oliphaunt.runtime" && row.artifactId === "oliphaunt-icu"; +} + +function hasDescriptor(row) { + return row.artifact.endsWith(".java") || isIcuDescriptor(row); +} + export function renderMavenArtifactPom(row) { const licenses = row.licenses.map((license) => ` ${xml(license.name)} @@ -190,7 +200,7 @@ export function renderMavenArtifactPom(row) { ${xml(row.groupId)} ${xml(row.artifactId)} ${xml(row.version)} - tar.gz + ${hasDescriptor(row) ? "jar" : "tar.gz"} ${xml(row.name)} ${xml(row.description)} https://github.com/f0rr0/oliphaunt @@ -210,7 +220,16 @@ ${licenses} scm:git:ssh://git@github.com:f0rr0/oliphaunt.git https://github.com/f0rr0/oliphaunt - ${runtimeProperties} +${hasDescriptor(row) ? ` + + dev.oliphaunt + oliphaunt-android + ${xml(currentProductVersionSync("oliphaunt-kotlin", TOOL))} + aar + compile + + +` : ""} ${runtimeProperties} ${xml(row.licenseSpdx)}     @@ -230,13 +249,15 @@ function exactFiles(directory, expected, label) { return names; } -async function writeCompanionJar(stageRoot, row, classifier) { +async function writeCompanionJar(stageRoot, row, classifier, descriptorSource) { const coordinate = `${row.groupId}:${row.artifactId}:${row.version}`; const root = path.join(stageRoot, `${classifier}-stage`); mkdirSync(path.join(root, "META-INF"), { recursive: true }); stageReleaseNotices(path.join(root, "META-INF"), { profile: "source-sdk" }); writeFileSync(path.join(root, "META-INF/MANIFEST.MF"), MANIFEST, { mode: 0o644 }); - if (classifier === "sources") { + if (classifier === "sources" && descriptorSource !== undefined) { + copyFileSync(descriptorSource, path.join(root, path.basename(descriptorSource))); + } else if (classifier === "sources") { writeFileSync( path.join(root, "README.md"), `# ${coordinate}\n\nThis binary carrier has no source API. See https://github.com/f0rr0/oliphaunt.\n`, @@ -245,18 +266,45 @@ async function writeCompanionJar(stageRoot, row, classifier) { } else { writeFileSync( path.join(root, "index.html"), - `${xml(coordinate)}

    This binary carrier has no Java API.

    \n`, + `${xml(coordinate)}

    ${descriptorSource === undefined ? "This binary carrier has no Java API." : "Versioned resource descriptor. See the sources archive."}

    \n`, { mode: 0o644 }, ); } return createDeterministicZip(root); } -/** - * Materialize the immutable, unsigned Maven Central input closure without - * Gradle, Java, registry access, credentials, or dependency resolution. - */ -export async function stageMavenArtifactManifest(manifest, outputRoot) { +let descriptorSdkJar; +function localDescriptorSdkJar() { + if (descriptorSdkJar !== undefined) return descriptorSdkJar; + const buildRoot = path.join(ROOT, "target/release/maven-descriptor-sdk"); + const result = spawnSync("./gradlew", [":oliphaunt:jvmJar", "--configuration-cache"], { + cwd: path.join(ROOT, "src/sdks/kotlin"), + env: { ...process.env, OLIPHAUNT_GRADLE_BUILD_ROOT: buildRoot }, + encoding: "utf8", maxBuffer: 4 * 1024 * 1024, + }); + if (result.status !== 0) throw error(`building descriptor SDK types failed: ${result.error ?? result.stderr ?? result.stdout}`); + const directory = path.join(buildRoot, "oliphaunt/libs"); + const jars = readdirSync(directory).filter(name => /^oliphaunt-jvm-[0-9].*\.jar$/u.test(name) && !/-sources|-javadoc/u.test(name)); + if (jars.length !== 1) throw error("descriptor SDK build must produce exactly one JVM jar"); + descriptorSdkJar = path.join(directory, jars[0]); + return descriptorSdkJar; +} + +async function writeDescriptorJar(root, source, sdkJar) { + const classes = path.join(root, "classes"); + mkdirSync(classes, { recursive: true }); + const result = spawnSync("javac", ["--release", "17", "-proc:none", "-classpath", sdkJar, "-d", classes, source], { + encoding: "utf8", maxBuffer: 1024 * 1024, + }); + if (result.status !== 0) throw error(`compiling ${source} failed: ${result.error ?? result.stderr}`); + stageReleaseNotices(path.join(classes, "META-INF"), { profile: "source-sdk" }); + writeFileSync(path.join(classes, "META-INF/MANIFEST.MF"), MANIFEST); + return createDeterministicZip(classes); +} + +/** Stage unsigned Maven files. Descriptor jars compile against the local SDK; + * binary-only carriers need no Gradle or Java invocation. */ +export async function stageMavenArtifactManifest(manifest, outputRoot, { sdkJar } = {}) { const rows = parseMavenArtifactManifest(path.resolve(manifest)); const destination = path.resolve(outputRoot); const stage = createSiblingStage(destination, "maven-artifacts"); @@ -266,18 +314,42 @@ export async function stageMavenArtifactManifest(manifest, outputRoot) { const directory = path.join(stage, ...row.groupId.split("."), row.artifactId, row.version); const prefix = `${row.artifactId}-${row.version}`; mkdirSync(directory, { recursive: true }); - const primary = path.join(directory, `${prefix}.tar.gz`); + const descriptor = hasDescriptor(row); + const primary = path.join(directory, `${prefix}.${descriptor ? "jar" : "tar.gz"}`); const pom = path.join(directory, `${prefix}.pom`); const sources = path.join(directory, `${prefix}-sources.jar`); const javadoc = path.join(directory, `${prefix}-javadoc.jar`); - copyFileSync(row.artifact, primary); + const companionRoot = path.join(stage, ".companion-stage", row.artifactId, row.version); + mkdirSync(companionRoot, { recursive: true }); + let descriptorSource; + const extraFiles = []; + if (descriptor) { + descriptorSource = row.artifact; + if (isIcuDescriptor(row)) { + descriptorSource = path.join(companionRoot, "ICU.java"); + writeFileSync(descriptorSource, `package dev.oliphaunt.icu; +/** Optional ICU data supplied by this package. */ +public final class ICU { + private ICU() {} + public static final dev.oliphaunt.IcuData data = new dev.oliphaunt.IcuData(${JSON.stringify(row.version)}); +} +`); + const payloadName = `${prefix}.tar.gz`; + copyFileSync(row.artifact, path.join(directory, payloadName)); + chmodSync(path.join(directory, payloadName), 0o644); + extraFiles.push(payloadName); + } + writeFileSync(primary, await writeDescriptorJar(companionRoot, descriptorSource, sdkJar ?? localDescriptorSdkJar())); + } else { + copyFileSync(row.artifact, primary); + } chmodSync(primary, 0o644); writeFileSync(pom, renderMavenArtifactPom(row), { mode: 0o644 }); - const companionRoot = path.join(stage, ".companion-stage", row.artifactId, row.version); - writeFileSync(sources, await writeCompanionJar(companionRoot, row, "sources"), { mode: 0o644 }); - writeFileSync(javadoc, await writeCompanionJar(companionRoot, row, "javadoc"), { mode: 0o644 }); + writeFileSync(sources, await writeCompanionJar(companionRoot, row, "sources", descriptorSource), { mode: 0o644 }); + writeFileSync(javadoc, await writeCompanionJar(companionRoot, row, "javadoc", descriptorSource), { mode: 0o644 }); rmSync(companionRoot, { recursive: true, force: true }); const files = exactFiles(directory, [ + ...extraFiles, path.basename(javadoc), path.basename(pom), path.basename(sources), diff --git a/tools/release/maven-artifact-staging.test.mjs b/tools/release/maven-artifact-staging.test.mjs index bc12ffe9c..7c1d61248 100644 --- a/tools/release/maven-artifact-staging.test.mjs +++ b/tools/release/maven-artifact-staging.test.mjs @@ -227,3 +227,35 @@ test("the release preflight freezes and bundles the exact locally staged Maven b } expect(lstatSync(result.bundle).size).toBeGreaterThan(0); }); + + +test("publishes importable external and ICU descriptors with their SDK dependency", async () => { + const value = fixture(); + const fields = readFileSync(value.manifest, "utf8").trimEnd().split("\t"); + const source = path.join(value.root, "Vector.java"); + writeFileSync(source, `package dev.oliphaunt.extensions.vector; +public final class Vector { + public static final dev.oliphaunt.ExtensionDescriptor descriptor = + new dev.oliphaunt.ExtensionDescriptor("vector", "oliphaunt-extension-vector", "1.2.3"); +} +`); + const vector = fields.with(1, "oliphaunt-extension-vector").with(3, source); + const icu = fields.with(0, "dev.oliphaunt.runtime").with(1, "oliphaunt-icu"); + writeFileSync(value.manifest, `${vector.join("\t")}\n${icu.join("\t")}\n`); + const staged = await stageMavenArtifactManifest(value.manifest, value.output); + for (const [index, classFile] of [ + "dev/oliphaunt/extensions/vector/Vector.class", "dev/oliphaunt/icu/ICU.class", + ].entries()) { + const row = staged[index]; + expect(row.packaging).toBe("jar"); + const prefix = `${row.artifactId}-${row.version}`; + const jar = path.join(row.directory, `${prefix}.jar`); + expect(readPortableArchiveEntries(jar).has(classFile)).toBe(true); + const pom = readFileSync(path.join(row.directory, `${prefix}.pom`), "utf8"); + expect(pom).toContain("oliphaunt-android"); + const sourceEntries = readPortableArchiveEntries(path.join(row.directory, `${prefix}-sources.jar`)); + expect([...sourceEntries.keys()].some(name => name.endsWith(".java"))).toBe(true); + } + expect(readFileSync(path.join(staged[1].directory, "oliphaunt-icu-1.2.3.tar.gz"), "utf8")) + .toBe("exact runtime carrier\n"); +}, 120_000); diff --git a/tools/release/native-icu-seeds.mjs b/tools/release/native-icu-seeds.mjs new file mode 100644 index 000000000..8f0ac98ea --- /dev/null +++ b/tools/release/native-icu-seeds.mjs @@ -0,0 +1,78 @@ +import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { createDeterministicTar } from "../../src/shared/artifact-packaging/archive-directory.mjs"; +import { canonicalGzipSync, readPortableArchiveEntries } from "../../src/shared/artifact-packaging/portable-archive.mjs"; +import { stageReleaseNotices } from "./release-notices.mjs"; +import { NATIVE_CLUSTER_SEED_TARGETS, validateNativeClusterSeedDirectory } from "./native-cluster-seed-contract.mjs"; + +export function nativeIcuSeedAsset(version, target) { + if (!NATIVE_CLUSTER_SEED_TARGETS.includes(target)) throw new Error(`unsupported ICU seed target ${target}`); + return `liboliphaunt-${version}-icu-seed-${target}.tar.gz`; +} + +export async function splitNativeIcuSeed(root, icuData, target, output) { + const seed = path.join(root, "cluster-seed-icu"); + validateNativeClusterSeedDirectory(seed, "icu", { target, icuData }); + stageReleaseNotices(seed, { profile: "native-runtime-resources" }); + const bytes = canonicalGzipSync(await createDeterministicTar(seed)); + mkdirSync(path.dirname(output), { recursive: true }); + writeFileSync(`${output}.partial`, bytes); + renameSync(`${output}.partial`, output); + rmSync(seed, { recursive: true }); + const report = path.join(root, "package-size.tsv"); + if (existsSync(report)) { + const rows = readFileSync(report, "utf8").split("\n"); + const removed = Number(rows.find(row => row.startsWith("package\tcluster-seed-icu\t"))?.split("\t")[4] ?? 0); + writeFileSync(report, rows.map(row => { + const fields = row.split("\t"); + if (fields[0] === "package" && fields[1] === "cluster-seed-icu") fields[4] = "0"; + if (fields[0] === "package" && fields[1] === "total") fields[4] = String(Number(fields[4]) - removed); + return fields.join("\t"); + }).join("\n")); + } +} + +export function stageNativeIcuSeeds(assetDir, version, destination, icuData, targets = NATIVE_CLUSTER_SEED_TARGETS) { + for (const target of targets) { + const archive = path.join(assetDir, nativeIcuSeedAsset(version, target)); + const seed = path.join(destination, target); + rmSync(seed, { recursive: true, force: true }); + mkdirSync(seed, { recursive: true }); + for (const entry of readPortableArchiveEntries(archive).values()) { + const file = path.join(seed, entry.name); + if (entry.isDirectory) mkdirSync(file, { recursive: true }); + else { + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, entry.data()); + } + } + validateNativeClusterSeedDirectory(seed, "icu", { target, icuData }); + } +} + +export async function stageNativeIcuArchive(assetDir, version, destination, targets) { + const stage = `${destination}.stage`; + rmSync(stage, { recursive: true, force: true }); + mkdirSync(stage, { recursive: true }); + try { + const source = path.join(assetDir, `liboliphaunt-${version}-icu-data.tar.gz`); + for (const entry of readPortableArchiveEntries(source).values()) { + const file = path.join(stage, entry.name); + if (entry.isDirectory) mkdirSync(file, { recursive: true }); + else { + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, entry.data()); + } + } + stageNativeIcuSeeds(assetDir, version, path.join(stage, "native-seeds"), path.join(stage, "share/icu"), targets); + writeFileSync(destination, canonicalGzipSync(await createDeterministicTar(stage))); + } finally { + rmSync(stage, { recursive: true, force: true }); + } +} + +if (import.meta.main) { + const [root, icuData, target, output, ...extra] = process.argv.slice(2); + if (!output || extra.length) throw new Error("usage: native-icu-seeds.mjs ROOT ICU_DATA TARGET OUTPUT.tar.gz"); + await splitNativeIcuSeed(root, icuData, target, output); +} diff --git a/tools/release/native-icu-seeds.test.mjs b/tools/release/native-icu-seeds.test.mjs new file mode 100644 index 000000000..a0ec10256 --- /dev/null +++ b/tools/release/native-icu-seeds.test.mjs @@ -0,0 +1,70 @@ +import { test, expect } from "bun:test"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync, cpSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { splitNativeIcuSeed, stageNativeIcuSeeds, nativeIcuSeedAsset } from "./native-icu-seeds.mjs"; +import { filesystemTreeRows, logicalTreeSha256 } from "./native-cluster-seed-contract.mjs"; +import { spawnSync } from "../test/fd-backed-spawn-sync.mjs"; + +test("optional native ICU carrier owns the matching seed and base retains only standard", async () => { + const root = mkdtempSync(path.join(os.tmpdir(), "native-icu-seeds-")); + try { + const target = "linux-x64-gnu"; + const data = path.join(root, "data"); + mkdirSync(data); + writeFileSync(path.join(data, "icudt76l.dat"), "ICU fixture"); + const seed = path.join(root, "cluster-seed-icu"); + mkdirSync(path.join(seed, "files/global"), { recursive: true }); + mkdirSync(path.join(seed, "files/pg_wal")); + writeFileSync(path.join(seed, "files/PG_VERSION"), "18\n"); + writeFileSync(path.join(seed, "files/global/pg_control"), "fixture"); + const fixture = readFileSync(new URL("../../src/shared/cluster-seed-contract/fixtures/native-icu.valid.properties", import.meta.url), "utf8"); + writeFileSync(path.join(seed, "manifest.properties"), fixture + .replace(/^target=.*$/mu, `target=${target}`) + .replace(/^compatibilityKey=.*$/mu, `compatibilityKey=native-pg18-${target}-v1`) + .replace(/^icuDataTreeSha256=.*$/mu, `icuDataTreeSha256=${logicalTreeSha256(filesystemTreeRows(data))}`)); + const output = path.join(root, nativeIcuSeedAsset("1.0.0", target)); + writeFileSync(path.join(root, "package-size.tsv"), "kind\tid\textensions\tfiles\tbytes\npackage\ttotal\t-\t-\t30\npackage\tcluster-seed-icu\t-\t-\t10\n"); + await splitNativeIcuSeed(root, data, target, output); + expect(existsSync(seed)).toBe(false); + expect(readFileSync(path.join(root, "package-size.tsv"), "utf8")).toContain("package\ttotal\t-\t-\t20"); + const destination = path.join(root, "optional"); + stageNativeIcuSeeds(root, "1.0.0", destination, data, [target]); + expect(readFileSync(path.join(destination, target, "files/global/pg_control"), "utf8")).toBe("fixture"); + if (process.platform === "linux" && process.arch === "x64") { + const repo = path.resolve(import.meta.dir, "../.."); + const crate = path.join(root, "icu-crate"); + cpSync(path.join(repo, "src/runtimes/liboliphaunt/icu"), crate, { recursive: true }); + const manifest = path.join(crate, "Cargo.toml"); + writeFileSync(manifest, readFileSync(manifest, "utf8").replace( + 'path = "../../../sdks/rust/crates/oliphaunt-resources"', + `path = ${JSON.stringify(path.join(repo, "src/sdks/rust/crates/oliphaunt-resources"))}`, + )); + cpSync(destination, path.join(crate, "payload/native-seeds"), { recursive: true }); + const app = path.join(root, "app"); + mkdirSync(path.join(app, "src"), { recursive: true }); + writeFileSync(path.join(app, "Cargo.toml"), `[package]\nname = "optional-native-icu-proof"\nversion = "0.0.0"\nedition = "2024"\n[dependencies]\noliphaunt-icu = { path = ${JSON.stringify(crate)} }\n[build-dependencies]\noliphaunt-build = { path = ${JSON.stringify(path.join(repo, "src/sdks/rust/crates/oliphaunt-build"))} }\n[workspace]\n`); + writeFileSync(path.join(app, "build.rs"), 'fn main() { oliphaunt_build::embed_resolved_artifacts().unwrap(); }'); + writeFileSync(path.join(app, "src/main.rs"), `fn main() { + let files = oliphaunt_icu::ICU.resources; + let embedded: &[(&str, &[u8], &str, bool)] = include!(concat!(env!("OUT_DIR"), "/embedded_resources.rs")); + assert_eq!(files.len(), embedded.len()); + for resource in files { assert!(embedded.contains(resource), "missing {}", resource.0); } + assert!(files.iter().any(|row| row.0 == "icu-data/oliphaunt-icu/native-seeds/linux-x64-gnu/files/global/pg_control" && row.1 == b"fixture")); + assert!(files.iter().any(|row| row.0.ends_with("/share/icu/icudt76l.dat"))); + assert!(files.iter().any(|row| row.0.ends_with("/files/pg_wal/") && row.1.is_empty())); + assert!(!files.iter().any(|row| row.0.contains("native-seeds/macos"))); +} +`); + const result = spawnSync("cargo", ["run", "--offline", "--manifest-path", path.join(app, "Cargo.toml")], { + cwd: repo, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], + env: { ...process.env, OLIPHAUNT_ICU_DATA_DIR: data, OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD: "1", CARGO_TARGET_DIR: path.join(repo, "target/optional-native-icu-proof") }, + }); + expect(result.status, result.stderr).toBe(0); + } + writeFileSync(path.join(data, "icudt76l.dat"), "different ICU"); + expect(() => stageNativeIcuSeeds(root, "1.0.0", destination, data, [target])).toThrow("icuDataTreeSha256"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}, 120_000); diff --git a/tools/release/native-runtime-carrier-contract.mjs b/tools/release/native-runtime-carrier-contract.mjs index b5ad01566..a5cb181d2 100644 --- a/tools/release/native-runtime-carrier-contract.mjs +++ b/tools/release/native-runtime-carrier-contract.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env bun -import { readFileSync, writeFileSync } from "node:fs"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; import { @@ -97,7 +97,9 @@ export function validateNativeRuntimeCarrier(root, { icuData } = {}) { throw new Error(`${runtimeManifestPath}: runtime resource manifest is not canonical for ${target}`); } validateNativeClusterSeedDirectory(path.join(root, "cluster-seed"), "standard", { target }); - validateNativeClusterSeedDirectory(path.join(root, "cluster-seed-icu"), "icu", { target, icuData }); + if (existsSync(path.join(root, "cluster-seed-icu"))) { + validateNativeClusterSeedDirectory(path.join(root, "cluster-seed-icu"), "icu", { target, icuData }); + } return Object.freeze({ target }); } diff --git a/tools/release/package-extension-cargo-facades.mjs b/tools/release/package-extension-cargo-facades.mjs index c9890776e..bb6ad3e02 100644 --- a/tools/release/package-extension-cargo-facades.mjs +++ b/tools/release/package-extension-cargo-facades.mjs @@ -4,6 +4,8 @@ import path from "node:path"; import { manualCargoPackageSource } from "./cargo-source-package.mjs"; import { exactExtensionProducts, + currentProductVersionSync, + extensionMetadata, extensionReleaseProduct, extensionReleaseVersion, extensionRegistryPackageTargetSets, @@ -13,11 +15,7 @@ import { compareText, ROOT } from "./release-graph.mjs"; import { nativeExtensionCargoPackageName, } from "./extension-registry-packages.mjs"; -import { - expectedExtensionAotTargets, - wasixExtensionAotPackageName, - wasixExtensionPackageName, -} from "./wasix-cargo-artifact-contract.mjs"; + import { renderUnsupportedNativeTargetGuard, rustNativeTargetCfg, @@ -36,10 +34,6 @@ function fail(message) { throw new Error(`package-extension-cargo-facades: ${message}`); } -function dependencyFeature(name) { - return `dep:${name}`; -} - function facadeLinksName(product) { return `oliphaunt_artifact_relay_extension_${product .replace(/^oliphaunt-extension-/u, "") @@ -54,6 +48,8 @@ const RELAY_PREFIX: &str = "DEP_OLIPHAUNT_ARTIFACT_RELAY_"; const SUFFIX: &str = "_MANIFEST"; fn main() { + let embedded = oliphaunt_build::embed_resolved_artifacts().expect("validate and embed extension resources"); + println!("cargo::rustc-env=OLIPHAUNT_EMBEDDED_RESOURCES_RS={}", embedded.display()); let mut manifests = BTreeMap::new(); for (key, value) in env::vars() { if value.is_empty() || key.starts_with(RELAY_PREFIX) { @@ -86,8 +82,6 @@ export function renderUnsupportedNativeGuard(product, nativeTargets, nativeCfgs) product, nativeTargets, nativeCfgs, - feature: "native", - featureLabel: "default native feature", guidance: "use a declared native target leaf, or depend on the WASIX carrier directly for WASIX builds.", }); } @@ -97,24 +91,15 @@ export function writeFacadeSource(product, outputRoot, { dependencyPaths = {} } fail(`${product} is not an exact extension product`); } const nativeOwner = extensionReleaseProduct(product, "native", "package-extension-cargo-facades"); - const wasixOwner = extensionReleaseProduct(product, "wasix", "package-extension-cargo-facades"); - const nativeOnly = nativeOwner !== wasixOwner; const version = extensionReleaseVersion(product, "native", "package-extension-cargo-facades"); const sqlNames = extensionSqlNames(product, "package-extension-cargo-facades"); + const sdkVersion = currentProductVersionSync("oliphaunt-rust"); + const buildVersion = sdkVersion; + const runtimeVersion = extensionMetadata(product).compatibility.nativeRuntimeVersion; const targets = extensionRegistryPackageTargetSets(product, "package-extension-cargo-facades"); - const wasixAotTargets = !nativeOnly && targets.includeWasixAot ? expectedExtensionAotTargets() : []; const sourceDir = path.join(outputRoot, "sources", product); mkdirSync(path.join(sourceDir, "src"), { recursive: true }); - const nativeNames = targets.nativeCargoTargets.map((target) => nativeExtensionCargoPackageName(product, target)); - const wasixName = nativeOnly ? null : wasixExtensionPackageName(product); - const aotNames = wasixAotTargets.map((target) => wasixExtensionAotPackageName(product, target)); - const features = [ - `default = ["native"]`, - `native = [${nativeNames.map((name) => JSON.stringify(dependencyFeature(name))).join(", ")}]`, - ...(wasixName === null ? [] : [`wasix = [${JSON.stringify(dependencyFeature(wasixName))}]`]), - ...aotNames.map((name, index) => `${JSON.stringify(`wasix-aot-${wasixAotTargets[index]}`)} = [${JSON.stringify(dependencyFeature(wasixName))}, ${JSON.stringify(dependencyFeature(name))}]`), - ]; const targetDependencies = []; const nativeCfgs = []; for (const target of targets.nativeCargoTargets) { @@ -122,12 +107,9 @@ export function writeFacadeSource(product, outputRoot, { dependencyPaths = {} } const name = nativeExtensionCargoPackageName(product, target); nativeCfgs.push(cfg); targetDependencies.push( - `[target.'cfg(${cfg})'.dependencies]\n${name} = { version = "=${version}", optional = true${dependencyPaths[name] ? `, path = ${JSON.stringify(dependencyPaths[name])}` : ""} }`, + `[target.'cfg(${cfg})'.dependencies]\n${name} = { version = "=${version}"${dependencyPaths[name] ? `, path = ${JSON.stringify(dependencyPaths[name])}` : ""} }`, ); } - const optionalDependencies = [...(wasixName === null ? [] : [wasixName]), ...aotNames] - .map((name) => `${name} = { version = "=${version}", optional = true${dependencyPaths[name] ? `, path = ${JSON.stringify(dependencyPaths[name])}` : ""} }`) - .join("\n"); const unsupportedNativeGuard = renderUnsupportedNativeGuard( product, targets.nativeCargoTargets, @@ -151,11 +133,11 @@ include = ${JSON.stringify(["Cargo.toml", "README.md", "build.rs", "src/**", ... [lib] path = "src/lib.rs" -[features] -${features.join("\n")} - [dependencies] -${optionalDependencies} +oliphaunt-resources = { version = ${JSON.stringify(sdkVersion)}${dependencyPaths["oliphaunt-resources"] ? `, path = ${JSON.stringify(dependencyPaths["oliphaunt-resources"])}` : ""} } + +[build-dependencies] +oliphaunt-build = { version = ${JSON.stringify(buildVersion)}${dependencyPaths["oliphaunt-build"] ? `, path = ${JSON.stringify(dependencyPaths["oliphaunt-build"])}` : ""} } ${targetDependencies.join("\n\n")} @@ -166,9 +148,8 @@ ${targetDependencies.join("\n\n")} Target-selecting Cargo facade for ${sqlNames.length === 1 ? `the \`${sqlNames[0]}\` PostgreSQL extension` : `the PostgreSQL 18 contrib bundle (${sqlNames.length} exact SQL members)`}. -The default \`native\` feature selects the matching native artifact leaf.${nativeOnly ? "" : ` Use -\`default-features = false, features = ["wasix"]\` (or a host-specific -\`wasix-aot-*\` feature) for WASIX artifacts.`} +Cargo selects the matching native artifact automatically. For WASIX, use +\`${product}-wasix\` and its exported descriptors. `); writeFileSync(path.join(sourceDir, "src/lib.rs"), `#![forbid(unsafe_code)] @@ -178,6 +159,12 @@ pub const PRODUCT: &str = ${JSON.stringify(product)}; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); pub const EXTENSION_SQL_NAMES: &[&str] = &[${sqlNames.map((sqlName) => JSON.stringify(sqlName)).join(", ")}]; ${sqlNames.length === 1 ? `pub const EXTENSION_SQL_NAME: &str = ${JSON.stringify(sqlNames[0])};` : ""} +const RESOURCES: &[oliphaunt_resources::EmbeddedResource] = include!(env!("OLIPHAUNT_EMBEDDED_RESOURCES_RS")); +${sqlNames.map((sqlName) => `pub const ${sqlName.replaceAll("-", "_").toUpperCase()}: oliphaunt_resources::ExtensionDescriptor = oliphaunt_resources::ExtensionDescriptor { + sql_name: ${JSON.stringify(sqlName)}, product: PRODUCT, version: Some(VERSION), + runtime_version: ${JSON.stringify(runtimeVersion)}, resources: RESOURCES, +};`).join("\n")} + `); stageReleaseNotices(sourceDir, FACADE_NOTICE_OPTIONS); assertReleaseNoticesInDirectory(sourceDir, FACADE_NOTICE_OPTIONS); diff --git a/tools/release/package-extension-cargo-facades.test.mjs b/tools/release/package-extension-cargo-facades.test.mjs index a9a3b7119..9b2320a70 100644 --- a/tools/release/package-extension-cargo-facades.test.mjs +++ b/tools/release/package-extension-cargo-facades.test.mjs @@ -96,14 +96,14 @@ function findFile(root, basename) { } describe("exact extension Cargo facade", () => { - test("fails closed for unsupported default-native targets while WASIX opt-out compiles", () => { + test("fails closed for unsupported native targets without a feature escape hatch", () => { const output = mkdtempSync(path.join(import.meta.dir, "../../target/extension-facade-test-")); directories.push(output); const [pkg] = packageExtensionCargoFacades(["oliphaunt-extension-pgtap"], output); const source = path.join(output, "sources/oliphaunt-extension-pgtap/src/lib.rs"); const text = readFileSync(source, "utf8"); expect(text).toContain("compile_error!"); - expect(text).toContain('feature = "native"'); + expect(text).not.toContain('feature = "native"'); expect(text).toContain('target_env = "gnu"'); expect(text).toContain('target_env = "msvc"'); @@ -120,7 +120,7 @@ pub const FIXTURE: bool = true; forcedUnsupportedSource, ], { encoding: "utf8" }); expect(unsupported.status).not.toBe(0); - expect(unsupported.stderr).toContain("default native feature supports only"); + expect(unsupported.stderr).toContain("supports only"); const wasixOnly = spawnSync("rustc", [ "--crate-name", "oliphaunt_extension_pgtap", @@ -131,11 +131,10 @@ pub const FIXTURE: bool = true; "-o", path.join(output, "wasix-only.rmeta"), forcedUnsupportedSource, ], { encoding: "utf8" }); - expect(wasixOnly.status).toBe(0); + expect(wasixOnly.status).not.toBe(0); const manifest = Bun.TOML.parse(readFileSync(pkg.manifestPath, "utf8")); - expect(manifest.features.default).toEqual(["native"]); - expect(manifest.features.wasix).toEqual([`dep:oliphaunt-extension-pgtap-wasix`]); + expect(manifest.features).toBeUndefined(); expect(pkg.cratePath.endsWith(".crate")).toBe(true); }); @@ -153,9 +152,8 @@ pub const FIXTURE: bool = true; "native", "package-extension-cargo-facades.test", )); - expect(manifest.features.default).toEqual(["native"]); - expect(manifest.features.wasix).toBeUndefined(); - expect(Object.keys(manifest.dependencies ?? {})).toHaveLength(0); + expect(manifest.features).toBeUndefined(); + expect(Object.keys(manifest.dependencies ?? {})).toEqual(["oliphaunt-resources"]); }); test("real Cargo metadata relays exact bundle and external manifests into an app build", { @@ -182,7 +180,11 @@ pub const FIXTURE: bool = true; const graph = loadGraph("package-extension-cargo-facades.test"); const nativeRuntimeVersion = graph.products["liboliphaunt-native"].version; const products = ["oliphaunt-extension-contrib-pg18", "oliphaunt-extension-vector"]; - const dependencyPaths = {}; + const dependencyPaths = { + oliphaunt: path.join(import.meta.dir, "../../src/sdks/rust"), + "oliphaunt-resources": path.join(import.meta.dir, "../../src/sdks/rust/crates/oliphaunt-resources"), + "oliphaunt-build": path.join(import.meta.dir, "../../src/sdks/rust/crates/oliphaunt-build"), + }; for (const product of products) { const productVersion = extensionReleaseVersion( product, @@ -284,5 +286,20 @@ oliphaunt-build = { path = ${JSON.stringify(path.join(import.meta.dir, "../../sr expect(text).toContain('extension = "pg_trgm"'); expect(text).toContain('extension = "vector"'); expect(text).not.toContain('extension = "hstore"'); + // The ordinary API needs only dependencies and imported descriptors. + rmSync(path.join(app, "build.rs")); + let plainManifest = readFileSync(path.join(app, "Cargo.toml"), "utf8") + .replace('build = "build.rs"\n', "") + .replace(/\[package\.metadata\.oliphaunt\][\s\S]*?(?=\[dependencies\])/u, "") + .replace(/\[build-dependencies\][\s\S]*?(?=\[workspace\])/u, "") + .replace("[dependencies]", `[dependencies]\noliphaunt = { path = ${JSON.stringify(dependencyPaths.oliphaunt)} }`); + writeFileSync(path.join(app, "Cargo.toml"), plainManifest); + writeFileSync(path.join(app, "src/lib.rs"), `pub fn configured() -> oliphaunt::OliphauntBuilder { + oliphaunt::Oliphaunt::builder().extensions([vector::VECTOR, oliphaunt::extensions::HSTORE]) + }\n`); + const plain = spawnSync("cargo", ["check", "--offline", "--target-dir", path.join(root, "cargo-target")], { + cwd: app, encoding: "utf8", maxBuffer: 20 * 1024 * 1024, + }); + expect(plain.status, `${plain.stdout}\n${plain.stderr}`).toBe(0); }); }); diff --git a/tools/release/package-extension-release-carriers.mjs b/tools/release/package-extension-release-carriers.mjs index 86412d73e..8c38cbe5c 100644 --- a/tools/release/package-extension-release-carriers.mjs +++ b/tools/release/package-extension-release-carriers.mjs @@ -38,11 +38,14 @@ import { extensionNpmPackageForProduct, extensionNpmTargetPackageForProduct, extensionNpmWasixPackageForProduct, + extensionNpmWasixAotTargets, nativeExtensionCargoLinksName, nativeExtensionCargoPackageName, nativeExtensionCargoPartPackageName, } from "./extension-registry-packages.mjs"; -import { CORE_RUNTIME_ARCHIVE_FILES } from "./wasix-cargo-artifact-contract.mjs"; +import { AOT_TARGET_TRIPLES, CORE_RUNTIME_ARCHIVE_FILES } from "./wasix-cargo-artifact-contract.mjs"; +import { assertCanonicalWasixAotManifest } from "./wasix-aot-manifest.mjs"; +import { assertWasixAotArtifactPayloads } from "./check-liboliphaunt-wasix-release-assets.mjs"; import { readPortableArchiveEntries, readPortableTarZstdBufferEntries, @@ -299,7 +302,7 @@ function tarballPackageJson(tarball) { } } -function pnpmPackForNpmPublish(packageDir, tarballRoot) { +export function pnpmPackForNpmPublish(packageDir, tarballRoot) { const packageJson = readJsonFile(path.join(packageDir, "package.json")); const packageName = packageJson.name; const packageVersion = packageJson.version; @@ -495,7 +498,7 @@ function sha256File(file) { return createHash("sha256").update(readFileSync(file)).digest("hex"); } -function npmPlatformConstraints(target) { +export function npmPlatformConstraints(target) { if (target === "linux-x64-gnu") { return { os: ["linux"], cpu: ["x64"], libc: ["glibc"] }; } @@ -1466,6 +1469,12 @@ function writeWasixExtensionNpmPackage(packageDir, { default: `./${descriptorPath}`, }; } + if (bundle) { + const index = members.map(sqlName => `export { default as ${sqlName.replaceAll("-", "_")} } from './descriptors/${sqlName}.js';`).join("\n") + "\n"; + writeFileSync(path.join(packageDir, "index.js"), index); + writeFileSync(path.join(packageDir, "index.d.ts"), index); + exports["."] = { types: "./index.d.ts", import: "./index.js", default: "./index.js" }; + } exports["./package.json"] = "./package.json"; writeWasixExtensionReadme(packageDir, packageName, members, bundle); @@ -1490,11 +1499,21 @@ function writeWasixExtensionNpmPackage(packageDir, { wasixRuntimeProduct: runtimeSet.compatibility.wasixRuntimeProduct, wasixRuntimeVersion: runtimeSet.compatibility.wasixRuntimeVersion, runtimeBound: runtimeSet.versioning === "runtime-bound", + carriers: Object.fromEntries(runtimeSet.members.map(member => [member.sqlName, { + path: `extensions/${member.sqlName}/extension.tar.zst`, + sha256: member.asset.sha256, + size: member.asset.bytes, + requiresAot: wasixMemberRequiresAot(member), + }])), }, + ...(runtimeSet.members.some(wasixMemberRequiresAot) ? { + optionalDependencies: Object.fromEntries(extensionNpmWasixAotTargets(product) + .map(target => [`${packageName}-${target}`, version])), + } : {}), publishConfig: { access: "public", provenance: true }, files: [ "README.md", - ...(bundle ? ["descriptors"] : ["index.js", "index.d.ts"]), + "index.js", "index.d.ts", ...(bundle ? ["descriptors"] : []), "extensions", ...carrierLegalMembers(legal), ], @@ -1504,6 +1523,47 @@ function writeWasixExtensionNpmPackage(packageDir, { return { legal, packageName }; } +function wasixMemberRequiresAot(member) { + return member.install.nativeModule !== null && member.install.nativeModule !== undefined + || (member.install.nativeModules?.length ?? 0) > 0; +} + +export function writeWasixExtensionAotNpmPackage(packageDir, { + product, version, runtimeVersion, sqlName, target, sourceDir, +}) { + const manifest = readJsonFile(path.join(sourceDir, "manifest.json")); + assertCanonicalWasixAotManifest(manifest, { expectedTarget: AOT_TARGET_TRIPLES[target] }); + const artifacts = assertWasixAotArtifactPayloads(manifest, { + readArtifact: relative => readFileSync(path.join(sourceDir, relative)), + }); + const prefix = `extension:${sqlName}`; + if (artifacts.some(row => row.name !== prefix && !row.name.startsWith(`${prefix}:`))) { + fail(TOOL, `${product} AOT package contains another extension's code`); + } + const legal = wasixExtensionCarrierLegal(product, [sqlName]); + mkdirSync(packageDir, { recursive: true }); + for (const artifact of artifacts) { + const output = path.join(packageDir, artifact.path); + mkdirSync(path.dirname(output), { recursive: true }); + copyFileSync(path.join(sourceDir, artifact.path), output); + } + writeJsonFile(path.join(packageDir, "aot-manifest.json"), manifest); + writeJsonFile(path.join(packageDir, "package.json"), { + name: `${extensionNpmWasixPackageForProduct(product)}-${target}`, + version, license: legal.packageSpdx, + ...npmPlatformConstraints(target), + repository: { type: "git", url: NPM_TRUSTED_PUBLISHING_REPOSITORY }, + publishConfig: { access: "public", provenance: true }, + oliphaunt: { + kind: "wasix-extension-aot", product, target, runtimeVersion, + manifestSha256: sha256File(path.join(packageDir, "aot-manifest.json")), + }, + files: ["aot-manifest.json", ...artifacts.map(row => row.path), ...carrierLegalMembers(legal)], + exports: { "./package.json": "./package.json" }, + }); + stageExtensionCarrierLegal(packageDir, legal); +} + function extensionModuleDirectory(runtimeDir) { for (const candidate of [ path.join(runtimeDir, "lib", "modules"), @@ -1540,7 +1600,34 @@ function writeExtensionReadme(packageDir, packageName, members, target) { ); } -function writeExtensionMetaPackage(packageDir, { +export function renderNativeExtensionDescriptor({ product, version, sqlName }) { + return { + schema: "oliphaunt-native-extension-v1", + sqlName, + product, + packageName: extensionNpmPackageForProduct(product), + version, + }; +} + +function writeNativeDescriptorEntrypoints(packageDir, descriptor, stem = "index") { + const json = JSON.stringify(descriptor, null, 2); + writeFileSync(path.join(packageDir, `${stem}.js`), `const descriptor = Object.freeze(${json});\nexport default descriptor;\n`); + writeFileSync(path.join(packageDir, `${stem}.cjs`), `module.exports = Object.freeze(${json});\n`); + writeFileSync(path.join(packageDir, `${stem}.node.js`), `const descriptor = Object.freeze({ ...${json}, packageJsonUrl: new URL("./package.json", import.meta.url).href });\nexport default descriptor;\n`); + writeFileSync(path.join(packageDir, `${stem}.node.cjs`), `module.exports = Object.freeze({ ...${json}, packageJsonUrl: require("node:url").pathToFileURL(require.resolve("./package.json")).href });\n`); + writeFileSync(path.join(packageDir, `${stem}.d.ts`), `declare const descriptor: Readonly<{\n schema: "oliphaunt-native-extension-v1";\n sqlName: ${JSON.stringify(descriptor.sqlName)};\n product: ${JSON.stringify(descriptor.product)};\n packageName: ${JSON.stringify(descriptor.packageName)};\n version: ${JSON.stringify(descriptor.version)};\n packageJsonUrl?: string;\n}>;\nexport default descriptor;\n`); + return { + types: `./${stem}.d.ts`, + "react-native": `./${stem}.js`, + node: { import: `./${stem}.node.js`, require: `./${stem}.node.cjs` }, + import: `./${stem}.js`, + require: `./${stem}.cjs`, + default: `./${stem}.js`, + }; +} + +export function writeExtensionMetaPackage(packageDir, { product, version, members, @@ -1561,7 +1648,15 @@ function writeExtensionMetaPackage(packageDir, { ); mkdirSync(packageDir, { recursive: true }); writeExtensionReadme(packageDir, packageName, members, null); - writeJsonFile(path.join(packageDir, IOS_CARRIER_FILENAME), iosCarrier); + if (iosCarrier !== undefined) writeJsonFile(path.join(packageDir, IOS_CARRIER_FILENAME), iosCarrier); + const descriptorExports = {}; + for (const sqlName of members) { + const stem = bundle ? sqlName : "index"; + descriptorExports[bundle ? `./${sqlName}` : "."] = writeNativeDescriptorEntrypoints( + packageDir, renderNativeExtensionDescriptor({ product, version, sqlName }), stem, + ); + } + writeJsonFile(path.join(packageDir, "package.json"), { name: packageName, version, @@ -1578,17 +1673,20 @@ function writeExtensionMetaPackage(packageDir, { ...(bundle ? {} : { sqlName: members[0] }), members, targetPackageNames, - iosCarrierManifest: `./${IOS_CARRIER_FILENAME}`, + ...(iosCarrier === undefined ? {} : { iosCarrierManifest: `./${IOS_CARRIER_FILENAME}` }), liboliphauntVersion, runtimeBound, }, publishConfig: { access: "public", provenance: true }, - files: ["README.md", IOS_CARRIER_FILENAME, ...carrierLegalMembers(legal)], + sideEffects: false, + files: ["README.md", "*.js", "*.cjs", "*.d.ts", ...(iosCarrier === undefined ? [] : [IOS_CARRIER_FILENAME]), ...carrierLegalMembers(legal)], exports: { - "./ios-carriers": `./${IOS_CARRIER_FILENAME}`, + ...descriptorExports, + ...(iosCarrier === undefined ? {} : { "./ios-carriers": `./${IOS_CARRIER_FILENAME}` }), "./package.json": "./package.json", }, }); + stageExtensionCarrierLegal(packageDir, legal); } function writeExtensionTargetPackage(packageDir, { @@ -1854,7 +1952,6 @@ export function stageExtensionNativeNpmPackages(roots, stagingRoot, target, resu memberModuleRelativePaths: runtimeSet.bundle ? memberModuleRelativePaths : null, legal: targetLegal, }); - stageExtensionCarrierLegal(metaDir, metaLegal); stageExtensionCarrierLegal(targetDir, targetLegal); const targetTarball = pnpmPackForNpmPublish(targetDir, tarballRoot); assertExtensionCarrierArchive(targetTarball, targetLegal, "package"); @@ -1972,6 +2069,18 @@ export function stageExtensionWasixNpmPackages(roots, stagingRoot, result) { } stagedIdentities.set(identity, digest); + for (const target of extensionNpmWasixAotTargets(product)) { + const sourceDir = path.join(extensionDir, "wasix-aot", target); + if (!isDirectory(sourceDir) || !runtimeSet.members.some(wasixMemberRequiresAot)) continue; + if (runtimeSet.members.length !== 1) fail(TOOL, "external WASIX AOT packages must have one SQL owner"); + const aotDir = path.join(packageRoot, safeNpmPackageFilenamePrefix(`${extensionNpmWasixPackageForProduct(product)}-${target}`)); + writeWasixExtensionAotNpmPackage(aotDir, { + product, version, runtimeVersion, target, sourceDir, sqlName: runtimeSet.members[0].sqlName, + }); + const aotTarball = pnpmPackForNpmPublish(aotDir, tarballRoot); + if (npmPackageSizeSafe(aotTarball, result)) result.staged.push(rel(aotTarball)); + } + const packageDir = path.join( packageRoot, safeNpmPackageFilenamePrefix(extensionNpmWasixPackageForProduct(product)), diff --git a/tools/release/package-liboliphaunt-cargo-artifacts.mjs b/tools/release/package-liboliphaunt-cargo-artifacts.mjs index 936ed0e2f..10a2b8a58 100644 --- a/tools/release/package-liboliphaunt-cargo-artifacts.mjs +++ b/tools/release/package-liboliphaunt-cargo-artifacts.mjs @@ -69,6 +69,7 @@ const TARGET: &str = __TARGET__; const PART_ROOTS: &[&str] = &[ __PART_ROOTS__ ]; +const DIRECTORIES: &[&str] = &[__DIRECTORIES__]; const FILE_SHA256: &[(&str, &str)] = &[ __FILE_SHA256__ ]; @@ -119,13 +120,16 @@ fn emit_manifest() { } } + for directory in DIRECTORIES { + fs::create_dir_all(payload.join(directory)).expect("restore packaged empty directory"); + } let files = collect_files(&payload).expect("collect reconstructed liboliphaunt payload files"); if files.is_empty() { panic!("liboliphaunt native payload part crates produced no files"); } let manifest = out_dir.join("oliphaunt-artifact.toml"); let mut text = format!( - "schema = {SCHEMA:?}\nproduct = {PRODUCT:?}\nversion = {VERSION:?}\nkind = {KIND:?}\ntarget = {TARGET:?}\n" + "schema = {SCHEMA:?}\nproduct = {PRODUCT:?}\nversion = {VERSION:?}\nkind = {KIND:?}\ntarget = {TARGET:?}\ndirectories = {DIRECTORIES:?}\n" ); if files.len() != FILE_SHA256.len() { panic!("reconstructed liboliphaunt payload file count does not match the frozen inventory"); @@ -418,6 +422,7 @@ function validateNativePayload(payloadRoot, target, { toolSet }) { } function validateNativeCargoRuntimeClosure(runtimeRoot, target, icuRoot) { + if (existsSync(path.join(runtimeRoot, "cluster-seed-icu"))) fail("base Cargo runtime must not bundle the optional ICU seed"); const icuData = path.join(icuRoot, "share/icu"); try { const { target: actualTarget } = validateNativeRuntimeCarrier(runtimeRoot, { icuData }); @@ -517,6 +522,7 @@ function writeAggregatorCrate( artifactKind, artifactLabel, payloadFiles, + payloadDirectories, }, ) { rmSync(crateDir, { recursive: true, force: true }); @@ -586,6 +592,7 @@ pub const LIBRARY_RELATIVE_PATH: &str = "${libraryRelativePath}"; .replace("__KIND__", tomlString(artifactKind)) .replace("__TARGET__", tomlString(target.triple)) .replace("__PART_ROOTS__", partRoots.join("\n")) + .replace("__DIRECTORIES__", payloadDirectories.map(tomlString).join(", ")) .replace("__FILE_SHA256__", payloadFiles.map(({ relative, sha256 }) => ` (${tomlString(relative)}, ${tomlString(sha256)}),`).join("\n")), ); stageReleaseNotices(crateDir, { profile: "code-facade" }); @@ -970,6 +977,10 @@ function packagePayload( artifactKind, artifactLabel, payloadFiles: frozenPayloadFiles(payloadRoot), + payloadDirectories: readdirSync(payloadRoot, { recursive: true, withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => path.relative(payloadRoot, path.join(entry.parentPath, entry.name)).split(path.sep).join("/")) + .sort(compareText), }); const packages = []; diff --git a/tools/release/package-liboliphaunt-cargo-artifacts.test.mjs b/tools/release/package-liboliphaunt-cargo-artifacts.test.mjs index 966422548..6670776ba 100644 --- a/tools/release/package-liboliphaunt-cargo-artifacts.test.mjs +++ b/tools/release/package-liboliphaunt-cargo-artifacts.test.mjs @@ -19,7 +19,6 @@ import { stageReleaseNotices, } from "./release-notices.mjs"; import { requiredCoreRuntimePaths } from "./optimize_native_runtime_payload.mjs"; -import { logicalTreeSha256 } from "./native-cluster-seed-contract.mjs"; import { nativeIcuDataManifest } from "./native-icu-data-contract.mjs"; import { nativeRuntimeCarrierManifest } from "./native-runtime-carrier-contract.mjs"; @@ -130,9 +129,7 @@ test("freezes .crate bytes for native parts, aggregators, and facade and rejects mkdirSync(icuData, { recursive: true }); const icuBytes = Buffer.from("fixture ICU data\n"); writeFileSync(path.join(icuData, "icudt76l.dat"), icuBytes); - const icuDigest = logicalTreeSha256([{ path: "icudt76l.dat", bytes: icuBytes }]); stageClusterSeed(runtime, "cluster-seed", "standard", "linux-x64-gnu"); - stageClusterSeed(runtime, "cluster-seed-icu", "icu", "linux-x64-gnu", icuDigest); writeFileSync(path.join(runtime, "manifest.properties"), nativeRuntimeCarrierManifest("linux-x64-gnu")); writeFileSync(path.join(runtime, "runtime/manifest.properties"), nativeRuntimeResourceManifestFixture({ cacheKey: "fixture-runtime", @@ -170,14 +167,38 @@ test("freezes .crate bytes for native parts, aggregators, and facade and rejects assert.ok(manifest.packages.every(({ cratePath }) => typeof cratePath === "string" && cratePath.endsWith(".crate"))); assert.equal(readdirSync(output).filter((name) => name.endsWith(".crate")).length, manifest.packages.length); const runtimeParts = manifest.packages.filter(({ role, kind }) => role === "part" && kind === "native-runtime"); - assert.ok(runtimeParts.some(({ cratePath, name }) => commandOutput("tar", [ + assert.ok(runtimeParts.every(({ cratePath, name }) => !commandOutput("tar", [ "-tzf", path.resolve(ROOT, cratePath), ]).includes(`${name}-9.8.7/payload/files/cluster-seed-icu/manifest.properties`))); for (const item of manifest.packages) { const expectedProfile = item.role === "part" ? item.kind : "code-facade"; assert.equal(item.noticeProfile, expectedProfile, `${item.name} must freeze its carrier notice profile`); - const packedManifest = commandOutput("tar", [ + if (process.platform === "linux" && process.arch === "x64") { + const consumer = path.join(root, "consumer"); + mkdirSync(path.join(consumer, "src"), { recursive: true }); + const runtimeCarrier = manifest.packages.find(row => row.role === "aggregator" && row.kind === "native-runtime"); + writeFileSync(path.join(consumer, "Cargo.toml"), `[package] +name = "native-seed-directory-proof" +version = "0.0.0" +edition = "2024" +[dependencies] +${runtimeCarrier.name} = { path = ${JSON.stringify(path.dirname(path.resolve(ROOT, runtimeCarrier.manifestPath)))} } +[workspace] +`); + writeFileSync(path.join(consumer, "build.rs"), `fn main() { + let manifest = std::env::vars().find(|(key, _)| key.starts_with("DEP_OLIPHAUNT_ARTIFACT_") && key.ends_with("_MANIFEST")).unwrap().1; + let path = std::path::Path::new(&manifest); + assert!(path.parent().unwrap().join("payload/cluster-seed/files/pg_wal").is_dir()); + let text = std::fs::read_to_string(manifest).unwrap(); + assert!(text.contains("cluster-seed/files/pg_wal")); +}`); + writeFileSync(path.join(consumer, "src/main.rs"), "fn main() {}"); + run("cargo", ["check", "--offline", "--manifest-path", path.join(consumer, "Cargo.toml")], { + env: { ...process.env, CARGO_TARGET_DIR: path.join(root, "consumer-target") }, + }); + } + const packedManifest = commandOutput("tar", [ "-xOzf", path.resolve(ROOT, item.cratePath), `${item.name}-9.8.7/Cargo.toml`, @@ -207,6 +228,30 @@ test("freezes .crate bytes for native parts, aggregators, and facade and rejects assert.doesNotThrow(() => assertLockedArtifactSet(lock, records, { product: "fixture", ecosystem: "cargo" })); const packedAggregator = manifest.packages.find(({ role }) => role === "aggregator"); + if (process.platform === "linux" && process.arch === "x64") { + const consumer = path.join(root, "consumer"); + mkdirSync(path.join(consumer, "src"), { recursive: true }); + const runtimeCarrier = manifest.packages.find(row => row.role === "aggregator" && row.kind === "native-runtime"); + writeFileSync(path.join(consumer, "Cargo.toml"), `[package] +name = "native-seed-directory-proof" +version = "0.0.0" +edition = "2024" +[dependencies] +${runtimeCarrier.name} = { path = ${JSON.stringify(path.dirname(path.resolve(ROOT, runtimeCarrier.manifestPath)))} } +[workspace] +`); + writeFileSync(path.join(consumer, "build.rs"), `fn main() { + let manifest = std::env::vars().find(|(key, _)| key.starts_with("DEP_OLIPHAUNT_ARTIFACT_") && key.ends_with("_MANIFEST")).unwrap().1; + let path = std::path::Path::new(&manifest); + assert!(path.parent().unwrap().join("payload/cluster-seed/files/pg_wal").is_dir()); + let text = std::fs::read_to_string(manifest).unwrap(); + assert!(text.contains("cluster-seed/files/pg_wal")); +}`); + writeFileSync(path.join(consumer, "src/main.rs"), "fn main() {}"); + run("cargo", ["check", "--offline", "--manifest-path", path.join(consumer, "Cargo.toml")], { + env: { ...process.env, CARGO_TARGET_DIR: path.join(root, "consumer-target") }, + }); + } const packedManifest = commandOutput("tar", [ "-xOzf", path.resolve(ROOT, packedAggregator.cratePath), diff --git a/tools/release/package-liboliphaunt-icu-data.test.mjs b/tools/release/package-liboliphaunt-icu-data.test.mjs index ce0ac7635..41b5436b8 100644 --- a/tools/release/package-liboliphaunt-icu-data.test.mjs +++ b/tools/release/package-liboliphaunt-icu-data.test.mjs @@ -67,7 +67,7 @@ test("packages the portable ICU payload deterministically outside platform relea expect(members).toContain("manifest.properties"); expect(members.some((member) => member.startsWith("cluster-seed"))).toBe(false); expect(members).toContain("package-size.tsv"); - expect(members).not.toContain("THIRD_PARTY_LICENSES/PostgreSQL-COPYRIGHT"); + expect(members).toContain("THIRD_PARTY_LICENSES/PostgreSQL-COPYRIGHT"); expect(members).toContain("THIRD_PARTY_LICENSES/ICU-LICENSE"); const receipt = spawnSync("tar", ["-xOzf", archive, "manifest.properties"], { encoding: "utf8" }); expect(receipt.status, receipt.stderr).toBe(0); diff --git a/tools/release/package-liboliphaunt-linux-assets.sh b/tools/release/package-liboliphaunt-linux-assets.sh index 93102d41d..a0087ddc3 100755 --- a/tools/release/package-liboliphaunt-linux-assets.sh +++ b/tools/release/package-liboliphaunt-linux-assets.sh @@ -140,6 +140,10 @@ env \ OLIPHAUNT_ICU_DATA_DIR="$work_root/icu/share/icu" \ node src/runtimes/liboliphaunt/native/tools/run-host-c-smoke.mjs --cluster-seeds +tools/dev/bun.sh tools/release/native-icu-seeds.mjs \ + "$stage" "$work_root/icu/share/icu" "$target_id" \ + "$out_dir/liboliphaunt-${version}-icu-seed-${target_id}.tar.gz" + src/shared/artifact-packaging/archive-directory.mjs "$stage" "$out_dir/$asset" src/shared/artifact-packaging/archive-directory.mjs "$tools_stage" "$out_dir/$tools_asset" tools/dev/bun.sh tools/release/release-notices.mjs check-archive "$out_dir/$asset" --profile native-runtime diff --git a/tools/release/package-liboliphaunt-macos-assets.sh b/tools/release/package-liboliphaunt-macos-assets.sh index 407934f5f..24ecaaf79 100755 --- a/tools/release/package-liboliphaunt-macos-assets.sh +++ b/tools/release/package-liboliphaunt-macos-assets.sh @@ -128,6 +128,10 @@ env \ OLIPHAUNT_ICU_DATA_DIR="$work_root/icu/share/icu" \ node src/runtimes/liboliphaunt/native/tools/run-host-c-smoke.mjs --cluster-seeds +tools/dev/bun.sh tools/release/native-icu-seeds.mjs \ + "$stage" "$work_root/icu/share/icu" "$target_id" \ + "$out_dir/liboliphaunt-${version}-icu-seed-${target_id}.tar.gz" + src/shared/artifact-packaging/archive-directory.mjs "$stage" "$out_dir/$asset" src/shared/artifact-packaging/archive-directory.mjs "$tools_stage" "$out_dir/$tools_asset" tools/dev/bun.sh tools/release/release-notices.mjs check-archive "$out_dir/$asset" --profile native-runtime diff --git a/tools/release/package-liboliphaunt-mobile-assets.sh b/tools/release/package-liboliphaunt-mobile-assets.sh index 981eae1da..083a5416e 100755 --- a/tools/release/package-liboliphaunt-mobile-assets.sh +++ b/tools/release/package-liboliphaunt-mobile-assets.sh @@ -87,6 +87,9 @@ stage_runtime_resource_closure() { --root "$closure" \ --target "$seed_target" \ --icu-data "$icu_data" + tools/dev/bun.sh tools/release/native-icu-seeds.mjs \ + "$closure" "$icu_data" "$seed_target" \ + "$out_dir/liboliphaunt-${version}-icu-seed-${seed_target}.tar.gz" } package_android() { diff --git a/tools/release/package-liboliphaunt-wasix-cargo-artifacts.test.mjs b/tools/release/package-liboliphaunt-wasix-cargo-artifacts.test.mjs index 8de430186..9821290ef 100644 --- a/tools/release/package-liboliphaunt-wasix-cargo-artifacts.test.mjs +++ b/tools/release/package-liboliphaunt-wasix-cargo-artifacts.test.mjs @@ -345,7 +345,7 @@ liboliphaunt-wasix-portable = { path = ${JSON.stringify(path.join(ROOT, "src/run mkdirSync(path.join(payload, "bin"), { recursive: true }); mkdirSync(path.join(payload, "cluster-seeds"), { recursive: true }); writeFileSync(path.join(payload, "bin/initdb.wasix.wasm"), "initdb-wasm\n"); - for (const profile of ["standard", "icu"]) { + for (const profile of ["standard"]) { writeFileSync(path.join(payload, `cluster-seeds/${profile}.tar.zst`), `${profile}\n`); writeFileSync(path.join(payload, `cluster-seeds/${profile}.json`), "{}\n"); } @@ -360,6 +360,9 @@ liboliphaunt-wasix-portable = { path = ${JSON.stringify(path.join(ROOT, "src/run }; writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); expect(() => validateRuntimePayload(payload)).not.toThrow(); + writeFileSync(path.join(payload, "cluster-seeds/icu.json"), "{}\n"); + expect(() => validateRuntimePayload(payload)).toThrow(/optional ICU seed/u); + rmSync(path.join(payload, "cluster-seeds/icu.json")); manifest.runtime.sha256 = "0".repeat(64); writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); diff --git a/tools/release/package-liboliphaunt-windows-assets.ps1 b/tools/release/package-liboliphaunt-windows-assets.ps1 index e83cbbbec..55dbadcd6 100644 --- a/tools/release/package-liboliphaunt-windows-assets.ps1 +++ b/tools/release/package-liboliphaunt-windows-assets.ps1 @@ -269,6 +269,9 @@ if ($LASTEXITCODE -ne 0) { Fail "failed to stage release notices in the Windows tools asset" } +bun tools/release/native-icu-seeds.mjs $Stage (Join-Path $WorkRoot "icu/share/icu") "windows-x64-msvc" (Join-Path $OutDir "liboliphaunt-${Version}-icu-seed-windows-x64-msvc.tar.gz") +if ($LASTEXITCODE -ne 0) { Fail "failed to split optional ICU seed" } + bun src/shared/artifact-packaging/archive-directory.mjs $Stage (Join-Path $OutDir $Asset) if ($LASTEXITCODE -ne 0) { Fail "failed to archive Windows liboliphaunt asset" diff --git a/tools/release/package-release-carriers.mjs b/tools/release/package-release-carriers.mjs index b36671e7d..68d36a9c7 100644 --- a/tools/release/package-release-carriers.mjs +++ b/tools/release/package-release-carriers.mjs @@ -1,4 +1,6 @@ #!/usr/bin/env bun +import { stageSwiftIcuPackage } from "./render_swiftpm_release_package.mjs"; +import { stageNativeIcuSeeds } from "./native-icu-seeds.mjs"; import { createHash } from "node:crypto"; import { chmodSync, @@ -983,7 +985,6 @@ function stageLiboliphauntNpmPayloads(version) { extractReleaseArchiveTree(archive, "lib/modules", path.join(stage, "lib/modules")); extractReleaseArchiveTree(archive, "runtime", path.join(stage, "runtime")); extractReleaseArchiveTree(archive, "cluster-seed", path.join(stage, "cluster-seed")); - extractReleaseArchiveTree(archive, "cluster-seed-icu", path.join(stage, "cluster-seed-icu")); extractReleaseArchiveFile(archive, "manifest.properties", path.join(stage, "manifest.properties")); const vcRuntimeMembers = [ ...stageWindowsVcRuntimeMembers(archive, stage, target.target, "bin", { profile: "provider" }), @@ -1066,13 +1067,13 @@ function stageLiboliphauntToolsNpmFacade(version) { return stage; } -function stageLiboliphauntIcuNpmPayload(version) { +export function stageLiboliphauntIcuNpmPayload(version, { seedTargets } = {}) { const stage = stageNpmPackageDescriptor( LIBOLIPHAUNT_ICU_PACKAGE_NAME, LIBOLIPHAUNT_ICU_PACKAGE_ROOT, version, { - extraDescriptors: [ICU_PODSPEC, ICU_REACT_NATIVE_CONFIG], + extraDescriptors: [ICU_PODSPEC, ICU_REACT_NATIVE_CONFIG, "index.js", "node.js", "index.d.ts"], target: "portable", }, ); @@ -1098,6 +1099,7 @@ function stageLiboliphauntIcuNpmPayload(version) { if (digest === undefined) { fail(`${rel(sourceArchive)} has no canonical ICU data tree digest`); } + stageNativeIcuSeeds(path.dirname(sourceArchive), version, path.join(stage, "OliphauntICU.bundle/native-seeds"), path.join(stage, ...ICU_DATA_RELATIVE_PATH.split("/")), seedTargets); packageJson.oliphaunt.icuDataTreeSha256 = digest; writeFileSync(manifestFile, `${JSON.stringify(packageJson, null, 2)}\n`); stageReleaseNotices(stage, { profile: "native-icu-data" }); @@ -1187,9 +1189,6 @@ export function liboliphauntNpmTarballs(version) { "package/cluster-seed/manifest.properties", "package/cluster-seed/files/PG_VERSION", "package/cluster-seed/files/global/pg_control", - "package/cluster-seed-icu/manifest.properties", - "package/cluster-seed-icu/files/PG_VERSION", - "package/cluster-seed-icu/files/global/pg_control", "package/manifest.properties", ...embeddedCoreModuleMembers(target.target, "package/lib/modules"), ...runtimeMembers, @@ -1564,6 +1563,11 @@ async function packageLiboliphauntNativeCarriers() { const version = currentProductVersionSync(LIBOLIPHAUNT_NATIVE_PRODUCT, TOOL); liboliphauntNativeCargoArtifactPackages(version); liboliphauntNpmTarballs(version); + await stageSwiftIcuPackage({ + assetDir: path.join(ROOT, "target/liboliphaunt/release-assets"), version, + baseSdkVersion: currentProductVersionSync("oliphaunt-swift", TOOL), + outputDir: path.join(ROOT, "target/release/swift-packages/oliphaunt-icu"), + }); const contribProduct = contribCarrierDescriptor(TOOL).artifactProduct; const manifest = await buildMavenArtifactManifest( "target/release/maven-manifests/liboliphaunt-native.tsv", @@ -1690,6 +1694,7 @@ export function liboliphauntWasixCargoArtifactPackages( ) { const outputDir = path.join(ROOT, "target/oliphaunt-wasix/cargo-artifacts"); ensureWasixReleaseAssets(); + ensureLiboliphauntReleaseAssets(); const args = [ process.execPath, "tools/release/package_liboliphaunt_wasix_cargo_artifacts.mjs", @@ -1850,6 +1855,30 @@ function packageExtensionFacade(product) { } } +function packageExtensionSwiftCarriers(product) { + ensureLiboliphauntReleaseAssets(); + const manifest = path.join(extensionPackageDir(product, "native"), "extension-artifacts.json"); + const carrier = buildIosCarrierManifest({ + baseAssetDir: path.join(ROOT, "target/liboliphaunt/release-assets"), + extensionManifests: [manifest], localUrls: true, + }); + const staging = path.join(ROOT, "target/release/extension-carriers/swift", product); + mkdirSync(staging, { recursive: true }); + const carrierFile = path.join(staging, "carrier-input.json"); + writeFileSync(carrierFile, `${JSON.stringify(carrier, null, 2)}\n`); + const releaseTree = path.join(ROOT, "target/release/swift-packages", product); + rmSync(releaseTree, { recursive: true, force: true }); + run(TOOL, [process.execPath, "src/sdks/swift/tools/render-extension-products.mjs", + "--carrier", carrierFile, "--extensions", carrier.extensions.map(row => row.sqlName).join(","), + "--release-product", product, "--base-package-version", currentProductVersionSync("oliphaunt-swift", TOOL), + "--allow-file-urls", "--local-binary-targets", "--cache-dir", path.join(staging, "cache"), + "--output-dir", releaseTree, + ]); + run(TOOL, [process.execPath, "src/shared/artifact-packaging/archive-directory.mjs", + releaseTree, path.join(staging, `${product}-swift-source.zip`), + ]); +} + async function packageExtensionCarriers(product) { requireExtensionAssets(product); await packageExtensionMavenCarriers(product); @@ -1857,6 +1886,7 @@ async function packageExtensionCarriers(product) { packageExtensionNativeCargoCarriers(product); packageExtensionWasixCargoCarriers(product); packageExtensionFacade(product); + packageExtensionSwiftCarriers(product); } async function packageContribNativeCarriers() { diff --git a/tools/release/package_liboliphaunt_wasix_cargo_artifacts.mjs b/tools/release/package_liboliphaunt_wasix_cargo_artifacts.mjs index 6820e70af..a3951d984 100755 --- a/tools/release/package_liboliphaunt_wasix_cargo_artifacts.mjs +++ b/tools/release/package_liboliphaunt_wasix_cargo_artifacts.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env bun +import { stageNativeIcuSeeds } from "./native-icu-seeds.mjs"; import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { @@ -291,7 +292,12 @@ function validateCanonicalAotManifest(manifest, manifestPath, expectedTarget) { } } -export function validateRuntimePayload(root) { +export function validateRuntimePayload(root, { producerClosure = false } = {}) { + if (!producerClosure) { + for (const file of ["icu.tar.zst", "icu.json"]) { + if (isFile(path.join(root, "cluster-seeds", file))) fail("base WASIX Cargo runtime must not bundle the optional ICU seed"); + } + } const extensionRoot = path.join(root, "extensions"); const extensionFiles = isDirectory(extensionRoot) ? payloadFiles(extensionRoot) : []; if (extensionFiles.length > 0) { @@ -312,8 +318,6 @@ export function validateRuntimePayload(root) { "bin/initdb.wasix.wasm", "cluster-seeds/standard.tar.zst", "cluster-seeds/standard.json", - "cluster-seeds/icu.tar.zst", - "cluster-seeds/icu.json", ]) { if (!isFile(path.join(root, required))) { fail(`WASIX runtime Cargo payload is missing ${required}`); @@ -445,6 +449,7 @@ function splitRuntimeToolsPayload(runtimeRoot, extractRoot) { rmSync(toolsRoot, { recursive: true, force: true }); cpSync(runtimeRoot, coreRoot, { recursive: true }); rmSync(path.join(coreRoot, "extensions"), { recursive: true, force: true }); + for (const file of ["icu.tar.zst", "icu.json"]) rmSync(path.join(coreRoot, "cluster-seeds", file), { force: true }); const missing = []; for (const relative of TOOLS_PAYLOAD_FILES) { const source = path.join(runtimeRoot, relative); @@ -677,6 +682,9 @@ function rewriteCargoManifest(manifest, { packageName, version, extensionSources `license = ${JSON.stringify(releaseProfilePackageLicense(noticeProfile).spdx)}`, ); text = injectCargoNoticeIncludes(text, noticeProfile); + if (packageName === ICU_PACKAGE) { + text = text.replace(/^oliphaunt-resources = .*$/mu, `oliphaunt-resources = { version = "${currentProductVersionSync("oliphaunt-rust", PREFIX)}", path = ${JSON.stringify(path.join(ROOT, "src/sdks/rust/crates/oliphaunt-resources"))} }`); + } if (packageName === RUNTIME_PACKAGE && extensionSources.length > 0) { text = injectRuntimeExtensionDependencies(text, extensionSources, extensionAotSources); } @@ -834,7 +842,7 @@ function cargoPackage(crateDir, targetDir, { noVerify = false } = {}) { } function packagedManifestText(text) { - return text.replace(/, path = "\.\.\/[^"]+"/gu, ""); + return text.replace(/, path = "[^"]+"/gu, ""); } function cargoPackageWithoutDependencyResolution(crateDir, targetDir) { @@ -1422,6 +1430,33 @@ function validateExtensionAotCoverage(extensionSpecs) { } } +export function renderWasixExtensionDescriptors(spec) { + const targets = spec.aotTargets ?? []; + const lines = [ + "pub const ARCHIVES: &[(&str, &[u8], &str)] = &[", + ...spec.members.map(member => ` (${JSON.stringify(member.sqlName)}, include_bytes!(concat!(env!("OUT_DIR"), "/payload/extensions/${member.sqlName}/extension.tar.zst")), ${JSON.stringify(member.sha256)}),`), + "];", + ]; + for (const target of targets) { + const crate = target.name.replaceAll("-", "_"); + lines.push(`#[${AOT_TARGET_CFGS[target.target]}]`, + `const AOT: (&str, &[(&str, &[u8])]) = (${crate}::MANIFEST_JSON, ${crate}::AOT_ARTIFACTS);`); + } + const cfgs = targets.map(target => AOT_TARGET_CFGS[target.target].slice(4, -1)); + lines.push(`#[cfg(not(any(${cfgs.join(", ")})))]`, 'const AOT: (&str, &[(&str, &[u8])]) = ("", &[]);', + "#[allow(unsafe_code)]", + "// SAFETY: these immutable artifacts and identities are emitted together by the verified release build.", + "pub const PACKAGE: oliphaunt_resources::WasixPackage = unsafe {", + " oliphaunt_resources::WasixPackage::from_trusted_release(", + ` ${JSON.stringify(spec.product)}, ${JSON.stringify(spec.version)}, ${JSON.stringify(spec.runtimeVersion)}, ARCHIVES, AOT.0, AOT.1,`, + " )", "};"); + for (const member of spec.members) { + const constant = member.sqlName.replaceAll("-", "_").toUpperCase(); + lines.push(`pub const ${constant}: oliphaunt_resources::WasixExtensionDescriptor = oliphaunt_resources::WasixExtensionDescriptor { sql_name: ${JSON.stringify(member.sqlName)}, package: Some(&PACKAGE) };`); + } + return `${lines.join("\n")}\n`; +} + function writeExtensionCargoSource(spec, sourceRoot, partBytes) { const crateDir = path.join(sourceRoot, spec.name); if (existsSync(crateDir)) { @@ -1489,6 +1524,12 @@ function writeExtensionCargoSource(spec, sourceRoot, partBytes) { "[lib]", 'path = "src/lib.rs"', "", + "[dependencies]", + `oliphaunt-resources = { version = "${currentProductVersionSync("oliphaunt-rust", PREFIX)}", path = ${JSON.stringify(path.join(ROOT, "src/sdks/rust/crates/oliphaunt-resources"))} }`, + ...(spec.aotTargets ?? []).flatMap(target => [ + `[target.'${AOT_TARGET_CFGS[target.target]}'.dependencies]`, + `${target.name} = { version = "=${spec.version}", path = "../${target.name}" }`, + ]), "[build-dependencies]", ...partSources.map((part) => `${part.name} = { version = "=${spec.version}", path = "../${part.name}" }`), "", @@ -1516,6 +1557,7 @@ function writeExtensionCargoSource(spec, sourceRoot, partBytes) { "}", "", ].join("\n")); + writeFileSync(path.join(crateDir, "src/lib.rs"), readFileSync(path.join(crateDir, "src/lib.rs"), "utf8") + renderWasixExtensionDescriptors(spec)); writeFileSync(path.join(crateDir, "build.rs"), extensionArtifactBuildRs({ ...spec, target: "portable" }, files, partSources)); return { spec, @@ -1533,9 +1575,15 @@ function writeExtensionAotCargoSource(spec, sourceRoot, partBytes) { } mkdirSync(path.join(crateDir, "src"), { recursive: true }); const artifacts = []; + let combinedManifest; for (const member of spec.members) { const manifestPath = path.join(member.sourceDir, "manifest.json"); const manifest = readJson(manifestPath); + if (combinedManifest === undefined) combinedManifest = { ...manifest, artifacts: [] }; + for (const key of ["target-triple", "engine", "wasmer-version", "wasmer-wasix-version", "source-fingerprint", "postgres-version"]) { + if (combinedManifest[key] !== manifest[key]) fail(`${spec.name} contains incompatible AOT manifests (${key})`); + } + combinedManifest.artifacts.push(...manifest.artifacts); const manifestDestination = path.join(crateDir, "manifests", `${member.sqlName}.json`); mkdirSync(path.dirname(manifestDestination), { recursive: true }); copyFileSync(manifestPath, manifestDestination); @@ -1627,6 +1675,10 @@ function writeExtensionAotCargoSource(spec, sourceRoot, partBytes) { `pub const SQL_NAMES: &[&str] = &[${spec.members.map((member) => JSON.stringify(member.sqlName)).join(", ")}];`, ...(spec.members.length === 1 ? [`pub const SQL_NAME: &str = ${JSON.stringify(spec.members[0].sqlName)};`] : []), `pub const TARGET_TRIPLE: &str = "${spec.target}";`, + `pub const MANIFEST_JSON: &str = ${JSON.stringify(JSON.stringify(combinedManifest))};`, + "pub const AOT_ARTIFACTS: &[(&str, &[u8])] = &[", + ...artifacts.map(artifact => ` (${JSON.stringify(artifact.name)}, include_bytes!(concat!(env!("OUT_DIR"), "/payload/${artifact.payloadRelative}"))),`), + "];", "", "pub fn aot_manifest_json(sql_name: &str) -> Option<&'static str> {", " match sql_name {", @@ -1755,7 +1807,7 @@ function packageSpecs(assetDir, extractRoot, version) { const runtimeExtract = path.join(extractRoot, "runtime-extracted"); extractTarZstd(runtimeArchive, runtimeExtract); const runtimeRoot = targetAssetRoot(runtimeExtract); - validateRuntimePayload(runtimeRoot); + validateRuntimePayload(runtimeRoot, { producerClosure: true }); const [runtimeCoreRoot, toolsRoot] = splitRuntimeToolsPayload(runtimeRoot, extractRoot); validateRuntimePayload(runtimeCoreRoot); validateToolsPayload(toolsRoot); @@ -1785,6 +1837,16 @@ function packageSpecs(assetDir, extractRoot, version) { const icuRoot = canonicalIcuRoot(targetIcuRoot(icuExtract)); validateIcuPayload(icuRoot); const icuPayloadRoot = writeIcuPayloadArchive(icuRoot, path.join(extractRoot, "icu-payload")); + mkdirSync(path.join(icuPayloadRoot, "cluster-seeds"), { recursive: true }); + for (const file of ["icu.tar.zst", "icu.json"]) { + copyFileSync(path.join(runtimeRoot, "cluster-seeds", file), path.join(icuPayloadRoot, "cluster-seeds", file)); + } + const nativeVersion = currentProductVersionSync("liboliphaunt-native", PREFIX); + writeFileSync(path.join(icuPayloadRoot, "native-runtime-version"), `${nativeVersion}\n`); + stageNativeIcuSeeds( + process.env.OLIPHAUNT_NATIVE_RELEASE_ASSET_DIR ?? path.join(ROOT, "target/liboliphaunt/release-assets"), + nativeVersion, path.join(icuPayloadRoot, "native-seeds"), icuRoot, + ); specs.push({ name: ICU_PACKAGE, target: "portable", diff --git a/tools/release/package_oliphaunt_wasix_sdk_crate.mjs b/tools/release/package_oliphaunt_wasix_sdk_crate.mjs index 8115cd40d..d324374e8 100755 --- a/tools/release/package_oliphaunt_wasix_sdk_crate.mjs +++ b/tools/release/package_oliphaunt_wasix_sdk_crate.mjs @@ -140,7 +140,8 @@ export async function prepareOliphauntWasixReleaseSource(version) { 'liboliphaunt-wasix', 'package_oliphaunt_wasix_sdk_crate.mjs', ); - const registryPackages = await wasixCargoRegistryPackages(); + // ICU is an independent descriptor package; the base SDK does not depend on it. + const registryPackages = (await wasixCargoRegistryPackages()).filter(name => name !== 'oliphaunt-icu'); const stageDir = path.join(root, 'target/release/cargo-package-sources/oliphaunt-wasix'); await stageWasixRustPackageSource(stageDir); const cargoToml = path.join(stageDir, 'Cargo.toml'); diff --git a/tools/release/preflight-swiftpm-source-tag.mjs b/tools/release/preflight-swiftpm-source-tag.mjs index c3b7369f5..a71f5d49c 100644 --- a/tools/release/preflight-swiftpm-source-tag.mjs +++ b/tools/release/preflight-swiftpm-source-tag.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env bun +import { appendFileSync } from "node:fs"; import path from "node:path"; import { @@ -6,7 +7,7 @@ import { loadPublicationLock, lockedProductArtifactPaths, } from "./publication-lock.mjs"; -import { ensureTag } from "./publish_swiftpm_source_tag.mjs"; +import { ensureIndependentSwiftpmTag, ensureTag } from "./publish_swiftpm_source_tag.mjs"; function error(message) { return new Error(`preflight-swiftpm-source-tag: ${message}`); @@ -33,6 +34,16 @@ export function parseSwiftpmPreflightArgs(argv) { export async function preflightLockedSwiftpmSourceTag({ lock, releaseCommit, ensureTagImpl = ensureTag }) { assertPublicationLockSource(lock, releaseCommit); + for (const product of lock.products) { + for (const input of lockedProductArtifactPaths(lock, product.id) + .filter(({ artifact }) => artifact.kind === "swiftpm-independent-package")) { + ensureIndependentSwiftpmTag({ + sourceTree: input.path, repository: `f0rr0/${input.artifact.identity}`, + version: product.version, target: releaseCommit, preflight: true, + }); + } + } + if (!lock.products.some(product => product.id === "oliphaunt-swift")) return; const inputs = lockedProductArtifactPaths(lock, "oliphaunt-swift"); const manifests = inputs.filter(({ artifact, type }) => artifact.kind === "swiftpm-release-manifest" && type === "file"); const trees = inputs.filter(({ artifact, type }) => artifact.kind === "swiftpm-release-tree" && type === "directory"); @@ -50,10 +61,16 @@ export async function preflightLockedSwiftpmSourceTag({ lock, releaseCommit, ens if (import.meta.main) { try { const args = parseSwiftpmPreflightArgs(process.argv.slice(2)); + const lock = loadPublicationLock(args.publicationLock); await preflightLockedSwiftpmSourceTag({ - lock: loadPublicationLock(args.publicationLock), + lock, releaseCommit: args.releaseCommit, }); + if (process.env.GITHUB_OUTPUT) { + const repositories = lock.productArtifacts.filter(row => row.kind === "swiftpm-independent-package") + .map(row => row.identity).sort().join(","); + appendFileSync(process.env.GITHUB_OUTPUT, `repositories=${repositories}\n`); + } } catch (cause) { console.error(cause instanceof Error ? cause.message : String(cause)); process.exitCode = 1; diff --git a/tools/release/prepare-rust-release-source.mjs b/tools/release/prepare-rust-release-source.mjs index bc8fc16cf..1f2383b51 100644 --- a/tools/release/prepare-rust-release-source.mjs +++ b/tools/release/prepare-rust-release-source.mjs @@ -112,6 +112,7 @@ function renderReleaseCargoToml(source, nativeVersion, brokerVersion, artifactTa for (const target of artifactTargets.nativeTargets) { const cfg = rustNativeTargetCfg(target); addTargetDependency(cfg, `${liboliphauntCargoPackageName(target.target)} = { version = "=${nativeVersion}" }`); + addTargetDependency(cfg, `oliphaunt-extension-contrib-pg18-${target.target} = { version = "=${nativeVersion}" }`); } for (const target of artifactTargets.brokerTargets) { const cfg = rustNativeTargetCfg(target); @@ -198,9 +199,11 @@ export function prepareRustReleaseSource({ stageDir = DEFAULT_STAGE_DIR, log = t export function prepareOliphauntBuildReleaseSource({ stageDir = DEFAULT_BUILD_STAGE_DIR, log = true, + packageName = "oliphaunt-build", } = {}) { + if (!["oliphaunt-build", "oliphaunt-resources"].includes(packageName)) throw new Error(`Unknown Rust helper ${packageName}`); const version = currentProductVersionSync(RUST_PRODUCT, TOOL); - const sourceDir = path.join(ROOT, "src/sdks/rust/crates/oliphaunt-build"); + const sourceDir = path.join(ROOT, "src/sdks/rust/crates", packageName); const outputDir = releaseStageDir(stageDir); rmSync(outputDir, { recursive: true, force: true }); cpSync(sourceDir, outputDir, { @@ -211,7 +214,7 @@ export function prepareOliphauntBuildReleaseSource({ const rendered = packagedCargoManifestText(readFileSync(cargoToml, "utf8")); writeFileSync(cargoToml, rendered, "utf8"); if (!packageSection(rendered).includes(`version = "${version}"`)) { - fail(`generated oliphaunt-build release source must keep SDK version ${version}`); + fail(`generated ${packageName} release source must keep SDK version ${version}`); } stageReleaseNotices(outputDir, SOURCE_NOTICE_OPTIONS); assertReleaseNoticesInDirectory(outputDir, SOURCE_NOTICE_OPTIONS); diff --git a/tools/release/public-consumer-smoke.mjs b/tools/release/public-consumer-smoke.mjs index c34901f1a..06b1431b9 100644 --- a/tools/release/public-consumer-smoke.mjs +++ b/tools/release/public-consumer-smoke.mjs @@ -20,6 +20,7 @@ import process from "node:process"; import { DEFAULT_PUBLICATION_LOCK, + directoryEnvelope, loadPublicationLock, } from "./publication-lock.mjs"; import { @@ -311,7 +312,14 @@ export function publicConsumerPlan(lock, products, graph, { repositoryUrl: repositoryUrl(repository), products: productRows.map(({ id }) => id).sort(compareText), surfaces, - github: { productTags, swift }, + github: { productTags, swift, swiftPackages: (lock.productArtifacts ?? []) + .filter(row => row.kind === "swiftpm-independent-package") + .map(row => ({ + product: row.product, repository: `${repository.split("/")[0]}/${row.identity}`, + tag: productRows.find(product => product.id === row.product).version, + sha256: row.sha256, size: row.size, + })).sort((a, b) => compareText(a.repository, b.repository)) }, + }; } @@ -1172,13 +1180,39 @@ async function runGithubSurface({ plan, root, deadlineMilliseconds, signal }) { proofScope: "anonymous-source-tag-and-manifest-only", }; } + const swiftPackages = []; + for (const row of plan.github.swiftPackages) { + const packageRoot = path.join(root, row.repository.split("/")[1]); + const packageGit = `${packageRoot}.git`; + await git(["init", "--bare", packageGit], { cwd: root, env, deadlineMilliseconds, signal }); + await git(["--git-dir", packageGit, "fetch", "--no-tags", repositoryUrl(row.repository), + `refs/tags/${row.tag}:refs/tags/${row.tag}`], { cwd: root, env, deadlineMilliseconds, signal }); + mkdirSync(packageRoot); + await git(["--git-dir", packageGit, "--work-tree", packageRoot, "checkout", row.tag, "--", "."], + { cwd: root, env, deadlineMilliseconds, signal }); + const observed = directoryEnvelope(packageRoot); + if (observed.sha256 !== row.sha256 || observed.size !== row.size) { + throw error(`SwiftPM ${row.repository}@${row.tag} source differs from the frozen package`); + } + const manifest = await runBoundedCommand("swift", ["package", "dump-package"], { + cwd: packageRoot, env: sanitizedPublicEnvironment({ + HOME: path.join(root, "swift-home"), + SWIFTPM_MODULECACHE_OVERRIDE: path.join(root, "swift-module-cache"), + CLANG_MODULE_CACHE_PATH: path.join(root, "swift-module-cache"), + }), deadlineMilliseconds, signal, + }); + const description = JSON.parse(manifest.stdout); + if (typeof description.name !== "string" || !description.name) throw error(`invalid SwiftPM package ${row.repository}`); + swiftPackages.push({ ...row, packageName: description.name, proofScope: "anonymous-source-tag-and-manifest-only" }); + } return { surface: "github", mode: "anonymous-public-exact-tag-resolution", repository: plan.repository, productTags: resolvedProductTags, swift, - limitation: plan.github.swift === null + swiftPackages, + limitation: plan.github.swift === null && swiftPackages.length === 0 ? null : "Draft GitHub binaryTarget assets are not anonymously public before promotion; their exact bytes are covered by the bound immutable GitHub receipt, not this source-tag probe.", }; @@ -1269,6 +1303,11 @@ export function validatePublicConsumerEvidence(evidence, lock, plan) { if (plan.github.swift === null ? github?.swift !== null : github?.swift?.tag !== plan.github.swift.tag) { throw error("GitHub public consumer evidence SwiftPM source-tag coverage mismatch"); } + const observedSwiftPackages = (github?.swiftPackages ?? []).map(({ product, repository, tag, sha256, size }) => + ({ product, repository, tag, sha256, size })); + if (stableJson(observedSwiftPackages) !== stableJson(plan.github.swiftPackages)) { + throw error("GitHub public consumer evidence independent SwiftPM coverage mismatch"); + } const withoutDigest = structuredClone(evidence); delete withoutDigest.evidenceDigest; const expectedDigest = sha256Bytes(stableJson(withoutDigest)); diff --git a/tools/release/public-consumer-smoke.test.mjs b/tools/release/public-consumer-smoke.test.mjs index 57170f9a9..2e5c454e7 100644 --- a/tools/release/public-consumer-smoke.test.mjs +++ b/tools/release/public-consumer-smoke.test.mjs @@ -95,6 +95,30 @@ test("supports source-only selections and records the exact Swift source tag sep }); }); +test("independent Swift sources are bound to their owning product version and frozen bytes", () => { + const products = [product("oliphaunt-extension-vector", ["github-release"], "2.3.4")]; + const frozen = lock(products, []); + frozen.productArtifacts = [{ + kind: "swiftpm-independent-package", product: products[0].id, + identity: "oliphaunt-extension-vector", sha256: "c".repeat(64), size: 123, + }]; + const plan = publicConsumerPlan(frozen, [products[0].id], graph(products)); + assert.deepEqual(plan.github.swiftPackages, [{ + product: products[0].id, repository: "f0rr0/oliphaunt-extension-vector", + tag: "2.3.4", sha256: "c".repeat(64), size: 123, + }]); + const evidence = publicConsumerEvidence({ + lock: frozen, plan, registryReceiptSha256: "e".repeat(64), githubReceiptDigest: "f".repeat(64), + surfaces: [{ + surface: "github", mode: "anonymous-public-exact-tag-resolution", + productTags: plan.github.productTags, swift: null, swiftPackages: plan.github.swiftPackages, + }], + }); + assert.equal(validatePublicConsumerEvidence(evidence, frozen, plan), evidence); + evidence.surfaces[0].swiftPackages = []; + assert.throws(() => validatePublicConsumerEvidence(evidence, frozen, plan), /independent SwiftPM coverage/); +}); + test("derives consumer closures from package-manager scopes instead of publication-only dev edges", () => { const products = [product("alpha", ["crates-io"])]; const leaf = carrier("cargo:leaf", "alpha", 0); diff --git a/tools/release/publication-catalog.test.mjs b/tools/release/publication-catalog.test.mjs index a038bfa8c..ae1034422 100644 --- a/tools/release/publication-catalog.test.mjs +++ b/tools/release/publication-catalog.test.mjs @@ -24,9 +24,9 @@ function catalogForArtifactProducts(products) { test("the live publication catalog includes PostGIS and the WASIX Node-API carriers", () => { const catalog = loadPublicationCatalog("publication-catalog.test"); expect(catalog.products).toHaveLength(20); - expect(catalog.carriers).toHaveLength(203); + expect(catalog.carriers).toHaveLength(239); expect(catalog.products.some(({ id }) => id === "oliphaunt-extension-postgis")).toBe(true); - expect(catalog.carriers.filter(({ product }) => product === "oliphaunt-extension-postgis")).toHaveLength(18); + expect(catalog.carriers.filter(({ product }) => product === "oliphaunt-extension-postgis")).toHaveLength(23); expect(catalog.products.some(({ id }) => id === "oliphaunt-wasix-napi")).toBe(true); expect(catalog.carriers.filter(({ product }) => product === "oliphaunt-wasix-napi")).toHaveLength(4); diff --git a/tools/release/publication-lock.mjs b/tools/release/publication-lock.mjs index 5e4fcec75..e41d2324e 100644 --- a/tools/release/publication-lock.mjs +++ b/tools/release/publication-lock.mjs @@ -451,7 +451,7 @@ function mavenManifestArtifacts(file) { return records; } -function directoryEnvelope(directory) { +export function directoryEnvelope(directory) { const files = walkFiles(directory, { ignoreBuildDirectories: true }); const hash = createHash("sha256"); let size = 0; @@ -1587,6 +1587,26 @@ function runtimeOwnedExtensionGithubReleaseArtifacts(files, product) { })); } +export function independentSwiftPackageName(product) { + if (product.id === "liboliphaunt-native") return "oliphaunt-icu"; + return product.kind === "exact-extension-artifact" ? product.id : null; +} + +function independentSwiftReleaseInput(files, product) { + const name = independentSwiftPackageName(product); + if (name === null) return []; + const matches = files.filter(file => file.split(path.sep).join("/") + .endsWith(`/swift-packages/${name}/Package.swift`)); + if (matches.length !== 1) { + throw error(`${product.id} requires exactly one standalone SwiftPM package ${name}, found ${matches.length}`); + } + return [productDirectoryArtifact({ + product: product.id, id: "release-input:swiftpm-independent-package", + role: "release-input", kind: "swiftpm-independent-package", target: "portable", + identity: name, name, directory: path.dirname(matches[0]), + })]; +} + function discoverProductArtifactsForSelection(roots, products, selectedProducts) { const files = [...new Set(roots.flatMap((root) => walkFiles(path.resolve(ROOT, root))))].sort(compareText); const artifacts = []; @@ -1605,6 +1625,7 @@ function discoverProductArtifactsForSelection(roots, products, selectedProducts) artifacts.push(...fixedGithubReleaseArtifacts(files, product)); artifacts.push(...runtimeOwnedExtensionGithubReleaseArtifacts(files, product)); } + artifacts.push(...independentSwiftReleaseInput(files, product)); if (product.id === "oliphaunt-swift") { artifacts.push(...swiftReleaseInputs(files, product, { requireExtensionFixture: hasSelectedExtensionProducts, @@ -1983,6 +2004,16 @@ function validateExtensionProductArtifactInventory(product, artifacts) { } function validateProductArtifactInventory(product, artifacts, { hasSelectedExtensionProducts }) { + const independentPackage = independentSwiftPackageName(product); + if (independentPackage !== null) { + const sources = artifacts.filter(artifact => artifact.id === "release-input:swiftpm-independent-package"); + if (sources.length !== 1 || sources[0].kind !== "swiftpm-independent-package" + || sources[0].identity !== independentPackage || sources[0].name !== independentPackage + || sources[0].role !== "release-input") { + throw error(`${product.id} requires its exact standalone SwiftPM package input`); + } + artifacts = artifacts.filter(artifact => artifact !== sources[0]); + } if (EXTENSION_PRODUCT_KINDS.has(product.kind)) { validateExtensionProductArtifactInventory(product, artifacts); return; diff --git a/tools/release/publication-lock.test.mjs b/tools/release/publication-lock.test.mjs index 2e70a4ade..2d2da7178 100644 --- a/tools/release/publication-lock.test.mjs +++ b/tools/release/publication-lock.test.mjs @@ -11,6 +11,7 @@ import { buildPublicationCandidate, discoverPublicationArtifacts, discoverProductArtifacts, + independentSwiftPackageName, freezePublicationCandidate, lockedCarrierFile, projectInternalDependencyIds, @@ -107,6 +108,7 @@ function selectionNeutralSwiftSourceCarrier(version = "1.2.3") { ["base-xcframework", `liboliphaunt-${version}-apple-spm-xcframework.zip`, "zip", "liboliphaunt.xcframework", "1"], ["runtime-resources", `liboliphaunt-${version}-runtime-resources-ios-datum64.tar.gz`, "tar.gz", "oliphaunt", "2"], ["icu-data", `liboliphaunt-${version}-icu-data.tar.gz`, "tar.gz", ".", "3"], + ["icu-seed", `liboliphaunt-${version}-icu-seed-ios-datum64.tar.gz`, "tar.gz", ".", "4"], ].map(([role, name, format, member, digestDigit], index) => ({ bytes: index + 1, format, @@ -188,7 +190,16 @@ function sha256File(file) { return createHash("sha256").update(readFileSync(file)).digest("hex"); } +function independentSwiftFixture(root, product) { + const name = independentSwiftPackageName(product); + if (name === null) return; + const directory = path.join(root, "swift-packages", name); + mkdirSync(directory, { recursive: true }); + writeFileSync(path.join(directory, "Package.swift"), "// swift-tools-version: 6.0\nimport PackageDescription\n"); +} + function githubReleaseFixture(root, product) { + independentSwiftFixture(root, product); const directory = path.join(root, product.id, "release-assets"); mkdirSync(directory, { recursive: true }); const rows = allArtifactTargets({ @@ -221,6 +232,7 @@ function extensionGithubReleaseFixture( bundleFixedFileMode = 0o644, } = {}, ) { + independentSwiftFixture(root, product); const productRoot = extensionArtifactProductRoot( artifactProduct, family ?? "native", @@ -557,13 +569,13 @@ describe("canonical publication catalog", () => { test("normalizes products and stable carriers without duplicate identities", () => { const catalog = loadPublicationCatalog("publication-lock.test"); expect(catalog.products).toHaveLength(20); - expect(catalog.carriers).toHaveLength(203); + expect(catalog.carriers).toHaveLength(239); expect(catalog.carriers.reduce((counts, { ecosystem }) => ({ ...counts, [ecosystem]: (counts[ecosystem] ?? 0) + 1, - }), {})).toEqual({ cargo: 103, npm: 77, maven: 23 }); + }), {})).toEqual({ cargo: 104, npm: 105, maven: 30 }); expect(catalog.products.some(({ id }) => id === "oliphaunt-extension-postgis")).toBe(true); - expect(catalog.carriers.filter(({ product }) => product === "oliphaunt-extension-postgis")).toHaveLength(18); + expect(catalog.carriers.filter(({ product }) => product === "oliphaunt-extension-postgis")).toHaveLength(23); expect(new Set(catalog.carriers.map((carrier) => carrier.id)).size).toBe(catalog.carriers.length); expect(catalog.carriers.every((carrier) => carrier.declared && carrier.product && carrier.version)).toBe(true); }); @@ -673,6 +685,7 @@ describe("publication artifact discovery and freezing", () => { const catalog = loadPublicationCatalog("publication-lock.test", { products: ["oliphaunt-rust"] }); const version = catalog.products[0].version; cargoFixture(root, "oliphaunt-build", version); + cargoFixture(root, "oliphaunt-resources", version); cargoFixture(root, "oliphaunt", version, { manifestSuffix: [ "", @@ -875,7 +888,7 @@ describe("publication artifact discovery and freezing", () => { const product = loadPublicationCatalog("publication-lock.test", { products: ["oliphaunt-extension-vector"] }).products[0]; const { assets, directory, manifestPath, swiftCarrierName } = extensionGithubReleaseFixture(root, product); const artifacts = discoverProductArtifacts([root], [product]); - expect(artifacts).toHaveLength(assets.length + 4); + expect(artifacts).toHaveLength(assets.length + 5); expect(new Set(artifacts.filter((artifact) => artifact.role === "github-release-asset").map((artifact) => artifact.target))).toEqual( new Set(extensionArtifactTargets({ product: product.id }, "publication-lock.test").map((target) => target.target)), ); @@ -929,7 +942,7 @@ describe("publication artifact discovery and freezing", () => { const { rows: runtimeAssets } = githubReleaseFixture(root, product); const { assets, manifestPath } = extensionGithubReleaseFixture(root, product, fixtureOptions); const artifacts = discoverProductArtifacts([root], [product]); - expect(artifacts).toHaveLength(runtimeAssets.length + assets.length + 4); + expect(artifacts).toHaveLength(runtimeAssets.length + assets.length + 5); expect(assets.every(({ name }) => /^oliphaunt-extension-contrib-pg18-[^-]+/u.test(name))).toBe(true); const candidate = buildPublicationCandidate({ products: [product.id], @@ -1118,12 +1131,12 @@ describe("publication artifact discovery and freezing", () => { expect(extensionProduct).toBeDefined(); const { manifestPath } = extensionGithubReleaseFixture(workspaceRoot, extensionProduct); const extensionRoot = path.dirname(manifestPath); - const selectedRoots = [sdk, fixture, extensionRoot]; + const selectedRoots = [sdk, fixture, extensionRoot, path.join(workspaceRoot, "swift-packages")]; expect(() => discoverProductArtifacts([sdk, fixture], [product])).toThrow( /selects no extension products and requires no frozen Swift consumer fixture/u, ); - expect(() => discoverProductArtifacts([sdk, extensionRoot], catalog.products)).toThrow( + expect(() => discoverProductArtifacts([sdk, extensionRoot, path.join(workspaceRoot, "swift-packages")], catalog.products)).toThrow( /selects extension products and requires exactly one frozen Swift consumer fixture/u, ); diff --git a/tools/release/publish_swiftpm_source_tag.mjs b/tools/release/publish_swiftpm_source_tag.mjs index a84dcf818..6855655bb 100644 --- a/tools/release/publish_swiftpm_source_tag.mjs +++ b/tools/release/publish_swiftpm_source_tag.mjs @@ -495,6 +495,63 @@ export async function ensureTag( return tag; } +/** Publish a complete standalone package without copying the monorepo history. */ +export function ensureIndependentSwiftpmTag({ + sourceTree, repository, version, target = "HEAD", preflight = false, push = false, +}, { + root = ROOT, remote = `https://github.com/${repository}.git`, + environment = process.env, reserveContentWrite = reserveGitHubContentWriteSync, +} = {}) { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository) + || !SEMVER_RE.test(version) || (preflight && push)) { + throw new TypeError("independent SwiftPM publication requires a repository, semantic version, and one mode"); + } + const source = commitForRef(target, root); + const date = commitTimestamp(source, root); + const packageRoot = path.resolve(root, sourceTree); + const manifest = readFileSync(path.join(packageRoot, "Package.swift"), "utf8"); + if (!manifest.startsWith("// swift-tools-version:") || manifest.includes("file://")) { + throw new Error("standalone SwiftPM manifest must declare a tools version and contain no file URLs"); + } + const scratch = mkdtempSync(path.join(tmpdir(), "oliphaunt-swiftpm-package.")); + try { + git(["init", "--quiet"], { root: scratch }); + git(["remote", "add", "origin", remote], { root: scratch }); + const env = { + ...environment, + GIT_AUTHOR_NAME: RELEASE_BOT_NAME, GIT_AUTHOR_EMAIL: RELEASE_BOT_EMAIL, + GIT_COMMITTER_NAME: RELEASE_BOT_NAME, GIT_COMMITTER_EMAIL: RELEASE_BOT_EMAIL, + GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date, + }; + git(["read-tree", "--empty"], { root: scratch, env }); + for (const file of iterTreeFiles(packageRoot)) { + const relative = path.relative(packageRoot, file).split(path.sep).join("/"); + if (relative.split("/").some(part => part === ".git" || part === ".build")) { + throw new Error(`standalone SwiftPM tree contains a forbidden path: ${relative}`); + } + addBlobToIndex(scratch, env, relative, readFileSync(file)); + } + const tree = git(["write-tree"], { root: scratch, env }).stdout; + const tagTarget = git(["commit-tree", tree, "-m", + `Release ${repository} ${version}\n\nSource: ${source}`], { root: scratch, env }).stdout; + const outcome = preflightSwiftpmSourceTagExactly({ + root: scratch, environment, tag: version, tagTarget, + }); + if (push && outcome.state === "absent") { + git(["tag", version, tagTarget], { root: scratch, env }); + pushSwiftpmSourceTagExactly({ + root: scratch, environment, tag: version, tagTarget, reserveContentWrite, + budget: createGitHubOperationBudget({ + defaultWindowMs: SWIFTPM_PUSH_OPERATION_WINDOW_MS, environment, + }), + }); + } + return { ...outcome, repository, source, tree, published: push }; + } finally { + rmSync(scratch, { recursive: true, force: true }); + } +} + if (import.meta.main) { await ensureTag(parseArgs(Bun.argv.slice(2))); } diff --git a/tools/release/publish_swiftpm_source_tag.test.mjs b/tools/release/publish_swiftpm_source_tag.test.mjs index 4f9c8173f..50e555e68 100644 --- a/tools/release/publish_swiftpm_source_tag.test.mjs +++ b/tools/release/publish_swiftpm_source_tag.test.mjs @@ -9,6 +9,7 @@ import { createSwiftpmManifestCommit, createSwiftpmReleaseTree, ensureTag, + ensureIndependentSwiftpmTag, preflightSwiftpmSourceTagExactly, pushSwiftpmSourceTagExactly, SWIFTPM_PUSH_ATTEMPT_TIMEOUT_MS, @@ -329,3 +330,39 @@ test("SwiftPM preflight and push modes are mutually exclusive", async () => { push: true, }, { version: "0.6.0" })).rejects.toThrow(/mutually exclusive/u); }); + + +test("independent SwiftPM versions publish only the exact package tree and reject replacement", () => { + const fixture = mkdtempSync(path.join(tmpdir(), "oliphaunt-swift-independent.")); + const source = path.join(fixture, "source"); + const remote = path.join(fixture, "remote.git"); + const packageTree = path.join(fixture, "package"); + try { + for (const dir of [source, remote, packageTree]) mkdirSync(dir); + git(source, ["init", "--quiet"]); + git(source, ["config", "user.name", "fixture"]); + git(source, ["config", "user.email", "fixture@example.invalid"]); + writeFileSync(path.join(source, "unrelated.txt"), "must not ship"); + git(source, ["add", "."]); + git(source, ["commit", "--quiet", "-m", "source"]); + git(remote, ["init", "--bare", "--quiet"]); + writeFileSync(path.join(packageTree, "Package.swift"), "// swift-tools-version: 6.0\nimport PackageDescription\n"); + writeFileSync(path.join(packageTree, "payload.dat"), "extension bytes"); + const args = { sourceTree: packageTree, repository: "f0rr0/oliphaunt-extension-vector", version: "0.1.0" }; + const options = { root: source, remote, reserveContentWrite: () => {} }; + const pending = ensureIndependentSwiftpmTag({ ...args, preflight: true }, options); + expect(pending.state).toBe("absent"); + expect(git(remote, ["tag", "--list"])).toBe(""); + const pushed = ensureIndependentSwiftpmTag({ ...args, push: true }, options); + expect(pushed.tagTarget).toBe(pending.tagTarget); + expect(git(remote, ["ls-tree", "--name-only", "0.1.0"])).toBe("Package.swift\npayload.dat"); + expect(git(remote, ["rev-list", "--count", "0.1.0"])).toBe("1"); + expect(ensureIndependentSwiftpmTag({ ...args, push: true }, options).state).toBe("exact"); + writeFileSync(path.join(packageTree, "payload.dat"), "new extension bytes"); + expect(() => ensureIndependentSwiftpmTag({ ...args, preflight: true }, options)).toThrow("not expected"); + ensureIndependentSwiftpmTag({ ...args, version: "0.2.0", push: true }, options); + expect(git(remote, ["tag", "--list"])).toBe("0.1.0\n0.2.0"); + } finally { + rmSync(fixture, { recursive: true, force: true }); + } +}); diff --git a/tools/release/release-artifact-targets.mjs b/tools/release/release-artifact-targets.mjs index 381c433f2..0ad7d44d7 100644 --- a/tools/release/release-artifact-targets.mjs +++ b/tools/release/release-artifact-targets.mjs @@ -1,3 +1,4 @@ +import { NATIVE_CLUSTER_SEED_TARGETS } from "./native-cluster-seed-contract.mjs"; import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; @@ -366,6 +367,13 @@ function liboliphauntNativeRows(prefix) { }; rows.push(row); } + for (const target of NATIVE_CLUSTER_SEED_TARGETS) { + rows.push({ + id: `${product}.icu-seed-${target}`, product, kind: "icu-seed", target, + asset: `liboliphaunt-{version}-icu-seed-${target}.tar.gz`, + surfaces: ["github-release"], _source_file: "Moon release metadata", + }); + } rows.push( { id: `${product}.apple-spm-xcframework`, diff --git a/tools/release/release-notices.mjs b/tools/release/release-notices.mjs index cf58e4e40..5d163f81c 100644 --- a/tools/release/release-notices.mjs +++ b/tools/release/release-notices.mjs @@ -31,12 +31,12 @@ export const RELEASE_CARRIER_PROFILES = Object.freeze({ "native-runtime": Object.freeze({ products: Object.freeze(["native"]), components: Object.freeze(["postgresql", "icu"]) }), "native-tools": Object.freeze({ products: Object.freeze(["native"]), components: Object.freeze(["postgresql"]) }), "native-runtime-resources": Object.freeze({ products: Object.freeze(["native"]), components: Object.freeze(["postgresql"]) }), - "native-icu-data": Object.freeze({ products: Object.freeze(["native"]), components: Object.freeze(["icu"]) }), + "native-icu-data": Object.freeze({ products: Object.freeze(["native"]), components: Object.freeze(["postgresql", "icu"]) }), "wasix-runtime": Object.freeze({ products: Object.freeze(["wasix"]), components: Object.freeze(["postgresql", "icu"]) }), "wasix-tools": Object.freeze({ products: Object.freeze(["wasix"]), components: Object.freeze(["postgresql", "icu"]) }), "wasix-aot": Object.freeze({ products: Object.freeze(["wasix"]), components: Object.freeze(["postgresql", "icu"]) }), "wasix-icu-data": Object.freeze({ products: Object.freeze(["wasix"]), components: Object.freeze(["postgresql", "icu"]) }), - "wasix-icu-data-crate": Object.freeze({ products: Object.freeze(["wasix"]), components: Object.freeze(["icu"]) }), + "wasix-icu-data-crate": Object.freeze({ products: Object.freeze(["wasix"]), components: Object.freeze(["postgresql", "icu"]) }), "contrib-native": Object.freeze({ products: Object.freeze([]), components: Object.freeze(["postgresql"]) }), "contrib-native-openssl": Object.freeze({ products: Object.freeze([]), components: Object.freeze(["postgresql", "openssl"]) }), "contrib-wasix": Object.freeze({ products: Object.freeze([]), components: Object.freeze(["postgresql"]) }), diff --git a/tools/release/release-publish.mjs b/tools/release/release-publish.mjs index 9b00a9e68..9b5d3162e 100755 --- a/tools/release/release-publish.mjs +++ b/tools/release/release-publish.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env bun +import { ensureIndependentSwiftpmTag } from "./publish_swiftpm_source_tag.mjs"; import { spawn } from "node:child_process"; import { mkdirSync, @@ -723,6 +724,21 @@ async function publishBootstrapCarrier(carrierId, headRef) { await npmPublishTarball(carrier.name, locked.file, carrier.version); } +function publishIndependentSwiftPackages(products, headRef) { + assertPublicationLockSource(ACTIVE_PUBLICATION_LOCK, headRef); + const lockedProducts = new Map(ACTIVE_PUBLICATION_LOCK.products.map(row => [row.id, row])); + for (const product of products) { + if (!lockedProducts.has(product)) fail(`unlocked SwiftPM package owner: ${product}`); + for (const input of lockedProductArtifactPaths(ACTIVE_PUBLICATION_LOCK, product) + .filter(({ artifact }) => artifact.kind === "swiftpm-independent-package")) { + ensureIndependentSwiftpmTag({ + sourceTree: input.path, repository: `f0rr0/${input.artifact.identity}`, + version: lockedProducts.get(product).version, target: headRef, push: true, + }); + } + } +} + function lockedSwiftSourceInputs(headRef) { const product = "oliphaunt-swift"; assertPublicationLockSource(ACTIVE_PUBLICATION_LOCK, headRef); @@ -1140,6 +1156,14 @@ if (command === "publish" && flagValue(argv.slice(1), "--step") === "github-rele } } +if (command === "publish" && flagValue(argv.slice(1), "--step") === "swift-package-source-tag" + && flagValue(argv.slice(1), "--product") === null) { + const products = parseProductsJson(argv.slice(1)); + if (products === null) fail("standalone SwiftPM publication requires --products-json"); + publishIndependentSwiftPackages(products, flagValue(argv.slice(1), "--head-ref") ?? "HEAD"); + process.exit(0); +} + if (publishProductStep?.product === "oliphaunt-swift" && publishProductStep.step === "github-release") { publishSwiftGithubRelease(publishProductStep.headRef); process.exit(0); diff --git a/tools/release/render_swiftpm_release_package.mjs b/tools/release/render_swiftpm_release_package.mjs index ecaaf92ce..12f50167d 100755 --- a/tools/release/render_swiftpm_release_package.mjs +++ b/tools/release/render_swiftpm_release_package.mjs @@ -1,11 +1,19 @@ #!/usr/bin/env bun +import { stageNativeIcuSeeds } from "./native-icu-seeds.mjs"; import { createHash } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; +import os from "node:os"; import { readPortableArchiveEntries } from "../../src/shared/artifact-packaging/portable-archive.mjs"; import { productCompatibilityVersion } from "./release-graph.mjs"; import { validateNativeIcuDataManifest } from "./native-icu-data-contract.mjs"; +import { buildIosCarrierManifest } from "./ios-carrier-manifest.mjs"; +import { contribCarrierDescriptor, extensionArtifactProductRoot, extensionSqlNames } from "./release-artifact-targets.mjs"; +import { resolveSwiftCarrierSelection } from "../../src/sdks/swift/tools/swift-carrier-resolver.mjs"; +import { validateSelection, writeBundledContrib, renderSwiftTargets } from "../../src/sdks/swift/tools/render-extension-products.mjs"; +import { loadSwiftExtensionInventoryCatalog, validateSwiftExtensionResourceArtifact } from "../../src/sdks/swift/tools/extension-resource-inventory.mjs"; +import { stageReleaseNotices } from "./release-notices.mjs"; const ROOT = path.resolve(import.meta.dir, "../.."); const REPOSITORY = "f0rr0/oliphaunt"; @@ -356,13 +364,38 @@ async function prepareIcuResourceTree(assetDir, version, generatedTree) { } catch (error) { fail(`SwiftPM ICU resource product did not extract its exact data receipt from ${archivePath}: ${error instanceof Error ? error.message : String(error)}`); } + stageNativeIcuSeeds(assetDir, version, path.join(target, "native-seeds"), path.join(target, "share/icu"), ["ios-datum64", "macos-arm64"]); + await fs.writeFile(path.join(target, "sdk-resources.properties"), `schema=oliphaunt-sdk-resources-v1\nicuVersion=${version}\n`); await fs.writeFile( path.join(target, "OliphauntICU.swift"), - "public enum OliphauntICUResources {\n public static let bundled = true\n}\n", + `import Foundation\nimport Oliphaunt\n\npublic enum OliphauntICU {\n public static let descriptor = OliphauntIcuData(version: "${version}", resourceDirectory: Bundle.module.resourceURL!)\n}\n`, "utf8", ); } +/** Build the independently installed ICU SwiftPM package. */ +export async function stageSwiftIcuPackage({ assetDir, version, outputDir, baseSdkVersion }) { + await fs.mkdir(outputDir, { recursive: true }); + await prepareIcuResourceTree(assetDir, version, outputDir); + const manifest = `// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "OliphauntICU", + platforms: [.iOS(.v17), .macOS(.v14)], + products: [.library(name: "OliphauntICU", targets: ["OliphauntICU"])], + dependencies: [.package(url: "https://github.com/f0rr0/oliphaunt.git", from: "${baseSdkVersion}")], + targets: [.target( + name: "OliphauntICU", dependencies: [.product(name: "Oliphaunt", package: "oliphaunt")], + path: "generated/swiftpm/OliphauntICU", + resources: [.copy("share"), .copy("native-seeds"), .copy("manifest.properties"), .copy("sdk-resources.properties")] + )] +) +`; + await fs.writeFile(path.join(outputDir, "Package.swift"), manifest); + stageReleaseNotices(outputDir, { profile: "native-icu-data" }); +} + export async function fetchText(url, { fetchImpl = fetch, timeoutMs = 20_000, @@ -450,7 +483,7 @@ async function resolveChecksum(assetDir, assetBaseUrl, asset, version) { return checksum; } -function renderManifest(assetBaseUrl, liboliphauntVersion, checksum) { +export function renderManifest(assetBaseUrl, liboliphauntVersion, checksum, contrib) { const asset = `liboliphaunt-${liboliphauntVersion}-apple-spm-xcframework.zip`; const url = `${assetBaseUrl.replace(/\/+$/u, "")}/${asset}`; return `// swift-tools-version: 6.0 @@ -460,8 +493,8 @@ import PackageDescription // Generated by tools/release/render_swiftpm_release_package.mjs. // This is the public SwiftPM release manifest. The source package under // src/sdks/swift remains the local development package. -// Exact PostgreSQL extensions are released as separate opt-in extension -// artifacts. The base Swift package must not require or publish extension files. +// PostgreSQL contrib ships with the SDK. External extensions and ICU are +// independently selected package dependencies. let package = Package( name: "Oliphaunt", platforms: [ @@ -471,8 +504,7 @@ let package = Package( products: [ .library(name: "COliphaunt", targets: ["COliphaunt"]), .library(name: "Oliphaunt", targets: ["Oliphaunt"]), - .library(name: "OliphauntExtensionSupport", targets: ["OliphauntExtensionSupport"]), - .library(name: "OliphauntICU", targets: ["OliphauntICU"]) + .library(name: "OliphauntExtensionSupport", targets: ["OliphauntExtensionSupport"]) ], targets: [ .binaryTarget( @@ -488,24 +520,44 @@ let package = Package( ), .target( name: "Oliphaunt", - dependencies: ["COliphaunt"], - path: "src/sdks/swift/Sources/Oliphaunt" + dependencies: ["COliphaunt", ${contrib.dependencies.map(JSON.stringify).join(", ")}], + path: "src/sdks/swift/Sources/Oliphaunt", + resources: [.copy("ContribResources")], + swiftSettings: [.define("OLIPHAUNT_PACKAGED_CONTRIB")] ), .target( name: "OliphauntExtensionSupport", dependencies: ["COliphaunt", "Oliphaunt"], path: "src/sdks/swift/Sources/OliphauntExtensionSupport" ), - .target( - name: "OliphauntICU", - path: "generated/swiftpm/OliphauntICU", - resources: [.copy("share"), .copy("manifest.properties")] - ) + ${renderSwiftTargets(contrib.targets)} ] ) `; } +async function prepareContrib(assetDir, generatedTree, contribManifest) { + if (generatedTree === undefined) fail("--generated-tree is required to package the base contrib distribution"); + const product = contribCarrierDescriptor().artifactProduct; + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "oliphaunt-swift-contrib-")); + try { + const carrier = buildIosCarrierManifest({ baseAssetDir: assetDir, localUrls: true, + extensionManifests: [contribManifest ?? path.join(ROOT, extensionArtifactProductRoot(product), "extension-artifacts.json")] }); + const carrierFile = path.join(scratch, "carrier.json"); + await fs.writeFile(carrierFile, JSON.stringify(carrier)); + const input = await resolveSwiftCarrierSelection({ carrierFile, allowFileUrls: true, localBinaryTargets: true, + cacheDir: path.join(scratch, "cache"), extensions: extensionSqlNames(product), + basePackageVersion: (await fs.readFile(path.join(ROOT, "src/sdks/swift/VERSION"), "utf8")).trim() }); + const selection = validateSelection(input, ROOT, { allowFileUrls: true, localBinaryTargets: true }); + const catalog = await loadSwiftExtensionInventoryCatalog(); + for (const extension of selection.extensions) { + extension.resources = await validateSwiftExtensionResourceArtifact({ extension, canonical: catalog.get(extension.sqlName), + nativeRuntime: selection.nativeRuntime, label: `${extension.sqlName} bundled contrib`, allowMobileCarrierArchives: true }); + } + return await writeBundledContrib(selection, generatedTree); + } finally { await fs.rm(scratch, { recursive: true, force: true }); } +} + function parseArgs(argv) { const usage = "usage: tools/release/render_swiftpm_release_package.mjs [--asset-dir DIR] [--asset-base-url URL] [--output FILE] [--generated-tree DIR]"; @@ -531,7 +583,7 @@ function parseArgs(argv) { } index += 1; } - if (!["--asset-dir", "--asset-base-url", "--output", "--generated-tree"].includes(arg)) { + if (!["--asset-dir", "--asset-base-url", "--output", "--generated-tree", "--contrib-manifest"].includes(arg)) { fail(`unknown argument ${arg}`); } args[arg.slice(2)] = value; @@ -540,6 +592,7 @@ function parseArgs(argv) { assetBaseUrl: args["asset-base-url"], assetDir: args["asset-dir"] ?? "target/liboliphaunt/release-assets", generatedTree: args["generated-tree"], + contribManifest: args["contrib-manifest"], output: args.output, }; } @@ -561,8 +614,8 @@ async function main(argv) { if (generatedTree !== undefined) { await fs.mkdir(generatedTree, { recursive: true }); } - await prepareIcuResourceTree(assetDir, liboliphauntVersion, generatedTree); - const manifest = renderManifest(assetBaseUrl, liboliphauntVersion, checksum); + const contrib = await prepareContrib(assetDir, generatedTree, args.contribManifest); + const manifest = renderManifest(assetBaseUrl, liboliphauntVersion, checksum, contrib); if (args.output) { const output = path.resolve(ROOT, args.output); await fs.mkdir(path.dirname(output), { recursive: true }); diff --git a/tools/release/render_swiftpm_release_package.test.mjs b/tools/release/render_swiftpm_release_package.test.mjs index adbe90477..3b654503b 100644 --- a/tools/release/render_swiftpm_release_package.test.mjs +++ b/tools/release/render_swiftpm_release_package.test.mjs @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; -import { fetchText, missingRequiredAppleArm64Slices } from "./render_swiftpm_release_package.mjs"; +import { fetchText, missingRequiredAppleArm64Slices, renderManifest } from "./render_swiftpm_release_package.mjs"; describe("SwiftPM Apple carrier architecture contract", () => { test("accepts the three published arm64 slices", () => { @@ -46,3 +46,15 @@ describe("SwiftPM remote checksum manifest", () => { })).rejects.toThrow("checksum manifest exceeds 1048576 bytes"); }); }); + + +test("the base Swift manifest owns contrib and leaves ICU in a separate package", () => { + const manifest = renderManifest("https://example.invalid/assets", "0.2.0", "a".repeat(64), { + dependencies: ["COliphauntExtensionHstore"], + targets: [{ kind: "binaryTarget", name: "OliphauntExtensionHstoreBinary", path: "generated/swiftpm/contrib/Artifacts/hstore.xcframework" }], + }); + expect(manifest).toContain('dependencies: ["COliphaunt", "COliphauntExtensionHstore"]'); + expect(manifest).toContain('resources: [.copy("ContribResources")]'); + expect(manifest).toContain('OLIPHAUNT_PACKAGED_CONTRIB'); + expect(manifest).not.toContain('OliphauntICU'); +}); diff --git a/tools/release/sdk-artifacts/rust.mjs b/tools/release/sdk-artifacts/rust.mjs index 9081b5f10..84ecd4afd 100644 --- a/tools/release/sdk-artifacts/rust.mjs +++ b/tools/release/sdk-artifacts/rust.mjs @@ -36,13 +36,15 @@ export function stageArtifacts(artifactRoot, workRoot) { }); copyFileSync(releaseCrate, path.join(artifactRoot, path.basename(releaseCrate))); + for (const packageName of ["oliphaunt-build", "oliphaunt-resources"]) { const buildManifest = prepareOliphauntBuildReleaseSource({ - stageDir: path.join(workRoot, "oliphaunt-build-release-source"), + stageDir: path.join(workRoot, `${packageName}-release-source`), log: false, + packageName, }); const buildCrate = manualCargoPackageSource( buildManifest, - path.join(workRoot, "oliphaunt-build-release-crate"), + path.join(workRoot, `${packageName}-release-crate`), { root: ROOT, fail, rel }, ); requireFile(buildCrate); @@ -51,5 +53,6 @@ export function stageArtifacts(artifactRoot, workRoot) { prefix: path.basename(buildCrate, ".crate"), }); copyFileSync(buildCrate, path.join(artifactRoot, path.basename(buildCrate))); + } copyFileSync(packageListing, path.join(artifactRoot, "cargo-package-files.txt")); } diff --git a/tools/release/sdk-artifacts/swift.mjs b/tools/release/sdk-artifacts/swift.mjs index 172057b20..617710fdc 100644 --- a/tools/release/sdk-artifacts/swift.mjs +++ b/tools/release/sdk-artifacts/swift.mjs @@ -31,7 +31,7 @@ import { const PREFIX = "build-sdk-ci-artifacts.mjs"; -export function stageArtifacts(artifactRoot, workRoot) { +export async function stageArtifacts(artifactRoot, workRoot) { requireCommand("swift"); const swiftSourceArchive = path.join( ROOT, diff --git a/tools/release/source-only-sdk-package.mjs b/tools/release/source-only-sdk-package.mjs index 9f87081ac..7edbb2f1a 100644 --- a/tools/release/source-only-sdk-package.mjs +++ b/tools/release/source-only-sdk-package.mjs @@ -141,6 +141,11 @@ function assertManifestContract(manifest, { name, scripts, optionalDependencyVer if (Object.hasOwn(manifest, "devDependencies")) { throw new Error(`${label} must not publish development-only dependencies`); } + const contribVersion = manifest.oliphaunt?.liboliphauntVersion; + if (!/^\d+[.]\d+[.]\d+$/u.test(contribVersion ?? "") + || manifest.dependencies?.["@oliphaunt/extension-contrib-pg18"] !== contribVersion) { + throw new Error(`${label} must depend on the runtime-compatible contrib distribution`); + } const expectedOptional = exactOptionalDependencies(manifest, optionalDependencyVersions, label); if ( expectedOptional !== undefined @@ -216,6 +221,11 @@ export function prepareSourceOnlyNpmPackage(packageDir, contract) { } manifest.optionalDependencies = exactOptional; } + const contribVersion = manifest.oliphaunt?.liboliphauntVersion; + if (!/^\d+[.]\d+[.]\d+$/u.test(contribVersion ?? "")) { + throw new Error("source SDK must identify its native runtime version for contrib"); + } + manifest.dependencies = { ...manifest.dependencies, "@oliphaunt/extension-contrib-pg18": contribVersion }; delete manifest.devDependencies; writeManifest(packageJsonFile, manifest); stageReleaseNotices(directory, SOURCE_NOTICE_OPTIONS); diff --git a/tools/release/source-only-sdk-package.test.mjs b/tools/release/source-only-sdk-package.test.mjs index 114d079c4..d34ffc784 100644 --- a/tools/release/source-only-sdk-package.test.mjs +++ b/tools/release/source-only-sdk-package.test.mjs @@ -55,6 +55,7 @@ function packageManifest(profile) { Object.keys(profile.optionalDependencyVersions).map((name) => [name, "workspace:*"]), ); } + manifest.oliphaunt = { ...manifest.oliphaunt, liboliphauntVersion: "1.2.0" }; return manifest; } diff --git a/tools/release/stage-react-native-resource-packages.mjs b/tools/release/stage-react-native-resource-packages.mjs new file mode 100644 index 000000000..c5fef17d2 --- /dev/null +++ b/tools/release/stage-react-native-resource-packages.mjs @@ -0,0 +1,82 @@ +#!/usr/bin/env bun +// Source qualification uses the same descriptor package writer as publication. +import { readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { nativeExtensionCarrierLegal, writeExtensionMetaPackage } from "./package-extension-release-carriers.mjs"; +import { currentProductVersionSync, extensionRegistryPackageTargetSets } from "./release-artifact-targets.mjs"; +import { stageLiboliphauntIcuNpmPayload } from "./package-release-carriers.mjs"; + +export function stageReactNativeResourcePackages({ carrier, selected, icu, outputDir, project, workspace, platform = "ios" }) { + if (platform !== "ios" && platform !== "android") throw new Error(`unsupported mobile platform: ${platform}`); + if (platform === "android") { + const metadata = JSON.parse(readFileSync(new URL("../../src/extensions/generated/sdk/extensions.json", import.meta.url), "utf8")); + carrier = { + base: { version: currentProductVersionSync("liboliphaunt-native", "mobile source qualification") }, + extensions: metadata.extensions.map(row => ({ + product: row["artifact-product"], sqlName: row["sql-name"], + version: currentProductVersionSync(row["release-product"], "mobile source qualification"), + })), + }; + } + const wanted = new Set(selected); + const products = new Map(); + for (const row of carrier.extensions) { + if (!wanted.has(row.sqlName) && row.product !== "oliphaunt-extension-contrib-pg18") continue; + const rows = products.get(row.product) ?? []; + rows.push(row); + products.set(row.product, rows); + } + for (const name of wanted) { + if (!carrier.extensions.some(row => row.sqlName === name)) throw new Error(`missing selected carrier: ${name}`); + } + if (!products.has("oliphaunt-extension-contrib-pg18")) throw new Error("source carrier is missing the base contrib bundle"); + const dependencies = {}; + for (const [product, rows] of products) { + // Bundle packages must retain every member, regardless of per-database selection. + const members = carrier.extensions.filter(row => row.product === product); + if (new Set(members.map(row => row.version)).size !== 1) throw new Error(`conflicting versions of ${product}`); + const version = rows[0].version; + const sqlNames = members.map(row => row.sqlName).sort(); + const directory = path.join(outputDir, product); + writeExtensionMetaPackage(directory, { + product, version, members: sqlNames, + targets: extensionRegistryPackageTargetSets(product, "mobile source qualification").npmTargets, + iosCarrier: platform === "android" ? undefined : { + ...carrier, extensions: members, + carriers: carrier.carriers.filter(row => row.product === product), + legal: { ...carrier.legal, extensions: carrier.legal.extensions.filter(row => sqlNames.includes(row.sqlName)) }, + }, + liboliphauntVersion: carrier.base.version, + runtimeBound: product === "oliphaunt-extension-contrib-pg18", + legal: nativeExtensionCarrierLegal(product, sqlNames, { carriesPayload: false }), + }); + dependencies[`@oliphaunt/${product.slice("oliphaunt-".length)}`] = `file:${directory}`; + } + if (icu) dependencies["@oliphaunt/icu"] = `file:${stageLiboliphauntIcuNpmPayload(carrier.base.version, { seedTargets: [`${platform}-datum64`] })}`; + const file = path.join(project, "package.json"); + const manifest = JSON.parse(readFileSync(file, "utf8")); + // Remove resource dependencies from a previous smoke selection in this scratch app. + for (const name of Object.keys(manifest.dependencies ?? {})) { + if (name.startsWith("@oliphaunt/extension-") || name === "@oliphaunt/icu") delete manifest.dependencies[name]; + } + manifest.dependencies = { ...manifest.dependencies, ...dependencies }; + writeFileSync(file, `${JSON.stringify(manifest, null, 2)}\n`); + const settings = Bun.YAML.parse(readFileSync(workspace, "utf8")); + for (const name of Object.keys(settings.overrides ?? {})) { + if (name.startsWith("@oliphaunt/extension-") || name === "@oliphaunt/icu") delete settings.overrides[name]; + } + settings.overrides = { ...settings.overrides, ...dependencies }; + writeFileSync(workspace, Bun.YAML.stringify(settings)); + return dependencies; +} + +if (import.meta.main) { + const [carrierFile, selection, icu, outputDir, project, workspace] = process.argv.slice(2); + if (!workspace) throw new Error("usage: stage-react-native-resource-packages.mjs CARRIER EXTENSIONS ICU OUTPUT PROJECT WORKSPACE_YAML"); + stageReactNativeResourcePackages({ + carrier: carrierFile === "android" ? undefined : JSON.parse(readFileSync(carrierFile, "utf8")), + platform: carrierFile === "android" ? "android" : "ios", + selected: selection.split(",").filter(Boolean), icu: ["1", "true", "yes", "on"].includes(icu.toLowerCase()), + outputDir: path.resolve(outputDir), project: path.resolve(project), workspace: path.resolve(workspace), + }); +} diff --git a/tools/release/stage-react-native-resource-packages.test.mjs b/tools/release/stage-react-native-resource-packages.test.mjs new file mode 100644 index 000000000..9228497d0 --- /dev/null +++ b/tools/release/stage-react-native-resource-packages.test.mjs @@ -0,0 +1,46 @@ +import { test, expect } from "bun:test"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { stageReactNativeResourcePackages } from "./stage-react-native-resource-packages.mjs"; +import { extensionSqlNames } from "./release-artifact-targets.mjs"; + +test("mobile source qualification stages installed package ownership independently of database selection", () => { + const root = mkdtempSync(path.join(os.tmpdir(), "mobile-resource-packages-")); + try { + const contrib = "oliphaunt-extension-contrib-pg18"; + const vector = "oliphaunt-extension-vector"; + const rows = product => extensionSqlNames(product).map(sqlName => ({ product, sqlName, version: "0.2.0" })); + const carrier = { + base: { version: "0.2.0" }, extensions: [...rows(contrib), ...rows(vector)], + carriers: [{ product: contrib }, { product: vector }], + legal: { extensions: [...rows(contrib), ...rows(vector)] }, + }; + const project = path.join(root, "app"); + mkdirSync(project); + const workspace = path.join(root, "pnpm-workspace.yaml"); + writeFileSync(workspace, 'packages: ["app"]\n'); + writeFileSync(path.join(project, "package.json"), JSON.stringify({ dependencies: { react: "19.0.0", "@oliphaunt/extension-pgtap": "0.1.0" } })); + const args = { carrier, selected: ["vector"], icu: false, outputDir: path.join(root, "packages"), project, workspace }; + const dependencies = stageReactNativeResourcePackages(args); + expect(Object.keys(dependencies).sort()).toEqual(["@oliphaunt/extension-contrib-pg18", "@oliphaunt/extension-vector"]); + const manifest = JSON.parse(readFileSync(path.join(project, "package.json"), "utf8")); + expect(manifest.dependencies.react).toBe("19.0.0"); + expect(manifest.dependencies["@oliphaunt/extension-pgtap"]).toBeUndefined(); + expect(Bun.YAML.parse(readFileSync(workspace, "utf8")).overrides).toEqual(dependencies); + const vectorPackage = JSON.parse(readFileSync(path.join(root, "packages", vector, "package.json"), "utf8")); + expect(vectorPackage.oliphaunt.members).toEqual(["vector"]); + expect(vectorPackage.exports["."]["react-native"]).toBeDefined(); + const installed = JSON.parse(readFileSync(path.join(root, "packages", contrib, "package.json"), "utf8")); + expect(installed.oliphaunt.members).toEqual(extensionSqlNames(contrib).sort()); + expect(Object.keys(stageReactNativeResourcePackages({ ...args, selected: [] }))).toEqual(["@oliphaunt/extension-contrib-pg18"]); + stageReactNativeResourcePackages({ ...args, carrier: undefined, platform: "android" }); + const androidPackage = JSON.parse(readFileSync(path.join(root, "packages", vector, "package.json"), "utf8")); + expect(androidPackage.exports["."]["react-native"]).toBeDefined(); + expect(androidPackage.exports["./ios-carriers"]).toBeUndefined(); + expect(() => stageReactNativeResourcePackages({ ...args, selected: ["missing"] })).toThrow("missing selected carrier"); + expect(() => stageReactNativeResourcePackages({ ...args, carrier: { ...carrier, extensions: rows(vector) } })).toThrow("missing the base contrib"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/release/swift-extension-release-consumer-inputs.test.mjs b/tools/release/swift-extension-release-consumer-inputs.test.mjs index 7a8848d4a..7901e0486 100644 --- a/tools/release/swift-extension-release-consumer-inputs.test.mjs +++ b/tools/release/swift-extension-release-consumer-inputs.test.mjs @@ -26,6 +26,7 @@ function sourceCarrier() { asset("base-xcframework", `liboliphaunt-${VERSION}-apple-spm-xcframework.zip`, "zip", "liboliphaunt.xcframework", 1), asset("runtime-resources", `liboliphaunt-${VERSION}-runtime-resources-ios-datum64.tar.gz`, "tar.gz", "oliphaunt", 2), asset("icu-data", `liboliphaunt-${VERSION}-icu-data.tar.gz`, "tar.gz", ".", 3), + asset("icu-seed", `liboliphaunt-${VERSION}-icu-seed-ios-datum64.tar.gz`, "tar.gz", ".", 4), ], product: "liboliphaunt-native", tag: BASE_TAG, diff --git a/tools/release/swift-source-carrier-contract.mjs b/tools/release/swift-source-carrier-contract.mjs index 1f766727a..6710c6fb2 100644 --- a/tools/release/swift-source-carrier-contract.mjs +++ b/tools/release/swift-source-carrier-contract.mjs @@ -140,6 +140,7 @@ export function validateSelectionNeutralSwiftSourceCarrier( name: `liboliphaunt-${base.version}-icu-data.tar.gz`, role: "icu-data", }, + { format: "tar.gz", member: ".", name: `liboliphaunt-${base.version}-icu-seed-ios-datum64.tar.gz`, role: "icu-seed" }, ]; if (!Array.isArray(base.assets) || base.assets.length !== assetContracts.length) { throw error( diff --git a/tools/release/verify-release-commit.mjs b/tools/release/verify-release-commit.mjs index 6c07b898f..0ec8a2cbd 100644 --- a/tools/release/verify-release-commit.mjs +++ b/tools/release/verify-release-commit.mjs @@ -4,11 +4,12 @@ import process from "node:process"; import { electronReleaseDependencies } from "../../examples/tools/example-release-dependencies.mjs"; import { captureCommandOutput } from "../dev/capture-command-output.mjs"; -import { exampleCargoReleaseVersionBindings } from "./example-cargo-policy.mjs"; +import { EXAMPLE_CARGO_POLICIES, exampleCargoReleaseVersionBindings } from "./example-cargo-policy.mjs"; import { nativeToolsOptionalPackageProducts, registryPackageRows, } from "./release-artifact-targets.mjs"; +import { loadPublicationCatalog } from "./publication-catalog.mjs"; import { compatibilityVersionEntries, loadGraph } from "./release-graph.mjs"; import { releaseDerivedPathInventory, @@ -397,6 +398,23 @@ function authorizedDerivedStructuredChange(context, rules) { const rule = rules.structured.get(structuredRuleKey(context.type, context.file, context.parts)); if (rule !== undefined) return productTransition(rule, context.before, context.after, context.transitions); if (context.type !== "toml") return false; + // Historical releases retain the dependency layout that existed in that commit. + if (EXAMPLE_CARGO_POLICIES.some(policy => context.file === `${policy.crateDir}/Cargo.toml`)) { + const parts = context.parts; + const entry = cargoDependencyEntryPath(parts); + if (entry !== undefined) { + const name = entry.at(-1); + const carrier = loadPublicationCatalog(TOOL).carriers.find(row => row.id === `cargo:${name}`); + if (carrier && productTransition({ sourceProduct: carrier.product, wrapped: true }, context.before, context.after, context.transitions)) return true; + } + if (pathKey(parts) === pathKey(["package", "metadata", "oliphaunt", "runtime-version"])) { + const runtimePath = ["package", "metadata", "oliphaunt", "runtime"]; + const product = valueAt(context.beforeRoot, runtimePath); + if (product === "liboliphaunt-native" && valueAt(context.afterRoot, runtimePath) === product) { + return productTransition({ sourceProduct: product, wrapped: false }, context.before, context.after, context.transitions); + } + } + } return cargoDependencyVersionChange(context) || cargoLockVersionChange(context); } diff --git a/tools/release/wasix-extension-features.mjs b/tools/release/wasix-extension-features.mjs index 21575f831..f8f72c465 100755 --- a/tools/release/wasix-extension-features.mjs +++ b/tools/release/wasix-extension-features.mjs @@ -10,6 +10,8 @@ function invariant(condition, message) { if (!condition) throw new Error(`${TOOL}: ${message}`); } +// Source-runtime evidence selects fixture bytes in the internal portable crate. +// Published SDK consumers install independent extension packages instead. export function extensionFeatures(manifest) { invariant(manifest !== null && typeof manifest === "object" && !Array.isArray(manifest), "asset manifest must be an object"); invariant(Array.isArray(manifest.extensions), "asset manifest must contain an extensions array"); @@ -22,7 +24,7 @@ export function extensionFeatures(manifest) { invariant(typeof sqlName === "string" && SQL_NAME_RE.test(sqlName), "extensions must have a portable sql-name"); invariant(!sqlNames.has(sqlName), `asset manifest repeats extension ${sqlName}`); sqlNames.add(sqlName); - features.push(`extension-${sqlName.replaceAll("_", "-")}`); + features.push(`liboliphaunt-wasix-portable/extension-${sqlName.replaceAll("_", "-")}`); } invariant(features.length > 0, "full WASIX evidence requires at least one extension"); diff --git a/tools/release/wasix-extension-features.test.mjs b/tools/release/wasix-extension-features.test.mjs index dc6f9265f..40be7ca18 100644 --- a/tools/release/wasix-extension-features.test.mjs +++ b/tools/release/wasix-extension-features.test.mjs @@ -24,11 +24,11 @@ test("the live WASIX public surface includes the PostGIS product", () => { "src/bindings/wasix-rust/crates/oliphaunt-wasix/Cargo.toml", ]) { const cargo = Bun.TOML.parse(readFileSync(path.join(ROOT, relative), "utf8")); - assert.equal(Object.hasOwn(cargo.features ?? {}, "extension-postgis"), true, relative); + assert.equal(Object.hasOwn(cargo.features ?? {}, "extension-postgis"), relative.includes("/assets/"), relative); } }); -test("full WASIX evidence enables every extension feature", () => { +test("source WASIX evidence enables every internal extension fixture", () => { const manifest = { extensions: [ { "sql-name": "vector" }, @@ -36,10 +36,10 @@ test("full WASIX evidence enables every extension feature", () => { ], }; - assert.deepEqual(extensionFeatures(manifest), ["extension-pg-trgm", "extension-vector"]); + assert.deepEqual(extensionFeatures(manifest), ["liboliphaunt-wasix-portable/extension-pg-trgm", "liboliphaunt-wasix-portable/extension-vector"]); assert.equal( fullEvidenceFeatures(manifest), - "extensions,tools,extension-pg-trgm,extension-vector", + "extensions,tools,liboliphaunt-wasix-portable/extension-pg-trgm,liboliphaunt-wasix-portable/extension-vector", ); }); diff --git a/tools/release/wasix-runtime-npm-carrier.test.mjs b/tools/release/wasix-runtime-npm-carrier.test.mjs index 5ca13cd86..0ce967508 100644 --- a/tools/release/wasix-runtime-npm-carrier.test.mjs +++ b/tools/release/wasix-runtime-npm-carrier.test.mjs @@ -15,7 +15,8 @@ import { afterAll, expect, test } from "bun:test"; import { createDeterministicTar } from "./cargo-source-package.mjs"; import { stageReleaseNotices } from "./release-notices.mjs"; -import { CORE_RUNTIME_ARCHIVE_FILES } from "./wasix-cargo-artifact-contract.mjs"; +import { canonicalWasixAotMetadata } from "./wasix-aot-manifest.mjs"; +import { AOT_TARGET_TRIPLES, CORE_RUNTIME_ARCHIVE_FILES } from "./wasix-cargo-artifact-contract.mjs"; import { packWasixRuntimeNpmCarrier, renderWasixRuntimeDescriptorModule, @@ -262,6 +263,25 @@ console.log(JSON.stringify({ test("packs split pg_dump and psql bytes from the same qualified release archive", () => { const root = temporaryRoot("oliphaunt-wasix-tools-npm-"); const fixture = portableReleaseFixture(root); + const canonical = canonicalWasixAotMetadata(); + for (const [target, triple] of Object.entries(AOT_TARGET_TRIPLES)) { + const stage = path.join(root, `aot-${target}`); + const artifacts = ["runtime", "tool:pg_dump", "tool:psql"].map((name, index) => { + const raw = Buffer.from(`fixture-${name}`); + const bytes = zstdCompressSync(raw); + const member = `${index}.bin.zst`; + writeMember(stage, member, bytes); + return { name, path: member, compressed: true, sha256: sha256(bytes), + "raw-sha256": sha256(raw), "raw-size": raw.length, "module-sha256": "a".repeat(64) }; + }); + writeMember(stage, "manifest.json", JSON.stringify({ + "format-version": 1, "source-lane": canonical.sourceLane, engine: canonical.engine, + "wasmer-version": canonical.wasmerVersion, "wasmer-wasix-version": canonical.wasmerWasixVersion, + "target-triple": triple, artifacts, + })); + writeFileSync(path.join(root, `liboliphaunt-wasix-7.8.9-runtime-aot-${target}.tar.zst`), + zstdCompressSync(deterministicTar(stage, "aot"))); + } const packed = packWasixToolsNpmCarrier({ version: "7.8.9", portableReleaseArchive: fixture.archive, diff --git a/tools/release/wasix-tools-aot-npm.test.mjs b/tools/release/wasix-tools-aot-npm.test.mjs new file mode 100644 index 000000000..cec14e31a --- /dev/null +++ b/tools/release/wasix-tools-aot-npm.test.mjs @@ -0,0 +1,43 @@ +import { test, expect } from 'bun:test'; +import { createHash } from 'node:crypto'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { zstdCompressSync } from 'node:zlib'; +import { createDeterministicTar } from './cargo-source-package.mjs'; +import { canonicalWasixAotMetadata } from './wasix-aot-manifest.mjs'; +import { stageWasixToolsAotNpmCarrier } from './wasix-tools-npm-carrier.mjs'; + +test('the optional tools host package contains only pg_dump and psql from the verified runtime release', () => { + const root = mkdtempSync(path.join(tmpdir(), 'oliphaunt-tools-aot-')); + try { + const source = path.join(root, 'source'); + mkdirSync(source); + const sha = bytes => createHash('sha256').update(bytes).digest('hex'); + const artifacts = ['runtime', 'tool:pg_dump', 'tool:psql'].map((name, index) => { + const raw = Buffer.from(`fixture-${name}`); + const bytes = zstdCompressSync(raw); + const file = `${index}.bin.zst`; + writeFileSync(path.join(source, file), bytes); + return { name, path: file, compressed: true, sha256: sha(bytes), 'raw-sha256': sha(raw), + 'raw-size': raw.length, 'module-sha256': 'a'.repeat(64) }; + }); + const canonical = canonicalWasixAotMetadata(); + writeFileSync(path.join(source, 'manifest.json'), JSON.stringify({ + 'format-version': 1, 'source-lane': canonical.sourceLane, engine: canonical.engine, + 'wasmer-version': canonical.wasmerVersion, 'wasmer-wasix-version': canonical.wasmerWasixVersion, + 'target-triple': 'x86_64-unknown-linux-gnu', artifacts, + })); + const archive = path.join(root, 'release.tar.zst'); + writeFileSync(archive, zstdCompressSync(createDeterministicTar(source, 'aot', { + fail: message => { throw new Error(message); }, fixedFileMode: 0o644, + }))); + const packageDir = path.join(root, 'package'); + const manifest = stageWasixToolsAotNpmCarrier({ version: '0.2.0', target: 'linux-x64-gnu', + packageDir, aotReleaseArchive: archive }); + expect(manifest.name).toBe('@oliphaunt/liboliphaunt-wasix-tools-linux-x64-gnu'); + expect(existsSync(path.join(packageDir, '0.bin.zst'))).toBe(false); + expect(JSON.parse(readFileSync(path.join(packageDir, 'aot-manifest.json'))).artifacts.map(row => row.name)) + .toEqual(['tool:pg_dump', 'tool:psql']); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/release/wasix-tools-npm-carrier.mjs b/tools/release/wasix-tools-npm-carrier.mjs index e540e93c9..34bdc9322 100644 --- a/tools/release/wasix-tools-npm-carrier.mjs +++ b/tools/release/wasix-tools-npm-carrier.mjs @@ -13,6 +13,10 @@ import path from 'node:path'; import { captureCommandOutput } from '../dev/capture-command-output.mjs'; import { validatePortableReleaseAsset } from './check-liboliphaunt-wasix-release-assets.mjs'; +import { assertWasixAotArtifactPayloads } from './check-liboliphaunt-wasix-release-assets.mjs'; +import { assertCanonicalWasixAotManifest } from './wasix-aot-manifest.mjs'; +import { AOT_TARGET_TRIPLES } from './wasix-cargo-artifact-contract.mjs'; +import { npmPlatformConstraints, pnpmPackForNpmPublish } from './package-extension-release-carriers.mjs'; import { NPM_TRUSTED_PUBLISHING_REPOSITORY, validateNpmTrustedPublishingManifest, @@ -118,7 +122,11 @@ export function stageWasixToolsNpmCarrier({ version, portableReleaseArchive, pac target: 'portable', runtimeVersion: version, descriptorSchema: DESCRIPTOR_SCHEMA, + tools: Object.fromEntries(Object.values(tools).map(tool => [tool.name, { + path: `assets/${tool.name}.wasix.wasm`, sha256: tool.sha256, size: tool.size, + }])), }, + optionalDependencies: Object.fromEntries(Object.keys(AOT_TARGET_TRIPLES).map(target => [`${PACKAGE_NAME}-${target}`, version])), publishConfig: { access: 'public', provenance: true }, files: ['README.md', 'index.js', 'index.d.ts', 'assets', ...notices], exports: { @@ -158,9 +166,53 @@ export function packWasixToolsNpmCarrier({ if (typeof filename !== 'string') fail('pnpm pack did not report its filename'); const tarball = path.isAbsolute(filename) ? filename : path.join(output, filename); assertWasixToolsNpmArchive(tarball, staged.descriptor); + for (const target of Object.keys(AOT_TARGET_TRIPLES)) { + const aotDir = `${packageDir}-${target}`; + stageWasixToolsAotNpmCarrier({ + version, target, packageDir: aotDir, + aotReleaseArchive: path.join(path.dirname(portableReleaseArchive), `liboliphaunt-wasix-${version}-runtime-aot-${target}.tar.zst`), + }); + pnpmPackForNpmPublish(aotDir, output); + } return Object.freeze({ ...staged, tarball }); } +export function stageWasixToolsAotNpmCarrier({ version, target, packageDir, aotReleaseArchive }) { + const entries = readPortableArchiveEntries(aotReleaseArchive); + const candidates = [...entries.keys()].filter(member => member.endsWith('/manifest.json')); + if (candidates.length !== 1) fail('tools AOT release must contain exactly one manifest'); + const manifestPath = candidates[0]; + const prefix = path.posix.dirname(manifestPath); + const manifest = JSON.parse(requiredEntry(entries, manifestPath, aotReleaseArchive).toString('utf8')); + assertCanonicalWasixAotManifest(manifest, { expectedTarget: AOT_TARGET_TRIPLES[target] }); + const artifacts = assertWasixAotArtifactPayloads(manifest, { + readArtifact: relative => requiredEntry(entries, `${prefix}/${relative}`, aotReleaseArchive), + }).filter(artifact => ['tool:pg_dump', 'tool:psql'].includes(artifact.name)); + if (artifacts.length !== 2) fail('tools AOT release is missing pg_dump or psql'); + rmSync(packageDir, { recursive: true, force: true }); + mkdirSync(packageDir, { recursive: true }); + for (const artifact of artifacts) { + const output = path.join(packageDir, artifact.path); + mkdirSync(path.dirname(output), { recursive: true }); + writeFileSync(output, requiredEntry(entries, `${prefix}/${artifact.path}`, aotReleaseArchive)); + } + const bytes = Buffer.from(`${JSON.stringify({ ...manifest, artifacts }, null, 2)}\n`); + writeFileSync(path.join(packageDir, 'aot-manifest.json'), bytes); + stageReleaseNotices(packageDir, NOTICE_OPTIONS); + const packageJson = { + name: `${PACKAGE_NAME}-${target}`, version, ...npmPlatformConstraints(target), + license: releaseProfilePackageLicense('wasix-runtime').spdx, + repository: { type: 'git', url: NPM_TRUSTED_PUBLISHING_REPOSITORY }, + publishConfig: { access: 'public', provenance: true }, + oliphaunt: { kind: 'wasix-tools-aot', target, runtimeVersion: version, manifestSha256: sha256(bytes) }, + files: ['aot-manifest.json', ...artifacts.map(artifact => artifact.path), ...releaseNoticeRows(NOTICE_OPTIONS).map(({ member }) => member)], + exports: { './package.json': './package.json' }, + }; + validateNpmTrustedPublishingManifest(packageJson, `${PACKAGE_NAME} tools AOT`); + writeFileSync(path.join(packageDir, 'package.json'), `${JSON.stringify(packageJson, null, 2)}\n`); + return packageJson; +} + export function assertWasixToolsNpmArchive(archive, descriptor) { if (statSync(archive).size > 20 * 1024 * 1024) fail(`${archive} exceeds the package size limit`); assertReleaseNoticesInArchive(archive, { ...NOTICE_OPTIONS, prefix: 'package' }); diff --git a/tools/release/wasix-typescript-package.mjs b/tools/release/wasix-typescript-package.mjs index 410be819c..827f15199 100644 --- a/tools/release/wasix-typescript-package.mjs +++ b/tools/release/wasix-typescript-package.mjs @@ -16,6 +16,7 @@ import { const TOOL = 'wasix-typescript-package.mjs'; const PACKAGE_NAME = '@oliphaunt/wasix-ts'; +const CONTRIB_PACKAGE = '@oliphaunt/extension-contrib-pg18-wasix'; const RUNTIME_PACKAGE = '@oliphaunt/liboliphaunt-wasix'; const FZSTD_PACKAGE = 'fzstd'; const FZSTD_VERSION = '0.1.1'; @@ -58,10 +59,11 @@ export function assertWasixTypescriptManifest(manifest, label = `${PACKAGE_NAME} } const dependencies = manifest.dependencies ?? {}; const optionalDependencies = manifest.optionalDependencies ?? {}; - const expectedDependencies = [FZSTD_PACKAGE, JS_CORE_PACKAGE, RUNTIME_PACKAGE].sort(compareText); + const expectedDependencies = [CONTRIB_PACKAGE, FZSTD_PACKAGE, JS_CORE_PACKAGE, RUNTIME_PACKAGE].sort(compareText); const nativeVersion = manifest.oliphaunt?.wasixNapiVersion; if ( JSON.stringify(sortedKeys(dependencies)) !== JSON.stringify(expectedDependencies) + || dependencies[CONTRIB_PACKAGE] !== dependencies[RUNTIME_PACKAGE] || typeof dependencies[RUNTIME_PACKAGE] !== 'string' || !/^\d+\.\d+\.\d+$/u.test(dependencies[RUNTIME_PACKAGE]) || dependencies[FZSTD_PACKAGE] !== FZSTD_VERSION diff --git a/tools/release/wasix-typescript-package.test.mjs b/tools/release/wasix-typescript-package.test.mjs index b282dd1d6..daca8b53d 100644 --- a/tools/release/wasix-typescript-package.test.mjs +++ b/tools/release/wasix-typescript-package.test.mjs @@ -12,6 +12,7 @@ function manifest() { dependencies: { '@oliphaunt/js-core': '0.0.0', '@oliphaunt/liboliphaunt-wasix': '1.2.3', + '@oliphaunt/extension-contrib-pg18-wasix': '1.2.3', fzstd: '0.1.1', }, bundledDependencies: ['@oliphaunt/js-core'], diff --git a/tools/test/create-liboliphaunt-release-fixture.mjs b/tools/test/create-liboliphaunt-release-fixture.mjs index 1da124ea7..91693ad0d 100644 --- a/tools/test/create-liboliphaunt-release-fixture.mjs +++ b/tools/test/create-liboliphaunt-release-fixture.mjs @@ -5,6 +5,7 @@ import path from 'node:path'; import { releaseNoticeRows } from '../release/release-notices.mjs'; import { + NATIVE_CLUSTER_SEED_TARGETS, logicalTreeSha256, nativeClusterSeedCompatibilityKey, } from '../release/native-cluster-seed-contract.mjs'; @@ -90,7 +91,7 @@ function nativeBinary(target, { provider = false } = {}) { throw new Error(`unsupported liboliphaunt release fixture target ${target}`); } -function nativeRuntimeEntries(target, icuDataTreeSha256) { +function nativeRuntimeEntries(target) { const windows = target === 'windows-x64-msvc'; const suffix = windows ? '.exe' : ''; const moduleSuffix = windows ? '.dll' : target === 'macos-arm64' ? '.dylib' : '.so'; @@ -120,7 +121,6 @@ function nativeRuntimeEntries(target, icuDataTreeSha256) { entries, nativeRuntimeCarrierReceipt(target), nativeClusterSeedEntries('standard', 'cluster-seed', target), - nativeClusterSeedEntries('icu', 'cluster-seed-icu', target, icuDataTreeSha256), ); return entries; } @@ -201,7 +201,7 @@ function nativeRuntimeCarrierReceipt(target, prefix = '') { }; } -function runtimeResourceEntries(target, icuDataTreeSha256) { +function runtimeResourceEntries(target) { const entries = { 'oliphaunt/runtime/files/share/postgresql/README.release-fixture': 'release-shaped runtime fixture\n', @@ -212,7 +212,6 @@ function runtimeResourceEntries(target, icuDataTreeSha256) { }), ...nativeRuntimeCarrierReceipt(target, 'oliphaunt/'), ...nativeClusterSeedEntries('standard', 'oliphaunt/cluster-seed', target), - ...nativeClusterSeedEntries('icu', 'oliphaunt/cluster-seed-icu', target, icuDataTreeSha256), }; entries['oliphaunt/runtime/files/share/postgresql/extension/plpgsql.control'] = "default_version = '1.0'\n"; @@ -255,6 +254,7 @@ function mobileAbiProofEntries(domain) { } function nativeClusterSeedEntries(profile, prefix, target, icuDataTreeSha256 = '') { + prefix = prefix ? `${prefix}/` : ''; const runtimeFeatures = profile === 'icu' ? 'icu' : ''; const manifest = [ 'schema=oliphaunt-runtime-resources-v1', @@ -274,10 +274,10 @@ function nativeClusterSeedEntries(profile, prefix, target, icuDataTreeSha256 = ' '', ].join('\n'); return { - [`${prefix}/manifest.properties`]: manifest, - [`${prefix}/files/PG_VERSION`]: '18\n', - [`${prefix}/files/global/pg_control`]: `${profile}-fixture-control\n`, - [`${prefix}/files/pg_wal/`]: '', + [`${prefix}manifest.properties`]: manifest, + [`${prefix}files/PG_VERSION`]: '18\n', + [`${prefix}files/global/pg_control`]: `${profile}-fixture-control\n`, + [`${prefix}files/pg_wal/`]: '', }; } @@ -418,9 +418,9 @@ async function writeProfiledArchive(output, entries, profile, modes = {}, notice async function writeFixtureAssets(assetDir, version) { await fs.mkdir(assetDir, { recursive: true }); const icu = icuClosure(); - const macosRuntimeResources = runtimeResourceEntries('macos-arm64', icu.icuDataTreeSha256); - const iosRuntimeResources = runtimeResourceEntries('ios-datum64', icu.icuDataTreeSha256); - const androidRuntimeResources = runtimeResourceEntries('android-datum64', icu.icuDataTreeSha256); + const macosRuntimeResources = runtimeResourceEntries('macos-arm64'); + const iosRuntimeResources = runtimeResourceEntries('ios-datum64'); + const androidRuntimeResources = runtimeResourceEntries('android-datum64'); const appleXcframeworkEntries = xcframeworkEntries({ macosRuntimeResources, iosRuntimeResources, @@ -441,13 +441,20 @@ async function writeFixtureAssets(assetDir, version) { icu.entries, 'native-icu-data', ); + for (const target of NATIVE_CLUSTER_SEED_TARGETS) { + await writeProfiledArchive( + path.join(assetDir, `liboliphaunt-${version}-icu-seed-${target}.tar.gz`), + nativeClusterSeedEntries('icu', '', target, icu.icuDataTreeSha256), + 'native-runtime-resources', + ); + } await writeProfiledArchive( path.join(assetDir, `liboliphaunt-${version}-macos-arm64.tar.gz`), { 'lib/liboliphaunt.dylib': nativeBinary('macos-arm64'), 'lib/modules/dict_snowball.dylib': nativeBinary('macos-arm64'), 'lib/modules/plpgsql.dylib': nativeBinary('macos-arm64'), - ...nativeRuntimeEntries('macos-arm64', icu.icuDataTreeSha256), + ...nativeRuntimeEntries('macos-arm64'), }, 'native-runtime', nativeRuntimeModes('macos-arm64'), @@ -464,7 +471,7 @@ async function writeFixtureAssets(assetDir, version) { 'lib/liboliphaunt.so': nativeBinary('linux-x64-gnu'), 'lib/modules/dict_snowball.so': nativeBinary('linux-x64-gnu'), 'lib/modules/plpgsql.so': nativeBinary('linux-x64-gnu'), - ...nativeRuntimeEntries('linux-x64-gnu', icu.icuDataTreeSha256), + ...nativeRuntimeEntries('linux-x64-gnu'), }, 'native-runtime', nativeRuntimeModes('linux-x64-gnu'), @@ -481,7 +488,7 @@ async function writeFixtureAssets(assetDir, version) { 'lib/liboliphaunt.so': nativeBinary('linux-arm64-gnu'), 'lib/modules/dict_snowball.so': nativeBinary('linux-arm64-gnu'), 'lib/modules/plpgsql.so': nativeBinary('linux-arm64-gnu'), - ...nativeRuntimeEntries('linux-arm64-gnu', icu.icuDataTreeSha256), + ...nativeRuntimeEntries('linux-arm64-gnu'), }, 'native-runtime', nativeRuntimeModes('linux-arm64-gnu'), @@ -515,7 +522,7 @@ async function writeFixtureAssets(assetDir, version) { 'lib/oliphaunt.lib': windowsImportLibraryFixture(), 'lib/modules/dict_snowball.dll': nativeBinary('windows-x64-msvc', { provider: true }), 'lib/modules/plpgsql.dll': nativeBinary('windows-x64-msvc', { provider: true }), - ...nativeRuntimeEntries('windows-x64-msvc', icu.icuDataTreeSha256), + ...nativeRuntimeEntries('windows-x64-msvc'), ...windowsIcuRuntimeEntries(), ...windowsVcRuntimeEntries(), }, diff --git a/tools/xtask/src/extension_catalog.rs b/tools/xtask/src/extension_catalog.rs index 44ffc602c..f2f5e500e 100644 --- a/tools/xtask/src/extension_catalog.rs +++ b/tools/xtask/src/extension_catalog.rs @@ -489,7 +489,6 @@ fn write_generated_extension_api(catalog: &ExtensionCatalog) -> Result<()> { for extension in extensions { let prefix = extension.rust_constant.as_str(); let definition_const = format!("DEFINITION_{prefix}"); - let feature = wasix_extension_feature(extension); let dependencies = api_dependencies(extension); let native_support_modules = api_native_support_modules(extension)?; let native_modules = native_support_modules @@ -507,7 +506,7 @@ fn write_generated_extension_api(catalog: &ExtensionCatalog) -> Result<()> { .as_ref() .map(|_| format!("extension:{}", extension.sql_name)); text.push_str(&format!( - "#[cfg(feature = {feature:?})]\nconst {definition_const}: Extension = Extension {{\n sql_name: {:?},\n native_support_modules: &[{native_modules}],\n native_module_file: {},\n aot_name: {},\n dependencies: &{},\n startup_config: &{},\n}};\n\n", + "const {definition_const}: Extension = Extension {{\n sql_name: {:?},\n native_support_modules: &[{native_modules}],\n native_module_file: {},\n aot_name: {},\n dependencies: &{},\n startup_config: &{},\n package: None,\n}};\n\n", extension.sql_name, option_string_literal(extension.native_module_file.as_deref()), option_string_literal(aot_name.as_deref()), @@ -519,22 +518,15 @@ fn write_generated_extension_api(catalog: &ExtensionCatalog) -> Result<()> { text.push_str("impl Extension {\n"); for extension in extensions { let prefix = extension.rust_constant.as_str(); - let feature = wasix_extension_feature(extension); text.push_str(&format!( - " /// Select the `{}` artifact.\n #[cfg(feature = {feature:?})]\n pub const {prefix}: Self = DEFINITION_{prefix};\n", + " /// Select the `{}` artifact.\n pub const {prefix}: Self = DEFINITION_{prefix};\n", extension.sql_name )); } let all = extensions .iter() - .map(|extension| { - format!( - " #[cfg(feature = {:?})]\n Self::{},", - wasix_extension_feature(extension), - extension.rust_constant - ) - }) + .map(|extension| format!(" Self::{},", extension.rust_constant)) .collect::>() .join("\n"); text.push_str(&format!( @@ -545,9 +537,8 @@ fn write_generated_extension_api(catalog: &ExtensionCatalog) -> Result<()> { "\n#[cfg(test)]\npub(super) fn creates_database_object_for_test(extension: Extension) -> bool {\n match extension.sql_name() {\n", ); for extension in extensions { - let feature = wasix_extension_feature(extension); text.push_str(&format!( - " #[cfg(feature = {feature:?})]\n {:?} => {},\n", + " {:?} => {},\n", extension.sql_name, extension.lifecycle.create_extension )); } @@ -557,10 +548,9 @@ fn write_generated_extension_api(catalog: &ExtensionCatalog) -> Result<()> { "\n#[cfg(test)]\npub(super) fn activation_sql_for_test(extension: Extension) -> &'static [&'static str] {\n match extension.sql_name() {\n", ); for extension in extensions { - let feature = wasix_extension_feature(extension); let activation_sql = api_test_activation_sql(extension); text.push_str(&format!( - " #[cfg(feature = {feature:?})]\n {:?} => &{},\n", + " {:?} => &{},\n", extension.sql_name, rust_string_array(&activation_sql) )); @@ -571,64 +561,51 @@ fn write_generated_extension_api(catalog: &ExtensionCatalog) -> Result<()> { "src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/generated_extensions.rs", ); fs::write(path, text).with_context(|| format!("write {}", path.display()))?; - format_rust_source(path) + format_rust_source(path)?; + let mut contrib = + String::from("// @generated by `cargo run -p xtask -- extensions generate`\n\n"); + for extension in extensions + .iter() + .filter(|extension| extension.source_kind == "postgres-contrib") + { + contrib.push_str(&format!( + "/// Select `{}`.\npub const {}: crate::ExtensionDescriptor = crate::ExtensionDescriptor {{ sql_name: {:?}, package: None }};\n", + extension.sql_name, extension.rust_constant, extension.sql_name + )); + } + let contrib_path = Path::new( + "src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/generated_contrib_descriptors.rs", + ); + fs::write(contrib_path, contrib)?; + format_rust_source(contrib_path) } fn validate_wasix_sdk_extension_features(catalog: &ExtensionCatalog) -> Result<()> { let manifest_path = Path::new("src/bindings/wasix-rust/crates/oliphaunt-wasix/Cargo.toml"); - let manifest_text = fs::read_to_string(manifest_path) - .with_context(|| format!("read {}", manifest_path.display()))?; - let manifest: toml::Value = toml::from_str(&manifest_text) - .with_context(|| format!("parse {}", manifest_path.display()))?; - let features = manifest - .get("features") - .and_then(toml::Value::as_table) - .ok_or_else(|| anyhow!("{} is missing [features]", manifest_path.display()))?; - - for extension in &catalog.extensions { - let feature = wasix_extension_feature(extension); - let members = features - .get(&feature) - .and_then(toml::Value::as_array) - .ok_or_else(|| { - anyhow!( - "{} is missing WASIX SDK feature {feature} for extension {}", - manifest_path.display(), - extension.sql_name - ) - })?; - let members = members - .iter() - .filter_map(toml::Value::as_str) - .collect::>(); - ensure!( - members.contains("extensions"), - "WASIX SDK feature {feature} must enable the extensions carrier" - ); - let portable_feature = format!("liboliphaunt-wasix-portable/{feature}"); - ensure!( - members.contains(portable_feature.as_str()), - "WASIX SDK feature {feature} must enable bundled asset feature {portable_feature}" - ); - - for dependency in api_dependencies(extension) { - let dependency_extension = catalog - .extensions - .iter() - .find(|candidate| candidate.sql_name == dependency || candidate.id == dependency) - .ok_or_else(|| { - anyhow!( - "extension {} has unknown WASIX API dependency {dependency}", - extension.sql_name - ) - })?; - let dependency_feature = wasix_extension_feature(dependency_extension); - ensure!( - members.contains(dependency_feature.as_str()), - "WASIX SDK feature {feature} must enable dependency feature {dependency_feature} so Extension::by_sql_name can resolve it" - ); - } - } + let manifest: toml::Value = toml::from_str(&fs::read_to_string(manifest_path)?)?; + let features = manifest["features"] + .as_table() + .context("WASIX SDK features missing")?; + ensure!( + !features.keys().any(|name| name.starts_with("extension-")), + "WASIX extensions must be selected through package descriptors, not SDK features" + ); + let selected = manifest["dependencies"]["liboliphaunt-wasix-portable"]["features"] + .as_array() + .context("WASIX base must include contrib")? + .iter() + .filter_map(toml::Value::as_str) + .collect::>(); + let expected = catalog + .extensions + .iter() + .filter(|extension| extension.source_kind == "postgres-contrib") + .map(wasix_extension_feature) + .collect::>(); + ensure!( + selected == expected.iter().map(String::as_str).collect(), + "WASIX base must include exactly the contrib artifact features" + ); Ok(()) } From d2c4d4da4ad7c87e532321cbb6b14f455b28f309 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 06:11:15 +0000 Subject: [PATCH 02/18] fix(napi): validate base-only release inputs in source checks --- src/runtimes/wasix-napi/moon.yml | 6 +- .../wasix-napi/tools/check-build-inputs.mjs | 17 ++--- .../tools/check-build-inputs.test.mjs | 74 +++++++++++++++++++ 3 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 src/runtimes/wasix-napi/tools/check-build-inputs.test.mjs diff --git a/src/runtimes/wasix-napi/moon.yml b/src/runtimes/wasix-napi/moon.yml index 5da3feeb1..c2c0866cd 100644 --- a/src/runtimes/wasix-napi/moon.yml +++ b/src/runtimes/wasix-napi/moon.yml @@ -51,7 +51,7 @@ tasks: tags: ["quality", "static", "requires-rust"] script: | set -e - node --check tools/smoke-packaged-addon.mjs + for file in tools/*.mjs; do node --check "$file"; done cargo check --manifest-path Cargo.toml --locked --no-default-features --features extensions,tools env: CARGO_TARGET_DIR: "../../../target/moon/oliphaunt-wasix-napi/compile" @@ -64,7 +64,7 @@ tasks: - "Cargo.toml" - "build.rs" - "src/**/*" - - "tools/smoke-packaged-addon.mjs" + - "tools/*.mjs" options: cache: true @@ -72,7 +72,7 @@ tasks: tags: ["quality", "unit", "requires-rust"] script: | set -e - bun test tools/detect-linux-libc.test.mjs tools/portable-command.test.mjs + bun test tools/detect-linux-libc.test.mjs tools/portable-command.test.mjs tools/check-build-inputs.test.mjs cargo test --manifest-path Cargo.toml --locked --no-default-features --features extensions,tools,test-noop --lib env: CARGO_TARGET_DIR: "../../../target/moon/oliphaunt-wasix-napi/unit" diff --git a/src/runtimes/wasix-napi/tools/check-build-inputs.mjs b/src/runtimes/wasix-napi/tools/check-build-inputs.mjs index 33aa99249..d80093e66 100755 --- a/src/runtimes/wasix-napi/tools/check-build-inputs.mjs +++ b/src/runtimes/wasix-napi/tools/check-build-inputs.mjs @@ -156,9 +156,6 @@ function portableInputs(portableRoot) { regularFile(path.join(portableRoot, seedManifest), `${profile} cluster seed manifest`); } - return { name, path: repoPath(file, `portable WASIX ${name} module`), sha256: sha256(file) }; - }); - return { manifest, provenance: { @@ -296,7 +293,7 @@ function extensionInputs(extensionRoot, target, targetTriple, sourceFingerprint) }).sort((left, right) => compareText(left.product, right.product)); } -function buildInventory(options) { +export function buildInventory(options) { const portableRoot = path.resolve(options["portable-root"]); const aotRoot = path.resolve(options["aot-root"]); const extensionRoot = path.resolve(options["extension-root"]); @@ -339,9 +336,11 @@ function main() { console.log(`WASIX Node-API build inputs validated: ${repoPath(destination, "build input inventory")}`); } -try { - main(); -} catch (error) { - console.error(`${PREFIX}: ${error instanceof Error ? error.message : String(error)}`); - process.exitCode = 1; +if (import.meta.main) { + try { + main(); + } catch (error) { + console.error(`${PREFIX}: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + } } diff --git a/src/runtimes/wasix-napi/tools/check-build-inputs.test.mjs b/src/runtimes/wasix-napi/tools/check-build-inputs.test.mjs new file mode 100644 index 000000000..5e63e56a8 --- /dev/null +++ b/src/runtimes/wasix-napi/tools/check-build-inputs.test.mjs @@ -0,0 +1,74 @@ +import { test, expect } from 'bun:test'; +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import path from 'node:path'; +import { buildInventory } from './check-build-inputs.mjs'; +import { canonicalWasixAotMetadata } from '../../../../tools/release/wasix-aot-manifest.mjs'; +import { + contribCarrierDescriptor, extensionArtifactProductRoot, + extensionSqlNames, extensionWasixAotMemberSqlNames, +} from '../../../../tools/release/release-artifact-targets.mjs'; + +const root = path.resolve(import.meta.dirname, '../../../..'); +const digest = bytes => createHash('sha256').update(bytes).digest('hex'); +function write(file, bytes) { + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, bytes); +} +function json(file, value) { write(file, JSON.stringify(value)); } + +test('base addon inputs require core, standard seed, and contrib without optional package payloads', () => { + mkdirSync(path.join(root, 'target'), { recursive: true }); + const scratch = mkdtempSync(path.join(root, 'target/napi-inputs-')); + try { + const portable = path.join(scratch, 'portable'); + const aot = path.join(scratch, 'aot'); + const extensions = path.join(scratch, 'extensions'); + const target = 'linux-x64-gnu'; + const triple = 'x86_64-unknown-linux-gnu'; + const bytes = Buffer.from('qualified fixture bytes'); + const fingerprint = 'fixture-source'; + write(path.join(portable, 'runtime.tar.zst'), bytes); + write(path.join(portable, 'bin/initdb.wasix.wasm'), bytes); + write(path.join(portable, 'cluster-seeds/standard.tar.zst'), bytes); + json(path.join(portable, 'cluster-seeds/standard.json'), { fixture: true }); + json(path.join(portable, 'manifest.json'), { + 'format-version': 2, 'source-fingerprint': fingerprint, + runtime: { archive: 'runtime.tar.zst', sha256: digest(bytes) }, + 'cluster-seeds': { standard: { + archive: 'cluster-seeds/standard.tar.zst', manifest: 'cluster-seeds/standard.json', sha256: digest(bytes), + } }, + }); + const canonical = canonicalWasixAotMetadata(); + const stageAot = (directory, name) => { + write(path.join(directory, 'module.bin.zst'), bytes); + json(path.join(directory, 'manifest.json'), { + 'format-version': 1, 'source-lane': canonical.sourceLane, + engine: canonical.engine, 'wasmer-version': canonical.wasmerVersion, + 'wasmer-wasix-version': canonical.wasmerWasixVersion, + 'target-triple': triple, 'source-fingerprint': fingerprint, + artifacts: [{ name, path: 'module.bin.zst', sha256: digest(bytes) }], + }); + }; + stageAot(path.join(aot, triple), 'runtime:oliphaunt'); + const product = contribCarrierDescriptor('napi inputs test').artifactProduct; + const productRoot = extensionArtifactProductRoot(product, 'wasix', extensions, 'napi inputs test'); + const members = extensionSqlNames(product).map(sqlName => { + write(path.join(productRoot, 'member-assets', sqlName, 'extension.tar.zst'), bytes); + return { sqlName, assets: [{ family: 'wasix', target: 'wasix-portable', kind: 'wasix-runtime', + name: 'extension.tar.zst', sha256: digest(bytes), bytes: bytes.length }] }; + }); + json(path.join(productRoot, 'extension-artifacts.json'), { + schema: 'oliphaunt-extension-ci-artifacts-v2', product, extensions: members, + }); + for (const sqlName of extensionWasixAotMemberSqlNames(product)) { + stageAot(path.join(productRoot, 'wasix-aot', target, sqlName), `extension:${sqlName}`); + } + const options = { target, 'target-triple': triple, 'portable-root': portable, 'aot-root': aot, 'extension-root': extensions }; + const inventory = buildInventory(options); + expect(inventory.inputs.extensionArtifacts.map(row => row.product)).toEqual([product]); + expect(Object.keys(inventory.inputs).sort()).toEqual(['extensionArtifacts', 'portableManifest', 'runtimeAotManifest']); + write(path.join(portable, 'cluster-seeds/standard.tar.zst'), 'tampered'); + expect(() => buildInventory(options)).toThrow('digest mismatch'); + } finally { rmSync(scratch, { recursive: true, force: true }); } +}); From 94c4b725ac589fd682bde1021b26a9b1c8344f35 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 06:27:50 +0000 Subject: [PATCH 03/18] fix(sdk): preserve bootstrap resources and seed-free initialization --- docs/architecture/cluster-seeds-and-icu.md | 37 ++++++++++- .../consumer-dx-release-blueprint.md | 6 ++ .../oliphaunt-wasix/src/oliphaunt/assets.rs | 2 +- .../oliphaunt-wasix/src/oliphaunt/base.rs | 53 +++++++++------- .../oliphaunt-wasix/src/oliphaunt/builder.rs | 4 +- .../oliphaunt-wasix/src/oliphaunt/data_dir.rs | 6 +- .../oliphaunt-wasix/tests/runtime_smoke.rs | 32 ++++++++++ .../liboliphaunt/wasix/crates/assets/build.rs | 5 +- src/runtimes/liboliphaunt/wasix/moon.yml | 6 ++ .../src/__tests__/native-initialize.test.ts | 32 +++++++++- src/sdks/js/src/native/initialize.ts | 22 ++++++- .../OliphauntAndroidRuntimeAssets.kt | 6 ++ .../OliphauntAndroidRuntimeAssetsTest.kt | 4 ++ src/sdks/rust/src/build_resources.rs | 29 ++++++--- src/sdks/rust/src/lib.rs | 2 + src/sdks/rust/src/liboliphaunt/ffi.rs | 42 ++++++------- src/sdks/rust/src/liboliphaunt/root.rs | 1 + .../rust/src/liboliphaunt/root/runtime.rs | 13 ++-- .../src/liboliphaunt/root/runtime/locate.rs | 63 +++++++++++++------ src/sdks/rust/tests/public_api.rs | 38 +++++++++++ .../cluster-seed-contract/contract.json | 28 ++++++++- tools/policy/check-cluster-seed-contract.mjs | 1 + .../check-liboliphaunt-release-assets.mjs | 1 + .../release/native-cluster-seed-contract.mjs | 39 ++++++++++-- .../native-cluster-seed-contract.test.mjs | 20 +++++- tools/release/native-icu-seeds.test.mjs | 8 ++- .../package-extension-cargo-facades.test.mjs | 12 ++-- ...kage-liboliphaunt-cargo-artifacts.test.mjs | 38 ++++------- tools/release/package-release-carriers.mjs | 1 + tools/release/stage-native-cluster-seed.mjs | 2 + .../create-liboliphaunt-release-fixture.mjs | 6 +- tools/xtask/moon.yml | 3 + tools/xtask/src/cluster_seed_runner.rs | 44 +++++++++++++ 33 files changed, 476 insertions(+), 130 deletions(-) diff --git a/docs/architecture/cluster-seeds-and-icu.md b/docs/architecture/cluster-seeds-and-icu.md index 207bbf1ac..845a15571 100644 --- a/docs/architecture/cluster-seeds-and-icu.md +++ b/docs/architecture/cluster-seeds-and-icu.md @@ -1,10 +1,45 @@ # Cluster seeds and ICU -Status: locked architecture and implemented contract, updated 2026-08-24. +Status: current distribution contract, with an optional-download transition +approved below; updated 2026-09-09. This document is the source of truth for preinitialized PostgreSQL clusters, ICU data, their public selection, and their release qualification. +## Optional-download transition + +The intended package contract is: + +- Ordinary installs download neither seeds nor ICU data. New databases run + `initdb`, retaining the caller's storage choice. +- An explicit standard-seed selection accelerates creation. +- One language-native ICU selection resolves both the compatible ICU seed and + common ICU data. Target-specific carriers may implement that selection. +- Existing databases are never reseeded. Adding ICU makes its data available; + importing collations remains an explicit application migration. + +This is not yet the shipped default. The distribution descriptions below remain +the current implementation until these prerequisites are complete: + +1. Native mobile needs seed-free initialization. The C ABI accepts prepared + PGDATA; Swift only runs packaged `initdb` on macOS, and Android requires a + seed. A shared native initialization implementation must be qualified before + mobile seeds can be removed. Calling PostgreSQL bootstrap entry points alone + does not provide the process isolation or state cleanup needed by `initdb`. +2. Browser WASIX needs a seed-free initialization path. Rust WASIX now supports + split `initdb` for both memory and directory storage when no seed is supplied. +3. Standard-seed bytes must leave default runtime archives and Cargo/npm/Maven/ + mobile resources. ICU data and ICU seeds already have optional carriers; + WASIX N-API still embeds the standard seed. Optional dependency declarations + alone do not remove downloads from a default carrier. +4. Separately downloadable standard and ICU seeds need artifact graph rows, + carrier identities, compatibility edges, and publication ordering under + their native/WASIX release owners. ICU selection must resolve the matching + seed plus shared data; a new carrier does not imply a new versioned product. +5. Fresh installed consumers must prove default download exclusion, seeded and + seed-free creation, ICU creation, and adding ICU to an existing database + without losing rows. Retain seed compatibility and incomplete-layout checks. + ## Names The public and manifest vocabulary is deliberately small: diff --git a/docs/maintainers/consumer-dx-release-blueprint.md b/docs/maintainers/consumer-dx-release-blueprint.md index 09907fef0..e3f007545 100644 --- a/docs/maintainers/consumer-dx-release-blueprint.md +++ b/docs/maintainers/consumer-dx-release-blueprint.md @@ -1,5 +1,11 @@ # SDK resource packaging contract +The approved next distribution contract makes standard seeds optional and uses +`initdb` by default. The [cluster-seed transition](../architecture/cluster-seeds-and-icu.md#optional-download-transition) +records the remaining native-mobile/browser initialization and carrier work. +The bundled-standard-seed descriptions below describe the current implementation, +not the target of that rollout. + Applications install a base SDK and explicitly select descriptors when opening a database. The base SDK includes PostgreSQL, supported contrib extensions, and the standard cluster seed. External extensions, ICU data with its matching seed, diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/assets.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/assets.rs index e29d45aed..81df2c3c6 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/assets.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/assets.rs @@ -123,7 +123,7 @@ pub(crate) fn asset_manifest_metadata_for( ) -> Result { let manifest = liboliphaunt_wasix_portable::manifest().context("parse oliphaunt-wasix asset manifest")?; - if liboliphaunt_wasix_portable::HAS_EMBEDDED_ASSETS { + if cluster_seed_manifest(selected_profile).is_some() { let seed = manifest .cluster_seeds .get(selected_profile.as_str()) diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/base.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/base.rs index 56b20c23f..95c6f358a 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/base.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/base.rs @@ -915,6 +915,7 @@ fn publish_cluster_seed_clone(source: &Path, pgdata: &Path) -> Result<()> { let result = (|| -> Result<()> { clone_cluster_seed_dir(source, &staging)?; remove_cluster_seed_runtime_state(&staging)?; + super::data_dir::apply_private_permissions(&staging, 0o700)?; promote_synced_directory(&staging, pgdata, root, "cluster seed")?; Ok(()) })(); @@ -1134,11 +1135,13 @@ fn collect_regular_files( fn validated_embedded_cluster_seed_manifest( profile: CatalogProfile, ) -> Result> { - let Some(seed_manifest) = assets::cluster_seed_manifest(profile) else { - return Ok(None); - }; - let Some(seed_archive) = assets::cluster_seed_archive(profile) else { - return Ok(None); + let (seed_manifest, seed_archive) = match ( + assets::cluster_seed_manifest(profile), + assets::cluster_seed_archive(profile), + ) { + (None, None) => return Ok(None), + (Some(manifest), Some(archive)) => (manifest, archive), + _ => bail!("packaged cluster seed requires both its manifest and archive"), }; let manifest = CLUSTER_SEED_MANIFEST @@ -1819,12 +1822,14 @@ fn prepare_memory_database(plan: DatabasePlan) -> Result { .memory_filesystem() .expect("memory storage has a virtual filesystem"); - let manifest = validated_embedded_cluster_seed_manifest(profile)? - .context("packaged cluster seed is unavailable")?; - ensure_module_matches_seed(&runtime_layout.module_path(), &manifest)?; - let archive = assets::cluster_seed_archive(profile) - .context("packaged cluster seed archive is unavailable")?; - unpack_cluster_seed_archive_virtual(archive, filesystem.as_ref())?; + if let Some(manifest) = validated_embedded_cluster_seed_manifest(profile)? { + ensure_module_matches_seed(&runtime_layout.module_path(), &manifest)?; + let archive = assets::cluster_seed_archive(profile) + .context("packaged cluster seed archive is unavailable")?; + unpack_cluster_seed_archive_virtual(archive, filesystem.as_ref())?; + } else { + PostgresMod::run_split_initdb(&runtime_layout, &pgdata_storage)?; + } remove_virtual_runtime_state(filesystem.as_ref())?; ensure!( @@ -1921,17 +1926,10 @@ fn prepare_pgdata( if try_install_embedded_cluster_seed(paths, &runtime_layout.module_path(), profile)? { return Ok(()); } - if std::env::var("OLIPHAUNT_WASIX_DEVELOPMENT_INITDB").as_deref() == Ok("1") { - PostgresMod::run_split_initdb( - runtime_layout, - &PgDataStorage::host_directory(paths.pgdata.clone()), - )?; - } else { - bail!( - "the selected packaged {} cluster seed is unavailable; published packages do not silently fall back to initdb", - profile.as_str() - ); - } + PostgresMod::run_split_initdb( + runtime_layout, + &PgDataStorage::host_directory(paths.pgdata.clone()), + )?; ensure!( cluster_is_complete(paths), "split WASIX initdb finished but did not create a complete PGDATA cluster at {}", @@ -2720,6 +2718,11 @@ mod tests { fs::write(source.path().join("PG_VERSION"), b"18\n")?; fs::write(source.path().join("global/pg_control"), b"control")?; fs::write(source.path().join("postmaster.pid"), b"stale")?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(source.path(), fs::Permissions::from_mode(0o755))?; + } let parent = TempDir::new()?; let root = parent.path().join("database"); @@ -2733,6 +2736,12 @@ mod tests { assert!(pgdata.join("PG_VERSION").is_file()); assert!(pgdata.join("global/pg_control").is_file()); + assert!(pgdata.join("pg_wal").is_dir()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!(fs::metadata(&pgdata)?.permissions().mode() & 0o777, 0o700); + } assert!(!pgdata.join("postmaster.pid").exists()); assert!(!staging.exists()); Ok(()) diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs index a7c513f7a..ccaac9a75 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs @@ -39,8 +39,8 @@ impl Default for OliphauntBuilder { } impl OliphauntBuilder { - /// Create a builder for a memory database initialized from the packaged - /// cluster seed. + /// Create a builder for a memory database initialized from a packaged + /// cluster seed when available, or with `initdb` otherwise. pub fn new() -> Self { Self::default() } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/data_dir.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/data_dir.rs index 47e6aa6ab..43a85b37c 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/data_dir.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/data_dir.rs @@ -974,15 +974,15 @@ fn archive_entry_plan(entry: &tar::Entry<'_, R>) -> Result Result<()> { +pub(super) fn apply_private_permissions(path: &Path, mode: u32) -> Result<()> { use std::os::unix::fs::PermissionsExt; fs::set_permissions(path, fs::Permissions::from_mode(mode)) - .with_context(|| format!("set restored PGDATA permissions on {}", path.display())) + .with_context(|| format!("set PGDATA permissions on {}", path.display())) } #[cfg(not(unix))] -fn apply_private_permissions(_path: &Path, _mode: u32) -> Result<()> { +pub(super) fn apply_private_permissions(_path: &Path, _mode: u32) -> Result<()> { Ok(()) } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/tests/runtime_smoke.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/tests/runtime_smoke.rs index 37966c7c7..629e00b54 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/tests/runtime_smoke.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/tests/runtime_smoke.rs @@ -26,6 +26,38 @@ fn synthetic_sdk_error() -> oliphaunt_wasix::Error { .expect_err("invalid archive creates a public SDK error") } +#[test] +fn initialization_preserves_storage_and_reopens_existing_rows() -> Result<()> { + let workspace = tempfile::TempDir::new()?; + let root = workspace.path().join("database"); + for storage in [ + DatabaseStorage::Memory, + DatabaseStorage::Directory(root.clone()), + ] { + let mut database = Oliphaunt::builder().storage(storage).open()?; + database.execute("CREATE TABLE initialization_probe(value integer)")?; + database.execute("INSERT INTO initialization_probe VALUES (42)")?; + assert_eq!( + database + .query("SELECT value FROM initialization_probe")? + .get_text(0, "value")?, + Some("42") + ); + database.close()?; + } + let mut reopened = Oliphaunt::builder() + .storage(DatabaseStorage::Directory(root)) + .open()?; + assert_eq!( + reopened + .query("SELECT value FROM initialization_probe")? + .get_text(0, "value")?, + Some("42") + ); + reopened.close()?; + Ok(()) +} + #[test] fn direct_api_query_transaction_persistence_and_backup() -> Result<()> { let workspace = tempfile::TempDir::new()?; diff --git a/src/runtimes/liboliphaunt/wasix/crates/assets/build.rs b/src/runtimes/liboliphaunt/wasix/crates/assets/build.rs index 96e878e32..d623d60e2 100644 --- a/src/runtimes/liboliphaunt/wasix/crates/assets/build.rs +++ b/src/runtimes/liboliphaunt/wasix/crates/assets/build.rs @@ -497,8 +497,9 @@ fn write_generated_assets(out: &Path, asset_dir: &Path, selected_extensions: &[S [("standard", &standard_seed_archive, &standard_seed_manifest)] { assert!( - archive.is_file() && seed_manifest.is_file(), - "generated asset directory {} is missing the required {profile} cluster seed; expected both {} and {}", + (archive.is_file() && seed_manifest.is_file()) + || (!archive.exists() && !seed_manifest.exists()), + "generated asset directory {} has an incomplete {profile} cluster seed; expected both {} and {} or neither", asset_dir.display(), archive.display(), seed_manifest.display() diff --git a/src/runtimes/liboliphaunt/wasix/moon.yml b/src/runtimes/liboliphaunt/wasix/moon.yml index 53ad8bbf0..3f341a4a2 100644 --- a/src/runtimes/liboliphaunt/wasix/moon.yml +++ b/src/runtimes/liboliphaunt/wasix/moon.yml @@ -97,6 +97,7 @@ tasks: - "tools/verify-committed-assets.sh" - "/src/bindings/wasix-rust/crates/oliphaunt-wasix/Cargo.toml" - "/tools/xtask/**/*" + - "/src/shared/cluster-seed-contract/**/*" - "/src/sources/tools/verify-source-tree.py" options: cache: false @@ -129,6 +130,7 @@ tasks: - "!assets/generated/**" - "/src/bindings/wasix-rust/crates/oliphaunt-wasix/Cargo.toml" - "/tools/xtask/**/*" + - "/src/shared/cluster-seed-contract/**/*" - "@group(release-archive-contract)" - "/src/sources/tools/verify-source-tree.py" outputs: @@ -168,6 +170,7 @@ tasks: - "/src/bindings/wasix-rust/crates/oliphaunt-wasix/Cargo.toml" - "/src/runtimes/liboliphaunt/wasix/tools/runtime-preflight.sh" - "/tools/xtask/**/*" + - "/src/shared/cluster-seed-contract/**/*" - "@group(release-archive-contract)" - "/src/sources/tools/verify-source-tree.py" outputs: @@ -201,6 +204,7 @@ tasks: - "/src/runtimes/liboliphaunt/wasix/crates/assets/**/*" - "/src/runtimes/liboliphaunt/wasix/crates/aot/**/*" - "/tools/xtask/**/*" + - "/src/shared/cluster-seed-contract/**/*" - "/tools/release/check-liboliphaunt-wasix-release-assets.mjs" - "@group(release-archive-contract)" - "/src/sources/tools/verify-source-tree.py" @@ -236,6 +240,7 @@ tasks: - "/src/bindings/wasix-rust/**/*" - "/src/runtimes/liboliphaunt/wasix/tools/runtime-preflight.sh" - "/tools/xtask/**/*" + - "/src/shared/cluster-seed-contract/**/*" - "/src/sources/tools/verify-source-tree.py" options: cache: local @@ -271,6 +276,7 @@ tasks: - "/tools/release/wasix-typescript-package.mjs" - "@group(release-archive-contract)" - "/tools/xtask/**/*" + - "/src/shared/cluster-seed-contract/**/*" - "/src/sources/tools/verify-source-tree.py" options: cache: local diff --git a/src/sdks/js/src/__tests__/native-initialize.test.ts b/src/sdks/js/src/__tests__/native-initialize.test.ts index 67758eb3a..d4871415c 100644 --- a/src/sdks/js/src/__tests__/native-initialize.test.ts +++ b/src/sdks/js/src/__tests__/native-initialize.test.ts @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, readdir, rm, symlink, writeFile } from 'node:fs/promises'; +import { chmod, mkdir, mkdtemp, readdir, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'vitest'; @@ -9,6 +9,7 @@ import { requireNativeClusterSeedTarget, } from '../native/cluster-seed.js'; import { + copyNativeClusterSeed, initializeNativePgdata, nativeInitdbArgs, nativePostgresChildEnvironment, @@ -212,3 +213,32 @@ async function writeCompletePgdata(pgdata: string): Promise { await writeFile(join(pgdata, 'PG_VERSION'), '18\n'); await writeFile(join(pgdata, 'global', 'pg_control'), new Uint8Array([1])); } + +test('packaged seed restores empty directories and private permissions without modifying the source', async () => { + const root = await mkdtemp(join(tmpdir(), 'oliphaunt-seed-copy-')); + const seed = join(root, 'seed'); + const staging = join(root, 'pgdata'); + try { + await mkdir(join(seed, 'files'), { recursive: true }); + await chmod(join(seed, 'files'), 0o755); + const inventory = join(seed, 'directories-v1.txt'); + await writeFile(inventory, 'pg_notify\npg_wal/archive_status\n'); + await copyNativeClusterSeed(seed, staging); + assert.deepEqual(await readdir(join(staging, 'pg_notify')), []); + assert.ok((await stat(join(staging, 'pg_wal', 'archive_status'))).isDirectory()); + assert.deepEqual(await readdir(join(seed, 'files')), []); + if (process.platform !== 'win32') assert.equal((await stat(staging)).mode & 0o777, 0o700); + for (const invalid of ['../escape', '/escape', 'pg_wal/../../escape', 'pg_wal\\escape']) { + await writeFile(inventory, `${invalid}\n`); + await rm(staging, { recursive: true }); + await assert.rejects(copyNativeClusterSeed(seed, staging), /unsafe seed directory/u); + } + if (process.platform !== 'win32') { + await symlink(root, join(seed, 'files', 'link')); + await rm(staging, { recursive: true }); + await assert.rejects(copyNativeClusterSeed(seed, staging), /regular file or directory/u); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/src/sdks/js/src/native/initialize.ts b/src/sdks/js/src/native/initialize.ts index 7e2092120..109eacfe3 100644 --- a/src/sdks/js/src/native/initialize.ts +++ b/src/sdks/js/src/native/initialize.ts @@ -1,5 +1,5 @@ import { randomUUID } from 'node:crypto'; -import { cp, lstat, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { chmod, cp, mkdir, lstat, readFile, rename, rm, writeFile } from 'node:fs/promises'; import { platform } from 'node:os'; import { basename, dirname, join } from 'node:path'; @@ -123,7 +123,27 @@ export async function copyNativeClusterSeed( await cp(join(clusterSeedDirectory, 'files'), stagingPgdata, { errorOnExist: true, recursive: true, + async filter(source) { + const metadata = await lstat(source); + if (!metadata.isFile() && !metadata.isDirectory()) { + throw new Error(`cluster seed member must be a regular file or directory: ${source}`); + } + return true; + }, }); + if (platform() !== 'win32') await chmod(stagingPgdata, 0o700); + const directories = await readFile(join(clusterSeedDirectory, 'directories-v1.txt'), 'utf8'); + if (!directories.endsWith('\n')) throw new Error('invalid seed directory inventory'); + for (const relative of directories.split(/\r?\n/u).slice(0, -1)) { + if ( + relative + .split('/') + .some((part) => !/^[A-Za-z0-9_.-]+$/u.test(part) || part === '.' || part === '..') + ) { + throw new Error(`unsafe seed directory path: ${JSON.stringify(relative)}`); + } + await mkdir(join(stagingPgdata, relative), { recursive: true }); + } await normalizeNativeClusterSeedForHost(stagingPgdata); } diff --git a/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssets.kt b/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssets.kt index 34cf5148d..cb598f51d 100644 --- a/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssets.kt +++ b/src/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssets.kt @@ -422,6 +422,12 @@ internal object OliphauntAndroidRuntimeAssets { ): AndroidPgdataPublication { validateCompleteAndroidPgdata(staging) if (isCompleteAndroidPgdata(destination)) return AndroidPgdataPublication.Existing + if (!staging.setReadable(false, false) || !staging.setWritable(false, false) || + !staging.setExecutable(false, false) || !staging.setReadable(true, true) || + !staging.setWritable(true, true) || !staging.setExecutable(true, true) + ) { + throw OliphauntException("failed to set private PGDATA permissions: ${staging.absolutePath}") + } syncPublicationTree(staging) if (destination.exists()) { diff --git a/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssetsTest.kt b/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssetsTest.kt index d386e0fb0..81bb2dd65 100644 --- a/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssetsTest.kt +++ b/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssetsTest.kt @@ -511,6 +511,10 @@ class OliphauntAndroidRuntimeAssetsTest { assertTrue(didPublish) assertFalse(staging.exists()) validateCompleteAndroidPgdata(destination) + assertEquals( + java.nio.file.attribute.PosixFilePermissions.fromString("rwx------"), + Files.getPosixFilePermissions(destination.toPath()), + ) } finally { parent.deleteRecursively() } diff --git a/src/sdks/rust/src/build_resources.rs b/src/sdks/rust/src/build_resources.rs index 01c9943f6..69d037016 100644 --- a/src/sdks/rust/src/build_resources.rs +++ b/src/sdks/rust/src/build_resources.rs @@ -226,8 +226,8 @@ pub(crate) fn materialize_embedded_resources(files: &[EmbeddedResource]) -> Resu /// Advanced preassembled deployments call [`crate::register_build_resources!`] /// after their `build.rs` has called `oliphaunt_build::configure()`. Ordinary /// applications use automatically embedded resources. The native -/// runtime locator uses this directory before falling back to explicit -/// environment variables and source-tree build layouts. +/// runtime locator uses this directory before source-tree layouts. Explicit +/// library and install-directory environment overrides take precedence. pub fn register_build_resources_dir(path: impl Into) -> Result<()> { let path = path.into(); if path.as_os_str().is_empty() { @@ -260,6 +260,23 @@ pub(crate) fn registered_build_resources_dir() -> Option { .and_then(|lock| lock.read().ok().and_then(|guard| guard.clone())) } +pub(crate) fn resources_dir_candidates() -> Vec { + registered_build_resources_dir() + .into_iter() + .chain(std::env::var_os("OLIPHAUNT_RESOURCES_DIR").map(PathBuf::from)) + .chain(embedded_base_resources_dir()) + .collect() +} + +/// Implementation of the exported registration macro. +#[doc(hidden)] +pub fn __register_build_resources(path: Option<&str>) -> Result<()> { + let path = path.ok_or_else(|| Error::InvalidConfig( + "OLIPHAUNT_RESOURCES_DIR was not emitted for this package; add oliphaunt-build as a build dependency and call oliphaunt_build::configure() from build.rs".to_owned(), + ))?; + register_build_resources_dir(path) +} + /// Register the resources staged by `oliphaunt-build` for the current package. /// /// The macro expands in the application crate, so it can read the @@ -268,13 +285,7 @@ pub(crate) fn registered_build_resources_dir() -> Option { #[macro_export] macro_rules! register_build_resources { () => { - match option_env!("OLIPHAUNT_RESOURCES_DIR") { - Some(path) => $crate::register_build_resources_dir(path), - None => Err($crate::Error::InvalidConfig( - "OLIPHAUNT_RESOURCES_DIR was not emitted for this package; add oliphaunt-build as a build dependency and call oliphaunt_build::configure() from build.rs" - .to_owned(), - )), - } + $crate::__register_build_resources(option_env!("OLIPHAUNT_RESOURCES_DIR")) }; } diff --git a/src/sdks/rust/src/lib.rs b/src/sdks/rust/src/lib.rs index feb0c7837..cc18113c4 100644 --- a/src/sdks/rust/src/lib.rs +++ b/src/sdks/rust/src/lib.rs @@ -61,6 +61,8 @@ mod session; mod storage; #[cfg(test)] mod test_fixtures; +#[doc(hidden)] +pub use build_resources::__register_build_resources; pub use build_resources::{EmbeddedResource, register_build_resources_dir}; pub use builder::{AsyncOliphauntBuilder, AsyncOliphauntServerBuilder}; pub use config::ServerListen; diff --git a/src/sdks/rust/src/liboliphaunt/ffi.rs b/src/sdks/rust/src/liboliphaunt/ffi.rs index 94ee3e727..21629038e 100644 --- a/src/sdks/rust/src/liboliphaunt/ffi.rs +++ b/src/sdks/rust/src/liboliphaunt/ffi.rs @@ -200,37 +200,31 @@ fn decode_error_text(length: usize, message: &[c_char]) -> Option { } fn resolve_library_path() -> Result { - resolve_library_path_candidates() - .into_iter() - .next() + if let Some(path) = std::env::var_os(ENV_OLIPHAUNT) { + return Ok(PathBuf::from(path)); + } + let candidates = resolve_library_path_candidates(); + candidates.iter().find(|path| path.is_file()).cloned() .ok_or_else(|| { Error::Engine(format!( - "{ENV_OLIPHAUNT} is not set; set it to a native liboliphaunt dynamic library" + "could not locate native liboliphaunt; register build resources or set {ENV_OLIPHAUNT}; searched {candidates:?}" )) }) } pub(super) fn resolve_library_path_candidates() -> Vec { - let mut candidates = env_path_candidates([ENV_OLIPHAUNT]); - if let Some(root) = crate::build_resources::registered_build_resources_dir() - .or_else(|| std::env::var_os("OLIPHAUNT_RESOURCES_DIR").map(PathBuf::from)) - .or_else(crate::build_resources::embedded_base_resources_dir) - { - let payload = root.join("native-runtime/liboliphaunt-native"); - let path = if cfg!(windows) { - payload.join("bin/oliphaunt.dll") - } else { - payload.join(format!( - "lib/{}oliphaunt{}", - std::env::consts::DLL_PREFIX, - std::env::consts::DLL_SUFFIX - )) - }; - if path.is_file() { - candidates.push(path); - } - } - candidates + env_path_candidates([ENV_OLIPHAUNT]) + .into_iter() + .chain( + crate::build_resources::resources_dir_candidates() + .into_iter() + .map(|root| { + root.join("native-runtime/liboliphaunt-native") + .join(if cfg!(windows) { "bin" } else { "lib" }) + .join(libloading::library_filename("oliphaunt")) + }), + ) + .collect() } pub(super) fn env_path_candidates(names: [&str; N]) -> Vec { diff --git a/src/sdks/rust/src/liboliphaunt/root.rs b/src/sdks/rust/src/liboliphaunt/root.rs index f05e707f8..1a72c2d9b 100644 --- a/src/sdks/rust/src/liboliphaunt/root.rs +++ b/src/sdks/rust/src/liboliphaunt/root.rs @@ -117,6 +117,7 @@ impl PreparedNativeRoot { None, config.resource_directory.as_deref(), Some(config.icu), + !initialized, )?; let runtime_dir = runtime_closure.runtime_dir; let mut pgdata_cleanup = CreatedPgdataCleanup::new(); diff --git a/src/sdks/rust/src/liboliphaunt/root/runtime.rs b/src/sdks/rust/src/liboliphaunt/root/runtime.rs index 8983be6a7..d25682d38 100644 --- a/src/sdks/rust/src/liboliphaunt/root/runtime.rs +++ b/src/sdks/rust/src/liboliphaunt/root/runtime.rs @@ -47,6 +47,7 @@ pub(super) fn resolve_runtime_closure( requested_catalog_profile, None, None, + false, ) } @@ -56,6 +57,7 @@ pub(super) fn resolve_runtime_closure_with_resources( requested_catalog_profile: Option, selected_resources: Option<&Path>, selected_icu: Option, + needs_initialization: bool, ) -> Result { let install_dir = locate_native_install_dir()?; let package_resources_root = package_resources_root_for_install(&install_dir); @@ -94,17 +96,20 @@ pub(super) fn resolve_runtime_closure_with_resources( )?; let package_closure_root = package_resources_root; // Packaging materializes the seeds after resolving the runtime closure. Only an - // ordinary SDK open consumes a seed that already belongs to a released carrier. - let cluster_seed = if requested_catalog_profile.is_none() { + // new SDK database consumes a seed; existing roots do not require seed resources. + let cluster_seed = if needs_initialization { package_closure_root .as_deref() - .map(|resources_root| locate_native_cluster_seed(resources_root, catalog_profile)) + .map(|resources_root| { + locate_native_cluster_seed(resources_root, catalog_profile, selected_resources) + }) .transpose()? .flatten() } else { None }; - if requested_catalog_profile.is_none() + if needs_initialization + && catalog_profile == NativeCatalogProfile::Icu && package_closure_root.is_some() && cluster_seed.is_none() { diff --git a/src/sdks/rust/src/liboliphaunt/root/runtime/locate.rs b/src/sdks/rust/src/liboliphaunt/root/runtime/locate.rs index da7e7bef4..9115b2030 100644 --- a/src/sdks/rust/src/liboliphaunt/root/runtime/locate.rs +++ b/src/sdks/rust/src/liboliphaunt/root/runtime/locate.rs @@ -4,11 +4,9 @@ use super::super::super::ffi::{ ENV_EMBEDDED_MODULE_DIR, ENV_INITDB, ENV_INSTALL_DIR, ENV_POSTGRES, env_path_candidates, resolve_library_path_candidates, }; -use crate::build_resources::registered_build_resources_dir; +use crate::build_resources::resources_dir_candidates; use crate::error::{Error, Result}; -const ENV_RESOURCES_DIR: &str = "OLIPHAUNT_RESOURCES_DIR"; - #[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct LocatedIcuData { pub(super) directory: PathBuf, @@ -136,20 +134,6 @@ fn native_tool_is_file(path: &Path, tool: &str) -> bool { path.join("bin").join(tool).is_file() || path.join("bin").join(format!("{tool}.exe")).is_file() } -pub(super) fn resources_dir_candidates() -> Vec { - let mut candidates = Vec::new(); - if let Some(path) = registered_build_resources_dir() { - candidates.push(path); - } - if let Some(path) = std::env::var_os(ENV_RESOURCES_DIR) { - candidates.push(PathBuf::from(path)); - } - if let Some(path) = crate::build_resources::embedded_base_resources_dir() { - candidates.push(path); - } - candidates -} - pub(super) fn locate_native_icu_data( selected_resources: Option<&Path>, ) -> Result> { @@ -186,12 +170,14 @@ pub(super) fn locate_native_icu_data( pub(super) fn locate_native_cluster_seed( resources_dir: &Path, profile: super::super::NativeCatalogProfile, + selected_resources: Option<&Path>, ) -> Result> { let payload = resources_dir.join("native-runtime/liboliphaunt-native"); let carrier_target = read_native_runtime_carrier(&payload.join("manifest.properties"))?; let directory = match profile { super::super::NativeCatalogProfile::Standard => payload.join("cluster-seed"), - super::super::NativeCatalogProfile::Icu => resources_dir + super::super::NativeCatalogProfile::Icu => selected_resources + .unwrap_or(resources_dir) .join("icu-data/oliphaunt-icu/native-seeds") .join(&carrier_target), }; @@ -529,6 +515,47 @@ mod tests { assert_eq!(located, modules_dir); } + #[test] + fn selected_icu_seed_uses_its_own_resource_directory() { + let temp = TempTree::new("selected-icu-seed"); + let base = temp.path().join("base"); + let selected = temp.path().join("selected"); + let payload = base.join("native-runtime/liboliphaunt-native"); + fs::create_dir_all(&payload).unwrap(); + let target = native_host_target_id().unwrap(); + write_runtime_carrier_receipt( + &payload.join("manifest.properties"), + target, + "cluster-seed", + "cluster-seed-icu", + "", + ); + let directory = selected + .join("icu-data/oliphaunt-icu/native-seeds") + .join(target); + write_cluster_seed_fixture(&directory, "native-icu.valid.properties"); + let manifest = directory.join("manifest.properties"); + let text = fs::read_to_string(&manifest) + .unwrap() + .replace("linux-x64-gnu", target); + fs::write(manifest, text).unwrap(); + assert!( + locate_native_cluster_seed(&base, NativeCatalogProfile::Icu, None) + .unwrap() + .is_none() + ); + let found = locate_native_cluster_seed(&base, NativeCatalogProfile::Icu, Some(&selected)) + .unwrap() + .unwrap(); + assert_eq!(found.directory, directory); + assert_eq!(found.target, target); + assert!( + locate_native_cluster_seed(&base, NativeCatalogProfile::Standard, Some(&selected)) + .unwrap() + .is_none() + ); + } + #[test] fn cluster_seed_manifest_accepts_shared_standard_and_icu_fixtures() { let temp = TempTree::new("target-qualified-seed"); diff --git a/src/sdks/rust/tests/public_api.rs b/src/sdks/rust/tests/public_api.rs index e472eb9cb..8e56d05ac 100644 --- a/src/sdks/rust/tests/public_api.rs +++ b/src/sdks/rust/tests/public_api.rs @@ -406,3 +406,41 @@ fn extension_catalog_is_exact_and_sorted() { ); } } + +#[test] +fn build_resources_macro_and_library_discovery_work_outside_the_sdk() { + let result = oliphaunt::register_build_resources!(); + match option_env!("OLIPHAUNT_RESOURCES_DIR") { + Some(path) if !path.is_empty() => result.unwrap(), + _ => assert_eq!(result.unwrap_err().kind(), ErrorKind::InvalidConfiguration), + } + if option_env!("OLIPHAUNT_RESOURCES_DIR").is_none() + && std::env::var_os("LIBOLIPHAUNT_PATH").is_none() + { + let root = std::env::temp_dir().join(format!( + "oliphaunt-library-discovery-{}", + std::process::id() + )); + std::fs::create_dir(&root).unwrap(); + let library = + root.join("native-runtime/liboliphaunt-native") + .join(if cfg!(target_os = "windows") { + "bin/oliphaunt.dll" + } else if cfg!(target_os = "macos") { + "lib/liboliphaunt.dylib" + } else { + "lib/liboliphaunt.so" + }); + std::fs::create_dir_all(library.parent().unwrap()).unwrap(); + std::fs::write(&library, "deliberately invalid dynamic library").unwrap(); + oliphaunt::register_build_resources_dir(&root).unwrap(); + let error = Oliphaunt::open() + .err() + .expect("invalid library must fail to load"); + std::fs::remove_dir_all(root).unwrap(); + assert!( + error.to_string().contains(&library.display().to_string()), + "{error}" + ); + } +} diff --git a/src/shared/cluster-seed-contract/contract.json b/src/shared/cluster-seed-contract/contract.json index 437d51942..64e55dd22 100644 --- a/src/shared/cluster-seed-contract/contract.json +++ b/src/shared/cluster-seed-contract/contract.json @@ -50,5 +50,31 @@ "physicalFormats": { "native": "native-pg18-v1", "wasix": "wasix-pg18-v1" - } + }, + "pgdataDirectories": [ + "global", + "pg_wal", + "pg_wal/archive_status", + "pg_wal/summaries", + "pg_commit_ts", + "pg_dynshmem", + "pg_notify", + "pg_serial", + "pg_snapshots", + "pg_subtrans", + "pg_twophase", + "pg_multixact", + "pg_multixact/members", + "pg_multixact/offsets", + "base", + "base/1", + "pg_replslot", + "pg_tblspc", + "pg_stat", + "pg_stat_tmp", + "pg_xact", + "pg_logical", + "pg_logical/snapshots", + "pg_logical/mappings" + ] } diff --git a/tools/policy/check-cluster-seed-contract.mjs b/tools/policy/check-cluster-seed-contract.mjs index 779d50b9d..e065f3a5a 100644 --- a/tools/policy/check-cluster-seed-contract.mjs +++ b/tools/policy/check-cluster-seed-contract.mjs @@ -16,6 +16,7 @@ assertExactKeys(contract, [ "icu", "icuDataSchema", "manifests", + "pgdataDirectories", "physicalFormats", "profiles", "schema", diff --git a/tools/release/check-liboliphaunt-release-assets.mjs b/tools/release/check-liboliphaunt-release-assets.mjs index 6ad9a0136..78c455541 100644 --- a/tools/release/check-liboliphaunt-release-assets.mjs +++ b/tools/release/check-liboliphaunt-release-assets.mjs @@ -322,6 +322,7 @@ function validateNativeRuntimeCarrierEntries( for (const required of [ "manifest.properties", "cluster-seed/manifest.properties", + "cluster-seed/directories-v1.txt", "cluster-seed/files/PG_VERSION", "cluster-seed/files/global/pg_control", ]) { diff --git a/tools/release/native-cluster-seed-contract.mjs b/tools/release/native-cluster-seed-contract.mjs index 3db4dceb7..a78402751 100644 --- a/tools/release/native-cluster-seed-contract.mjs +++ b/tools/release/native-cluster-seed-contract.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env bun import { createHash } from "node:crypto"; -import { lstatSync, readFileSync, readdirSync } from "node:fs"; +import { lstatSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -13,6 +13,32 @@ const SHA256 = /^[0-9a-f]{64}$/u; const CACHE_KEY = new RegExp(CONTRACT.manifests.native.cacheKeyPattern, "u"); const DISALLOWED_CACHE_KEYS = new Set(CONTRACT.manifests.native.cacheKeyDisallowedValues); +// PostgreSQL 18 initdb.c: subdirs[], plus its separately created pg_wal. +export const NATIVE_PGDATA_DIRECTORIES = Object.freeze(CONTRACT.pgdataDirectories); + +// A regular file preserves directory metadata through Cargo and npm file inventories. +function nativeSeedDirectoryInventory(seed) { + const directories = []; + function visit(relative) { + for (const entry of readdirSync(path.join(seed, "files", relative), { withFileTypes: true })) { + if (entry.isDirectory()) { + const child = relative ? `${relative}/${entry.name}` : entry.name; + if (child.split("/").some((part) => !/^[A-Za-z0-9_.-]+$/u.test(part) || part === "." || part === "..")) { + throw new Error(`unsafe seed directory path: ${JSON.stringify(child)}`); + } + directories.push(child); + visit(child); + } + } + } + visit(""); + return `${directories.sort().join("\n")}\n`; +} + +export function writeNativeSeedDirectories(seed) { + writeFileSync(path.join(seed, "directories-v1.txt"), nativeSeedDirectoryInventory(seed)); +} + export function validNativeCacheKey(value) { return CACHE_KEY.test(value) && !DISALLOWED_CACHE_KEYS.has(value); } @@ -209,16 +235,14 @@ function visitRegularFileTree(root, label, onFile) { export function validateNativeClusterSeedDirectory(seed, profile, options = {}) { for (const relative of [ "files", - "files/global", - "files/pg_wal", + ...NATIVE_PGDATA_DIRECTORIES.map((relative) => `files/${relative}`), "files/PG_VERSION", "files/global/pg_control", "manifest.properties", ]) { const file = path.join(seed, ...relative.split("/")); const expectedDirectory = relative === "files" - || relative === "files/global" - || relative === "files/pg_wal"; + || NATIVE_PGDATA_DIRECTORIES.some((directory) => relative === `files/${directory}`); const metadata = lstatSync(file); if (metadata.isSymbolicLink() || (expectedDirectory ? !metadata.isDirectory() : !metadata.isFile())) { @@ -236,6 +260,11 @@ export function validateNativeClusterSeedDirectory(seed, profile, options = {}) if (rootEntries.has(transient)) throw new Error(`${seed} contains transient ${transient}`); } visitRegularFileTree(files, "native cluster seed", () => {}); + const inventory = path.join(seed, "directories-v1.txt"); + if (!lstatSync(inventory).isFile() + || readFileSync(inventory, "utf8") !== nativeSeedDirectoryInventory(seed)) { + throw new Error(`${seed} directory inventory does not match its PGDATA tree`); + } const icuDataTreeSha256 = options.icuData === undefined ? undefined : logicalTreeSha256(filesystemTreeRows(options.icuData)); diff --git a/tools/release/native-cluster-seed-contract.test.mjs b/tools/release/native-cluster-seed-contract.test.mjs index 5cb527a66..d01a5bf33 100644 --- a/tools/release/native-cluster-seed-contract.test.mjs +++ b/tools/release/native-cluster-seed-contract.test.mjs @@ -1,3 +1,4 @@ +import { NATIVE_PGDATA_DIRECTORIES, writeNativeSeedDirectories } from "./native-cluster-seed-contract.mjs"; import { expect, test } from "bun:test"; import { mkdirSync, @@ -118,8 +119,8 @@ test("requires a complete regular native PGDATA seed tree", () => { const root = mkdtempSync(path.join(os.tmpdir(), "oliphaunt-native-seed-contract-")); const seed = path.join(root, "seed"); try { - mkdirSync(path.join(seed, "files/global"), { recursive: true }); - mkdirSync(path.join(seed, "files/pg_wal")); + for (const directory of NATIVE_PGDATA_DIRECTORIES) mkdirSync(path.join(seed, "files", directory), { recursive: true }); + writeNativeSeedDirectories(seed); writeFileSync(path.join(seed, "files/PG_VERSION"), "18\n"); writeFileSync(path.join(seed, "files/global/pg_control"), "control\n"); writeFileSync(path.join(seed, "manifest.properties"), fixture("native-standard.valid.properties")); @@ -127,6 +128,21 @@ test("requires a complete regular native PGDATA seed tree", () => { target: "linux-x64-gnu", })).not.toThrow(); + expect(readFileSync(path.join(seed, "directories-v1.txt"), "utf8").split("\n")).toContain("pg_notify"); + for (const directory of ["pg_notify", "pg_wal/archive_status", "pg_multixact/offsets"]) { + rmSync(path.join(seed, "files", directory), { recursive: true }); + expect(() => validateNativeClusterSeedDirectory(seed, "standard", { + target: "linux-x64-gnu", + })).toThrow(); + mkdirSync(path.join(seed, "files", directory)); + } + + writeFileSync(path.join(seed, "directories-v1.txt"), "pg_wal\n"); + expect(() => validateNativeClusterSeedDirectory(seed, "standard", { + target: "linux-x64-gnu", + })).toThrow(/directory inventory/u); + writeNativeSeedDirectories(seed); + writeFileSync(path.join(seed, "files/postmaster.pid"), "1\n"); expect(() => validateNativeClusterSeedDirectory(seed, "standard", { target: "linux-x64-gnu", diff --git a/tools/release/native-icu-seeds.test.mjs b/tools/release/native-icu-seeds.test.mjs index a0ec10256..b648002b4 100644 --- a/tools/release/native-icu-seeds.test.mjs +++ b/tools/release/native-icu-seeds.test.mjs @@ -3,7 +3,7 @@ import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync import os from "node:os"; import path from "node:path"; import { splitNativeIcuSeed, stageNativeIcuSeeds, nativeIcuSeedAsset } from "./native-icu-seeds.mjs"; -import { filesystemTreeRows, logicalTreeSha256 } from "./native-cluster-seed-contract.mjs"; +import { filesystemTreeRows, logicalTreeSha256, NATIVE_PGDATA_DIRECTORIES, writeNativeSeedDirectories } from "./native-cluster-seed-contract.mjs"; import { spawnSync } from "../test/fd-backed-spawn-sync.mjs"; test("optional native ICU carrier owns the matching seed and base retains only standard", async () => { @@ -14,8 +14,10 @@ test("optional native ICU carrier owns the matching seed and base retains only s mkdirSync(data); writeFileSync(path.join(data, "icudt76l.dat"), "ICU fixture"); const seed = path.join(root, "cluster-seed-icu"); - mkdirSync(path.join(seed, "files/global"), { recursive: true }); - mkdirSync(path.join(seed, "files/pg_wal")); + for (const directory of NATIVE_PGDATA_DIRECTORIES) { + mkdirSync(path.join(seed, "files", directory), { recursive: true }); + } + writeNativeSeedDirectories(seed); writeFileSync(path.join(seed, "files/PG_VERSION"), "18\n"); writeFileSync(path.join(seed, "files/global/pg_control"), "fixture"); const fixture = readFileSync(new URL("../../src/shared/cluster-seed-contract/fixtures/native-icu.valid.properties", import.meta.url), "utf8"); diff --git a/tools/release/package-extension-cargo-facades.test.mjs b/tools/release/package-extension-cargo-facades.test.mjs index 9b2320a70..1f0331a79 100644 --- a/tools/release/package-extension-cargo-facades.test.mjs +++ b/tools/release/package-extension-cargo-facades.test.mjs @@ -230,14 +230,13 @@ pub const FIXTURE: bool = true; const genericCarrier = (name, product, version, kind, files) => fakeCarrier(leaves, { name, version, - header: `schema = "oliphaunt-artifact-manifest-v1"\nproduct = ${JSON.stringify(product)}\nversion = ${JSON.stringify(version)}\nkind = ${JSON.stringify(kind)}\ntarget = ${JSON.stringify(host)}`, - members: [{ files: files.map((relative) => ({ relative, contents: `${name}:${relative}` })) }], + header: `schema = "oliphaunt-artifact-manifest-v1"\nproduct = ${JSON.stringify(product)}\nversion = ${JSON.stringify(version)}\nkind = ${JSON.stringify(kind)}\ntarget = ${JSON.stringify(host)}${kind === "native-runtime" ? '\ndirectories = ["cluster-seed/files/pg_notify", "cluster-seed/files/pg_wal/archive_status"]' : ""}`, + members: [{ files: files.map((relative) => ({ relative, contents: relative.endsWith("/directories-v1.txt") ? "pg_notify\npg_wal/archive_status\n" : `${name}:${relative}` })) }], }); const runtime = genericCarrier("fixture-native-runtime", "liboliphaunt-native", nativeRuntimeVersion, "native-runtime", [ "runtime/bin/postgres", "runtime/bin/initdb", "runtime/bin/pg_ctl", - "cluster-seed/manifest.properties", "cluster-seed/files/PG_VERSION", - "cluster-seed/files/global/pg_control", "cluster-seed-icu/manifest.properties", - "cluster-seed-icu/files/PG_VERSION", "cluster-seed-icu/files/global/pg_control", + "cluster-seed/manifest.properties", "cluster-seed/directories-v1.txt", "cluster-seed/files/PG_VERSION", + "cluster-seed/files/global/pg_control", ]); const tools = genericCarrier("fixture-native-tools", "oliphaunt-tools", nativeRuntimeVersion, "native-tools", [ "runtime/bin/pg_basebackup", "runtime/bin/pg_dump", "runtime/bin/psql", @@ -281,6 +280,9 @@ oliphaunt-build = { path = ${JSON.stringify(path.join(import.meta.dir, "../../sr expect(cargo.status, `${cargo.stdout}\n${cargo.stderr}`).toBe(0); const lock = findFile(path.join(root, "cargo-target"), "oliphaunt-assets.lock"); expect(lock).not.toBeNull(); + for (const seed of ["cluster-seed"]) { + expect(statSync(path.join(path.dirname(lock), "resources/native-runtime/liboliphaunt-native", seed, "files/pg_notify")).isDirectory()).toBe(true); + } const text = readFileSync(lock, "utf8"); expect(text).toContain('extension = "cube"'); expect(text).toContain('extension = "pg_trgm"'); diff --git a/tools/release/package-liboliphaunt-cargo-artifacts.test.mjs b/tools/release/package-liboliphaunt-cargo-artifacts.test.mjs index 6670776ba..c3ad1a1d3 100644 --- a/tools/release/package-liboliphaunt-cargo-artifacts.test.mjs +++ b/tools/release/package-liboliphaunt-cargo-artifacts.test.mjs @@ -19,6 +19,7 @@ import { stageReleaseNotices, } from "./release-notices.mjs"; import { requiredCoreRuntimePaths } from "./optimize_native_runtime_payload.mjs"; +import { NATIVE_PGDATA_DIRECTORIES, writeNativeSeedDirectories } from "./native-cluster-seed-contract.mjs"; import { nativeIcuDataManifest } from "./native-icu-data-contract.mjs"; import { nativeRuntimeCarrierManifest } from "./native-runtime-carrier-contract.mjs"; @@ -53,8 +54,9 @@ function sha256(file) { function stageClusterSeed(root, directory, profile, target, icuDataTreeSha256 = "") { const seed = path.join(root, directory); - mkdirSync(path.join(seed, "files/global"), { recursive: true }); + for (const directory of NATIVE_PGDATA_DIRECTORIES) mkdirSync(path.join(seed, "files", directory), { recursive: true }); mkdirSync(path.join(seed, "files/pg_wal"), { recursive: true }); + writeNativeSeedDirectories(seed); writeFileSync(path.join(seed, "files/PG_VERSION"), "18\n"); writeFileSync(path.join(seed, "files/global/pg_control"), `${profile}\n`); writeFileSync(path.join(seed, "manifest.properties"), [ @@ -171,33 +173,19 @@ test("freezes .crate bytes for native parts, aggregators, and facade and rejects "-tzf", path.resolve(ROOT, cratePath), ]).includes(`${name}-9.8.7/payload/files/cluster-seed-icu/manifest.properties`))); + for (const seed of ["cluster-seed"]) { + const member = `payload/files/${seed}/directories-v1.txt`; + const part = runtimeParts.find(({ cratePath, name }) => commandOutput("tar", [ + "-tzf", path.resolve(ROOT, cratePath), + ]).includes(`${name}-9.8.7/${member}`)); + assert.ok(part, `${seed} directory inventory must survive Cargo packaging`); + assert.ok(commandOutput("tar", [ + "-xOzf", path.resolve(ROOT, part.cratePath), `${part.name}-9.8.7/${member}`, + ]).split("\n").includes("pg_notify")); + } for (const item of manifest.packages) { const expectedProfile = item.role === "part" ? item.kind : "code-facade"; assert.equal(item.noticeProfile, expectedProfile, `${item.name} must freeze its carrier notice profile`); - if (process.platform === "linux" && process.arch === "x64") { - const consumer = path.join(root, "consumer"); - mkdirSync(path.join(consumer, "src"), { recursive: true }); - const runtimeCarrier = manifest.packages.find(row => row.role === "aggregator" && row.kind === "native-runtime"); - writeFileSync(path.join(consumer, "Cargo.toml"), `[package] -name = "native-seed-directory-proof" -version = "0.0.0" -edition = "2024" -[dependencies] -${runtimeCarrier.name} = { path = ${JSON.stringify(path.dirname(path.resolve(ROOT, runtimeCarrier.manifestPath)))} } -[workspace] -`); - writeFileSync(path.join(consumer, "build.rs"), `fn main() { - let manifest = std::env::vars().find(|(key, _)| key.starts_with("DEP_OLIPHAUNT_ARTIFACT_") && key.ends_with("_MANIFEST")).unwrap().1; - let path = std::path::Path::new(&manifest); - assert!(path.parent().unwrap().join("payload/cluster-seed/files/pg_wal").is_dir()); - let text = std::fs::read_to_string(manifest).unwrap(); - assert!(text.contains("cluster-seed/files/pg_wal")); -}`); - writeFileSync(path.join(consumer, "src/main.rs"), "fn main() {}"); - run("cargo", ["check", "--offline", "--manifest-path", path.join(consumer, "Cargo.toml")], { - env: { ...process.env, CARGO_TARGET_DIR: path.join(root, "consumer-target") }, - }); - } const packedManifest = commandOutput("tar", [ "-xOzf", path.resolve(ROOT, item.cratePath), diff --git a/tools/release/package-release-carriers.mjs b/tools/release/package-release-carriers.mjs index 68d36a9c7..be2d09e73 100644 --- a/tools/release/package-release-carriers.mjs +++ b/tools/release/package-release-carriers.mjs @@ -1187,6 +1187,7 @@ export function liboliphauntNpmTarballs(version) { const requiredMembers = [ `package/${libraryRelativePath}`, "package/cluster-seed/manifest.properties", + "package/cluster-seed/directories-v1.txt", "package/cluster-seed/files/PG_VERSION", "package/cluster-seed/files/global/pg_control", "package/manifest.properties", diff --git a/tools/release/stage-native-cluster-seed.mjs b/tools/release/stage-native-cluster-seed.mjs index 7808f5efd..cc3cf8590 100644 --- a/tools/release/stage-native-cluster-seed.mjs +++ b/tools/release/stage-native-cluster-seed.mjs @@ -15,6 +15,7 @@ import { NATIVE_CLUSTER_SEED_TARGETS, bindNativeClusterSeedManifest, validateNativeClusterSeedDirectory, + writeNativeSeedDirectories, } from "./native-cluster-seed-contract.mjs"; const TOOL = "stage-native-cluster-seed.mjs"; @@ -131,6 +132,7 @@ export function stageNativeClusterSeed(argv) { const manifestPath = path.join(source, "manifest.properties"); const manifest = bindNativeClusterSeedManifest(readFileSync(manifestPath), args.target, args.profile); writeFileSync(manifestPath, manifest); + writeNativeSeedDirectories(source); validateNativeClusterSeedDirectory(source, args.profile, { target: args.target, icuData: args.icuData, diff --git a/tools/test/create-liboliphaunt-release-fixture.mjs b/tools/test/create-liboliphaunt-release-fixture.mjs index 91693ad0d..c61d1e3d5 100644 --- a/tools/test/create-liboliphaunt-release-fixture.mjs +++ b/tools/test/create-liboliphaunt-release-fixture.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env bun +import { NATIVE_PGDATA_DIRECTORIES } from '../release/native-cluster-seed-contract.mjs'; import fs from 'node:fs/promises'; import { createHash } from 'node:crypto'; import path from 'node:path'; @@ -277,7 +278,10 @@ function nativeClusterSeedEntries(profile, prefix, target, icuDataTreeSha256 = ' [`${prefix}manifest.properties`]: manifest, [`${prefix}files/PG_VERSION`]: '18\n', [`${prefix}files/global/pg_control`]: `${profile}-fixture-control\n`, - [`${prefix}files/pg_wal/`]: '', + ...Object.fromEntries( + NATIVE_PGDATA_DIRECTORIES.map((directory) => [`${prefix}files/${directory}/`, '']), + ), + [`${prefix}directories-v1.txt`]: `${[...NATIVE_PGDATA_DIRECTORIES].sort().join('\n')}\n`, }; } diff --git a/tools/xtask/moon.yml b/tools/xtask/moon.yml index 73e9c1323..563b8f2fe 100644 --- a/tools/xtask/moon.yml +++ b/tools/xtask/moon.yml @@ -26,6 +26,7 @@ tasks: inputs: - "@group(cargo-workspace)" - "**/*" + - "/src/shared/cluster-seed-contract/**/*" options: cache: true runFromWorkspaceRoot: true @@ -37,6 +38,7 @@ tasks: inputs: - "@group(cargo-workspace)" - "**/*" + - "/src/shared/cluster-seed-contract/**/*" options: cache: true runFromWorkspaceRoot: true @@ -49,6 +51,7 @@ tasks: inputs: - "@group(cargo-workspace)" - "**/*" + - "/src/shared/cluster-seed-contract/**/*" options: cache: true runFromWorkspaceRoot: true diff --git a/tools/xtask/src/cluster_seed_runner.rs b/tools/xtask/src/cluster_seed_runner.rs index 262007eb4..4e81febbe 100644 --- a/tools/xtask/src/cluster_seed_runner.rs +++ b/tools/xtask/src/cluster_seed_runner.rs @@ -42,6 +42,21 @@ pub(crate) fn default_initdb_profile() -> &'static str { } pub(crate) fn clean_generated_cluster_seed(pgdata: &Path) -> Result<()> { + #[derive(serde::Deserialize)] + #[serde(rename_all = "camelCase")] + struct Contract { + pgdata_directories: Vec, + } + let contract: Contract = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../src/shared/cluster-seed-contract/contract.json" + )))?; + for directory in contract.pgdata_directories { + anyhow::ensure!( + pgdata.join(&directory).is_dir(), + "cluster seed is missing required PostgreSQL directory {directory}" + ); + } for name in ["postmaster.pid", "postmaster.opts"] { let path = pgdata.join(name); if path.exists() { @@ -562,3 +577,32 @@ impl wasmer_wasix::runtime::package_loader::PackageLoader for LocalOnlyPackageLo .await } } + +#[cfg(test)] +mod layout_tests { + use super::*; + + #[test] + fn rejects_missing_empty_pgdata_directory() -> Result<()> { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "oliphaunt-seed-layout-{}-{nonce}", + std::process::id() + )); + let contract: serde_json::Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../src/shared/cluster-seed-contract/contract.json" + )))?; + for directory in contract["pgdataDirectories"].as_array().unwrap() { + fs::create_dir_all(root.join(directory.as_str().unwrap()))?; + } + clean_generated_cluster_seed(&root)?; + fs::remove_dir(root.join("pg_notify"))?; + let error = clean_generated_cluster_seed(&root).unwrap_err(); + fs::remove_dir_all(&root)?; + assert!(error.to_string().contains("pg_notify")); + Ok(()) + } +} From 8e70b4bc5d83cae9cc8ab4b0633cc12f3b2eafe7 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 07:16:27 +0000 Subject: [PATCH 04/18] fix(sdk): complete resource packaging and consumer qualification --- .github/workflows/ci.yml | 9 +- coverage/baseline.toml | 14 ++ docs/architecture/cluster-seeds-and-icu.md | 5 +- docs/architecture/wasix-typescript-napi.md | 187 +----------------- .../consumer-dx-release-blueprint.md | 2 +- .../maintainers/extension-packaging-policy.md | 6 +- examples/browser-wasix/tsconfig.json | 1 + .../native/smoke/liboliphaunt_smoke.c | 22 ++- src/sdks/js/src/__tests__/client.test.ts | 9 +- src/sdks/kotlin/oliphaunt/build.gradle.kts | 4 + .../OliphauntAndroidRuntimeAssetsTest.kt | 75 +++++++ .../kotlin/dev/oliphaunt/ResourcesTest.kt | 49 +++++ src/sdks/rust/tools/check-release-consumer.sh | 11 +- .../ExtensionResourceCompositionTests.swift | 38 ++++ .../Tests/OliphauntTests/OliphauntTests.swift | 38 ++++ tools/policy/ci-plan-node-products.test.mjs | 8 + .../broker-dependency-license-contract.mjs | 1 + ...roker-dependency-license-contract.test.mjs | 6 + tools/release/check_artifact_targets.mjs | 2 +- tools/release/moon.yml | 2 + tools/release/sdk-artifacts/swift.mjs | 3 + 21 files changed, 295 insertions(+), 197 deletions(-) create mode 100644 src/sdks/kotlin/oliphaunt/src/commonTest/kotlin/dev/oliphaunt/ResourcesTest.kt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37506c616..91b2107f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1471,6 +1471,7 @@ jobs: needs: - affected - liboliphaunt-native-ios-abi + - mobile-extension-packages if: ${{ contains(fromJson(needs.affected.outputs.jobs), 'swift-sdk-package') }} runs-on: macos-26 timeout-minutes: 90 @@ -1494,10 +1495,16 @@ jobs: name: liboliphaunt-native-abi-compatible-release-assets-ios-datum64 path: target/liboliphaunt/abi-compatible-release-assets/ios-datum64 + - name: Download mobile extension package artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: oliphaunt-mobile-extension-package-artifacts + path: target/mobile-extension-artifacts + - name: Build Swift SDK package artifacts env: OLIPHAUNT_SWIFT_RELEASE_ASSET_DIR: ${{ github.workspace }}/target/liboliphaunt/abi-compatible-release-assets/ios-datum64 - OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-native:finalize-runtime-ios-abi"]' + OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-native:finalize-runtime-ios-abi", "extension-packages:package-mobile"]' run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh swift-sdk-package - name: Upload Swift SDK package artifacts diff --git a/coverage/baseline.toml b/coverage/baseline.toml index 1b372a1c1..cfb8e1e10 100644 --- a/coverage/baseline.toml +++ b/coverage/baseline.toml @@ -39,6 +39,13 @@ evidence = "oliphaunt-rust broker tests plus src/shared/fixtures/protocol/query- owner = "@oliphaunt/core" expires = "before-0.2.0" +[[products.oliphaunt-rust.waivers]] +path = "src/sdks/rust/src/extensions.rs" +reason = "generated const descriptors contain no executable lines for LLVM instrumentation" +evidence = "extensions:lint validates the generated model; sdk_extensions tests validate descriptor selection" +owner = "@oliphaunt/core" +expires = "before-0.2.0" + [[products.oliphaunt-rust.waivers]] path = "src/sdks/rust/src/lib.rs" reason = "crate root is a re-export surface with no durable executable behavior to line-cover" @@ -76,6 +83,13 @@ exclude_globs = [ ] per_file_line_warning = 50.0 +[[products.oliphaunt-swift.waivers]] +path = "src/sdks/swift/Sources/Oliphaunt/OliphauntExtensions.swift" +reason = "generated static descriptor and dependency tables have no Swift coverage instrumentation" +evidence = "extensions:lint validates generated metadata; explicit resource selection tests cover dependency closure" +owner = "@oliphaunt/core" +expires = "before-0.2.0" + [[products.oliphaunt-swift.waivers]] path = "src/sdks/swift/Sources/Oliphaunt/OliphauntNativeDirect.swift" reason = "native direct FFI shell is validated by runtime smoke/XCTest paths rather than pure Swift line coverage" diff --git a/docs/architecture/cluster-seeds-and-icu.md b/docs/architecture/cluster-seeds-and-icu.md index 845a15571..fb8387865 100644 --- a/docs/architecture/cluster-seeds-and-icu.md +++ b/docs/architecture/cluster-seeds-and-icu.md @@ -1,13 +1,16 @@ # Cluster seeds and ICU Status: current distribution contract, with an optional-download transition -approved below; updated 2026-09-09. +deferred below; updated 2026-09-09. This document is the source of truth for preinitialized PostgreSQL clusters, ICU data, their public selection, and their release qualification. ## Optional-download transition +This transition is deferred from the SDK consistency change. Standard seeds +remain bundled; ICU data and its matching seeds remain optional. + The intended package contract is: - Ordinary installs download neither seeds nor ICU data. New databases run diff --git a/docs/architecture/wasix-typescript-napi.md b/docs/architecture/wasix-typescript-napi.md index 82c4ee747..b05dca14f 100644 --- a/docs/architecture/wasix-typescript-napi.md +++ b/docs/architecture/wasix-typescript-napi.md @@ -1,8 +1,7 @@ -# WASIX TypeScript Node-API architecture and implementation checklist +# WASIX TypeScript Node-API architecture -Status: implemented locally; exact-commit hosted qualification pending -Reviewed against: `origin/main` at `4384d1bdfafee07e4e1963ac68027b4bcf002a1e` -Last reviewed: 2026-08-30 +The execution architecture below also reflects explicit resource packages. +Standard seeds remain bundled; the optional-standard-seed rollout is deferred. This document is the decision record and delivery checklist for replacing the Node, Bun, Deno, and Electron Wasmer-JS execution path in @@ -118,6 +117,7 @@ string/lifecycle contract; only capability discovery moves to an import. The public shape is exactly one host-only conditional subpath: ```ts +import { directory } from '@oliphaunt/wasix-ts'; import { openServer } from '@oliphaunt/wasix-ts/server'; await using server = await openServer({ @@ -355,186 +355,13 @@ separate intermediate PRs with knowingly incomplete package contracts: 6. Review Rust catalog selection and caches as an immutable per-builder standard/ICU profile and prove mixed-profile process behavior. 7. Review the one profile-selecting addon per target, carrier topology, - cross-runtime smoke tests, release policy, licenses, and provenance. Keep the - frozen extension/tool catalog. + cross-runtime smoke tests, release policy, licenses, and provenance. Keep + external extensions, ICU resources, and tools in their separate packages. Public cancellation, wire `CancelRequest`, a multi-client server, and new carrier targets remain separate proposals. Feature PRs do not edit versions or changelogs; release automation owns those changes. -## Implementation checklist - -### Phase 0 — preserve and separate the existing work - -- [x] Confirm the implementation base is current `origin/main`. -- [x] Audit the dirty tree and treat all existing changes as user-owned WIP. -- [x] Record the final architecture and scope in this document. -- [x] Keep the integrated change reviewable through the bounded slices above - and co-locate each slice's tests, generated contracts, and documentation. -- [x] Retain useful structured storage errors and exact tool stdout/stderr from - the provisional implementation. - -### Phase 1 — minimal functional Node-API placements - -Rust shared owner and Node-API bridge: - -- [x] Extend the existing `AsyncOliphaunt` owner core with immediate - callback-completion admission for open, buffered raw protocol, backup, and - close. Its bounded FIFO, transaction ownership, terminal close, owner-loss - handling, and panic quarantine remain the single source of truth. -- [x] Gate the completion seam behind an exact-purpose private - `__internal-napi` Cargo feature (plus tests). The Node-API crate enables it; - ordinary Rust WASIX users and generated public API docs do not gain adapter - methods. -- [x] Retain the Future API's fair, waiting async admission and oneshot replies. - Share the typed owner command, execution, close, and owner-loss state machine - with immediate callback admission; do not turn Rust Future calls into - immediate busy rejection or fork lifecycle semantics for Node-API. -- [x] Export separate async actor and synchronous direct native classes. Keep - creator affinity only on direct. -- [x] Settle one napi-rs `JsDeferred` directly from each owner callback, giving - exactly one cross-thread completion dispatch per non-streaming operation and - napi-rs-owned environment cleanup. Streaming additionally uses its bounded - per-chunk rendezvous. Do not route replies through `napi_async_work`, a - blocking receiver, or a custom runtime. -- [x] Map structured Rust errors at the ABI boundary. Remove text/source-chain - inference once every Rust error carries its classification. -- [x] Return V8-owned public output bytes; retain external buffers only for - proven internal nonescaping use. -- [x] Add focused Rust unit tests for callback ordering/admission, exact-once - rejection, callback panic, queue/owner loss, close cutoff/retry/reentry, - shared close, and terminal replay; keep the full Rust library suite green. -- [x] Add the live Node-API integration roundtrip for actor open, recoverable - raw protocol work and reuse, backup, repeated close, and environment cleanup. -- [x] Remove the rejected standalone N-API actor prototype; do not ship two - lifecycle state machines. - -TypeScript integration: - -- [x] Root Node/Bun/Deno/Electron exports use the actor class while preserving the - Promise-shaped database API. -- [x] Add `/direct` as the explicit synchronous-placement import and wire it to - the direct class. -- [x] Restore/adapt the real Node/Bun/Deno/Electron Worker implementation and load the - direct class inside it. -- [x] Remove `node-child.ts`, child ports/options, liveness watchdogs, - `child_process` permissions, fixtures, tests, and documentation after the - Worker replacement is proven. -- [x] Keep the existing logical-operation scheduler and transaction ownership; - do not replace `#tail` as part of placement migration. -- [x] Keep browser root and Worker module graphs byte-for-byte behaviorally on - Wasmer-JS and prove that they cannot resolve a native carrier. -- [x] Add root/direct/Worker parity tests covering query, raw protocol, - streaming callback failure, transaction, backup/restore, storage errors, - close, and async disposal. - -Server surface: - -- [x] Replace `/server/{node,bun,deno}` with one conditional `/server` export. -- [x] Consolidate `ServerListen`, `ServerOpenConfig`, and `OliphauntServer` in - one TS module. -- [x] Adapt the existing Rust `AsyncOliphauntServer` owner and its exact - open/close memoization for Node-API lifecycle. Do not create a database-actor - command or a second N-API/server owner state machine for this cold path. -- [x] Run N-API server open/close lifecycle off the importing event loop while - Rust continues to own all socket traffic. -- [x] Remove duplicated JavaScript socket/filesystem policy and TOCTOU checks; - snapshot/resolve options in TS and let Rust atomically create and own the - listener path. -- [x] Correct manifest/docs ownership from child process to Rust owner/native - listener. -- [x] Test TCP automatic/fixed port, Unix ownership-safe cleanup, close during - an active client, recoverable SQL error, reconnect, exact public shape, and - browser export exclusion. - -Phase 1 acceptance: - -- [x] No Node/Bun/Deno/Electron root or `/worker` import loads Wasmer-JS. -- [x] No normal host-runtime path spawns a child process. -- [x] Root heartbeat remains responsive during a long query; `/direct` blocks; - `/worker` remains responsive. -- [x] Add a reproducible direct-versus-actor-versus-Worker harness which reports - tiny-operation overhead, p50/p95/p99, event-loop delay, throughput, copies, - fan-out, and overload/RSS instead of assuming the result. -- [x] Repeated close/finalization and environment exit produce no hang, abort, - use-after-free, or late Node-API call. -- [x] Pin the addon image before a direct-only Worker can initialize the - process-wide WASIX runtime, and exercise a fresh process whose parent never - loads the addon while repeated direct Workers open, query, close, and - self-exit. -- [x] Prove raw direct streaming callback reentry is rejected by napi-rs's - generated native borrow guard and leaves the database reusable. - -### Phase 2 — safe Worker shutdown - -- [x] Make Worker close stop admission, settle the active operation and queue, - quiesce stream callbacks, close/release the direct native handle, acknowledge - the complete shutdown state, and self-exit. -- [x] Ensure every error and finalization path tracks that full shutdown state. - Treat an observed clean self-exit as final without a redundant terminate; - reserve forced termination for startup/fatal cleanup before self-exit and - never use an idle native frame alone as proof that active termination is safe. -- [x] Add regressions for close during a long query, idle termination, - environment teardown during open/query/stream/close, and the reproduced - active-termination process abort. -- [x] Document that a hung guest can leave close pending and that `/worker` is - scheduling/realm isolation, not process containment. - -### Phase 3 — packaging optimization before first release - -- [x] Refactor standard/ICU seed selection from compile-time global selection - to an immutable per-builder/database profile so one addon binary per target - supports both without changing default catalog or storage behavior. -- [x] Thread that profile through manifest identity, seed selection, runtime - materialization, ICU-data installation, database and server builders, and - compatibility checks. Key every reusable seed/runtime/materialization cache - by profile rather than sharing singleton `OnceLock` state. -- [x] Test standard and ICU databases/servers in both construction orders and - concurrently in one process; prove their receipts, directories, manifests, - and extension catalogs cannot contaminate each other. -- [x] Keep the first release's qualified extension/tool catalog frozen inside - that addon. Preserve descriptor identity checks; do not introduce dynamic - native extension loading in this migration. -- [x] Update the private build package, carrier manifests, loader identities, - artifact provenance, notices/licenses, checksum aggregation, and publication - catalog together. -- [x] Preserve the exact four-target release matrix and existing Linux ABI - policy. -- [x] Verify clean install, optional-dependency pruning, unsupported target, - missing carrier, wrong ABI/version/runtime identity, pnpm/npm, and archive - contents without network-time install hooks. -- [x] Wire packaged Node, Bun, Deno, and Electron smoke tests into every matching - carrier job; document and test Deno permissions and the Electron ASAR-unpacked - layout, including its missing-companion failure. -- [ ] Observe those packaged smokes on all four physical hosted carrier jobs for - the exact candidate commit. - -### Phase 4 — documentation, cleanup, and release qualification - -- [x] Update `README.md`, `ARCHITECTURE.md`, API reference, SDK parity/product - policies, package surface inventory, examples, and migration notes. -- [x] Describe root as actor-backed, `/direct` as blocking, `/worker` as a real - Worker, and `/server` as a Rust listener. Never call realm isolation crash - containment. -- [x] Remove stale child-process, Wasmer-on-server, external-zero-copy output, - duplicate-addon, and runtime-named server-subpath claims. -- [x] Remove dead modules only after `rg`, TypeScript build output, packed - package inspection, browser bundle inspection, and Moon graph checks prove - they are unreachable. -- [x] Update focused CI first; add the full carrier/runtime matrix only with the - packaging PR. Avoid duplicating Moon-owned checks in workflows. -- [x] Run `moon run oliphaunt-wasix-napi:qualify`, - `moon run oliphaunt-wasix-napi:qualify`, - `moon run oliphaunt-wasix-rust:compile`, - `moon run oliphaunt-wasix-ts:unit`, - `moon run oliphaunt-wasix-ts:compile`, and the product package checks. -- [x] Run `moon run sdk-contracts:check` for public-surface changes. -- [x] Run workflow-policy, release-check, committed-asset, extension-model, - WASIX source/patch, portable/AOT, carrier, license, provenance, and Linux ABI - checks selected by the repository qualification graph. -- [ ] Before merge/release, prove the exact commit with the repository - `Qualified` gate and all required WASIX lifecycle evidence. - ## Performance proof Maintain a reproducible benchmark rather than a single favorable number. @@ -572,8 +399,6 @@ These items are not prerequisites for the Node-API migration: when undertaken, they reuse the lower cancellation handle and do not add `server.cancel()`; - extracting the duplicate JavaScript query codec into a new shared package; -- independently loaded native extension carriers or a user-visible native - profile package split; - new native carrier targets or a new glibc policy; and - unrelated storage, extension-catalog, SDK, or release-system redesigns. diff --git a/docs/maintainers/consumer-dx-release-blueprint.md b/docs/maintainers/consumer-dx-release-blueprint.md index e3f007545..bfcf2ff4d 100644 --- a/docs/maintainers/consumer-dx-release-blueprint.md +++ b/docs/maintainers/consumer-dx-release-blueprint.md @@ -1,6 +1,6 @@ # SDK resource packaging contract -The approved next distribution contract makes standard seeds optional and uses +A separate, deferred distribution change would make standard seeds optional and use `initdb` by default. The [cluster-seed transition](../architecture/cluster-seeds-and-icu.md#optional-download-transition) records the remaining native-mobile/browser initialization and carrier work. The bundled-standard-seed descriptions below describe the current implementation, diff --git a/docs/maintainers/extension-packaging-policy.md b/docs/maintainers/extension-packaging-policy.md index a685ab9fb..eb1efd5cf 100644 --- a/docs/maintainers/extension-packaging-policy.md +++ b/docs/maintainers/extension-packaging-policy.md @@ -77,8 +77,10 @@ and changelog. An npm WASIX leaf is host-neutral: browser, Node, Bun, Deno, and Electron WASIX hosts consume the same descriptor package. Its ESM descriptor selects one exact SQL extension and carries the verified browser byte closure required to -materialize it. Native hosts validate that identity and resolve the SQL name -against the frozen catalog embedded in their Node-API addon. Contrib +materialize it. Node-API hosts validate the installed package identity, runtime +compatibility, payload hashes, and matching host AOT package before passing its +bytes to Rust. The addon embeds contrib; external payloads come from the +selected packages. Contrib members use exact package subpaths so importing one member does not create an implicit selector group. Each extension product freezes its own archive identity and `oliphaunt-wasix-extension-install-v1` projection: dependencies, diff --git a/examples/browser-wasix/tsconfig.json b/examples/browser-wasix/tsconfig.json index 4e6454003..e47547229 100644 --- a/examples/browser-wasix/tsconfig.json +++ b/examples/browser-wasix/tsconfig.json @@ -5,6 +5,7 @@ "moduleResolution": "Bundler", "noEmit": true, "paths": { + "@oliphaunt/extension-contrib-pg18-wasix": ["../../src/bindings/wasix-ts/src/contrib-carrier-shim.d.ts"], "@oliphaunt/liboliphaunt-wasix": ["../../src/bindings/wasix-ts/src/runtime-carrier-shim.d.ts"], "@oliphaunt/liboliphaunt-wasix-tools": ["../../src/bindings/wasix-ts/tools-package/src/runtime-carrier-shim.d.ts"], "@oliphaunt/wasix-ts": ["../../src/bindings/wasix-ts/src/index.ts"], diff --git a/src/runtimes/liboliphaunt/native/smoke/liboliphaunt_smoke.c b/src/runtimes/liboliphaunt/native/smoke/liboliphaunt_smoke.c index 5e4e61af9..b5519ab4e 100644 --- a/src/runtimes/liboliphaunt/native/smoke/liboliphaunt_smoke.c +++ b/src/runtimes/liboliphaunt/native/smoke/liboliphaunt_smoke.c @@ -1238,11 +1238,25 @@ static int exec_static_extension_registry_smoke(OliphauntHandle *db) { liboliphaunt_smoke_static_init_calls); return 1; } + if (oliphaunt_register_static_extensions(NULL, 0) != 0 || + register_static_extension_fixture() != 0) { + fprintf(stderr, "identical static registration after startup failed\n"); + return 1; + } + const OliphauntStaticExtension conflicting = { + .abi_version = OLIPHAUNT_STATIC_EXTENSION_ABI_VERSION, + .name = "liboliphaunt_smoke_static", + .magic = (const void *(*)(void))liboliphaunt_smoke_static_magic, + }; if (expect_static_extension_registration_fails( - NULL, - 0, - "static extension registry freeze", - "static extension registry cannot be changed after backend startup") != 0) { + &conflicting, + 1, + "static extension replacement after startup", + "conflicting static extension registration") != 0) { + return 1; + } + if (liboliphaunt_smoke_static_init_calls != before + 1) { + fprintf(stderr, "repeated static registration reran module initialization\n"); return 1; } return 0; diff --git a/src/sdks/js/src/__tests__/client.test.ts b/src/sdks/js/src/__tests__/client.test.ts index 9dbe8e834..94ac52636 100644 --- a/src/sdks/js/src/__tests__/client.test.ts +++ b/src/sdks/js/src/__tests__/client.test.ts @@ -5,6 +5,7 @@ import { join } from 'node:path'; import { test } from 'vitest'; import { createOliphauntClient } from '../client.js'; +import { extensions as contrib } from '../extensions.js'; import type { NativeBinding, NativeBindingOptions, @@ -96,7 +97,7 @@ test('snapshots open configuration before asynchronous storage work', async () = return { state: 'closed' }; }; const startupGUCs: Record = { work_mem: '8MB' }; - const extensions: string[] = []; + const extensions: Array = []; const config: OpenConfig = { topology: 'broker', storage: { kind: 'directory', path: root }, @@ -113,7 +114,7 @@ test('snapshots open configuration before asynchronous storage work', async () = config.username = 'after'; config.database = 'after'; startupGUCs.work_mem = '64MB'; - extensions.push('vector'); + extensions.push(contrib.pg_trgm); const database = await opening; assert.equal(direct.openCalls.length, 0); @@ -196,7 +197,7 @@ test('snapshots server storage and nested configuration before asynchronous work const storage = { kind: 'directory' as const, path: root }; const listen = { transport: 'tcp' as const, port: 15432 }; const startupGUCs: Record = { work_mem: '8MB' }; - const extensions: string[] = []; + const extensions: Array = []; const config: ServerOpenConfig = { storage, listen, startupGUCs, extensions }; const client = createOliphauntClient(() => new FakeBinding(), { server: serverRuntime }); @@ -205,7 +206,7 @@ test('snapshots server storage and nested configuration before asynchronous work storage.path = movedRoot; listen.port = 25432; startupGUCs.work_mem = '64MB'; - extensions.push('vector'); + extensions.push(contrib.pg_trgm); const database = await opening; assert.equal(database.connectionString, 'postgresql://postgres@127.0.0.1:15432/postgres'); diff --git a/src/sdks/kotlin/oliphaunt/build.gradle.kts b/src/sdks/kotlin/oliphaunt/build.gradle.kts index dcb6a1a25..1dd78663e 100644 --- a/src/sdks/kotlin/oliphaunt/build.gradle.kts +++ b/src/sdks/kotlin/oliphaunt/build.gradle.kts @@ -95,6 +95,10 @@ kover { includes { classes( "dev.oliphaunt.GeneratedExtensionsKt", + "dev.oliphaunt.ExtensionDescriptor", + "dev.oliphaunt.Extensions", + "dev.oliphaunt.IcuData", + "dev.oliphaunt.ResourcesKt", "dev.oliphaunt.Oliphaunt*", "dev.oliphaunt.Query*", ) diff --git a/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssetsTest.kt b/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssetsTest.kt index 81bb2dd65..ce8db8cb0 100644 --- a/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssetsTest.kt +++ b/src/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/OliphauntAndroidRuntimeAssetsTest.kt @@ -11,6 +11,81 @@ import kotlin.test.assertFalse import kotlin.test.assertTrue class OliphauntAndroidRuntimeAssetsTest { + @Test + fun runtimeAndSeedManifestsRejectCrossProfileResources() { + for ((key, value) in listOf( + "artifactRole" to "cluster-seed-standard", + "catalogProfile" to "standard", + "clusterSeedTarget" to "other-target", + "icuDataTreeSha256" to "a".repeat(64), + "extensions" to "vector", + )) { + val manifest = manifestProperties().apply { setProperty(key, value) } + assertFailsWith { + OliphauntAndroidRuntimeAssets.parseManifestProperties("oliphaunt/runtime", manifest) + } + } + val icuRuntime = manifestProperties("runtimeFeatures" to "icu").apply { setProperty("icuDataTreeSha256", "bad") } + assertFailsWith { + OliphauntAndroidRuntimeAssets.parseManifestProperties("oliphaunt/runtime", icuRuntime) + } + for (profile in listOf("standard", "icu")) { + val base = OliphauntAndroidRuntimeAssets.parseManifestText( + retargetNativeClusterSeedFixture("native-$profile.valid.properties", "android-datum64"), + "fixture", + ) + for ((key, value) in listOf( + "postgresMajor" to "17", + "runtimeFeatures" to if (profile == "icu") "" else "icu", + "icuDataVersion" to "99.0", + "icuDataForm" to "wrong", + "icuDataTreeSha256" to "invalid", + )) { + val manifest = Properties().apply { + putAll(base) + setProperty(key, value) + } + assertFailsWith { + OliphauntAndroidRuntimeAssets.parseManifestProperties( + if (profile == "icu") "oliphaunt/cluster-seed-icu" else "oliphaunt/cluster-seed", + manifest, + ) + } + } + } + } + + @Test + fun resourceReceiptChecksSelectedProductVersionAndIcu() { + val vector = ExtensionDescriptor("vector", "oliphaunt-extension-vector", "0.8.2") + val selected = listOf(vector, Extensions.PG_TRGM, vector) + val icu = IcuData("0.2.0") + val receipt = Properties().apply { + setProperty("schema", "oliphaunt-sdk-resources-v1") + setProperty("extension.vector.product", vector.product) + setProperty("extension.vector.version", "0.8.2") + setProperty("extension.pg_trgm.product", Extensions.PG_TRGM.product) + setProperty("extension.pg_trgm.version", "0.2.0") + setProperty("icuVersion", "0.2.0") + } + OliphauntAndroidRuntimeAssets.validateSelectedResourceReceipt(receipt, selected, icu) + OliphauntAndroidRuntimeAssets.validateSelectedResourceReceipt(receipt, emptyList(), null) + for ((key, value) in listOf( + "schema" to "unknown", + "extension.vector.product" to "oliphaunt-extension-pgtap", + "extension.vector.version" to "0.8.3", + "icuVersion" to "0.3.0", + )) { + val invalid = Properties().apply { + putAll(receipt) + setProperty(key, value) + } + assertFailsWith { + OliphauntAndroidRuntimeAssets.validateSelectedResourceReceipt(invalid, selected, icu) + } + } + } + @Test fun rejectsDuplicateManifestProperties() { val error = diff --git a/src/sdks/kotlin/oliphaunt/src/commonTest/kotlin/dev/oliphaunt/ResourcesTest.kt b/src/sdks/kotlin/oliphaunt/src/commonTest/kotlin/dev/oliphaunt/ResourcesTest.kt new file mode 100644 index 000000000..0f93608a5 --- /dev/null +++ b/src/sdks/kotlin/oliphaunt/src/commonTest/kotlin/dev/oliphaunt/ResourcesTest.kt @@ -0,0 +1,49 @@ +package dev.oliphaunt + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class ResourcesTest { + @Test + fun descriptorsValidateIdentityAndDeduplicateExactVersions() { + val vector = ExtensionDescriptor("vector", "oliphaunt-extension-vector", "0.8.2") + val contrib = Extensions.PG_TRGM + assertEquals(listOf(vector, contrib), selectedExtensionDescriptors(listOf(vector, contrib, vector))) + assertEquals("0.2.0-rc.1+build.2", IcuData("0.2.0-rc.1+build.2").version) + assertFailsWith { + selectedExtensionDescriptors(listOf(vector, vector.copy(version = "0.8.3"))) + } + for (sqlName in listOf("", "../vector", "Vector", "vector;")) { + assertFailsWith { vector.copy(sqlName = sqlName) } + } + for (product in listOf("", "vector", "oliphaunt-extension-../vector")) { + assertFailsWith { vector.copy(product = product) } + } + assertFailsWith { vector.copy(version = null) } + for (version in listOf("", "latest", "0.2", "../0.2.0")) { + assertFailsWith { vector.copy(version = version) } + assertFailsWith { IcuData(version) } + } + } + + @Test + fun selectionIncludesDependenciesAndFiltersOnlyUnselectedResources() { + val selected = selectedExtensionClosure(listOf("earthdistance", "vector", "earthdistance")) + assertEquals(setOf("earthdistance", "cube", "vector"), selected) + assertFailsWith { selectedExtensionClosure(listOf("unknown")) } + for (file in listOf("vector.so", "cube.dylib", "plpgsql.so")) { + assertTrue(includeSelectedRuntimeFile("lib/postgresql/$file", selected, false)) + } + for (file in listOf("vector.control", "cube--1.5.sql", "earthdistance.sql", "unknown.control")) { + assertTrue(includeSelectedRuntimeFile("share/postgresql/extension/$file", selected, false)) + } + for (file in listOf("lib/postgresql/hstore.so", "share/postgresql/extension/hstore.control", "share/icu", "share/icu/icudt76l.dat")) { + assertFalse(includeSelectedRuntimeFile(file, selected, false)) + } + assertTrue(includeSelectedRuntimeFile("share/icu/icudt76l.dat", selected, true)) + assertTrue(includeSelectedRuntimeFile("share/postgresql/postgres.bki", selected, false)) + } +} diff --git a/src/sdks/rust/tools/check-release-consumer.sh b/src/sdks/rust/tools/check-release-consumer.sh index dc36708e3..8d3bf648b 100755 --- a/src/sdks/rust/tools/check-release-consumer.sh +++ b/src/sdks/rust/tools/check-release-consumer.sh @@ -70,19 +70,26 @@ build_consumer() { OLIPHAUNT_CARGO_METADATA="$metadata" tools/dev/bun.sh -e ' const metadata = await Bun.file(process.env.OLIPHAUNT_CARGO_METADATA).json(); for (const dependency of metadata.packages[0].dependencies) { - if (dependency.name.startsWith("liboliphaunt-native-") || dependency.name.startsWith("oliphaunt-broker-")) { + if (dependency.name.startsWith("liboliphaunt-native-") || dependency.name.startsWith("oliphaunt-broker-") || dependency.name.startsWith("oliphaunt-extension-contrib-pg18-")) { const version = dependency.req.match(/^=([0-9A-Za-z.+-]+)$/)?.[1]; if (!version) throw new Error(`artifact dependency ${dependency.name} must use an exact version`); console.log(`${dependency.name}\t${version}`); } } ' | sort -u >"$dependency_rows" - for pattern in '^liboliphaunt-native-' '^oliphaunt-broker-'; do + for pattern in '^liboliphaunt-native-' '^oliphaunt-broker-' '^oliphaunt-extension-contrib-pg18-'; do rg -q "$pattern" "$dependency_rows" || fail "packed crate is missing artifact dependency $pattern" done { printf '[net]\noffline = true\n\n[patch.crates-io]\n' + for name in oliphaunt-build oliphaunt-resources; do + crate="$(find_one "$sdk_artifacts" "$name-[0-9]*.crate")" + mkdir -p "$scratch/dependencies/$name" + tar -xzf "$crate" -C "$scratch/dependencies/$name" + packed_manifest="$(find_one "$scratch/dependencies/$name" Cargo.toml)" + printf '"%s" = { path = "%s" }\n' "$name" "$(dirname "$packed_manifest")" + done while IFS=$'\t' read -r name version; do stub="$scratch/stubs/$name" mkdir -p "$stub/src" diff --git a/src/sdks/swift/Tests/OliphauntTests/ExtensionResourceCompositionTests.swift b/src/sdks/swift/Tests/OliphauntTests/ExtensionResourceCompositionTests.swift index bbc98889c..031c18b02 100644 --- a/src/sdks/swift/Tests/OliphauntTests/ExtensionResourceCompositionTests.swift +++ b/src/sdks/swift/Tests/OliphauntTests/ExtensionResourceCompositionTests.swift @@ -1,3 +1,4 @@ +import COliphaunt import Foundation @testable @_spi(ExtensionSupport) import Oliphaunt import Testing @@ -692,3 +693,40 @@ func explicitResourceSelectionKeepsDependenciesAndRejectsConflictingVersions() t ]) #expect(throws: OliphauntError.self) { try configuration.prepareExtensionResources() } } + +@Test +func staticExtensionRegistrationValidatesDescriptorsAndRollsBackRejectedResources() throws { + let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { + OliphauntRuntimeResources.unregisterPackagedExtensionResource(sqlName: "vector", resourceRoot: root) + try? FileManager.default.removeItem(at: root) + } + try makeExtensionCompositionFragment( + at: root, product: "oliphaunt-extension-vector", sqlName: "vector", version: "0.8.2", + createsExtension: true, dependencies: [], nativeModuleStem: "vector", + nativeDependencies: [], sharedPreloadLibraries: [] + ) + func register(_ stem: String?, _ descriptor: UnsafePointer?) throws { + try OliphauntStaticExtensionRegistry.register( + product: "oliphaunt-extension-vector", sqlName: "vector", version: "0.8.2", + dependencies: [], nativeDependencies: [], sharedPreloadLibraries: [], + nativeModuleStem: stem, resourceRoot: root, descriptor: descriptor + ) + } + #expect(throws: OliphauntError.self) { try register("vector", nil) } + try "vector".withCString { name throws in + // A missing magic callback must be rejected even when liboliphaunt is installed. + var descriptor = OliphauntStaticExtension(abi_version: UInt32(OLIPHAUNT_STATIC_EXTENSION_ABI_VERSION), name: name, magic: nil, init: nil, symbols: nil, symbol_count: 0, reserved_flags: 0) + try withUnsafePointer(to: &descriptor) { pointer throws in + #expect(throws: OliphauntError.self) { try register(nil, pointer) } + #expect(throws: OliphauntError.self) { try register("wrong", pointer) } + #expect(throws: OliphauntError.self) { try register("vector", pointer) } + // Native rejection must remove the newly inserted resource as well as its descriptor. + #expect(try OliphauntRuntimeResources.registerPackagedExtensionResource( + product: "oliphaunt-extension-vector", version: "0.8.2", sqlName: "vector", + dependencies: [], nativeDependencies: [], nativeModuleStem: "vector", + sharedPreloadLibraries: [], resourceRoot: root + )) + } + } +} diff --git a/src/sdks/swift/Tests/OliphauntTests/OliphauntTests.swift b/src/sdks/swift/Tests/OliphauntTests/OliphauntTests.swift index e3fd14e69..7438a6b53 100644 --- a/src/sdks/swift/Tests/OliphauntTests/OliphauntTests.swift +++ b/src/sdks/swift/Tests/OliphauntTests/OliphauntTests.swift @@ -2634,3 +2634,41 @@ private func makeCompletePgdata(at pgdata: URL) throws { try Data("18\n".utf8).write(to: pgdata.appendingPathComponent("PG_VERSION")) try Data("control".utf8).write(to: pgdata.appendingPathComponent("global/pg_control")) } + +@Test +func explicitResourceDescriptorsPrepareOnceAndRejectConflictingVersions() throws { + let calls = ChunkBox() + let vector = OliphauntExtension( + sqlName: "vector", product: "oliphaunt-extension-vector", version: "0.8.2", + prepare: { calls.append(Data("vector".utf8)) } + ) + let icu = OliphauntIcuData(version: "0.2.0", resourceDirectory: URL(fileURLWithPath: "/resources/icu")) + let configuration = OliphauntConfiguration(extensions: [vector, .init( + sqlName: "pg_trgm", product: "oliphaunt-extension-contrib-pg18", + prepare: { calls.append(Data("pg_trgm".utf8)) } + ), vector], icu: icu) + try configuration.prepareExtensionResources() + #expect(configuration.extensionSqlNames == ["vector", "pg_trgm", "vector"]) + #expect(configuration.icu == icu) + #expect(calls.snapshot() == [Data("pg_trgm".utf8), Data("vector".utf8)]) + #expect(vector == OliphauntExtension(sqlName: "vector", product: vector.product, version: "0.8.2")) + let conflicting = OliphauntExtension(sqlName: "vector", product: vector.product, version: "0.8.3") + #expect(vector != conflicting) + #expect(throws: OliphauntError.self) { + try OliphauntConfiguration(extensions: [vector, conflicting]).prepareExtensionResources() + } + #expect(calls.snapshot().count == 2) + for invalid in [ + OliphauntExtension(sqlName: "../vector", product: vector.product, version: "0.8.2"), + OliphauntExtension(sqlName: "vector", product: "unrelated", version: "0.8.2"), + OliphauntExtension(sqlName: "vector", product: vector.product), + OliphauntExtension(sqlName: "vector", product: vector.product, version: ""), + ] { + #expect(throws: OliphauntError.self) { try invalid.prepare() } + } + let unavailable = OliphauntExtension(sqlName: "vector", product: vector.product, version: "0.8.2") { + throw OliphauntError.engine("missing selected package") + } + #expect(throws: OliphauntError.self) { try unavailable.prepare() } + #expect(throws: OliphauntError.self) { try selectedOliphauntExtensions(["unknown"]) } +} diff --git a/tools/policy/ci-plan-node-products.test.mjs b/tools/policy/ci-plan-node-products.test.mjs index fecf0d1f4..5c022f2b3 100644 --- a/tools/policy/ci-plan-node-products.test.mjs +++ b/tools/policy/ci-plan-node-products.test.mjs @@ -443,3 +443,11 @@ test("extension package tooling invalidates packaging without changing builders" assert.equal(result.tasks.includes(target), false, `${target} does not consume package tooling`); } }); + + +test("Swift package staging receives contrib artifacts through the mobile package producer", () => { + const result = effects("tools/release/sdk-artifacts/swift.mjs"); + for (const job of ["swift-sdk-package", "mobile-extension-packages", "extension-artifacts-native"]) { + assert.equal(result.jobs.includes(job), true, job); + } +}); diff --git a/tools/release/broker-dependency-license-contract.mjs b/tools/release/broker-dependency-license-contract.mjs index 111b47f3e..a897b88d2 100644 --- a/tools/release/broker-dependency-license-contract.mjs +++ b/tools/release/broker-dependency-license-contract.mjs @@ -51,6 +51,7 @@ const PAYLOAD_LICENSE_ATOMS = Object.freeze([ ]); const PATH_PACKAGE_MANIFESTS = new Map([ ["oliphaunt", path.join(ROOT, "src/sdks/rust/Cargo.toml")], + ["oliphaunt-resources", path.join(ROOT, "src/sdks/rust/crates/oliphaunt-resources/Cargo.toml")], ["oliphaunt-broker", path.join(ROOT, "src/runtimes/broker/Cargo.toml")], ]); const LEGAL_BASENAME_PREFIXES = Object.freeze([ diff --git a/tools/release/broker-dependency-license-contract.test.mjs b/tools/release/broker-dependency-license-contract.test.mjs index 85283486f..733735e08 100644 --- a/tools/release/broker-dependency-license-contract.test.mjs +++ b/tools/release/broker-dependency-license-contract.test.mjs @@ -159,6 +159,12 @@ test("production audit prefetches the exact locked all-target closure into a cle test("broker path-package validation follows exact manifests without pinning release versions", () => { const sdkManifest = path.join(ROOT, "src/sdks/rust/Cargo.toml"); + for (const manifest of ["src/sdks/rust/crates/oliphaunt-resources/Cargo.toml", "Cargo.toml"]) { + assert.equal(isAllowedBrokerPathPackageMetadataRow({ + name: "oliphaunt-resources", version: "0.2.0", source: null, + manifest_path: path.join(ROOT, manifest), + }), manifest !== "Cargo.toml"); + } assert.equal(isAllowedBrokerPathPackageMetadataRow({ name: "oliphaunt", version: "17.23.401", diff --git a/tools/release/check_artifact_targets.mjs b/tools/release/check_artifact_targets.mjs index 340b28657..59c489339 100644 --- a/tools/release/check_artifact_targets.mjs +++ b/tools/release/check_artifact_targets.mjs @@ -1080,7 +1080,7 @@ export function validateCiArtifactCoverage(workflow, inventory) { validateWorkflowConsumer(workflow, "extension-packages", ["extension-artifacts-native", "extension-artifacts-wasix", "liboliphaunt-wasix-aot"], [...nativeExtensionArtifacts, ...wasixExtensionArtifacts, ...extensionAot]); validateWorkflowConsumer(workflow, "mobile-extension-packages", ["extension-artifacts-native"], nativeExtensionArtifacts); const abiCompatibleIosRelease = ["liboliphaunt-native-abi-compatible-release-assets-ios-datum64"]; - validateWorkflowConsumer(workflow, "swift-sdk-package", ["liboliphaunt-native-ios-abi"], abiCompatibleIosRelease); + validateWorkflowConsumer(workflow, "swift-sdk-package", ["liboliphaunt-native-ios-abi", "mobile-extension-packages"], [...abiCompatibleIosRelease, "oliphaunt-mobile-extension-package-artifacts"]); validateWorkflowConsumer(workflow, "react-native-sdk-package", ["liboliphaunt-native-ios-abi"], abiCompatibleIosRelease); validateWorkflowConsumer(workflow, "mobile-build-android", ["liboliphaunt-native-android", "liboliphaunt-native-android-abi", "mobile-extension-packages", "kotlin-sdk-package", "react-native-sdk-package"], [ ...matrixRows.reactNativeAndroid.map(({ target }) => `liboliphaunt-native-target-${target}`), diff --git a/tools/release/moon.yml b/tools/release/moon.yml index 5208756d7..16b4c0455 100644 --- a/tools/release/moon.yml +++ b/tools/release/moon.yml @@ -161,6 +161,7 @@ tasks: deps: - "oliphaunt-swift:package" - "liboliphaunt-native:finalize-runtime-ios-abi" + - "extension-packages:package-mobile" inputs: - "@group(legal-files)" - "@group(release-archive-contract)" @@ -177,6 +178,7 @@ tasks: - "/Package.swift" - "/src/extensions/generated/sdk/extensions.json" - "/target/liboliphaunt/abi-compatible-release-assets/ios-datum64/**/*" + - "/target/mobile-extension-artifacts/**/*" - "/tools/dev/bun.sh" - "/src/sources/tools/source-fetch-core.mjs" outputs: diff --git a/tools/release/sdk-artifacts/swift.mjs b/tools/release/sdk-artifacts/swift.mjs index 617710fdc..d840cf831 100644 --- a/tools/release/sdk-artifacts/swift.mjs +++ b/tools/release/sdk-artifacts/swift.mjs @@ -16,6 +16,7 @@ import { assertReleaseNoticesInDirectory, stageReleaseNotices, } from "../release-notices.mjs"; +import { contribCarrierDescriptor, extensionArtifactProductRoot } from "../release-artifact-targets.mjs"; import { productCompatibilityVersion } from "../release-graph.mjs"; import { validateSwiftSourceReleaseContract } from "../swift-source-carrier-contract.mjs"; import { @@ -49,6 +50,8 @@ export async function stageArtifacts(artifactRoot, workRoot) { "tools/release/render_swiftpm_release_package.mjs", "--asset-dir", assetDir, + "--contrib-manifest", + path.join(ROOT, extensionArtifactProductRoot(contribCarrierDescriptor().artifactProduct, "native", "target/mobile-extension-artifacts"), "extension-artifacts.json"), "--output", path.join(artifactRoot, "Package.swift.release"), "--generated-tree", From 24796b7f49e17ec9a0f4ccded354966db4afb076 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 07:20:18 +0000 Subject: [PATCH 05/18] docs(napi): remove obsolete bundled resource contracts --- src/runtimes/wasix-napi/README.md | 85 ++++++++----------- .../packages/darwin-arm64/README.md | 5 +- .../packages/linux-arm64-gnu/README.md | 7 +- .../packages/linux-x64-gnu/README.md | 7 +- .../packages/win32-x64-msvc/README.md | 10 +-- .../wasix-napi/tests/native.integration.mjs | 25 +++--- 6 files changed, 65 insertions(+), 74 deletions(-) diff --git a/src/runtimes/wasix-napi/README.md b/src/runtimes/wasix-napi/README.md index 681abaa76..dcbd12343 100644 --- a/src/runtimes/wasix-napi/README.md +++ b/src/runtimes/wasix-napi/README.md @@ -46,59 +46,46 @@ Ordinary frontend nonzero exits therefore retain stdout and stderr. A `PostgresToolError` is still thrown with its structured diagnostics even if it reports exit code zero; unrelated runtime failures remain thrown errors. -`extensionIdentity(sqlName)` and `toolIdentity(name)` expose each embedded -archive/module as canonical `sha256:size`. The TypeScript adapter compares -these identities with its validated public descriptors, so a same-name but -different payload fails before database startup or tool execution. +`extensionIdentity(sqlName)` exposes an embedded contrib archive as canonical +`sha256:size`. `toolIdentity(name)` reports a tool from an explicitly registered +installed tools package. The TypeScript adapter compares these identities with +its validated public descriptors before use. -`payloadIdentity(component)` exposes the same identity form for the runtime, -standard seed, ICU data, and ICU seed payloads embedded in the single addon. +`payloadIdentity(component)` identifies the embedded runtime archive and +standard seed archive/manifest. ICU payloads come from the selected ICU package. ## Standard and ICU profiles Each platform carrier contains one stable addon subpath, -`oliphaunt_wasix_napi.node`. The release feature embeds both the standard and -ICU payloads in that binary, and database open options select the requested -profile. `supportedProfiles()` reports the exact `['standard', 'icu']` contract. - -The existing TypeScript `icu` option therefore changes the selected database -profile, not the package or binary that gets loaded. The default remains the -standard profile. - -Release builds enable the `release` Cargo feature, which includes packaged -PostgreSQL tools and all extension features supported by the WASIX catalog. -The TypeScript API continues to accept extension descriptors, but the addon -receives the validated SQL names and resolves them against this compile-time -catalog. It never loads arbitrary extension bytes from JavaScript. A new or -updated server extension, or a changed frontend tool, therefore needs a new -N-API carrier release. This makes each carrier larger, but removes portable -archive expansion, WebAssembly compilation, and dynamic side-module linking -from server startup. - -Source-only `cargo check` intentionally leaves those payload features disabled. -The artifact build validates every staged runtime, tool, extension, cluster -seed, and AOT payload before embedding it. - -`tools/build-native.sh` fails closed unless the same-run producer outputs are -available through the dependency build-script contract: - -- `OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR` points at the portable runtime and - split `pg_dump`/`psql` payload root; -- `OLIPHAUNT_WASM_GENERATED_AOT_DIR` points at the root containing the current - Rust target triple's core and tool AOT manifest; -- `OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT` points at the exact portable and - per-target AOT extension inventory; -- `OLIPHAUNT_ICU_DATA_DIR` points at the portable ICU data tree; and -- `OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD=1` prevents every dependency crate - from selecting its source-only fallback. - -The build records and rechecks a deterministic inventory before packaging. -Its portable manifest, split tools, host AOT manifest, every selected extension -manifest/archive/AOT manifest, and ICU tree digest are embedded under -`artifact-provenance.json.buildInputs` in both distribution forms. Its `build` -object also records the release Cargo profile, disabled incremental compilation, -single codegen unit, thin LTO, symbol stripping, exact `release` feature, and -Rust target triple. +`oliphaunt_wasix_napi.node`. It embeds the runtime, initdb, standard seed, and +contrib extensions. External extensions, ICU data with its matching seed, and +frontend tools are separate dependencies. Standard seeds remain bundled; making +those optional is a separate rollout. + +`supportedProfiles()` reports `['standard', 'icu']`. Standard is the default. +Selecting ICU requires the installed ICU descriptor and its bytes; the same +addon supports both profiles without embedding the optional data. + +Release builds enable the `release` feature, which enables extension and tool +APIs. It does not pull optional payload crates into the addon. TypeScript passes +contrib SQL names to Rust and resolves external descriptors to their installed +portable and host AOT package manifests. Rust validates owner, version, target, +runtime compatibility, containment, and payload hashes before loading them. +The tools package follows the same installed-package registration contract. +Compatible external package releases do not require rebuilding the addon. + +Source-only `cargo check` leaves release features disabled. The artifact builder +validates its base payload closure through these inputs: + +- `OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR`: portable runtime, initdb, and standard seed; +- `OLIPHAUNT_WASM_GENERATED_AOT_DIR`: the current target's core AOT manifest; +- `OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT`: contrib portable and target AOT inventory; +- `OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD=1`: reject source-only payload fallbacks. + +The build records and rechecks the runtime and contrib inventories in +`artifact-provenance.json.buildInputs`. The `build` object records the release +Cargo profile, disabled incremental compilation, single codegen unit, thin LTO, +symbol stripping, exact `release` feature, and Rust target triple. The addon's `runtimeVersion()` identity comes directly from the selected `liboliphaunt-wasix-portable` crate. Workspace builds therefore report the local runtime while released carriers retain exact product compatibility pins. @@ -117,7 +104,7 @@ managers to install only the matching target: - `@oliphaunt/wasix-napi-win32-x64-msvc` Carrier packages have no install scripts and never download executable code. -`tools/build-native.sh` creates the single profile-complete addon and +`tools/build-native.sh` creates the single base addon and `tools/package-platform.mjs` stages the matching carrier and portable release archive with source/artifact provenance before `pnpm pack`. Per-target jobs do not write the shared checksum filename; the aggregate release-assets task diff --git a/src/runtimes/wasix-napi/packages/darwin-arm64/README.md b/src/runtimes/wasix-napi/packages/darwin-arm64/README.md index d8784e1b2..896529a59 100644 --- a/src/runtimes/wasix-napi/packages/darwin-arm64/README.md +++ b/src/runtimes/wasix-napi/packages/darwin-arm64/README.md @@ -1,5 +1,6 @@ # `@oliphaunt/wasix-napi-darwin-arm64` Prebuilt Node-API 8 addon for Oliphaunt WASIX on macOS arm64. This optional -carrier embeds both the standard and ICU database profiles in one binary and -has no install script. +carrier embeds the runtime, standard seed, and contrib. External extensions, +ICU data with its matching seed, and tools come from separate packages. It has +no install script. diff --git a/src/runtimes/wasix-napi/packages/linux-arm64-gnu/README.md b/src/runtimes/wasix-napi/packages/linux-arm64-gnu/README.md index 8114af97a..9441150fa 100644 --- a/src/runtimes/wasix-napi/packages/linux-arm64-gnu/README.md +++ b/src/runtimes/wasix-napi/packages/linux-arm64-gnu/README.md @@ -1,5 +1,6 @@ # `@oliphaunt/wasix-napi-linux-arm64-gnu` -Prebuilt Node-API 8 addon for Oliphaunt WASIX on Linux arm64 with glibc. This -optional carrier embeds both the standard and ICU database profiles in one -binary and has no install script. +Prebuilt Node-API 8 addon for Oliphaunt WASIX on Linux arm64 with glibc. This optional +carrier embeds the runtime, standard seed, and contrib. External extensions, +ICU data with its matching seed, and tools come from separate packages. It has +no install script. diff --git a/src/runtimes/wasix-napi/packages/linux-x64-gnu/README.md b/src/runtimes/wasix-napi/packages/linux-x64-gnu/README.md index 1e3be9901..381769cb1 100644 --- a/src/runtimes/wasix-napi/packages/linux-x64-gnu/README.md +++ b/src/runtimes/wasix-napi/packages/linux-x64-gnu/README.md @@ -1,5 +1,6 @@ # `@oliphaunt/wasix-napi-linux-x64-gnu` -Prebuilt Node-API 8 addon for Oliphaunt WASIX on Linux x64 with glibc. This -optional carrier embeds both the standard and ICU database profiles in one -binary and has no install script. +Prebuilt Node-API 8 addon for Oliphaunt WASIX on Linux x64 with glibc. This optional +carrier embeds the runtime, standard seed, and contrib. External extensions, +ICU data with its matching seed, and tools come from separate packages. It has +no install script. diff --git a/src/runtimes/wasix-napi/packages/win32-x64-msvc/README.md b/src/runtimes/wasix-napi/packages/win32-x64-msvc/README.md index 59a1e9078..93e7f529e 100644 --- a/src/runtimes/wasix-napi/packages/win32-x64-msvc/README.md +++ b/src/runtimes/wasix-napi/packages/win32-x64-msvc/README.md @@ -1,8 +1,6 @@ # `@oliphaunt/wasix-napi-win32-x64-msvc` -Prebuilt Node-API 8 addon for Oliphaunt WASIX on Windows x64 with MSVC. This -optional carrier embeds both the standard and ICU database profiles in one -binary and has no install script. Its `prebuilds` directory also carries the -import-derived app-local Visual C++ runtime closure and its SHA-256 receipt. The -unmodified DLLs remain subject to the [Microsoft Visual Studio redistribution -terms](https://learn.microsoft.com/visualstudio/releases/2026/redistribution). +Prebuilt Node-API 8 addon for Oliphaunt WASIX on Windows x64 with MSVC. This optional +carrier embeds the runtime, standard seed, and contrib. External extensions, +ICU data with its matching seed, and tools come from separate packages. It has +no install script. diff --git a/src/runtimes/wasix-napi/tests/native.integration.mjs b/src/runtimes/wasix-napi/tests/native.integration.mjs index 62bbd6f01..0e881beaf 100644 --- a/src/runtimes/wasix-napi/tests/native.integration.mjs +++ b/src/runtimes/wasix-napi/tests/native.integration.mjs @@ -7,7 +7,7 @@ import { Worker } from 'node:worker_threads'; const addonPath = process.argv[2]; if (addonPath === undefined) { - throw new Error('usage: node native.integration.mjs /absolute/path/to/addon.node [--tools]'); + throw new Error('usage: node native.integration.mjs /absolute/path/to/release-addon.node'); } const addon = createRequire(import.meta.url)(addonPath); @@ -19,6 +19,7 @@ const expectedExports = [ 'extensionIdentity', 'nodeApiVersion', 'payloadIdentity', + 'registerTools', 'restore', 'restoreDirect', 'runtimeVersion', @@ -119,12 +120,8 @@ assert.equal( ); assertResponse(await actor.execProtocolRaw(queryMessage('select 4204')), 4204); -if (process.argv.includes('--tools')) { - const dump = await actor.pgDump([]); - assert.equal(dump.status, 0); - assert(dump.stdout.byteLength > 0); - assertTransferable(dump.stdout); - assertTransferable(dump.stderr); +for (const name of ['pg_dump', 'psql']) { + assert.throws(() => addon.toolIdentity(name), /not installed|not embedded|missing|unavailable/iu); } await Promise.all([actor.close(), actor.close()]); @@ -143,10 +140,16 @@ assert.match(server.connectionString, /^postgresql:\/\//u); await Promise.all([server.close(), server.close()]); assert.equal(server.closed, true); -assert.match(addon.payloadIdentity('icuDataArchive'), /^[0-9a-f]{64}:\d+$/u); -const icu = await addon.NativeWasixActorDatabase.open(openOptions('icu')); -assertResponse(await icu.execProtocolRaw(queryMessage('select 4301')), 4301); -await icu.close(); +for (const component of ['icuDataArchive', 'icuSeedArchive', 'icuSeedManifest']) { + assert.throws(() => addon.payloadIdentity(component), /unsupported WASIX payload component/u); +} +for (const sqlName of ['vector', 'pgtap']) { + assert.throws(() => addon.extensionIdentity(sqlName), /not embedded/u); +} +assert.throws( + () => addon.NativeWasixActorDatabase.open(openOptions('icu')), + /requires the optional ICU package/u, +); const temporaryRoot = mkdtempSync(join(tmpdir(), 'oliphaunt-wasix-napi-')); try { From aa8ca5e6a3a2778990c73f3e73bf0a9ae044dd2b Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 07:56:26 +0000 Subject: [PATCH 06/18] fix(sdk): qualify resource loaders and cold consumer builds --- .../src/oliphaunt/extensions.rs | 29 ++++--- .../src/oliphaunt/postgres_mod.rs | 78 +++++++++---------- src/bindings/wasix-rust/moon.yml | 2 + .../liboliphaunt/wasix/crates/assets/build.rs | 20 ++--- .../wasix-napi/src/extension_package.rs | 16 ++-- src/runtimes/wasix-napi/src/lib.rs | 11 ++- .../rust/crates/oliphaunt-build/README.md | 9 ++- .../rust/crates/oliphaunt-build/src/lib.rs | 8 +- tools/release/maven-artifact-staging.test.mjs | 3 +- 9 files changed, 90 insertions(+), 86 deletions(-) diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/extensions.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/extensions.rs index a2d2a151a..a49eb2ed9 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/extensions.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/extensions.rs @@ -132,16 +132,16 @@ pub(crate) fn resolve_extension_set(extensions: &[Extension]) -> Result, + package: Option<&super::extensions::ExtensionPackage>, + label: &str, + ) -> Result<()> { + let Some(aot_name) = aot_name else { + return Ok(()); + }; + let path = Path::new("/").join(runtime_path); + let wasm = self + .runtime_storage + .read(&path) + .with_context(|| format!("{label} is not installed at {}", path.display()))?; + if let Some(package) = package { + let module = aot::load_package_module(&self.engine, package, aot_name, &wasm)?; + block_on_tokio_runtime( + &self.tokio_runtime, + self.wasix_module_cache + .save(ModuleHash::new(&wasm), &self.engine, &module), + ) + .with_context(|| format!("seed Wasmer module cache for {label}"))?; + return Ok(()); + } + seed_wasix_module_cache_bytes( + &self.tokio_runtime, + &self.engine, + &self.wasix_module_cache, + &wasm, + aot_name, + label, + ) + } + #[cfg(feature = "extensions")] pub fn preload_extension_module(&self, extension: Extension) -> Result<()> { for module in extension.native_support_modules() { - seed_extension_side_module( - &self.tokio_runtime, - &self.engine, - &self.wasix_module_cache, - &self.runtime_storage, + self.seed_extension_side_module( module.runtime_path(), module.aot_name(), extension.package(), @@ -433,11 +465,7 @@ impl PostgresMod { let Some(module_file) = extension.native_module_file() else { return Ok(()); }; - seed_extension_side_module( - &self.tokio_runtime, - &self.engine, - &self.wasix_module_cache, - &self.runtime_storage, + self.seed_extension_side_module( &format!("lib/postgresql/{module_file}"), extension.aot_name(), extension.package(), @@ -1357,36 +1385,6 @@ fn preload_runtime_side_modules( Ok(()) } -#[cfg(feature = "extensions")] -fn seed_extension_side_module( - runtime: &TokioRuntime, - engine: &Engine, - module_cache: &Arc, - runtime_root: &StorageRoot, - runtime_path: &str, - aot_name: Option<&'static str>, - package: Option<&super::extensions::ExtensionPackage>, - label: &str, -) -> Result<()> { - let Some(aot_name) = aot_name else { - return Ok(()); - }; - let path = Path::new("/").join(runtime_path); - let wasm = runtime_root - .read(&path) - .with_context(|| format!("{label} is not installed at {}", path.display()))?; - if let Some(package) = package { - let module = aot::load_package_module(engine, package, aot_name, &wasm)?; - block_on_tokio_runtime( - runtime, - module_cache.save(ModuleHash::new(&wasm), engine, &module), - ) - .with_context(|| format!("seed Wasmer module cache for {label}"))?; - return Ok(()); - } - seed_wasix_module_cache_bytes(runtime, engine, module_cache, &wasm, aot_name, label) -} - fn seed_wasix_module_cache( runtime: &TokioRuntime, engine: &Engine, diff --git a/src/bindings/wasix-rust/moon.yml b/src/bindings/wasix-rust/moon.yml index 5e27beabd..b5660a9e0 100644 --- a/src/bindings/wasix-rust/moon.yml +++ b/src/bindings/wasix-rust/moon.yml @@ -101,6 +101,8 @@ tasks: inputs: [] options: cache: false + # Both suites share Cargo outputs; feature rebuilds must not race rustdoc. + runDepsInParallel: false runFromWorkspaceRoot: true runInCI: false diff --git a/src/runtimes/liboliphaunt/wasix/crates/assets/build.rs b/src/runtimes/liboliphaunt/wasix/crates/assets/build.rs index d623d60e2..0a19e04f4 100644 --- a/src/runtimes/liboliphaunt/wasix/crates/assets/build.rs +++ b/src/runtimes/liboliphaunt/wasix/crates/assets/build.rs @@ -493,18 +493,14 @@ fn write_generated_assets(out: &Path, asset_dir: &Path, selected_extensions: &[S required.display() ); } - for (profile, archive, seed_manifest) in - [("standard", &standard_seed_archive, &standard_seed_manifest)] - { - assert!( - (archive.is_file() && seed_manifest.is_file()) - || (!archive.exists() && !seed_manifest.exists()), - "generated asset directory {} has an incomplete {profile} cluster seed; expected both {} and {} or neither", - asset_dir.display(), - archive.display(), - seed_manifest.display() - ); - } + assert!( + (standard_seed_archive.is_file() && standard_seed_manifest.is_file()) + || (!standard_seed_archive.exists() && !standard_seed_manifest.exists()), + "generated asset directory {} has an incomplete standard cluster seed; expected both {} and {} or neither", + asset_dir.display(), + standard_seed_archive.display(), + standard_seed_manifest.display() + ); let standard_seed_archive_body = optional_include_bytes_body(&standard_seed_archive); let standard_seed_manifest_body = optional_include_bytes_body(&standard_seed_manifest); diff --git a/src/runtimes/wasix-napi/src/extension_package.rs b/src/runtimes/wasix-napi/src/extension_package.rs index 5f11f49d0..a7ea2c029 100644 --- a/src/runtimes/wasix-napi/src/extension_package.rs +++ b/src/runtimes/wasix-napi/src/extension_package.rs @@ -9,20 +9,15 @@ use std::path::{Component, Path, PathBuf}; use std::sync::{Mutex, OnceLock}; use napi::{Error, Result}; +#[cfg(feature = "tools")] use napi_derive::napi; -use oliphaunt_wasix::{Extension, ExtensionPackage}; +use oliphaunt_wasix::ExtensionPackage; use serde_json::Value; use sha2::{Digest, Sha256}; +#[cfg(any(feature = "extensions", test))] +use {super::NativeExtensionPackage, oliphaunt_wasix::Extension}; -#[napi(object)] -pub struct NativeExtensionPackage { - pub sql_name: String, - pub product: String, - pub version: String, - pub package_json: String, - pub aot_package_json: Option, -} - +#[cfg(feature = "tools")] #[napi(object)] pub struct NativeToolPackage { pub package_json: String, @@ -244,6 +239,7 @@ fn target() -> &'static str { "unsupported" } +#[cfg(any(feature = "extensions", test))] pub(super) fn load(selection: NativeExtensionPackage) -> Result { let extension = Extension::by_sql_name(&selection.sql_name) .ok_or_else(|| fail("unknown selected WASIX extension"))?; diff --git a/src/runtimes/wasix-napi/src/lib.rs b/src/runtimes/wasix-napi/src/lib.rs index a21167511..3eb00560b 100644 --- a/src/runtimes/wasix-napi/src/lib.rs +++ b/src/runtimes/wasix-napi/src/lib.rs @@ -4,9 +4,9 @@ //! owners directly. Promise settlement is the only owner-to-JavaScript hop; //! no Tokio runtime or Node async-work queue participates in database work. +#[cfg(any(feature = "extensions", feature = "tools", test))] mod extension_package; mod icu_data; -use extension_package::NativeExtensionPackage; use icu_data::NativeIcuData; use std::collections::BTreeMap; @@ -62,6 +62,15 @@ pub struct NativeStorageOptions { pub path: Option, } +#[napi(object)] +pub struct NativeExtensionPackage { + pub sql_name: String, + pub product: String, + pub version: String, + pub package_json: String, + pub aot_package_json: Option, +} + #[napi(object)] pub struct NativeOpenOptions { pub profile: String, diff --git a/src/sdks/rust/crates/oliphaunt-build/README.md b/src/sdks/rust/crates/oliphaunt-build/README.md index d28fafd4a..856b72680 100644 --- a/src/sdks/rust/crates/oliphaunt-build/README.md +++ b/src/sdks/rust/crates/oliphaunt-build/README.md @@ -1,9 +1,12 @@ # oliphaunt-build -`oliphaunt-build` is the Cargo build-script helper for Rust applications that -bundle Oliphaunt runtime artifacts. +`oliphaunt-build` stages Cargo-resolved Oliphaunt artifacts. The native SDK and +resource packages call `embed_resolved_artifacts()` in their own builds, so +ordinary applications need no build script, build dependency, or +`package.metadata.oliphaunt` configuration. See the [Rust SDK setup](../../README.md). -Applications add it as a build dependency and call it from `build.rs`: +Custom bundles can instead use the explicit staging API from their own +`build.rs`: ```rust fn main() { diff --git a/src/sdks/rust/crates/oliphaunt-build/src/lib.rs b/src/sdks/rust/crates/oliphaunt-build/src/lib.rs index 6373deafd..c446908a0 100644 --- a/src/sdks/rust/crates/oliphaunt-build/src/lib.rs +++ b/src/sdks/rust/crates/oliphaunt-build/src/lib.rs @@ -1,8 +1,8 @@ -//! Cargo build-script integration for Oliphaunt applications. +//! Cargo build-script integration for Oliphaunt SDKs and custom bundles. //! -//! `configure()` is intended to be called from an application `build.rs`. -//! Cargo resolves target-specific artifact crates; this crate stages the -//! already-resolved files into `OUT_DIR`. +//! SDK and resource crates call `embed_resolved_artifacts()` internally; ordinary +//! applications need no build script. Custom bundles can call `configure()` to +//! stage Cargo-resolved artifacts into `OUT_DIR`. use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; diff --git a/tools/release/maven-artifact-staging.test.mjs b/tools/release/maven-artifact-staging.test.mjs index 7c1d61248..928a6027c 100644 --- a/tools/release/maven-artifact-staging.test.mjs +++ b/tools/release/maven-artifact-staging.test.mjs @@ -229,6 +229,7 @@ test("the release preflight freezes and bundles the exact locally staged Maven b }); +// A cold runner downloads Gradle dependencies and compiles the real SDK jar. test("publishes importable external and ICU descriptors with their SDK dependency", async () => { const value = fixture(); const fields = readFileSync(value.manifest, "utf8").trimEnd().split("\t"); @@ -258,4 +259,4 @@ public final class Vector { } expect(readFileSync(path.join(staged[1].directory, "oliphaunt-icu-1.2.3.tar.gz"), "utf8")) .toBe("exact runtime carrier\n"); -}, 120_000); +}, 600_000); From dec9c9b5124a9b42ec90cdbc18f2c7b76e0cf6ac Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 08:29:31 +0000 Subject: [PATCH 07/18] fix(sdk): clean up stale carriers and activate packaging checks --- .github/workflows/ci.yml | 1 - docs/maintainers/assets.md | 47 +-- .../crates/oliphaunt-wasix/src/async_api.rs | 8 +- .../oliphaunt-wasix/src/oliphaunt/builder.rs | 4 +- .../src/oliphaunt/extensions.rs | 4 +- .../oliphaunt-wasix/src/oliphaunt/server.rs | 4 +- .../native/packages/darwin-arm64/package.json | 4 +- .../packages/linux-arm64-gnu/package.json | 4 +- .../packages/linux-x64-gnu/package.json | 4 +- .../packages/win32-x64-msvc/package.json | 4 +- .../wasix-napi/tests/native.integration.mjs | 1 - .../wasix-napi/tools/smoke-packaged-addon.mjs | 8 + .../js/src/__tests__/native-bindings.test.ts | 42 +- src/sdks/js/src/native/assets-deno.ts | 12 +- src/sdks/js/src/native/assets-node.ts | 1 - .../build.gradle.kts | 379 ------------------ src/sdks/kotlin/settings.gradle.kts | 1 - src/sdks/rust/src/direct.rs | 8 +- src/sdks/swift/moon.yml | 5 + .../swift/tools/render-extension-products.mjs | 7 +- .../render-extension-products.test-driver.mjs | 90 ----- .../tools/swift-carrier-resolver.test.mjs | 10 + ...=> build-maven-artifact-manifest.test.mjs} | 20 +- tools/release/check_artifact_targets.mjs | 8 +- tools/release/moon.yml | 5 +- .../native-npm-archive-extraction.test.mjs | 4 +- tools/release/publication-lock.mjs | 117 ------ 27 files changed, 125 insertions(+), 677 deletions(-) delete mode 100644 src/sdks/kotlin/oliphaunt-maven-artifacts/build.gradle.kts delete mode 100644 src/sdks/swift/tools/render-extension-products.test-driver.mjs rename tools/release/{maven-artifact-manifest-publication-lock.test.mjs => build-maven-artifact-manifest.test.mjs} (92%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91b2107f5..6b590915c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1405,7 +1405,6 @@ jobs: shell: bash env: OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD: "1" - OLIPHAUNT_ICU_DATA_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/wasix-build/work/icu-wasix/share/icu OLIPHAUNT_WASM_GENERATED_AOT_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/aot OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT: ${{ github.workspace }}/target/extension-artifacts OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/assets diff --git a/docs/maintainers/assets.md b/docs/maintainers/assets.md index 5e4d17fe6..1985b013e 100644 --- a/docs/maintainers/assets.md +++ b/docs/maintainers/assets.md @@ -16,43 +16,27 @@ artifact products staged by the language build integration. The WASIX artifact products contain: - the portable Oliphaunt/Postgres WASIX runtime tree; -- `standard` and `icu` cluster seeds for faster new databases; -- bundled extension archives for supported SQL extensions; +- the bundled `standard` cluster seed for faster new databases; +- an optional ICU data package containing the matching `icu` cluster seed; +- bundled contrib archives and independently installed external extension archives; - the packaged `initdb` module used by asset CI and explicit fresh-initdb paths; - the packaged `pg_dump` and `psql` modules used by the optional tools APIs and maintenance CLI; - a target-specific Wasmer AOT pack when the current host target is supported. -Application code depends on `oliphaunt-wasix` plus the selected artifact -packages. The build integration stages only selected package-manager artifacts -into the application output. +Application code depends on `oliphaunt-wasix` plus its selected resource packages. +Those packages supply their own bytes; the SDK installs the selected resources. -## Feature Flags +## Application dependencies -Default SDK dependency after the first public release (use the exact version -selected by the application lockfile): +The SDK resolves the base runtime, standard seed, contrib archives, and host AOT +carrier through Cargo. External extensions have independent versioned packages; +applications pass their exported descriptors alongside the SDK's contrib values. +See the [WASIX Rust guide](../../src/docs/content/sdk/wasix-rust/guide.mdx) for setup. -```toml -oliphaunt-wasix = "0.1" -``` - -Enable only the extension selectors the application uses: - -```toml -oliphaunt-wasix = { version = "0.1", features = [ - "extension-vector", - "extension-pg-trgm", -] } -``` - -The repository source version remains `0.0.0` until Release Please creates the -first `0.1.0` release PR. Do not copy the repository source version into a -consumer manifest and do not reuse the legacy repository-wide `0.5.x` tags; -they predate the independently versioned Oliphaunt products. - -The crate exposes no `bundled` feature. Runtime and AOT assets enter the -application through package-manager artifact products, not through SDK default -features or public archive environment variables. +The optional `tools` feature selects the split tool and tools-AOT carriers. +ICU data and its matching seed come from `oliphaunt-icu`. Neither ICU nor external +extensions use SDK feature flags. Ordinary applications need no build script. ## Cache Behavior @@ -86,10 +70,11 @@ generated startup configuration, including `shared_preload_libraries`, before PostgreSQL starts: ```rust,no_run -use oliphaunt_wasix::{Extension, Oliphaunt}; +use oliphaunt_wasix::{extensions, Oliphaunt}; +use oliphaunt_extension_vector_wasix::VECTOR; let mut db = Oliphaunt::builder() - .extensions([Extension::VECTOR, Extension::PG_TRGM]) + .extensions([VECTOR, extensions::PG_TRGM]) .open()?; # Ok::<_, Box>(()) ``` diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/async_api.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/async_api.rs index 19ede7937..3129b83df 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/async_api.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/async_api.rs @@ -1801,7 +1801,7 @@ impl AsyncOliphauntBuilder { } #[cfg(feature = "extensions")] - /// Make one bundled PostgreSQL extension artifact available to the database. + /// Make one explicitly selected PostgreSQL extension artifact available to the database. /// Database-local installation remains the application's migration concern. pub fn extension(mut self, extension: impl Into) -> Self { self.inner = self.inner.extension(extension); @@ -1809,7 +1809,7 @@ impl AsyncOliphauntBuilder { } #[cfg(feature = "extensions")] - /// Make bundled PostgreSQL extension artifacts available to the database. + /// Make explicitly selected PostgreSQL extension artifacts available to the database. /// Database-local installation remains the application's migration concern. pub fn extensions>( mut self, @@ -2444,7 +2444,7 @@ impl AsyncOliphauntServerBuilder { } #[cfg(feature = "extensions")] - /// Make one bundled PostgreSQL extension artifact available to clients. + /// Make one explicitly selected PostgreSQL extension artifact available to clients. /// Database-local installation remains the application's migration concern. pub fn extension(mut self, extension: impl Into) -> Self { self.inner = self.inner.extension(extension); @@ -2452,7 +2452,7 @@ impl AsyncOliphauntServerBuilder { } #[cfg(feature = "extensions")] - /// Make bundled PostgreSQL extension artifacts available to clients. + /// Make explicitly selected PostgreSQL extension artifacts available to clients. /// Database-local installation remains the application's migration concern. pub fn extensions>( mut self, diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs index ccaac9a75..6b6fc01f9 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs @@ -98,7 +98,7 @@ impl OliphauntBuilder { self } - /// Make one bundled PostgreSQL extension artifact available to the database. + /// Make one explicitly selected PostgreSQL extension artifact available to the database. /// Database-local installation remains the application's migration concern. #[cfg(feature = "extensions")] pub fn extension(mut self, extension: impl Into) -> Self { @@ -106,7 +106,7 @@ impl OliphauntBuilder { self } - /// Make bundled PostgreSQL extension artifacts available to the database. + /// Make explicitly selected PostgreSQL extension artifacts available to the database. /// Database-local installation remains the application's migration concern. #[cfg(feature = "extensions")] pub fn extensions>( diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/extensions.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/extensions.rs index a49eb2ed9..d450cf1ed 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/extensions.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/extensions.rs @@ -30,7 +30,7 @@ impl ExtensionNativeModule { use oliphaunt_resources::WasixExtensionDescriptor; pub use oliphaunt_resources::WasixPackage as ExtensionPackage; -/// A bundled PostgreSQL extension artifact that Oliphaunt can make available. +/// A selected PostgreSQL extension artifact that Oliphaunt can make available. /// /// Selecting an extension does not run `CREATE EXTENSION`, `LOAD`, or other /// database-local SQL. Applications retain ordinary migration ownership. @@ -300,7 +300,7 @@ fn visit_extension( } if !visiting.insert(extension.sql_name()) { return Err(crate::error::invalid_configuration(format!( - "cyclic bundled extension dependency involving '{}'", + "cyclic extension dependency involving '{}'", extension.sql_name() ))); } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/server.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/server.rs index c968be304..0d822123a 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/server.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/server.rs @@ -349,7 +349,7 @@ impl OliphauntServerBuilder { self } - /// Make one bundled PostgreSQL extension artifact available to clients. + /// Make one explicitly selected PostgreSQL extension artifact available to clients. /// Database-local installation remains the application's migration concern. #[cfg(feature = "extensions")] pub fn extension(mut self, extension: impl Into) -> Self { @@ -357,7 +357,7 @@ impl OliphauntServerBuilder { self } - /// Make bundled PostgreSQL extension artifacts available to clients. + /// Make explicitly selected PostgreSQL extension artifacts available to clients. /// Database-local installation remains the application's migration concern. #[cfg(feature = "extensions")] pub fn extensions>( diff --git a/src/runtimes/liboliphaunt/native/packages/darwin-arm64/package.json b/src/runtimes/liboliphaunt/native/packages/darwin-arm64/package.json index e08352ec0..2b2c32ebd 100644 --- a/src/runtimes/liboliphaunt/native/packages/darwin-arm64/package.json +++ b/src/runtimes/liboliphaunt/native/packages/darwin-arm64/package.json @@ -21,8 +21,7 @@ "clusterSeedTarget": "macos-arm64", "libraryRelativePath": "lib/liboliphaunt.dylib", "runtimeRelativePath": "runtime", - "clusterSeedRelativePath": "cluster-seed", - "icuClusterSeedRelativePath": "cluster-seed-icu" + "clusterSeedRelativePath": "cluster-seed" }, "publishConfig": { "access": "public", @@ -37,7 +36,6 @@ "lib", "runtime", "cluster-seed", - "cluster-seed-icu", "manifest.properties", "README.md", "LICENSE", diff --git a/src/runtimes/liboliphaunt/native/packages/linux-arm64-gnu/package.json b/src/runtimes/liboliphaunt/native/packages/linux-arm64-gnu/package.json index 8cc20b12f..6eca728f1 100644 --- a/src/runtimes/liboliphaunt/native/packages/linux-arm64-gnu/package.json +++ b/src/runtimes/liboliphaunt/native/packages/linux-arm64-gnu/package.json @@ -24,8 +24,7 @@ "clusterSeedTarget": "linux-arm64-gnu", "libraryRelativePath": "lib/liboliphaunt.so", "runtimeRelativePath": "runtime", - "clusterSeedRelativePath": "cluster-seed", - "icuClusterSeedRelativePath": "cluster-seed-icu" + "clusterSeedRelativePath": "cluster-seed" }, "publishConfig": { "access": "public", @@ -40,7 +39,6 @@ "lib", "runtime", "cluster-seed", - "cluster-seed-icu", "manifest.properties", "README.md", "LICENSE", diff --git a/src/runtimes/liboliphaunt/native/packages/linux-x64-gnu/package.json b/src/runtimes/liboliphaunt/native/packages/linux-x64-gnu/package.json index fb1f483f4..69ea54e5d 100644 --- a/src/runtimes/liboliphaunt/native/packages/linux-x64-gnu/package.json +++ b/src/runtimes/liboliphaunt/native/packages/linux-x64-gnu/package.json @@ -24,8 +24,7 @@ "clusterSeedTarget": "linux-x64-gnu", "libraryRelativePath": "lib/liboliphaunt.so", "runtimeRelativePath": "runtime", - "clusterSeedRelativePath": "cluster-seed", - "icuClusterSeedRelativePath": "cluster-seed-icu" + "clusterSeedRelativePath": "cluster-seed" }, "publishConfig": { "access": "public", @@ -40,7 +39,6 @@ "lib", "runtime", "cluster-seed", - "cluster-seed-icu", "manifest.properties", "README.md", "LICENSE", diff --git a/src/runtimes/liboliphaunt/native/packages/win32-x64-msvc/package.json b/src/runtimes/liboliphaunt/native/packages/win32-x64-msvc/package.json index 337609de5..01359be5f 100644 --- a/src/runtimes/liboliphaunt/native/packages/win32-x64-msvc/package.json +++ b/src/runtimes/liboliphaunt/native/packages/win32-x64-msvc/package.json @@ -21,8 +21,7 @@ "clusterSeedTarget": "windows-x64-msvc", "libraryRelativePath": "bin/oliphaunt.dll", "runtimeRelativePath": "runtime", - "clusterSeedRelativePath": "cluster-seed", - "icuClusterSeedRelativePath": "cluster-seed-icu" + "clusterSeedRelativePath": "cluster-seed" }, "publishConfig": { "access": "public", @@ -38,7 +37,6 @@ "lib", "runtime", "cluster-seed", - "cluster-seed-icu", "manifest.properties", "README.md", "LICENSE", diff --git a/src/runtimes/wasix-napi/tests/native.integration.mjs b/src/runtimes/wasix-napi/tests/native.integration.mjs index 0e881beaf..488f5de30 100644 --- a/src/runtimes/wasix-napi/tests/native.integration.mjs +++ b/src/runtimes/wasix-napi/tests/native.integration.mjs @@ -70,7 +70,6 @@ assertTransferable(directResponse); const directChunks = []; assert.equal( direct.execProtocolRawStream(queryMessage('select 4102'), (chunk) => { - assertResponse(chunk, 4102); directChunks.push(assertTransferable(chunk)); }), 'complete', diff --git a/src/runtimes/wasix-napi/tools/smoke-packaged-addon.mjs b/src/runtimes/wasix-napi/tools/smoke-packaged-addon.mjs index a01f346d9..9dd49c892 100644 --- a/src/runtimes/wasix-napi/tools/smoke-packaged-addon.mjs +++ b/src/runtimes/wasix-napi/tools/smoke-packaged-addon.mjs @@ -330,6 +330,13 @@ async function main() { } await runWorkerUnloadSmoke(scratch, carrierManifest, options.runtime); + if (options.runtime === "node" && options.packageManager === "npm") { + const require = createRequire(path.join(scratch, "package.json")); + const addonPath = require.resolve(`${carrierManifest.name}/${BINARY}`); + const { stdout } = await run(process.execPath, + [path.join(PACKAGE_ROOT, "tests/native.integration.mjs"), addonPath], scratch); + process.stdout.write(stdout); + } const verification = path.join(scratch, "verify.mjs"); await writeFile( @@ -371,6 +378,7 @@ for (const name of [ 'extensionIdentity', 'nodeApiVersion', 'payloadIdentity', + 'registerTools', 'restore', 'restoreDirect', 'runtimeVersion', diff --git a/src/sdks/js/src/__tests__/native-bindings.test.ts b/src/sdks/js/src/__tests__/native-bindings.test.ts index 78ef6128c..18445bf10 100644 --- a/src/sdks/js/src/__tests__/native-bindings.test.ts +++ b/src/sdks/js/src/__tests__/native-bindings.test.ts @@ -12,7 +12,7 @@ import { import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import { test, vi } from 'vitest'; import * as publicEntrypoint from '../index.js'; import Oliphaunt, { type OliphauntClient } from '../index.js'; @@ -1112,6 +1112,46 @@ async function testDenoPackageManagedResolverUsesStandardCarrierRuntime(): Promi assert.equal(install.libraryPath, join(runtimePackageRoot, target.libraryRelativePath)); assert.equal(install.packageManaged, true); assert.equal(install.runtimeDirectory, join(runtimePackageRoot, target.runtimeRelativePath)); + assert.equal(install.clusterSeedDirectory, join(runtimePackageRoot, 'cluster-seed')); + + const icuRoot = join(root, 'optional-icu'); + const icuBundle = join(icuRoot, 'OliphauntICU.bundle'); + const versions = await readTypeScriptPackageVersions(); + await writeFixtureFile( + join(icuRoot, 'package.json'), + JSON.stringify({ + name: '@oliphaunt/icu', + version: versions.icuVersion, + oliphaunt: { + product: 'oliphaunt-icu', + kind: 'icu-data', + target: 'portable', + dataRelativePath: 'OliphauntICU.bundle/share/icu', + manifestRelativePath: 'OliphauntICU.bundle/manifest.properties', + icuDataTreeSha256: 'a'.repeat(64), + }, + }), + createdFiles, + ); + await writeFixtureFile(join(icuBundle, 'share/icu/icudt76l.dat'), 'ICU', createdFiles); + await writeFixtureFile( + join(icuBundle, 'manifest.properties'), + `schema=oliphaunt-icu-data-v1\nartifactRole=icu-data\nicuDataVersion=76.1\nicuDataForm=files-le\nicuDataTreeSha256=${'a'.repeat(64)}\n`, + createdFiles, + ); + const seed = join(icuBundle, 'native-seeds', target.id); + await writeClusterSeedFixture(seed, 'icu', target.id, createdFiles); + const descriptor = { + schema: 'oliphaunt-native-icu-v1' as const, + packageName: '@oliphaunt/icu' as const, + version: versions.icuVersion, + packageJsonUrl: pathToFileURL(join(icuRoot, 'package.json')).href, + }; + const withIcu = await resolveDenoNativeInstall(undefined, descriptor); + assert.equal(withIcu.clusterSeedDirectory, seed); + assert.equal(withIcu.catalogProfile, 'icu'); + await rm(join(seed, 'manifest.properties')); + await assert.rejects(resolveDenoNativeInstall(undefined, descriptor), /ICU cluster seed/); } finally { if (previousDeno === undefined) { delete (globalThis as { Deno?: unknown }).Deno; diff --git a/src/sdks/js/src/native/assets-deno.ts b/src/sdks/js/src/native/assets-deno.ts index 3100dafb7..fe64478c3 100644 --- a/src/sdks/js/src/native/assets-deno.ts +++ b/src/sdks/js/src/native/assets-deno.ts @@ -58,7 +58,6 @@ type LiboliphauntPackageMetadata = { libraryRelativePath?: string; runtimeRelativePath?: string; clusterSeedRelativePath?: string; - icuClusterSeedRelativePath?: string; clusterSeedTarget?: string; }; }; @@ -77,6 +76,7 @@ type IcuPackageMetadata = { }; type ResolvedDenoIcuResources = { + seedDirectory: URL; dataDirectory: string; dataTreeSha256: string; }; @@ -196,11 +196,6 @@ async function resolvePackageNativeInstall( 'cluster-seed', `${target.packageName} clusterSeedRelativePath`, ); - const icuClusterSeedRelativePath = requireNativeClusterSeedPath( - packageJson.oliphaunt.icuClusterSeedRelativePath, - 'cluster-seed-icu', - `${target.packageName} icuClusterSeedRelativePath`, - ); const packageRoot = new URL('.', packageJsonUrl); const carrierManifestUrl = new URL('manifest.properties', packageRoot); await requireFile(deno, carrierManifestUrl, `${target.packageName} runtime carrier receipt`); @@ -244,8 +239,8 @@ async function resolvePackageNativeInstall( icu === undefined ? standardClusterSeedUrl : resolvePackageRelativeUrl( - packageRoot, - icuClusterSeedRelativePath, + icu.seedDirectory, + clusterSeedTarget, `${target.packageName} ICU cluster seed metadata`, ); let icuDataTreeSha256: string | undefined; @@ -319,6 +314,7 @@ async function resolveDenoIcuResources( throw new Error(`${packageName} ICU data receipt does not match package metadata`); } return { + seedDirectory: new URL('native-seeds/', manifestUrl), dataDirectory: fileURLToPath(dataUrl), dataTreeSha256, }; diff --git a/src/sdks/js/src/native/assets-node.ts b/src/sdks/js/src/native/assets-node.ts index abce92b0d..05c10d72a 100644 --- a/src/sdks/js/src/native/assets-node.ts +++ b/src/sdks/js/src/native/assets-node.ts @@ -67,7 +67,6 @@ type LiboliphauntPackageMetadata = { libraryRelativePath?: string; runtimeRelativePath?: string; clusterSeedRelativePath?: string; - icuClusterSeedRelativePath?: string; clusterSeedTarget?: string; }; }; diff --git a/src/sdks/kotlin/oliphaunt-maven-artifacts/build.gradle.kts b/src/sdks/kotlin/oliphaunt-maven-artifacts/build.gradle.kts deleted file mode 100644 index e1787914a..000000000 --- a/src/sdks/kotlin/oliphaunt-maven-artifacts/build.gradle.kts +++ /dev/null @@ -1,379 +0,0 @@ -import groovy.json.JsonSlurper -import org.gradle.api.GradleException -import org.gradle.api.publish.maven.MavenPublication -import org.gradle.api.tasks.bundling.Jar -import org.gradle.api.tasks.compile.JavaCompile -import java.util.Locale - -plugins { - `maven-publish` - alias(libs.plugins.maven.publish) -} - -data class OliphauntMavenLicense( - val name: String, - val url: String, - val distribution: String, -) - -data class OliphauntMavenArtifact( - val groupId: String, - val artifactId: String, - val version: String, - val file: File, - val name: String, - val description: String, - val runtimeProduct: String?, - val runtimeVersion: String?, - val licenseSpdx: String, - val licenses: List, -) - -val manifestPath = - providers - .gradleProperty("oliphauntMavenArtifactsManifest") - .orElse(providers.environmentVariable("OLIPHAUNT_MAVEN_ARTIFACTS_MANIFEST")) -val repositoryRoot = rootDir.toPath().resolve("../../..").normalize().toFile() -val baseReleaseNoticeFiles = - files( - repositoryRoot.resolve("LICENSE"), - repositoryRoot.resolve("THIRD_PARTY_NOTICES.md"), - ) - -fun manifestFilePath(value: String): File { - val path = File(value) - return if (path.isAbsolute) path else repositoryRoot.resolve(value) -} - -fun parseLicenses(value: String, label: String): List { - val parsed = - try { - JsonSlurper().parseText(value) - } catch (cause: Exception) { - throw GradleException("$label must be valid JSON", cause) - } - if (parsed !is List<*> || parsed.isEmpty()) { - throw GradleException("$label must be a non-empty JSON array") - } - return parsed.mapIndexed { index, raw -> - if (raw !is Map<*, *>) { - throw GradleException("$label entry ${index + 1} must be a JSON object") - } - val expectedKeys = setOf("name", "url", "distribution") - val actualKeys = raw.keys.map { it?.toString() }.toSet() - if (actualKeys != expectedKeys) { - throw GradleException("$label entry ${index + 1} must contain exactly $expectedKeys") - } - fun requiredString(key: String): String = - (raw[key] as? String)?.takeIf { it.isNotBlank() } - ?: throw GradleException("$label entry ${index + 1}.$key must be a non-empty string") - OliphauntMavenLicense( - name = requiredString("name"), - url = requiredString("url"), - distribution = requiredString("distribution"), - ) - } -} - -fun parseArtifactManifest(path: File): List { - if (!path.isFile) { - throw GradleException("Oliphaunt Maven artifact manifest is missing: $path") - } - val rows = - path.readLines(Charsets.UTF_8) - .filter { it.isNotBlank() && !it.startsWith("#") } - if (rows.isEmpty()) { - throw GradleException("Oliphaunt Maven artifact manifest is empty: $path") - } - val artifacts = - rows.mapIndexed { index, line -> - val parts = line.split('\t') - if (parts.size != 10) { - throw GradleException( - "Oliphaunt Maven artifact manifest ${path.relativeToOrSelf(rootDir)} line ${index + 1} must have 10 tab-separated fields", - ) - } - val file = manifestFilePath(parts[3]) - OliphauntMavenArtifact( - groupId = parts[0], - artifactId = parts[1], - version = parts[2], - file = file, - name = parts[4], - description = parts[5], - runtimeProduct = parts[6].ifBlank { null }, - runtimeVersion = parts[7].ifBlank { null }, - licenseSpdx = parts[8], - licenses = parseLicenses(parts[9], "Oliphaunt Maven artifact manifest line ${index + 1} licenses"), - ) - } - val duplicateCoordinates = - artifacts - .groupBy { "${it.groupId}:${it.artifactId}:${it.version}" } - .filterValues { it.size > 1 } - .keys - .sorted() - if (duplicateCoordinates.isNotEmpty()) { - throw GradleException("Oliphaunt Maven artifact manifest contains duplicate coordinates: $duplicateCoordinates") - } - return artifacts -} - -fun publicationName(artifact: OliphauntMavenArtifact): String = - artifact.artifactId - .split('-', '_', '.') - .filter { it.isNotBlank() } - .joinToString("") { segment -> - segment.replaceFirstChar { char -> - if (char.isLowerCase()) char.titlecase(Locale.ROOT) else char.toString() - } - } - .replaceFirstChar { char -> char.lowercase(Locale.ROOT) } - -val oliphauntArtifacts = manifestPath.orNull?.let { parseArtifactManifest(file(it)) }.orEmpty() - -mavenPublishing { - publishToMavenCentral(automaticRelease = true) - if ( - gradle.startParameter.taskNames.any { it.contains("MavenCentral", ignoreCase = true) } || - providers.gradleProperty("signAllPublications").map { - it.equals("true", ignoreCase = true) || it.equals("yes", ignoreCase = true) || it == "1" - }.orElse(false).get() - ) { - signAllPublications() - } -} - -publishing { - publications { - oliphauntArtifacts.forEach { artifact -> - val publicationName = publicationName(artifact) - val icuDescriptor = artifact.groupId == "dev.oliphaunt.runtime" && artifact.artifactId == "oliphaunt-icu" - val descriptorSource = if (icuDescriptor) { - layout.buildDirectory.file("generated/oliphaunt-descriptors/$publicationName/src/ICU.java").get().asFile - } else artifact.file.takeIf { it.extension == "java" } - val generateIcuDescriptor = if (icuDescriptor) { - tasks.register("${publicationName}GenerateDescriptor") { - inputs.property("version", artifact.version) - outputs.file(descriptorSource!!) - doLast { - descriptorSource.parentFile.mkdirs() - descriptorSource.writeText(""" - package dev.oliphaunt.icu; - /** Optional ICU data supplied by this package. */ - public final class ICU { - private ICU() {} - public static final dev.oliphaunt.IcuData data = new dev.oliphaunt.IcuData("${artifact.version}"); - } - """.trimIndent() + "\n") - } - } - } else null - val descriptorClasses = layout.buildDirectory.dir("generated/oliphaunt-descriptors/$publicationName/classes") - val compileDescriptor = descriptorSource?.let { source -> - tasks.register("${publicationName}CompileDescriptor") { - val sdkJar = project(":oliphaunt").tasks.named("jvmJar") - dependsOn(sdkJar) - if (generateIcuDescriptor != null) dependsOn(generateIcuDescriptor) - source(source) - classpath = files(sdkJar) - destinationDirectory.set(descriptorClasses) - sourceCompatibility = "17" - targetCompatibility = "17" - } - } - val descriptorJar = compileDescriptor?.let { compile -> - tasks.register("${publicationName}DescriptorJar") { - dependsOn(compile) - archiveBaseName.set(artifact.artifactId) - archiveVersion.set(artifact.version) - destinationDirectory.set(layout.buildDirectory.dir("oliphaunt-maven-artifacts/$publicationName")) - from(descriptorClasses) - from(baseReleaseNoticeFiles) { into("META-INF") } - isPreserveFileTimestamps = false - isReproducibleFileOrder = true - } - } - val placeholderRoot = layout.buildDirectory.dir("generated/oliphaunt-maven-artifacts/$publicationName") - val placeholderSources = placeholderRoot.map { it.file("sources/README.md") } - val placeholderJavadocs = placeholderRoot.map { it.file("javadoc/index.html") } - val generatePlaceholders = - tasks.register("${publicationName}GenerateCentralPlaceholders") { - outputs.files(placeholderSources, placeholderJavadocs) - doLast { - val coordinate = "${artifact.groupId}:${artifact.artifactId}:${artifact.version}" - placeholderSources.get().asFile.apply { - parentFile.mkdirs() - writeText("# $coordinate\n\nThis binary carrier has no source API. See https://github.com/f0rr0/oliphaunt.\n") - } - placeholderJavadocs.get().asFile.apply { - parentFile.mkdirs() - writeText("$coordinate

    ${if (descriptorSource != null) "Versioned resource descriptor for Kotlin and Java. See the sources archive." else "This binary carrier has no Java API."}

    \n") - } - } - } - val sourcesJar = - tasks.register("${publicationName}SourcesJar") { - dependsOn(generatePlaceholders) - archiveBaseName.set(artifact.artifactId) - archiveVersion.set(artifact.version) - archiveClassifier.set("sources") - destinationDirectory.set(layout.buildDirectory.dir("oliphaunt-maven-artifacts/$publicationName")) - isPreserveFileTimestamps = false - isReproducibleFileOrder = true - if (descriptorSource != null) { - if (generateIcuDescriptor != null) dependsOn(generateIcuDescriptor) - from(descriptorSource) - } else from(placeholderSources) - from(baseReleaseNoticeFiles) { - into("META-INF") - filePermissions { - unix("0644") - } - } - } - val javadocJar = - tasks.register("${publicationName}JavadocJar") { - dependsOn(generatePlaceholders) - archiveBaseName.set(artifact.artifactId) - archiveVersion.set(artifact.version) - archiveClassifier.set("javadoc") - destinationDirectory.set(layout.buildDirectory.dir("oliphaunt-maven-artifacts/$publicationName")) - isPreserveFileTimestamps = false - isReproducibleFileOrder = true - from(placeholderJavadocs) - from(baseReleaseNoticeFiles) { - into("META-INF") - filePermissions { - unix("0644") - } - } - } - create(publicationName) { - groupId = artifact.groupId - artifactId = artifact.artifactId - version = artifact.version - if (descriptorJar != null) artifact(descriptorJar) - if (descriptorJar == null || icuDescriptor) artifact(artifact.file) { extension = "tar.gz" } - artifact(sourcesJar) - artifact(javadocJar) - pom { - name.set(artifact.name) - description.set(artifact.description) - val publicationProperties = mutableMapOf("oliphaunt.license.spdx" to artifact.licenseSpdx) - if (artifact.runtimeProduct != null && artifact.runtimeVersion != null) { - publicationProperties["oliphaunt.runtime.product"] = artifact.runtimeProduct - publicationProperties["oliphaunt.runtime.version"] = artifact.runtimeVersion - } - properties.set(publicationProperties) - if (descriptorSource != null) { - withXml { - val dependencies = asNode().appendNode("dependencies") - val dependency = dependencies.appendNode("dependency") - dependency.appendNode("groupId", "dev.oliphaunt") - dependency.appendNode("artifactId", "oliphaunt-android") - dependency.appendNode("version", project(":oliphaunt").version.toString()) - dependency.appendNode("type", "aar") - dependency.appendNode("scope", "compile") - } - } - inceptionYear.set("2026") - url.set("https://github.com/f0rr0/oliphaunt") - licenses { - artifact.licenses.forEach { declaredLicense -> - license { - name.set(declaredLicense.name) - url.set(declaredLicense.url) - distribution.set(declaredLicense.distribution) - } - } - } - developers { - developer { - id.set("f0rr0") - name.set("Oliphaunt Maintainers") - url.set("https://github.com/f0rr0") - } - } - scm { - url.set("https://github.com/f0rr0/oliphaunt") - connection.set("scm:git:https://github.com/f0rr0/oliphaunt.git") - developerConnection.set("scm:git:ssh://git@github.com:f0rr0/oliphaunt.git") - } - } - } - } - } -} - -tasks.register("validateOliphauntMavenArtifacts") { - if (manifestPath.isPresent) { - inputs.file(manifestPath) - } - oliphauntArtifacts.forEach { artifact -> - inputs.file(artifact.file) - } - doLast { - if (oliphauntArtifacts.isEmpty()) { - throw GradleException( - "Set -PoliphauntMavenArtifactsManifest or OLIPHAUNT_MAVEN_ARTIFACTS_MANIFEST before publishing Oliphaunt Maven artifact packages.", - ) - } - for (artifact in oliphauntArtifacts) { - if (!artifact.groupId.matches(Regex("[A-Za-z0-9_.-]+"))) { - throw GradleException("Invalid Maven groupId: ${artifact.groupId}") - } - if (!artifact.artifactId.matches(Regex("[A-Za-z0-9_.-]+"))) { - throw GradleException("Invalid Maven artifactId: ${artifact.artifactId}") - } - if (!artifact.version.matches(Regex("[A-Za-z0-9_.-]+"))) { - throw GradleException("Invalid Maven version for ${artifact.groupId}:${artifact.artifactId}: ${artifact.version}") - } - if (!artifact.file.isFile) { - throw GradleException("Missing Maven artifact file for ${artifact.groupId}:${artifact.artifactId}: ${artifact.file}") - } - if (!artifact.file.name.endsWith(".tar.gz") && artifact.file.extension != "java") { - throw GradleException("Oliphaunt Maven artifact ${artifact.file} must be a carrier .tar.gz or descriptor .java file") - } - if ((artifact.runtimeProduct == null) != (artifact.runtimeVersion == null)) { - throw GradleException( - "Oliphaunt Maven artifact ${artifact.groupId}:${artifact.artifactId} must declare both runtime product and version or neither", - ) - } - if (artifact.licenseSpdx.isBlank() || artifact.licenseSpdx.any { it.isISOControl() }) { - throw GradleException( - "Oliphaunt Maven artifact ${artifact.groupId}:${artifact.artifactId} must declare a non-empty SPDX expression", - ) - } - for (license in artifact.licenses) { - if (license.name.isBlank() || license.name.any { it.isISOControl() }) { - throw GradleException( - "Oliphaunt Maven artifact ${artifact.groupId}:${artifact.artifactId} has an invalid license name", - ) - } - if (!license.url.startsWith("https://") || license.url.any { it.isISOControl() }) { - throw GradleException( - "Oliphaunt Maven artifact ${artifact.groupId}:${artifact.artifactId} license URLs must use HTTPS", - ) - } - if (license.distribution != "repo") { - throw GradleException( - "Oliphaunt Maven artifact ${artifact.groupId}:${artifact.artifactId} licenses must use distribution=repo", - ) - } - } - if (artifact.groupId == "dev.oliphaunt.extensions" && - (artifact.runtimeProduct != "liboliphaunt-native" || - artifact.runtimeVersion?.matches(Regex("(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)")) != true) - ) { - throw GradleException( - "Oliphaunt Maven extension artifact ${artifact.artifactId} must bind an exact stable liboliphaunt-native runtime version", - ) - } - } - } -} - -tasks.matching { it.name.startsWith("publish") }.configureEach { - dependsOn("validateOliphauntMavenArtifacts") -} diff --git a/src/sdks/kotlin/settings.gradle.kts b/src/sdks/kotlin/settings.gradle.kts index 15bcfa028..6536cbddf 100644 --- a/src/sdks/kotlin/settings.gradle.kts +++ b/src/sdks/kotlin/settings.gradle.kts @@ -39,4 +39,3 @@ if (oliphauntBuildRoot != null) { rootProject.name = "oliphaunt-kotlin" include(":oliphaunt") include(":oliphaunt-android-gradle-plugin") -include(":oliphaunt-maven-artifacts") diff --git a/src/sdks/rust/src/direct.rs b/src/sdks/rust/src/direct.rs index f4260e2a3..f6482ddad 100644 --- a/src/sdks/rust/src/direct.rs +++ b/src/sdks/rust/src/direct.rs @@ -118,14 +118,14 @@ impl OliphauntBuilder { self } - /// Make one bundled PostgreSQL extension artifact available to the database. + /// Make one explicitly selected PostgreSQL extension artifact available to the database. /// Database-local installation remains the application's migration concern. pub fn extension(mut self, extension: impl Into) -> Self { self.inner = self.inner.extension(extension); self } - /// Make bundled PostgreSQL extension artifacts available to the database. + /// Make explicitly selected PostgreSQL extension artifacts available to the database. /// Database-local installation remains the application's migration concern. pub fn extensions( mut self, @@ -206,14 +206,14 @@ impl OliphauntServerBuilder { self } - /// Make one bundled PostgreSQL extension artifact available to clients. + /// Make one explicitly selected PostgreSQL extension artifact available to clients. /// Database-local installation remains the application's migration concern. pub fn extension(mut self, extension: impl Into) -> Self { self.inner = self.inner.extension(extension); self } - /// Make bundled PostgreSQL extension artifacts available to clients. + /// Make explicitly selected PostgreSQL extension artifacts available to clients. /// Database-local installation remains the application's migration concern. pub fn extensions( mut self, diff --git a/src/sdks/swift/moon.yml b/src/sdks/swift/moon.yml index 03aa94817..2748cb807 100644 --- a/src/sdks/swift/moon.yml +++ b/src/sdks/swift/moon.yml @@ -43,10 +43,15 @@ tasks: tags: ["quality", "unit"] script: | set -e + node tools/extension-resource-inventory.test.mjs ../../../target/moon/oliphaunt-swift/inventory-test + node tools/swift-carrier-resolver.test.mjs ../../../target/moon/oliphaunt-swift/carrier-test sh tools/test-c-bridge.sh swift test --scratch-path ../../../target/moon/oliphaunt-swift/unit inputs: - "@group(code)" + - "@group(release-archive-contract)" + - "/src/extensions/generated/sdk/extensions.json" + - "/src/shared/extension-runtime-contract/extension-artifact-archive-policy.properties" - "!/src/sdks/swift/.build" - "!/src/sdks/swift/.build/**" options: diff --git a/src/sdks/swift/tools/render-extension-products.mjs b/src/sdks/swift/tools/render-extension-products.mjs index 8c6482aa5..8dba618d8 100755 --- a/src/sdks/swift/tools/render-extension-products.mjs +++ b/src/sdks/swift/tools/render-extension-products.mjs @@ -1015,12 +1015,15 @@ async function writeGeneratedTree(selection, outputDir, basePackagePath, localBi products.push({ name: extension.swiftTarget, targets: [extension.swiftTarget], type: "library" }); targets.push(...targetIR(extension, selection.bySqlName, localBinaryTargets)); selected.push({ - asset: releaseProduct === undefined || extension.asset === undefined + asset: releaseProduct === undefined || extension.asset == null ? extension.asset : { name: extension.asset.name, checksum: extension.asset.checksum }, createsExtension: extension.resources.createsExtension, dependencies: extension.dependencies, - nativeDependencies: extension.nativeDependencies, + nativeDependencies: extension.nativeDependencies.map(({ asset, ...dependency }) => ({ + ...dependency, + asset: releaseProduct === undefined ? asset : { name: asset.name, checksum: asset.checksum }, + })), nativeModuleStem: extension.nativeModuleStem, product: extension.product, releaseProduct: extension.releaseProduct, diff --git a/src/sdks/swift/tools/render-extension-products.test-driver.mjs b/src/sdks/swift/tools/render-extension-products.test-driver.mjs deleted file mode 100644 index b8d0afcbd..000000000 --- a/src/sdks/swift/tools/render-extension-products.test-driver.mjs +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env node - -import fs from "node:fs/promises"; -import path from "node:path"; - -import { - loadSwiftExtensionInventoryCatalog, - validateSwiftExtensionResourceArtifact, -} from "./extension-resource-inventory.mjs"; -import { - validateSelection, - writeGenerated, -} from "./render-extension-products.mjs"; - -function fail(message) { - throw new Error(`render-extension-products.test-driver.mjs: ${message}`); -} - -function parseArgs(argv) { - const args = { allowFileUrls: false, localBinaryTargets: false }; - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - if (arg === "--allow-file-urls" || arg === "--local-binary-targets") { - if (arg === "--allow-file-urls") args.allowFileUrls = true; - else args.localBinaryTargets = true; - continue; - } - if (!["--selection", "--output-dir", "--base-package-path"].includes(arg)) { - fail(`unknown argument ${arg}`); - } - const value = argv[index + 1]; - if (value === undefined || value.startsWith("--")) fail(`${arg} requires a value`); - index += 1; - if (arg === "--selection") args.selection = path.resolve(value); - if (arg === "--output-dir") args.outputDir = path.resolve(value); - if (arg === "--base-package-path") args.basePackagePath = path.resolve(value); - } - if (!args.selection || !args.outputDir) fail("--selection and --output-dir are required"); - return args; -} - -async function main() { - const args = parseArgs(process.argv.slice(2)); - const input = JSON.parse(await fs.readFile(args.selection, "utf8")); - const selection = validateSelection(input, path.dirname(args.selection), { - allowFileUrls: args.allowFileUrls, - localBinaryTargets: args.localBinaryTargets, - }); - const catalog = await loadSwiftExtensionInventoryCatalog(); - if (args.basePackagePath !== undefined) { - const manifest = path.join(args.basePackagePath, "Package.swift"); - if ((await fs.stat(manifest).catch(() => null))?.isFile() !== true) { - fail(`--base-package-path does not contain Package.swift: ${args.basePackagePath}`); - } - } - for (const extension of selection.extensions) { - extension.resources = await validateSwiftExtensionResourceArtifact({ - extension, - canonical: catalog.get(extension.sqlName), - nativeRuntime: selection.nativeRuntime, - label: `${extension.sqlName} resource artifact`, - allowMobileCarrierArchives: false, - }); - } - await writeGenerated( - selection, - args.outputDir, - args.basePackagePath, - args.localBinaryTargets, - [ - { label: "working directory", mode: "containment", path: process.cwd() }, - { label: "selection fixture", mode: "containment", path: args.selection }, - { label: "base package", mode: "disjoint", path: args.basePackagePath }, - ...selection.extensions.flatMap((extension) => [ - { label: `${extension.sqlName} resource root`, mode: "disjoint", path: extension.resourceRoot }, - { label: `${extension.sqlName} XCFramework`, mode: "disjoint", path: extension.asset?.localPath }, - ...extension.nativeDependencies.map((dependency) => ({ - label: `${dependency.name} XCFramework`, - mode: "disjoint", - path: dependency.asset.localPath, - })), - ]), - ].filter(({ path: protectedPath }) => protectedPath !== undefined), - ); -} - -main().catch((error) => { - console.error(error instanceof Error ? error.message : String(error)); - process.exit(1); -}); diff --git a/src/sdks/swift/tools/swift-carrier-resolver.test.mjs b/src/sdks/swift/tools/swift-carrier-resolver.test.mjs index 9aebef11f..0e02973c0 100755 --- a/src/sdks/swift/tools/swift-carrier-resolver.test.mjs +++ b/src/sdks/swift/tools/swift-carrier-resolver.test.mjs @@ -699,6 +699,16 @@ async function main() { assert.match(standaloneManifest, /from: "0.1.0"/u); assert.doesNotMatch(standaloneManifest, /OliphauntSelectedExtensions/u); + const standalonePostgis = path.join(root, "standalone-postgis"); + run(process.execPath, [generator, "--carrier", carrier, + "--extensions", "postgis", "--release-product", "oliphaunt-extension-postgis", + "--cache-dir", cache, "--allow-file-urls", "--local-binary-targets", + "--base-package-version", "0.1.0", "--output-dir", standalonePostgis]); + const standaloneProducts = await fs.readFile(path.join(standalonePostgis, "extension-products.json"), "utf8"); + assert.doesNotMatch(standaloneProducts, /file:|"localPath"/u); + assert.deepEqual(JSON.parse(standaloneProducts).selected[0].nativeDependencies.map(({ name }) => name), + postgisNativeDependencies.map(([name]) => name)); + const pgtapRuntime = manifest.extensions.find(({ sqlName }) => sqlName === "pgtap").assets[0]; const cachedPgtap = path.join(cache, "extracted", pgtapRuntime.sha256); await fs.writeFile(path.join(cachedPgtap, "manifest.properties"), "tampered-cache-entry\n"); diff --git a/tools/release/maven-artifact-manifest-publication-lock.test.mjs b/tools/release/build-maven-artifact-manifest.test.mjs similarity index 92% rename from tools/release/maven-artifact-manifest-publication-lock.test.mjs rename to tools/release/build-maven-artifact-manifest.test.mjs index 25bc93329..20a2de548 100644 --- a/tools/release/maven-artifact-manifest-publication-lock.test.mjs +++ b/tools/release/build-maven-artifact-manifest.test.mjs @@ -8,6 +8,7 @@ import { createDeterministicTar } from "./cargo-source-package.mjs"; import { stageExtensionUpstreamLicenses } from "./extension-upstream-licenses.mjs"; import { canonicalGzipSync } from "../../src/shared/artifact-packaging/portable-archive.mjs"; import { discoverPublicationArtifacts } from "./publication-lock.mjs"; +import { parseMavenArtifactManifest } from "./maven-artifact-staging.mjs"; import { currentProductVersionSync, extensionReleaseVersion, @@ -88,7 +89,7 @@ function singletonAndroidRuntime(root, target, { mutateUpstream = false, upstrea return output; } -test("the real Maven manifest builder feeds the canonical ten-field schema into publication locking", { +test("the Maven manifest builder feeds the canonical staging parser, not the publication lock", { timeout: 30_000, }, () => { const root = temporaryDirectory(); @@ -117,8 +118,9 @@ test("the real Maven manifest builder feeds the canonical ten-field schema into expect(rows).toHaveLength(4); expect(rows.every((row) => row.split("\t").length === 10)).toBe(true); - const records = discoverPublicationArtifacts([manifest]); - expect(records.map(({ name }) => name).sort()).toEqual([ + expect(discoverPublicationArtifacts([manifest])).toEqual([]); + const records = parseMavenArtifactManifest(manifest); + expect(records.map(({ groupId, artifactId }) => `${groupId}:${artifactId}`).sort()).toEqual([ "dev.oliphaunt.runtime:liboliphaunt-android-arm64-v8a", "dev.oliphaunt.runtime:liboliphaunt-android-x86_64", "dev.oliphaunt.runtime:liboliphaunt-runtime-resources-android-datum64", @@ -126,16 +128,14 @@ test("the real Maven manifest builder feeds the canonical ten-field schema into ]); expect(records.every((record) => record.version === version - && record.artifacts.length === 1 - && record.artifacts[0].path.endsWith(".tar.gz") - && record.artifacts[0].sha256.length === 64)).toBe(true); + && record.artifact.endsWith(".tar.gz"))).toBe(true); const first = rows[0].split("\t"); const mutations = [ - ["legacy field count", first.slice(0, 8), /ten Maven publication fields/u], - ["missing display name", first.with(4, ""), /display name/u], + ["legacy field count", first.slice(0, 8), /ten.*fields/u], + ["missing display name", first.with(4, ""), /name must/u], ["half runtime binding", first.with(6, "liboliphaunt-native"), /both runtime product and version/u], - ["missing SPDX expression", first.with(8, ""), /SPDX expression/u], + ["missing SPDX expression", first.with(8, ""), /SPDX/u], ["non-array licenses", first.with(9, "{}"), /non-empty JSON array/u], [ "non-canonical license entry", @@ -150,7 +150,7 @@ test("the real Maven manifest builder feeds the canonical ten-field schema into ]; for (const [label, mutated, pattern] of mutations) { writeFileSync(manifest, `${mutated.join("\t")}\n${rows.slice(1).join("\n")}\n`); - expect(() => discoverPublicationArtifacts([manifest]), label).toThrow(pattern); + expect(() => parseMavenArtifactManifest(manifest), label).toThrow(pattern); } }); diff --git a/tools/release/check_artifact_targets.mjs b/tools/release/check_artifact_targets.mjs index 59c489339..2bf51c9ac 100644 --- a/tools/release/check_artifact_targets.mjs +++ b/tools/release/check_artifact_targets.mjs @@ -1050,9 +1050,9 @@ export function validateCiArtifactCoverage(workflow, inventory) { && wasixNapiBuild.env.OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR === "${{ github.workspace }}/target/oliphaunt-wasix/assets" && wasixNapiBuild.env.OLIPHAUNT_WASM_GENERATED_AOT_DIR === "${{ github.workspace }}/target/oliphaunt-wasix/aot" && wasixNapiBuild.env.OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT === "${{ github.workspace }}/target/extension-artifacts" - && wasixNapiBuild.env.OLIPHAUNT_ICU_DATA_DIR === "${{ github.workspace }}/target/oliphaunt-wasix/wasix-build/work/icu-wasix/share/icu" + && wasixNapiBuild.env.OLIPHAUNT_ICU_DATA_DIR === undefined && wasixNapiBuild.env.OLIPHAUNT_WASIX_NAPI_ARTIFACT_SOURCE_SHA === "${{ github.event.pull_request.head.sha || github.sha }}", - "WASIX Node-API builds must fail closed on exact same-run portable, ICU, AOT, and extension payload roots", + "WASIX Node-API builds must fail closed on exact same-run portable, AOT, and contrib payload roots", ); const releaseTasks = object( Bun.YAML.parse(readFileSync(path.join(ROOT, "tools/release/moon.yml"), "utf8")), @@ -1060,8 +1060,8 @@ export function validateCiArtifactCoverage(workflow, inventory) { ).tasks; invariant( String(releaseTasks?.["wasix-napi-runtime"]?.command ?? "").startsWith("bash ") - && String(releaseTasks["wasix-napi-runtime"].command).includes("build-extension-ci-artifacts.mjs --all --family wasix --require-wasix"), - "WASIX Node-API builds must stage complete exact-extension portable and target AOT inputs", + && String(releaseTasks["wasix-napi-runtime"].command).includes("build-extension-ci-artifacts.mjs oliphaunt-extension-contrib-pg18 --family wasix --require-wasix"), + "WASIX Node-API builds must stage only contrib portable and target AOT inputs", ); const wasixNapiAotRestore = namedStep(workflow, "wasix-napi", "Restore exact target core and tool AOT layout"); invariant( diff --git a/tools/release/moon.yml b/tools/release/moon.yml index 16b4c0455..3cd39888a 100644 --- a/tools/release/moon.yml +++ b/tools/release/moon.yml @@ -609,7 +609,7 @@ tasks: wasix-napi-runtime: tags: ["release", "artifact", "in-place-finalizer-input", "ci-wasix-napi"] - command: "bash -c 'set -e; tools/dev/bun.sh tools/release/build-extension-ci-artifacts.mjs --all --family wasix --require-wasix; exec bash src/runtimes/wasix-napi/tools/build-native.sh'" + command: "bash -c 'set -e; tools/dev/bun.sh tools/release/build-extension-ci-artifacts.mjs oliphaunt-extension-contrib-pg18 --family wasix --require-wasix; exec bash src/runtimes/wasix-napi/tools/build-native.sh'" deps: - "liboliphaunt-wasix:runtime-aot" - "extension-artifacts-wasix:build-target" @@ -633,15 +633,14 @@ tasks: - "/src/runtimes/wasix-napi/tools/package-platform.mjs" - "/src/runtimes/wasix-napi/tools/portable-command.mjs" - "/src/runtimes/wasix-napi/tools/smoke-packaged-addon.mjs" + - "/src/runtimes/wasix-napi/tests/native.integration.mjs" - "/src/bindings/wasix-ts/tools/pgwire-client.mjs" - project: "oliphaunt-wasix-rust" group: "code" - project: "liboliphaunt-wasix" group: "crates" - - "/src/runtimes/liboliphaunt/icu/**/*" - "/target/oliphaunt-wasix/assets/**/*" - "/target/oliphaunt-wasix/aot/**/*" - - "/target/oliphaunt-wasix/wasix-build/work/icu-wasix/share/icu/**/*" - "/target/extensions/wasix/release-assets/**/*" - "/target/extensions/wasix/aot-artifacts/**/*" - "/target/extension-artifacts/**/*" diff --git a/tools/release/native-npm-archive-extraction.test.mjs b/tools/release/native-npm-archive-extraction.test.mjs index 327ddcb9a..33ad5e78d 100644 --- a/tools/release/native-npm-archive-extraction.test.mjs +++ b/tools/release/native-npm-archive-extraction.test.mjs @@ -90,7 +90,8 @@ test("native npm descriptors publish every staged payload root", () => { const libraryRoot = packageJson.oliphaunt?.libraryRelativePath?.split("/")[0]; const runtimeRoot = packageJson.oliphaunt?.runtimeRelativePath?.split("/")[0]; const clusterSeedRoot = packageJson.oliphaunt?.clusterSeedRelativePath?.split("/")[0]; - const icuClusterSeedRoot = packageJson.oliphaunt?.icuClusterSeedRelativePath?.split("/")[0]; + expect(packageJson.oliphaunt?.icuClusterSeedRelativePath).toBeUndefined(); + expect(packageJson.files).not.toContain("cluster-seed-icu"); expect(packageJson.oliphaunt?.clusterSeedTarget).toBe(packageJson.oliphaunt?.target); const expected = [ ...new Set([ @@ -98,7 +99,6 @@ test("native npm descriptors publish every staged payload root", () => { "lib", runtimeRoot, clusterSeedRoot, - icuClusterSeedRoot, "manifest.properties", "README.md", ...REQUIRED_LEGAL_FILES, diff --git a/tools/release/publication-lock.mjs b/tools/release/publication-lock.mjs index e41d2324e..23e03cb77 100644 --- a/tools/release/publication-lock.mjs +++ b/tools/release/publication-lock.mjs @@ -338,119 +338,6 @@ function mavenArtifact(file) { }; } -const MAVEN_MANIFEST_TOKEN = /^[A-Za-z0-9_.-]+$/u; -const STABLE_SEMVER = /^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$/u; -const CONTROL_CHARACTER = /[\u0000-\u001f\u007f]/u; - -function requiredMavenManifestText(value, label) { - if (typeof value !== "string" || value.length === 0 || CONTROL_CHARACTER.test(value)) { - throw error(`${label} must be a non-empty string without control characters`); - } - return value; -} - -function mavenManifestLicenses(value, label) { - let licenses; - try { - licenses = JSON.parse(value); - } catch (cause) { - throw error(`${label} must be valid JSON: ${cause.message}`); - } - if (!Array.isArray(licenses) || licenses.length === 0) { - throw error(`${label} must be a non-empty JSON array`); - } - const expectedKeys = ["name", "url", "distribution"]; - for (const [index, license] of licenses.entries()) { - const entryLabel = `${label} entry ${index + 1}`; - if (license === null || Array.isArray(license) || typeof license !== "object") { - throw error(`${entryLabel} must be an object`); - } - if (stableJson(Object.keys(license)) !== stableJson(expectedKeys)) { - throw error(`${entryLabel} must contain exactly ${expectedKeys.join(", ")} in canonical order`); - } - requiredMavenManifestText(license.name, `${entryLabel}.name`); - const url = requiredMavenManifestText(license.url, `${entryLabel}.url`); - if (!url.startsWith("https://")) { - throw error(`${entryLabel}.url must use HTTPS`); - } - if (license.distribution !== "repo") { - throw error(`${entryLabel}.distribution must be repo`); - } - } - if (value !== JSON.stringify(licenses)) { - throw error(`${label} must use canonical compact JSON`); - } - return licenses; -} - -function mavenManifestArtifacts(file) { - if (!rel(file).includes("/maven-artifacts/")) { - return []; - } - const records = []; - const coordinates = new Set(); - for (const [index, line] of readFileSync(file, "utf8").split(/\r?\n/u).filter(Boolean).entries()) { - const values = line.split("\t"); - const label = `${rel(file)} line ${index + 1}`; - if (values.length !== 10) { - throw error(`${label} must contain ten Maven publication fields`); - } - const [ - group, - name, - version, - artifactPath, - displayName, - description, - runtimeProduct, - runtimeVersion, - licenseSpdx, - licensesJson, - ] = values; - for (const [field, value] of [["groupId", group], ["artifactId", name], ["version", version]]) { - requiredMavenManifestText(value, `${label} ${field}`); - if (!MAVEN_MANIFEST_TOKEN.test(value)) { - throw error(`${label} ${field} contains non-portable Maven characters`); - } - } - requiredMavenManifestText(artifactPath, `${label} artifact path`); - requiredMavenManifestText(displayName, `${label} display name`); - requiredMavenManifestText(description, `${label} description`); - if ((runtimeProduct.length === 0) !== (runtimeVersion.length === 0)) { - throw error(`${label} must declare both runtime product and version or neither`); - } - if (runtimeProduct.length > 0) { - requiredMavenManifestText(runtimeProduct, `${label} runtime product`); - requiredMavenManifestText(runtimeVersion, `${label} runtime version`); - } - requiredMavenManifestText(licenseSpdx, `${label} SPDX expression`); - mavenManifestLicenses(licensesJson, `${label} licenses`); - if ( - group === "dev.oliphaunt.extensions" - && (runtimeProduct !== "liboliphaunt-native" || !STABLE_SEMVER.test(runtimeVersion)) - ) { - throw error(`${label} extension carrier must bind an exact stable liboliphaunt-native runtime version`); - } - const coordinate = `${group}:${name}:${version}`; - if (coordinates.has(coordinate)) { - throw error(`${rel(file)} contains duplicate Maven coordinate ${coordinate}`); - } - coordinates.add(coordinate); - const artifact = path.resolve(ROOT, artifactPath); - if (!artifactPath.endsWith(".tar.gz") || !isFile(artifact)) { - throw error(`${label} references a missing or non-tar.gz Maven artifact ${artifactPath}`); - } - records.push({ - ecosystem: "maven", - name: `${group}:${name}`, - version, - dependencies: [], - artifacts: [{ path: rel(artifact), sha256: sha256File(artifact), size: statSync(artifact).size }], - }); - } - return records; -} - export function directoryEnvelope(directory) { const files = walkFiles(directory, { ignoreBuildDirectories: true }); const hash = createHash("sha256"); @@ -528,10 +415,6 @@ function discoverPublicationArtifactsMatching(roots, includeRecord) { mavenPoms.add(key); addRecord(record); } - } else if (file.endsWith(".tsv")) { - for (const record of mavenManifestArtifacts(file)) { - addRecord(record); - } } } return [...records.values()].sort((left, right) => compareText(`${left.ecosystem}:${left.name}`, `${right.ecosystem}:${right.name}`)); From 7567ffa27edfdfd93bc66f84b7fd3656794f59e7 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 08:59:10 +0000 Subject: [PATCH 08/18] fix(sdk): verify consumer builds and preserve Android JNI callbacks --- src/bindings/wasix-ts/ARCHITECTURE.md | 30 +++++++------- src/bindings/wasix-ts/README.md | 9 ++-- src/bindings/wasix-ts/tools-package/README.md | 5 ++- src/docs/content/sdk/kotlin/guide.mdx | 2 +- src/docs/content/sdk/kotlin/index.mdx | 2 +- src/docs/content/sdk/swift/guide.mdx | 4 +- src/docs/content/sdk/swift/index.mdx | 4 +- src/sdks/kotlin/README.md | 2 +- src/sdks/kotlin/gradle/libs.versions.toml | 1 + src/sdks/kotlin/moon.yml | 6 ++- src/sdks/kotlin/oliphaunt/build.gradle.kts | 1 + src/sdks/kotlin/oliphaunt/consumer-rules.pro | 4 ++ .../public-api-consumer/build.gradle.kts | 22 +++++++++- .../tests/public-api-consumer/consumer.pro | 3 ++ .../public-api-consumer/gradle.properties | 1 + .../src/main/AndroidManifest.xml | 3 ++ .../consumer/JavaPublicApiConsumer.java | 4 +- src/sdks/react-native/app.plugin.js | 12 +++--- .../src/__tests__/config-plugin.test.ts | 1 + src/sdks/swift/README.md | 4 +- .../tools/swift-carrier-resolver.test.mjs | 22 +++++++++- .../package-extension-cargo-facades.test.mjs | 41 ++++++++++++++++--- .../package-extension-release-carriers.mjs | 23 ++++++++--- 23 files changed, 153 insertions(+), 53 deletions(-) create mode 100644 src/sdks/kotlin/oliphaunt/consumer-rules.pro create mode 100644 src/sdks/kotlin/tests/public-api-consumer/consumer.pro create mode 100644 src/sdks/kotlin/tests/public-api-consumer/gradle.properties create mode 100644 src/sdks/kotlin/tests/public-api-consumer/src/main/AndroidManifest.xml diff --git a/src/bindings/wasix-ts/ARCHITECTURE.md b/src/bindings/wasix-ts/ARCHITECTURE.md index b468a7ee7..b7ad2809e 100644 --- a/src/bindings/wasix-ts/ARCHITECTURE.md +++ b/src/bindings/wasix-ts/ARCHITECTURE.md @@ -20,8 +20,9 @@ portable liboliphaunt-wasix Rust actor, direct, Worker, server The browser adapter owns the portable runtime/seed descriptors and dynamic extension carrier installation. The server adapter owns no Wasmer JavaScript fallback: it loads one exact, prebuilt platform carrier whose Rust dependency -embeds the runtime, AOT objects, cluster seed, tools, and supported extension -catalog. Both execute the canonical WASIX guest and preserve its physical +embeds the core runtime, core AOT objects, standard cluster seed, and contrib. +External extensions, ICU, and tools come from separately installed packages. +Both execute the canonical WASIX guest and preserve its physical database and backup formats. This boundary deliberately does not depend on `src/sdks/js`, @@ -411,13 +412,11 @@ carrier-owned installed-file inventory. The core manifest is required to have That byte-closure processing is the browser implementation. Node.js, Bun, Deno, and Electron retain the same public descriptor and perform its structural/runtime -validation, but pass only the validated, dependency-ordered SQL names across -the N-API boundary. The Rust runtime resolves those names against the exact -extension features compiled into the release carrier. Unknown names fail; the -addon never treats arbitrary descriptor bytes as native code. A new or upgraded -extension can ship independently for browsers, but it becomes available to -native-host consumers only after the N-API product is rebuilt and released -with that feature. +validation. Contrib uses the embedded payloads; external selections pass installed +package identities and locations across the N-API boundary. Rust verifies package +ownership, version, runtime compatibility, target, and payload hashes before +loading portable resources and host AOT modules. External extensions can ship +independently of N-API when they match its runtime and AOT compatibility contract. ## Host compatibility @@ -579,12 +578,13 @@ Linux carriers are GNU/glibc-only. The adapter identifies libc from the runtime diagnostic report before resolving package-adjacent, optional, or explicit addon paths; known musl and unknown libc identities fail closed. -Native release builds embed the runtime, seed, AOT objects, frontend tools, and -complete currently supported extension feature set. Optional extensions remain -exact, separately imported `-wasix` packages at the public TypeScript boundary, -but native hosts use their descriptor identity to select compiled-in artifacts -instead of copying the carrier bytes. Their availability is consequently a -release-time N-API contract. +Native release builds embed the core runtime, standard seed, core AOT objects, +and contrib. External extensions use independently versioned `-wasix` packages +for their portable bytes and target-specific packages for their AOT objects. +ICU data with matching seeds and frontend tools are separate packages too. +The adapter resolves the selected installed packages, and N-API validates their +identity and bytes before loading them. Compatible extension releases do not +require rebuilding N-API. The source workspace manifest deliberately does not resolve that generated carrier from npm: the carrier exists only after same-candidate runtime assets diff --git a/src/bindings/wasix-ts/README.md b/src/bindings/wasix-ts/README.md index 373a52610..85b56cf5f 100644 --- a/src/bindings/wasix-ts/README.md +++ b/src/bindings/wasix-ts/README.md @@ -346,11 +346,10 @@ await psql(target, { script: sql }); `pgDump()` runs with the database's existing owner, so it supports root, `/direct`, and `/worker` entrypoints where available. In browsers, `psql()` requires `/worker` because restoring COPY input is full duplex. Node.js, Bun, Deno, and Electron route both -tools through the frontend binaries compiled into the native carrier, so -`psql()` works with root, `/direct`, and `/worker` on those hosts. The optional -`@oliphaunt/wasix-tools` package remains the public opt-in API even though the -native carrier includes the tool code at build time. Adding or changing a tool -requires a matching N-API carrier release. +tools through the portable modules and matching host AOT artifacts supplied by +the optional tools package. `psql()` works with root, `/direct`, and `/worker` +on those hosts. The N-API carrier validates and loads those installed packages; +it does not embed the tool payloads. The package preserves PostgreSQL's normal plain SQL and COPY output. It does not support interactive psql, custom dump archives, parallel jobs, or diff --git a/src/bindings/wasix-ts/tools-package/README.md b/src/bindings/wasix-ts/tools-package/README.md index 4d941cef8..e28b578ce 100644 --- a/src/bindings/wasix-ts/tools-package/README.md +++ b/src/bindings/wasix-ts/tools-package/README.md @@ -3,8 +3,9 @@ Optional standard PostgreSQL `pg_dump` and non-interactive `psql` runners for an open `@oliphaunt/wasix-ts` database. This package remains the public opt-in facade on every host. Browsers load separately carried portable tool binaries; -Node.js, Bun, Deno, and Electron call the copies compiled into the matching Node-API -platform carrier. +Node.js, Bun, Deno, and Electron use the same installed portable modules with +matching host AOT packages. The Node-API carrier validates and loads those +optional resources. `pgDump()` returns PostgreSQL's ordinary plain SQL dump, including normal `COPY` data. `psql()` accepts a command or script and can restore that output. diff --git a/src/docs/content/sdk/kotlin/guide.mdx b/src/docs/content/sdk/kotlin/guide.mdx index 86b185da7..4a0a31cdc 100644 --- a/src/docs/content/sdk/kotlin/guide.mdx +++ b/src/docs/content/sdk/kotlin/guide.mdx @@ -28,7 +28,7 @@ packages native runtime artifacts, Android ABIs, and exact extension files. ```kotlin plugins { id("com.android.application") - id("dev.oliphaunt.android") version "0.1.1" + id("dev.oliphaunt.android") version "0.2.0" } dependencies { diff --git a/src/docs/content/sdk/kotlin/index.mdx b/src/docs/content/sdk/kotlin/index.mdx index a880b2b33..661827013 100644 --- a/src/docs/content/sdk/kotlin/index.mdx +++ b/src/docs/content/sdk/kotlin/index.mdx @@ -20,7 +20,7 @@ Add the Android package to your app: ```kotlin plugins { id("com.android.application") - id("dev.oliphaunt.android") version "0.1.1" + id("dev.oliphaunt.android") version "0.2.0" } dependencies { diff --git a/src/docs/content/sdk/swift/guide.mdx b/src/docs/content/sdk/swift/guide.mdx index e3be78322..01d327aa6 100644 --- a/src/docs/content/sdk/swift/guide.mdx +++ b/src/docs/content/sdk/swift/guide.mdx @@ -24,7 +24,9 @@ Add the Swift package in Xcode or `Package.swift`. The package includes the Swift API plus the platform runtime artifacts required for the selected target. ```swift -// Package.swift +// swift-tools-version: 6.0 +import PackageDescription + let package = Package( name: "Example", platforms: [.iOS(.v17), .macOS(.v14)], diff --git a/src/docs/content/sdk/swift/index.mdx b/src/docs/content/sdk/swift/index.mdx index ba5a9b208..b669ae002 100644 --- a/src/docs/content/sdk/swift/index.mdx +++ b/src/docs/content/sdk/swift/index.mdx @@ -18,7 +18,9 @@ this SDK, so the Swift lifecycle model is the Apple behavior source. Add the Swift package from Xcode or `Package.swift`: ```swift -// Package.swift +// swift-tools-version: 6.0 +import PackageDescription + let package = Package( name: "Example", platforms: [.iOS(.v17), .macOS(.v14)], diff --git a/src/sdks/kotlin/README.md b/src/sdks/kotlin/README.md index 3f026838b..e81222491 100644 --- a/src/sdks/kotlin/README.md +++ b/src/sdks/kotlin/README.md @@ -11,7 +11,7 @@ versioned dependency; contrib ships with the SDK. ```kotlin plugins { - id("dev.oliphaunt.android") version "0.1.1" + id("dev.oliphaunt.android") version "0.2.0" } dependencies { diff --git a/src/sdks/kotlin/gradle/libs.versions.toml b/src/sdks/kotlin/gradle/libs.versions.toml index f2f9ff180..7315c8b2e 100644 --- a/src/sdks/kotlin/gradle/libs.versions.toml +++ b/src/sdks/kotlin/gradle/libs.versions.toml @@ -16,6 +16,7 @@ kotlinx-coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-t kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinx-serialization" } [plugins] +android-application = { id = "com.android.application", version.ref = "android-gradle-plugin" } android-library = { id = "com.android.library", version.ref = "android-gradle-plugin" } kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } diff --git a/src/sdks/kotlin/moon.yml b/src/sdks/kotlin/moon.yml index fa4e6abb9..9b7c63671 100644 --- a/src/sdks/kotlin/moon.yml +++ b/src/sdks/kotlin/moon.yml @@ -82,7 +82,7 @@ tasks: options: cache: true package: - tags: ["package"] + tags: ["package", "requires-android-sdk"] script: | set -e ./gradlew \ @@ -91,6 +91,10 @@ tasks: "-Dmaven.repo.local=$MOON_WORKSPACE_ROOT/target/moon/oliphaunt-kotlin/package/maven" \ "-PoliphauntAndroidAbiFilters=arm64-v8a,x86_64" \ --no-configuration-cache + ./gradlew -p tests/public-api-consumer checkMinifiedCallbacks \ + "-PoliphauntConsumerAar=$MOON_WORKSPACE_ROOT/target/moon/oliphaunt-kotlin/package/oliphaunt/outputs/aar/oliphaunt-release.aar" \ + "-PoliphauntConsumerBuildRoot=$MOON_WORKSPACE_ROOT/target/moon/oliphaunt-kotlin/package/consumer" \ + --configuration-cache env: OLIPHAUNT_GRADLE_BUILD_ROOT: "../../../target/moon/oliphaunt-kotlin/package" inputs: diff --git a/src/sdks/kotlin/oliphaunt/build.gradle.kts b/src/sdks/kotlin/oliphaunt/build.gradle.kts index 1dd78663e..34964dfa7 100644 --- a/src/sdks/kotlin/oliphaunt/build.gradle.kts +++ b/src/sdks/kotlin/oliphaunt/build.gradle.kts @@ -921,6 +921,7 @@ android { defaultConfig { minSdk = 24 + consumerProguardFiles("consumer-rules.pro") if (androidAbiFilters.isNotEmpty()) { ndk { abiFilters.addAll(androidAbiFilters) diff --git a/src/sdks/kotlin/oliphaunt/consumer-rules.pro b/src/sdks/kotlin/oliphaunt/consumer-rules.pro new file mode 100644 index 000000000..f77ea2d2e --- /dev/null +++ b/src/sdks/kotlin/oliphaunt/consumer-rules.pro @@ -0,0 +1,4 @@ +# JNI looks up this callback by name and signature during streamed queries. +-keep interface dev.oliphaunt.OliphauntAndroidProtocolStreamSink { + public int onChunk(byte[]); +} diff --git a/src/sdks/kotlin/tests/public-api-consumer/build.gradle.kts b/src/sdks/kotlin/tests/public-api-consumer/build.gradle.kts index 9da706fd3..644a36b05 100644 --- a/src/sdks/kotlin/tests/public-api-consumer/build.gradle.kts +++ b/src/sdks/kotlin/tests/public-api-consumer/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - alias(libs.plugins.android.library) + alias(libs.plugins.android.application) alias(libs.plugins.kotlin.android) } @@ -20,6 +20,13 @@ android { defaultConfig { minSdk = 24 } + + buildTypes { + release { + isMinifyEnabled = true + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "consumer.pro") + } + } } kotlin { @@ -28,4 +35,17 @@ kotlin { dependencies { implementation(files(consumerAar)) + implementation(libs.kotlinx.coroutines.core) +} + +tasks.register("checkMinifiedCallbacks") { + val callbackMapping = layout.buildDirectory.file("outputs/mapping/release/mapping.txt") + dependsOn("assembleRelease") + inputs.file(callbackMapping) + doLast { + val callbacks = callbackMapping.get().asFile.readLines().filter { "int " in it && it.endsWith(" -> onChunk") } + check(callbacks.isNotEmpty()) { + "R8 removed or renamed the JNI stream callback: $callbacks" + } + } } diff --git a/src/sdks/kotlin/tests/public-api-consumer/consumer.pro b/src/sdks/kotlin/tests/public-api-consumer/consumer.pro new file mode 100644 index 000000000..c78f5820f --- /dev/null +++ b/src/sdks/kotlin/tests/public-api-consumer/consumer.pro @@ -0,0 +1,3 @@ +# Keep the consumer entry point so shrinking exercises its calls into the SDK. +-keep class dev.oliphaunt.consumer.PublicApiConsumerKt { *; } +-keep class dev.oliphaunt.consumer.JavaPublicApiConsumer { *; } diff --git a/src/sdks/kotlin/tests/public-api-consumer/gradle.properties b/src/sdks/kotlin/tests/public-api-consumer/gradle.properties new file mode 100644 index 000000000..49760fd52 --- /dev/null +++ b/src/sdks/kotlin/tests/public-api-consumer/gradle.properties @@ -0,0 +1 @@ +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=1536m -Dfile.encoding=UTF-8 diff --git a/src/sdks/kotlin/tests/public-api-consumer/src/main/AndroidManifest.xml b/src/sdks/kotlin/tests/public-api-consumer/src/main/AndroidManifest.xml new file mode 100644 index 000000000..34cd8cda7 --- /dev/null +++ b/src/sdks/kotlin/tests/public-api-consumer/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + + diff --git a/src/sdks/kotlin/tests/public-api-consumer/src/main/java/dev/oliphaunt/consumer/JavaPublicApiConsumer.java b/src/sdks/kotlin/tests/public-api-consumer/src/main/java/dev/oliphaunt/consumer/JavaPublicApiConsumer.java index c1d91b9f3..03170d3bb 100644 --- a/src/sdks/kotlin/tests/public-api-consumer/src/main/java/dev/oliphaunt/consumer/JavaPublicApiConsumer.java +++ b/src/sdks/kotlin/tests/public-api-consumer/src/main/java/dev/oliphaunt/consumer/JavaPublicApiConsumer.java @@ -8,14 +8,14 @@ import dev.oliphaunt.OliphauntConfig; import dev.oliphaunt.OliphauntJava; import java.io.File; -import java.util.Map; +import java.util.Collections; /** Compile-only proof of the Java API against the packaged Android AAR. */ public final class JavaPublicApiConsumer { public static void useDatabase(Context context, File directory, ExtensionDescriptor vector, IcuData icu) { var config = OliphauntConfig.builder() .storage(new DatabaseStorage.Directory(directory)) - .startupGucs(Map.of("application_name", "java-consumer")) + .startupGucs(Collections.singletonMap("application_name", "java-consumer")) .extensions(vector, Extensions.HSTORE) .icu(icu) .build(); diff --git a/src/sdks/react-native/app.plugin.js b/src/sdks/react-native/app.plugin.js index 747c04a94..b70bb9366 100644 --- a/src/sdks/react-native/app.plugin.js +++ b/src/sdks/react-native/app.plugin.js @@ -92,13 +92,11 @@ function resolveInstalledResources(projectRoot) { queue.push(resolve(`${name}/package.json`)); } catch (error) { if (error.code === 'ERR_PACKAGE_PATH_NOT_EXPORTED') { - let directory = path.dirname(resolve(name)); - while (!fs.existsSync(path.join(directory, 'package.json'))) { - const parent = path.dirname(directory); - if (parent === directory) throw error; - directory = parent; - } - queue.push(path.join(directory, 'package.json')); + const packageJson = (resolve.paths(name) ?? []) + .map((directory) => path.join(directory, name, 'package.json')) + .find((file) => fs.statSync(file, { throwIfNoEntry: false })?.isFile()); + if (packageJson === undefined) throw error; + queue.push(packageJson); } else if (error.code !== 'MODULE_NOT_FOUND' || (manifest.dependencies?.[name] && !manifest.optionalDependencies?.[name])) { throw error; diff --git a/src/sdks/react-native/src/__tests__/config-plugin.test.ts b/src/sdks/react-native/src/__tests__/config-plugin.test.ts index 60363fef2..a08b12baf 100644 --- a/src/sdks/react-native/src/__tests__/config-plugin.test.ts +++ b/src/sdks/react-native/src/__tests__/config-plugin.test.ts @@ -610,6 +610,7 @@ test('shipping resources follow resolved dependencies, aliases, and nested versi const feature = path.join(root, 'node_modules/feature'); writeJson(path.join(feature, 'package.json'), { name: 'feature', + exports: { './feature': { import: './dist/feature.js' } }, dependencies: { vectors: 'npm:@oliphaunt/extension-vector@0.8.2', '@oliphaunt/icu': '0.2.0' }, }); const vector = path.join(feature, 'node_modules/vectors/package.json'); diff --git a/src/sdks/swift/README.md b/src/sdks/swift/README.md index b72f6aa7e..485823db6 100644 --- a/src/sdks/swift/README.md +++ b/src/sdks/swift/README.md @@ -7,7 +7,9 @@ base `Oliphaunt` product includes PostgreSQL, supported contrib extensions, and the standard cluster seed. External extensions and ICU are separate packages. ```swift -// Package.swift +// swift-tools-version: 6.0 +import PackageDescription + let package = Package( name: "Example", platforms: [.iOS(.v17), .macOS(.v14)], diff --git a/src/sdks/swift/tools/swift-carrier-resolver.test.mjs b/src/sdks/swift/tools/swift-carrier-resolver.test.mjs index 0e02973c0..93f8bbfc6 100755 --- a/src/sdks/swift/tools/swift-carrier-resolver.test.mjs +++ b/src/sdks/swift/tools/swift-carrier-resolver.test.mjs @@ -1247,8 +1247,7 @@ async function main() { ], { expectFailure: true }); assert.match(diagnostic, /checksum mismatch/u); - // Recreate only the SQL-only archive and leave a buildable consumer package - // for check-sdk's clean Swift compile/link lane. + // Compile an app against an independent package and the public descriptor API. const pgtap = await extension("pgtap", null); const sqlOnly = carrierize([pgtap]); const sqlCarrier = path.join(root, "sql-only-carrier.json"); @@ -1260,10 +1259,29 @@ async function main() { const sqlOutput = path.join(root, "sql-only"); run(process.execPath, [ generator, "--carrier", sqlCarrier, "--extensions", "pgtap", "--cache-dir", path.join(root, "sql-cache"), + "--release-product", "oliphaunt-extension-pgtap", "--allow-file-urls", "--base-package-version", "0.1.0", "--base-package-path", sdk, "--output-dir", sqlOutput, ]); const sqlPackage = await fs.readFile(path.join(sqlOutput, "Package.swift"), "utf8"); assert.doesNotMatch(sqlPackage, /binaryTarget/u); + const consumer = path.join(root, "consumer"); + await fs.mkdir(path.join(consumer, "Sources", "Consumer"), { recursive: true }); + await fs.writeFile(path.join(consumer, "Package.swift"), `// swift-tools-version: 6.0 +import PackageDescription +let package = Package(name: "Consumer", platforms: [.macOS(.v14)], dependencies: [ + .package(name: "oliphaunt", path: ${JSON.stringify(sdk)}), + .package(name: "pgtap", path: ${JSON.stringify(sqlOutput)}) +], targets: [.executableTarget(name: "Consumer", dependencies: [ + .product(name: "Oliphaunt", package: "oliphaunt"), + .product(name: "OliphauntExtensionPgtap", package: "pgtap") +])]) +`); + await fs.writeFile(path.join(consumer, "Sources", "Consumer", "main.swift"), `import Oliphaunt +import OliphauntExtensionPgtap +let configuration = OliphauntConfiguration(extensions: [OliphauntExtensionPgtap.descriptor, OliphauntExtensions.hstore]) +precondition(configuration.extensions.map(\\.sqlName) == ["pgtap", "hstore"]) +`); + run("swift", ["run", "--package-path", consumer, "--scratch-path", path.join(root, "../consumer-build"), "Consumer"], { timeout: 180_000 }); console.log(`swift-carrier-resolver.test.mjs: metadata, malicious ZIP, cache-tamper, and consumer checks passed; sql-only-package=${sqlOutput}`); } diff --git a/tools/release/package-extension-cargo-facades.test.mjs b/tools/release/package-extension-cargo-facades.test.mjs index 1f0331a79..1cf24aa54 100644 --- a/tools/release/package-extension-cargo-facades.test.mjs +++ b/tools/release/package-extension-cargo-facades.test.mjs @@ -14,6 +14,7 @@ import { extensionRegistryPackageTargetSets, } from "./release-artifact-targets.mjs"; import { loadGraph } from "./release-graph.mjs"; +import { stageRustPackageSource } from "../../src/sdks/rust/tools/package-source.mjs"; import { nativeExtensionCargoPackageName, } from "./extension-registry-packages.mjs"; @@ -290,12 +291,28 @@ oliphaunt-build = { path = ${JSON.stringify(path.join(import.meta.dir, "../../sr expect(text).not.toContain('extension = "hstore"'); // The ordinary API needs only dependencies and imported descriptors. rmSync(path.join(app, "build.rs")); - let plainManifest = readFileSync(path.join(app, "Cargo.toml"), "utf8") - .replace('build = "build.rs"\n', "") - .replace(/\[package\.metadata\.oliphaunt\][\s\S]*?(?=\[dependencies\])/u, "") - .replace(/\[build-dependencies\][\s\S]*?(?=\[workspace\])/u, "") - .replace("[dependencies]", `[dependencies]\noliphaunt = { path = ${JSON.stringify(dependencyPaths.oliphaunt)} }`); - writeFileSync(path.join(app, "Cargo.toml"), plainManifest); + const sdk = path.join(root, "sdk"); + const sdkManifest = stageRustPackageSource(sdk); + let sdkText = readFileSync(sdkManifest, "utf8"); + for (const name of ["oliphaunt-resources", "oliphaunt-build"]) { + sdkText = sdkText.replace(new RegExp(`^${name} = .+$`, "m"), `${name} = { path = ${JSON.stringify(dependencyPaths[name])} }`); + } + const hostTarget = Object.entries(targetTriples).find(([, triple]) => triple === host)[0]; + const contribName = nativeExtensionCargoPackageName("oliphaunt-extension-contrib-pg18", hostTarget); + sdkText = sdkText.replace("[dependencies]", `[dependencies] +fixture-native-runtime = { path = ${JSON.stringify(runtime)} } +fixture-broker = { path = ${JSON.stringify(broker)} } +${contribName} = { path = ${JSON.stringify(dependencyPaths[contribName])} }`); + writeFileSync(sdkManifest, sdkText); + writeFileSync(path.join(app, "Cargo.toml"), `[package] +name = "facade-app" +version = "0.0.0" +edition = "2024" +[dependencies] +oliphaunt = { path = ${JSON.stringify(sdk)} } +vector = { package = "oliphaunt-extension-vector", path = ${JSON.stringify(path.join(generated, "sources/oliphaunt-extension-vector"))} } +[workspace] +`); writeFileSync(path.join(app, "src/lib.rs"), `pub fn configured() -> oliphaunt::OliphauntBuilder { oliphaunt::Oliphaunt::builder().extensions([vector::VECTOR, oliphaunt::extensions::HSTORE]) }\n`); @@ -303,5 +320,17 @@ oliphaunt-build = { path = ${JSON.stringify(path.join(import.meta.dir, "../../sr cwd: app, encoding: "utf8", maxBuffer: 20 * 1024 * 1024, }); expect(plain.status, `${plain.stdout}\n${plain.stderr}`).toBe(0); + const buildRoot = path.join(root, "cargo-target/debug/build"); + const embedded = readdirSync(buildRoot) + .filter(name => /^oliphaunt-[0-9a-f]+$/u.test(name)) + .map(name => findFile(path.join(buildRoot, name), "embedded_resources.rs")) + .filter(file => file !== null) + .map(file => readFileSync(file, "utf8")) + .find(source => source.includes("native-runtime/liboliphaunt-native/")); + expect(embedded).toContain("runtime/bin/postgres"); + expect(embedded).toContain("cluster-seed/files/pg_notify/"); + expect(embedded).toContain("extension/oliphaunt-extension-contrib-pg18/"); + expect(embedded).not.toContain("native-tools/"); + expect(embedded).not.toContain("extension/oliphaunt-extension-vector/"); }); }); diff --git a/tools/release/package-extension-release-carriers.mjs b/tools/release/package-extension-release-carriers.mjs index 8c38cbe5c..eb3ff42a1 100644 --- a/tools/release/package-extension-release-carriers.mjs +++ b/tools/release/package-extension-release-carriers.mjs @@ -1580,12 +1580,12 @@ function extensionModuleDirectory(runtimeDir) { return null; } -function writeExtensionReadme(packageDir, packageName, members, target) { +function writeExtensionReadme(packageDir, packageName, product, members, target) { const targetText = target === null ? "" : ` for \`${target}\``; const memberText = members.length === 1 ? `the \`${members[0]}\` PostgreSQL extension` : `${members.length} PostgreSQL contrib extensions`; - const selectionExample = members.length === 1 ? members[0] : members.slice(0, 2).join("', '"); + const ownerPackage = extensionNpmPackageForProduct(product); writeFileSync( path.join(packageDir, "README.md"), [ @@ -1593,8 +1593,19 @@ function writeExtensionReadme(packageDir, packageName, members, target) { "", `Oliphaunt registry package for ${memberText}${targetText}.`, "", - "This package is consumed by `@oliphaunt/ts` when an application opens a database with", - `\`extensions: ['${selectionExample}']\`.`, + ...(members.length > 1 + ? ["Contrib ships with the base SDK. Select its descriptors explicitly, for example", "`extensions: [extensions.hstore]` after importing `{ extensions }` from `@oliphaunt/ts`."] + : [ + `Install \`${ownerPackage}\` alongside \`@oliphaunt/ts\` or \`@oliphaunt/react-native\`.`, + "", + "```typescript", + "import Oliphaunt from '@oliphaunt/ts';", + `import extension from '${ownerPackage}';`, + "", + "const db = await Oliphaunt.open({ extensions: [extension] });", + "```", + ]), + "Installing the package supplies resources; select them per database and use SQL migrations to create extensions.", "", ].join("\n"), ); @@ -1647,7 +1658,7 @@ export function writeExtensionMetaPackage(packageDir, { .map((item) => [item, extensionNpmTargetPackageForProduct(product, item)]), ); mkdirSync(packageDir, { recursive: true }); - writeExtensionReadme(packageDir, packageName, members, null); + writeExtensionReadme(packageDir, packageName, product, members, null); if (iosCarrier !== undefined) writeJsonFile(path.join(packageDir, IOS_CARRIER_FILENAME), iosCarrier); const descriptorExports = {}; for (const sqlName of members) { @@ -1732,7 +1743,7 @@ function writeExtensionTargetPackage(packageDir, { metadata.moduleRelativePath = path.relative(packageDir, moduleDir).split(path.sep).join("/"); } mkdirSync(packageDir, { recursive: true }); - writeExtensionReadme(packageDir, packageName, members, target); + writeExtensionReadme(packageDir, packageName, product, members, target); writeJsonFile( path.join(packageDir, NPM_EXTENSION_CONTRACT_FILENAME), renderNpmExtensionContractManifest({ product, version, target, members: memberContracts }), From 3275006f047332e01114b9e0b5726c4a99530915 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 10:23:36 +0000 Subject: [PATCH 09/18] fix(release): verify installed SDKs and repair consumer artifact staging --- .github/workflows/ci.yml | 5 + docs/maintainers/release.md | 22 ++- src/sdks/rust/tools/check-release-consumer.sh | 7 +- .../OliphauntExtensionResources.swift | 3 +- .../tools/check-extension-release-consumer.sh | 132 ++---------------- .../prepare-extension-release-consumer.mjs | 94 +++++++++++++ .../tools/swift-carrier-resolver.test.mjs | 27 ++-- tools/release/moon.yml | 5 + ...tive-extension-lifecycle-receipts.test.mjs | 51 +++++++ .../native-npm-archive-extraction.test.mjs | 8 +- .../package-extension-cargo-facades.test.mjs | 2 + tools/release/package-release-carriers.mjs | 1 + tools/release/public-consumer-smoke.mjs | 95 ++++++++++++- tools/release/public-consumer-smoke.test.mjs | 90 ++++++++++-- .../stage-native-extension-lifecycle.mjs | 7 +- tools/xtask/moon.yml | 2 + tools/xtask/src/asset_checks.rs | 69 ++++++--- 17 files changed, 428 insertions(+), 192 deletions(-) create mode 100644 src/sdks/swift/tools/prepare-extension-release-consumer.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b590915c..1fb92668e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2569,6 +2569,11 @@ jobs: const { target } = validateNativeRuntimeCarrier(root, { icuData }); if (target !== "android-datum64") throw new Error(`expected android-datum64, got ${target}`); ' "$closure" "target/liboliphaunt-mobile-host/$OLIPHAUNT_NATIVE_TARGET/icu/share/icu" + # Assemble the optional npm ICU package inputs from these same-run bytes. + tools/release/package-liboliphaunt-icu-data.sh \ + "target/liboliphaunt-mobile-host/$OLIPHAUNT_NATIVE_TARGET/icu/share/icu" \ + target/liboliphaunt/release-assets + cp "$1" target/liboliphaunt/release-assets/ echo "OLIPHAUNT_EXPO_ANDROID_SEED_CLOSURE_DIR=$GITHUB_WORKSPACE/$closure" >> "$GITHUB_ENV" - name: Download Kotlin SDK package artifacts diff --git a/docs/maintainers/release.md b/docs/maintainers/release.md index 4b4a2e705..80d39934b 100644 --- a/docs/maintainers/release.md +++ b/docs/maintainers/release.md @@ -450,7 +450,7 @@ The post-publication consumer gate is an anonymous public-endpoint probe. It derives its products, ecosystem lanes, dependency roots, full carrier closure, versions, Maven coordinates, and Git tags from the same frozen lock. In parallel clean temporary homes/caches it resolves each Cargo consumer root -in an independent scratch manifest without compiling payloads, installs each +in an independent scratch manifest with all published features, installs each npm dependency root in an independent project, resolves each Maven entry in an isolated Gradle configuration without an Android build, and anonymously fetches every product tag. Each lane requires @@ -458,8 +458,14 @@ the resolver's platform-independent lock graphs to cover every carrier in the corresponding frozen dependency closure; a missing carrier cannot be silently relabelled as receipt-only. It never invents one all-platform consumer graph. Evidence separately identifies npm carriers not installed on the macOS host -and Cargo payloads intentionally not fetched/compiled; immutable receipts -prove those bytes. When Swift is +and records receipt coverage for the complete Cargo closure. If the native Rust +SDK is selected, a separate application compiles its exact public crate with +default features. Installed native and WASIX TypeScript SDKs also run through +their public Node APIs. Each application discovers its packaged runtime without +workspace overrides, creates bundled hstore, queries it, closes, and reopens +the same directory. Missing execution evidence blocks promotion. Cargo WASIX +and non-host payloads remain covered by same-SHA CI and immutable receipts; +this macOS probe does not compile every Cargo target. When Swift is selected it also fetches the unscoped source tag, requires its synthetic commit to have the release SHA as its only parent, and evaluates that tagged `Package.swift` with `swift package dump-package`. @@ -592,8 +598,8 @@ Target packages are required where package managers select by OS/CPU/libc/ABI or the lower macOS 11.0 floor. SwiftPM consumes the XCFramework with runtime resources. Every native exact-extension and native-dependency XCFramework carries the same three Apple platform slices and is rejected before packaging - if any slice is missing. The base Swift package remains extension-free; - exact-extension products are generated from their separately released, + if any slice is missing. The base Swift package includes contrib descriptors + and payloads; external-extension products are generated from their separately released, checksum-covered carrier assets. Every extension release publishes one immutable `*-swift-extension-carrier.json`: the contrib bundle carrier owns exactly 32 SQL-member rows, while each independently versioned external @@ -603,8 +609,10 @@ Target packages are required where package managers select by OS/CPU/libc/ABI or repeatable `--extension-carrier` composition lets an external-only release be consumed without a Swift version bump; base mismatch, dependency skew, duplicate SQL ownership, or native-dependency byte conflicts fail closed. - Hosted macOS qualification final-links and - runs a generated native-extension Swift executable against the produced assets. + Hosted macOS qualification consumes separate external SwiftPM packages, + final-links against the produced assets, and runs the public descriptor API + through database open, contrib and external extension migrations, query, + close, and reopening persistent storage. - WASIX: portable runtime/extension carriers plus native AOT carriers for Linux x64/arm64 GNU, macOS arm64, and Windows x64 MSVC. - SDK façades: Rust/Cargo, npm, Maven/Gradle, and SwiftPM entry points select diff --git a/src/sdks/rust/tools/check-release-consumer.sh b/src/sdks/rust/tools/check-release-consumer.sh index 8d3bf648b..cd100ad14 100755 --- a/src/sdks/rust/tools/check-release-consumer.sh +++ b/src/sdks/rust/tools/check-release-consumer.sh @@ -115,13 +115,14 @@ build_consumer() { run_consumer() { local consumer="$1" local native_assets="$2" - local runtime_archive tools_archive install_dir tools_dir + local runtime_archive tools_archive install_dir tools_dir native_version require_linux_x64 require_file "$consumer" [ -x "$consumer" ] || fail "release consumer is not executable: $consumer" [ -d "$native_assets" ] || fail "native asset directory is missing: $native_assets" - runtime_archive="$(find_one "$native_assets" 'liboliphaunt-*-linux-x64-gnu.tar.gz')" - tools_archive="$(find_one "$native_assets" 'oliphaunt-tools-*-linux-x64-gnu.tar.gz')" + native_version="$(tools/dev/bun.sh tools/release/product-version.mjs version liboliphaunt-native)" + runtime_archive="$(find_one "$native_assets" "liboliphaunt-$native_version-linux-x64-gnu.tar.gz")" + tools_archive="$(find_one "$native_assets" "oliphaunt-tools-$native_version-linux-x64-gnu.tar.gz")" scratch="$(mktemp -d "${TMPDIR:-/tmp}/oliphaunt-rust-release-consumer-run.XXXXXX")" mkdir -p "$scratch/native" "$scratch/tools" "$scratch/runtime-cache" diff --git a/src/sdks/swift/Sources/Oliphaunt/OliphauntExtensionResources.swift b/src/sdks/swift/Sources/Oliphaunt/OliphauntExtensionResources.swift index 4c00b513e..a8f10e5c5 100644 --- a/src/sdks/swift/Sources/Oliphaunt/OliphauntExtensionResources.swift +++ b/src/sdks/swift/Sources/Oliphaunt/OliphauntExtensionResources.swift @@ -62,8 +62,7 @@ extension OliphauntRuntimeResources { } /// Registers a generated SwiftPM exact-extension resource fragment. - /// Applications normally call the generated `OliphauntExtension*.register()` - /// wrapper rather than invoking this packaging API directly. + /// The generated extension descriptor invokes registration when opening a database. @discardableResult @_spi(ExtensionSupport) public static func registerPackagedExtensionResource( product: String, diff --git a/src/sdks/swift/tools/check-extension-release-consumer.sh b/src/sdks/swift/tools/check-extension-release-consumer.sh index 69ca97d8e..369d65430 100755 --- a/src/sdks/swift/tools/check-extension-release-consumer.sh +++ b/src/sdks/swift/tools/check-extension-release-consumer.sh @@ -199,131 +199,17 @@ node src/sdks/swift/tools/render-extension-products.mjs \ --base-package-path "$release_package" \ --output-dir "$selected_package" -products="$selected_package/extension-products.json" -require_file "$products" consumer="$scratch/consumer" -mkdir -p "$consumer/Sources/OliphauntExtensionReleaseConsumer" -# JavaScript template interpolation is evaluated by Bun. # shellcheck disable=SC2016 -OLIPHAUNT_CARRIER_PLAN="$carrier_plan" \ - OLIPHAUNT_EXTENSION_PRODUCTS="$products" \ - OLIPHAUNT_RELEASE_PACKAGE="$release_package" \ - OLIPHAUNT_SELECTED_PACKAGE="$selected_package" \ - OLIPHAUNT_EXTENSION_CONSUMER="$consumer" \ - tools/dev/bun.sh -e ' - import path from "node:path"; - const plan = JSON.parse(process.env.OLIPHAUNT_CARRIER_PLAN); - const products = JSON.parse(await Bun.file(process.env.OLIPHAUNT_EXTENSION_PRODUCTS).text()); - if (!Array.isArray(products.selected) || products.selected.length === 0) { - throw new Error("generated extension package selected no products"); - } - const selected = products.selected.map((row, index) => { - const swiftProduct = row?.swiftProduct; - if (typeof swiftProduct !== "string" || !/^[A-Za-z][A-Za-z0-9]*$/u.test(swiftProduct)) { - throw new Error(`generated extension package selected[${index}] has an invalid Swift product name`); - } - if (typeof row.sqlName !== "string" || !/^[A-Za-z0-9._-]+$/u.test(row.sqlName)) { - throw new Error(`generated extension package selected[${index}] has an invalid SQL name`); - } - if (typeof row.product !== "string" || !/^oliphaunt-extension-[A-Za-z0-9._-]+$/u.test(row.product)) { - throw new Error(`generated extension package selected[${index}] has an invalid release product`); - } - if ( - row.nativeModuleStem !== null - && (typeof row.nativeModuleStem !== "string" || !/^[A-Za-z0-9._-]+$/u.test(row.nativeModuleStem)) - ) { - throw new Error(`generated extension package selected[${index}] has an invalid native module stem`); - } - return { - nativeModuleStem: row.nativeModuleStem, - product: row.product, - sqlName: row.sqlName, - swiftProduct, - }; - }); - if (new Set(selected.map(({ swiftProduct }) => swiftProduct)).size !== selected.length) { - throw new Error("generated extension package repeats a Swift product name"); - } - if (new Set(selected.map(({ sqlName }) => sqlName)).size !== selected.length) { - throw new Error("generated extension package repeats an extension SQL name"); - } - const actualExtensions = selected.map(({ sqlName }) => sqlName).sort(); - if (JSON.stringify(actualExtensions) !== JSON.stringify(plan.extensions)) { - throw new Error("generated extension package does not exactly cover the carrier-planned extension set"); - } - const actualProducts = [...new Set(selected.map(({ product }) => product))].sort(); - if (JSON.stringify(actualProducts) !== JSON.stringify(plan.extensionProducts)) { - throw new Error("generated extension package does not exactly cover the carrier-planned release products"); - } - if ( - products.nativeRuntime?.product !== plan.finalLink.runtimeProduct - || products.nativeRuntime?.version !== plan.finalLink.runtimeVersion - ) { - throw new Error("generated extension package native runtime identity differs from the final-link plan"); - } - let finalLink = null; - if (plan.finalLink.kind === "native-extension") { - finalLink = selected.find(({ sqlName }) => sqlName === plan.finalLink.nativeExtension) ?? null; - if (finalLink === null || finalLink.nativeModuleStem !== plan.finalLink.nativeModuleStem) { - throw new Error( - `generated extension package is missing the planned native final-link extension ${plan.finalLink.nativeExtension}/${plan.finalLink.nativeModuleStem}`, - ); - } - } else if (plan.finalLink.kind === "base-runtime") { - if ( - plan.finalLink.nativeExtension !== null - || plan.finalLink.nativeModuleStem !== null - || selected.some(({ nativeModuleStem }) => nativeModuleStem !== null) - ) { - throw new Error("base-runtime final-link proof requires an entirely SQL-only extension selection"); - } - } else { - throw new Error(`unknown final-link proof kind ${plan.finalLink.kind}`); - } - const packagePath = JSON.stringify(path.resolve(process.env.OLIPHAUNT_SELECTED_PACKAGE)); - const releasePackagePath = JSON.stringify(path.resolve(process.env.OLIPHAUNT_RELEASE_PACKAGE)); - const dependencies = [ - `.product(name: "COliphaunt", package: "oliphaunt")`, - ...selected.map(({ swiftProduct }) => - `.product(name: ${JSON.stringify(swiftProduct)}, package: "selectedExtensions")`), - ].join(", "); - const packageFile = `// swift-tools-version: 6.0\n\n` + - `import PackageDescription\n\n` + - `let package = Package(\n` + - ` name: "OliphauntExtensionReleaseConsumer",\n` + - ` platforms: [.macOS(.v14)],\n` + - ` dependencies: [\n` + - ` .package(name: "oliphaunt", path: ${releasePackagePath}),\n` + - ` .package(name: "selectedExtensions", path: ${packagePath})\n` + - ` ],\n` + - ` targets: [\n` + - ` .executableTarget(\n` + - ` name: "OliphauntExtensionReleaseConsumer",\n` + - ` dependencies: [${dependencies}]\n` + - ` )\n` + - ` ]\n` + - `)\n`; - const runtimeVersion = JSON.stringify(plan.finalLink.runtimeVersion); - const nativeAssertion = finalLink === null - ? `print("OLIPHAUNT_SWIFT_BASE_RUNTIME_LINK_PASS runtime=\\(linkedNativeRuntimeVersion!) products=${selected.length}")\n` - : `precondition(${finalLink.swiftProduct}.sqlName == ${JSON.stringify(finalLink.sqlName)} && ${finalLink.swiftProduct}.product == ${JSON.stringify(finalLink.product)}, "planned native extension identity mismatch")\n` + - `print("OLIPHAUNT_SWIFT_NATIVE_EXTENSION_LINK_PASS extension=${finalLink.sqlName} native_module=${finalLink.nativeModuleStem} runtime=\\(linkedNativeRuntimeVersion!) products=${selected.length}")\n`; - const main = `import COliphaunt\n${selected.map(({ swiftProduct }) => `import ${swiftProduct}`).join("\n")}\n\n` + - `${selected.map(({ swiftProduct }) => `try ${swiftProduct}.register()`).join("\n")}\n` + - `let linkedNativeRuntimeVersion = oliphaunt_version().map { String(cString: $0) }\n` + - `precondition(linkedNativeRuntimeVersion == ${runtimeVersion}, "linked liboliphaunt runtime version mismatch")\n` + - nativeAssertion; - await Bun.write(path.join(process.env.OLIPHAUNT_EXTENSION_CONSUMER, "Package.swift"), packageFile); - await Bun.write( - path.join( - process.env.OLIPHAUNT_EXTENSION_CONSUMER, - "Sources", - "OliphauntExtensionReleaseConsumer", - "main.swift", - ), - main, - ); - ' +tools/dev/bun.sh -e ' + await Bun.write(Bun.argv[1], JSON.stringify({ + plan: JSON.parse(Bun.argv[2]), productsFile: Bun.argv[3], + releasePackage: Bun.argv[4], carrier: Bun.argv[5], cache: Bun.argv[6], + output: Bun.argv[7], extensionCarriers: Bun.argv.slice(8), + })); +' "$scratch/consumer.json" "$carrier_plan" "$selected_package/extension-products.json" \ + "$release_package" "$source_carrier" "$cache" "$consumer" "${extension_carriers[@]}" +node src/sdks/swift/tools/prepare-extension-release-consumer.mjs "$scratch/consumer.json" echo "==> Building and running a macOS exact-extension Swift consumer (proof=$final_link_kind${extension:+ extension=$extension})" swift package \ diff --git a/src/sdks/swift/tools/prepare-extension-release-consumer.mjs b/src/sdks/swift/tools/prepare-extension-release-consumer.mjs new file mode 100644 index 000000000..cf5826614 --- /dev/null +++ b/src/sdks/swift/tools/prepare-extension-release-consumer.mjs @@ -0,0 +1,94 @@ +import assert from "node:assert/strict"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { pathToFileURL } from "node:url"; + +export function prepareExtensionReleaseConsumer({ plan, productsFile, releasePackage, carrier, extensionCarriers, cache, output }) { + const products = JSON.parse(readFileSync(productsFile, "utf8")); + const selected = products.selected; + assert(Array.isArray(selected) && selected.length > 0, "consumer requires selected extensions"); + assert.deepEqual(selected.map(row => row.sqlName).sort(), plan.extensions); + assert.deepEqual([...new Set(selected.map(row => row.product))].sort(), plan.extensionProducts); + assert.equal(products.nativeRuntime.product, plan.finalLink.runtimeProduct); + assert.equal(products.nativeRuntime.version, plan.finalLink.runtimeVersion); + if (plan.finalLink.kind === "native-extension") { + const native = selected.find(row => row.sqlName === plan.finalLink.nativeExtension); + assert(native && native.nativeModuleStem === plan.finalLink.nativeModuleStem, "missing planned native extension"); + } else { + assert.equal(plan.finalLink.kind, "base-runtime"); + assert(selected.every(row => row.nativeModuleStem === null), "SQL-only proof contains a native extension"); + } + const packageDependencies = [`.package(name: "oliphaunt", path: ${JSON.stringify(releasePackage)})`]; + const targetDependencies = [`.product(name: "Oliphaunt", package: "oliphaunt")`]; + const imports = ["import Foundation", "import Oliphaunt"]; + const descriptors = []; + const sqlNames = []; + for (const row of selected) { + assert.match(row.sqlName, /^[A-Za-z0-9_-]+$/u); + assert.match(row.product, /^oliphaunt-extension-[A-Za-z0-9-]+$/u); + assert.match(row.swiftProduct, /^OliphauntExtension[A-Za-z0-9]+$/u); + assert.equal(typeof row.createsExtension, "boolean"); + if (row.product === "oliphaunt-extension-contrib-pg18") { + const stem = row.swiftProduct.slice("OliphauntExtension".length); + descriptors.push(`OliphauntExtensions.${stem[0].toLowerCase()}${stem.slice(1)}`); + } else { + const directory = path.join(output, "packages", row.product); + const result = spawnSync(process.execPath, [ + path.join(import.meta.dirname, "render-extension-products.mjs"), + "--carrier", carrier, + ...extensionCarriers.flatMap(file => ["--extension-carrier", file]), + "--extensions", row.sqlName, "--release-product", row.product, + "--base-package-path", releasePackage, + "--base-package-version", products.basePackage.version, + "--cache-dir", cache, "--offline", "--allow-file-urls", "--local-binary-targets", "--output-dir", directory, + ], { stdio: "inherit" }); + assert.equal(result.status, 0, `generate independent package ${row.product}: ${result.error ?? result.signal ?? result.status}`); + packageDependencies.push(`.package(name: ${JSON.stringify(row.product)}, path: ${JSON.stringify(directory)})`); + targetDependencies.push(`.product(name: ${JSON.stringify(row.swiftProduct)}, package: ${JSON.stringify(row.product)})`); + imports.push(`import ${row.swiftProduct}`); + descriptors.push(`${row.swiftProduct}.descriptor`); + } + if (row.createsExtension) sqlNames.push(row.sqlName); + } + // Always exercise a bundled contrib descriptor alongside standalone packages. + if (!selected.some(row => row.sqlName === "hstore")) { + descriptors.push("OliphauntExtensions.hstore"); + sqlNames.push("hstore"); + } + const source = path.join(output, "Sources", "OliphauntExtensionReleaseConsumer"); + mkdirSync(source, { recursive: true }); + writeFileSync(path.join(output, "Package.swift"), `// swift-tools-version: 6.0 +import PackageDescription +let package = Package(name: "OliphauntExtensionReleaseConsumer", platforms: [.macOS(.v14)], + dependencies: [${packageDependencies.join(",\n ")}], + targets: [.executableTarget(name: "OliphauntExtensionReleaseConsumer", dependencies: [${targetDependencies.join(", ")}])]) +`); + const migration = sqlNames.map(name => `CREATE EXTENSION IF NOT EXISTS "${name}" CASCADE;`).join(" "); + writeFileSync(path.join(source, "main.swift"), `${imports.join("\n")} +let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) +defer { try? FileManager.default.removeItem(at: root) } +let configuration = OliphauntConfiguration(storage: .directory(root), extensions: [${descriptors.join(", ")}]) +for attempt in 0..<2 { + let database = try await OliphauntDatabase.open(configuration: configuration) + do { + if attempt == 0 { try await database.exec(${JSON.stringify(migration)}) } + let installed = try await database.query("SELECT extname::text AS name FROM pg_extension") + let names = try Set(installed.rows.map { row -> String in + guard let name: String = try row.value(named: "name") else { fatalError("null extension name") } + return name + }) + precondition(Set(${JSON.stringify(sqlNames)}).isSubset(of: names), "extension migrations did not survive reopen") + try await database.close() + } catch { + try? await database.close() + throw error + } +} +print("OLIPHAUNT_SWIFT_EXTENSION_DATABASE_PASS checks=independent-packages,descriptors,contrib,migrations,reopen,close") +`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + prepareExtensionReleaseConsumer(JSON.parse(readFileSync(process.argv[2], "utf8"))); +} diff --git a/src/sdks/swift/tools/swift-carrier-resolver.test.mjs b/src/sdks/swift/tools/swift-carrier-resolver.test.mjs index 93f8bbfc6..e50b8538c 100755 --- a/src/sdks/swift/tools/swift-carrier-resolver.test.mjs +++ b/src/sdks/swift/tools/swift-carrier-resolver.test.mjs @@ -13,6 +13,7 @@ import { resolveSwiftCarrierSelection, } from "./swift-carrier-resolver.mjs"; +import { prepareExtensionReleaseConsumer } from "./prepare-extension-release-consumer.mjs"; import { validateSelection, writeBundledContrib, renderSwiftTargets } from "./render-extension-products.mjs"; import { loadSwiftExtensionInventoryCatalog, validateSwiftExtensionResourceArtifact } from "./extension-resource-inventory.mjs"; @@ -1265,23 +1266,15 @@ async function main() { const sqlPackage = await fs.readFile(path.join(sqlOutput, "Package.swift"), "utf8"); assert.doesNotMatch(sqlPackage, /binaryTarget/u); const consumer = path.join(root, "consumer"); - await fs.mkdir(path.join(consumer, "Sources", "Consumer"), { recursive: true }); - await fs.writeFile(path.join(consumer, "Package.swift"), `// swift-tools-version: 6.0 -import PackageDescription -let package = Package(name: "Consumer", platforms: [.macOS(.v14)], dependencies: [ - .package(name: "oliphaunt", path: ${JSON.stringify(sdk)}), - .package(name: "pgtap", path: ${JSON.stringify(sqlOutput)}) -], targets: [.executableTarget(name: "Consumer", dependencies: [ - .product(name: "Oliphaunt", package: "oliphaunt"), - .product(name: "OliphauntExtensionPgtap", package: "pgtap") -])]) -`); - await fs.writeFile(path.join(consumer, "Sources", "Consumer", "main.swift"), `import Oliphaunt -import OliphauntExtensionPgtap -let configuration = OliphauntConfiguration(extensions: [OliphauntExtensionPgtap.descriptor, OliphauntExtensions.hstore]) -precondition(configuration.extensions.map(\\.sqlName) == ["pgtap", "hstore"]) -`); - run("swift", ["run", "--package-path", consumer, "--scratch-path", path.join(root, "../consumer-build"), "Consumer"], { timeout: 180_000 }); + prepareExtensionReleaseConsumer({ + plan: { + extensions: ["pgtap"], extensionProducts: ["oliphaunt-extension-pgtap"], + finalLink: { kind: "base-runtime", runtimeProduct: "liboliphaunt-native", runtimeVersion: manifest.base.version }, + }, + productsFile: path.join(sqlOutput, "extension-products.json"), releasePackage: sdk, + carrier: sqlCarrier, extensionCarriers: [], cache: path.join(root, "sql-cache"), output: consumer, + }); + run("swift", ["build", "--package-path", consumer, "--scratch-path", path.join(root, "../consumer-build")], { timeout: 180_000 }); console.log(`swift-carrier-resolver.test.mjs: metadata, malicious ZIP, cache-tamper, and consumer checks passed; sql-only-package=${sqlOutput}`); } diff --git a/tools/release/moon.yml b/tools/release/moon.yml index 3cd39888a..5e0c00d27 100644 --- a/tools/release/moon.yml +++ b/tools/release/moon.yml @@ -414,6 +414,11 @@ tasks: - "/tools/release/extension-upstream-licenses.mjs" - "/tools/release/native-extension-asset-index-contract.mjs" - "/tools/release/native-extension-lifecycle-receipts.test.mjs" + - "/tools/release/native-icu-seeds.mjs" + - "/tools/release/native-cluster-seed-contract.mjs" + - "/tools/release/product-version.mjs" + - "/src/shared/artifact-packaging/archive-directory.mjs" + - "/src/sdks/rust/tools/check-release-consumer.sh" - "/tools/release/optimize_native_runtime_payload.mjs" - "/tools/release/platform-compatibility-policy.mjs" - "/tools/release/product-task-model.test.mjs" diff --git a/tools/release/native-extension-lifecycle-receipts.test.mjs b/tools/release/native-extension-lifecycle-receipts.test.mjs index a253f481b..60bc660b6 100644 --- a/tools/release/native-extension-lifecycle-receipts.test.mjs +++ b/tools/release/native-extension-lifecycle-receipts.test.mjs @@ -4,6 +4,7 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { existsSync, + mkdirSync, mkdtempSync, readFileSync, rmSync, @@ -20,6 +21,7 @@ import { } from "../graph/ci_plan.mjs"; import { compareText, + currentProductVersionSync, exactExtensionProducts, extensionSqlNames, } from "./release-artifact-targets.mjs"; @@ -27,10 +29,17 @@ import { assertExactFiles, selectedExtensionDependencies, stageExtensionCarrier, + stageBaseRuntime, } from "./stage-native-extension-lifecycle.mjs"; import { verifyReceipts } from "./verify-native-extension-lifecycle-receipts.mjs"; import { writeReceipt } from "./write-native-extension-lifecycle-receipt.mjs"; +import { spawnSync } from "node:child_process"; +import { createDeterministicTar } from "../../src/shared/artifact-packaging/archive-directory.mjs"; +import { canonicalGzipSync } from "../../src/shared/artifact-packaging/portable-archive.mjs"; +import { requiredRuntimeMemberPaths, requiredToolsMemberPaths } from "./optimize_native_runtime_payload.mjs"; +import { nativeIcuSeedAsset } from "./native-icu-seeds.mjs"; + const CANDIDATE_SHA = "a".repeat(40); const CANDIDATE_TREE = "b".repeat(40); @@ -350,3 +359,45 @@ test("aggregate verification rejects candidate, shard, and PASS-record drift eve } } }); + +test("native consumers distinguish the runtime from its independently shipped ICU seed", async () => { + const root = mkdtempSync(path.join(os.tmpdir(), "oliphaunt-native-runtime-inputs-")); + try { + const version = currentProductVersionSync("liboliphaunt-native", "test"); + const target = "linux-x64-gnu"; + const assets = path.join(root, "assets"); + mkdirSync(assets); + for (const [name, members] of [ + [`liboliphaunt-${version}-${target}.tar.gz`, ["lib/liboliphaunt.so", "lib/modules/dict_snowball.so", "lib/modules/plpgsql.so", ...requiredRuntimeMemberPaths(target, "runtime/bin")]], + [`oliphaunt-tools-${version}-${target}.tar.gz`, requiredToolsMemberPaths(target, "runtime/bin")], + [nativeIcuSeedAsset(version, target), ["manifest.properties", "files/PG_VERSION"]], + ]) { + const source = path.join(root, name); + for (const member of members) { + mkdirSync(path.dirname(path.join(source, member)), { recursive: true }); + writeFileSync(path.join(source, member), "fixture"); + } + writeFileSync(path.join(assets, name), canonicalGzipSync(await createDeterministicTar(source))); + } + const output = path.join(root, "staged"); + const records = stageBaseRuntime(assets, output, []); + assert.deepEqual(records.map(row => row.identity), ["native-runtime", "native-tools"]); + assert.equal(existsSync(path.join(output, "resources/native-runtime/liboliphaunt-native/cluster-seed-icu")), false); + if (process.platform === "linux" && process.arch === "x64") { + const consumer = path.join(root, "consumer.sh"); + writeFileSync(consumer, '#!/bin/sh\nset -eu\ntest -s "$OLIPHAUNT_INSTALL_DIR/bin/postgres"\ntest -s "$OLIPHAUNT_TOOLS_DIR/bin/psql"\necho EXACT_RUNTIME_PASS\n', { mode: 0o755 }); + const result = spawnSync(process.env.OLIPHAUNT_TEST_BASH || "bash", [ + "src/sdks/rust/tools/check-release-consumer.sh", "run", consumer, assets, + ], { cwd: path.resolve(import.meta.dirname, "../.."), encoding: "utf8" }); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.match(result.stdout, /EXACT_RUNTIME_PASS/u); + } + writeFileSync(path.join(assets, "unindexed.tar.gz"), "unexpected"); + assert.throws(() => stageBaseRuntime(assets, output, []), /unindexed files/u); + rmSync(path.join(assets, "unindexed.tar.gz")); + rmSync(path.join(assets, nativeIcuSeedAsset(version, target))); + assert.throws(() => stageBaseRuntime(assets, output, []), /must contain exactly one.*icu-seed/u); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/release/native-npm-archive-extraction.test.mjs b/tools/release/native-npm-archive-extraction.test.mjs index 33ad5e78d..df79ebd45 100644 --- a/tools/release/native-npm-archive-extraction.test.mjs +++ b/tools/release/native-npm-archive-extraction.test.mjs @@ -31,11 +31,11 @@ function writeFixtureFile(root, relativePath, contents) { writeFileSync(file, contents); } -test("native npm ZIP assembly preserves complete nested runtime trees", () => { +for (const format of ["zip", "tar.gz"]) test(`native npm ${format} assembly preserves complete nested runtime and ICU trees`, () => { const root = mkdtempSync(path.join(os.tmpdir(), "oliphaunt-native-npm-zip-tree-")); try { const source = path.join(root, "source"); - const archive = path.join(root, "native.zip"); + const archive = path.join(root, `native.${format}`); const runtimeFiles = new Map([ ["bin/initdb.exe", "initdb\n"], ["bin/pg_ctl.exe", "pg_ctl\n"], @@ -50,6 +50,7 @@ test("native npm ZIP assembly preserves complete nested runtime trees", () => { writeFixtureFile(source, "lib/modules/dict_snowball.dll", "embedded dict_snowball\n"); writeFixtureFile(source, "lib/modules/plpgsql.dll", "embedded plpgsql\n"); writeFixtureFile(source, "outside/not-packaged.txt", "outside\n"); + writeFixtureFile(source, "share/icu/icudt76l/root.res", "icu\n"); const packed = spawnSync(process.execPath, [ARCHIVER, source, archive], { encoding: "utf8", @@ -64,6 +65,9 @@ test("native npm ZIP assembly preserves complete nested runtime trees", () => { for (const [relativePath, contents] of runtimeFiles) { expect(readFileSync(path.join(stage, ...relativePath.split("/")), "utf8")).toBe(contents); } + const icu = path.join(root, name, "icu"); + extract(archive, "share/icu", icu); + expect(readFileSync(path.join(icu, "icudt76l/root.res"), "utf8")).toBe("icu\n"); const modules = path.join(root, name, "lib/modules"); extract(archive, "lib/modules", modules); expect(readFileSync(path.join(modules, "dict_snowball.dll"), "utf8")).toBe( diff --git a/tools/release/package-extension-cargo-facades.test.mjs b/tools/release/package-extension-cargo-facades.test.mjs index 1cf24aa54..cf03335f3 100644 --- a/tools/release/package-extension-cargo-facades.test.mjs +++ b/tools/release/package-extension-cargo-facades.test.mjs @@ -1,3 +1,4 @@ +import { cargoDatabaseSmokeSource } from "./public-consumer-smoke.mjs"; import { afterEach, describe, expect, test } from "bun:test"; import { spawnSync } from "../test/fd-backed-spawn-sync.mjs"; import { createHash } from "node:crypto"; @@ -316,6 +317,7 @@ vector = { package = "oliphaunt-extension-vector", path = ${JSON.stringify(path. writeFileSync(path.join(app, "src/lib.rs"), `pub fn configured() -> oliphaunt::OliphauntBuilder { oliphaunt::Oliphaunt::builder().extensions([vector::VECTOR, oliphaunt::extensions::HSTORE]) }\n`); + writeFileSync(path.join(app, "src/main.rs"), cargoDatabaseSmokeSource().replace("use locked_entry::", "use oliphaunt::")); const plain = spawnSync("cargo", ["check", "--offline", "--target-dir", path.join(root, "cargo-target")], { cwd: app, encoding: "utf8", maxBuffer: 20 * 1024 * 1024, }); diff --git a/tools/release/package-release-carriers.mjs b/tools/release/package-release-carriers.mjs index be2d09e73..2b45b04e1 100644 --- a/tools/release/package-release-carriers.mjs +++ b/tools/release/package-release-carriers.mjs @@ -735,6 +735,7 @@ function copyExtractedTree(source, destination) { } export function extractReleaseArchiveTree(archive, sourcePrefix, destination) { + if (!isFile(archive)) fail(`missing release archive: ${rel(archive)}`); const temp = archiveTempDir(); const prefix = sourcePrefix.replace(/\/+$/u, ""); try { diff --git a/tools/release/public-consumer-smoke.mjs b/tools/release/public-consumer-smoke.mjs index 06b1431b9..8eb71951e 100644 --- a/tools/release/public-consumer-smoke.mjs +++ b/tools/release/public-consumer-smoke.mjs @@ -31,7 +31,7 @@ import { ROOT, compareText, loadGraph } from "./release-graph.mjs"; import { validateRegistryReceiptEvidence } from "./registry-integrity.mjs"; import { validateGithubAttestationReceipt } from "./verify_github_release_attestations.mjs"; -export const PUBLIC_CONSUMER_EVIDENCE_SCHEMA = "oliphaunt-public-consumer-smoke-v1"; +export const PUBLIC_CONSUMER_EVIDENCE_SCHEMA = "oliphaunt-public-consumer-smoke-v2"; const TOOL = "public-consumer-smoke"; const REGISTRY_ECOSYSTEMS = ["cargo", "maven", "npm"]; @@ -342,10 +342,13 @@ export function sanitizedPublicEnvironment(overrides = {}, inherited = process.e for (const name of Object.keys(env)) { if ( /(?:^|_)(?:AUTH|PASSWORD|PASSPHRASE|SECRET|TOKEN|USERNAME)(?:_|$)/iu.test(name) - || /^CARGO_(?:REGISTRIES|REGISTRY|SOURCE)_/iu.test(name) + || /^CARGO_/iu.test(name) + || /^(?:RUSTC|RUSTC_WRAPPER|RUSTC_WORKSPACE_WRAPPER|RUSTFLAGS|RUSTDOC|RUSTDOCFLAGS)$/u.test(name) || /^GIT_/iu.test(name) || /^NPM_CONFIG_/iu.test(name) || /^ORG_GRADLE_PROJECT_/iu.test(name) + || /^(?:OLIPHAUNT_|LIBOLIPHAUNT_|DYLD_)/u.test(name) + || /^(?:NODE_OPTIONS|NODE_PATH|LD_LIBRARY_PATH)$/u.test(name) || /^(?:DENO_CONFIG|DENO_DIR|DENO_IMPORT_MAP|DENO_LOCK|GRADLE_OPTS|JAVA_OPTS|JAVA_TOOL_OPTIONS|JDK_JAVA_OPTIONS|_JAVA_OPTIONS)$/iu.test(name) ) delete env[name]; } @@ -832,6 +835,26 @@ export function validateCargoResolution(lockText, carriers, requiredCarrierIds = return rows.sort((left, right) => compareText(left.id, right.id)); } +export function cargoDatabaseSmokeSource() { + return `use locked_entry::{DatabaseStorage, Oliphaunt, extensions}; +fn main() -> Result<(), Box> { + let root = std::path::PathBuf::from(std::env::args_os().nth(1).expect("database directory")); + for attempt in 0..2 { + let mut db = Oliphaunt::builder() + .storage(DatabaseStorage::Directory(root.clone())) + .extensions([extensions::HSTORE]) + .open()?; + if attempt == 0 { db.exec("CREATE EXTENSION hstore")?; } + let result = db.query("SELECT 'answer=>ready'::hstore -> 'answer' AS value")?; + assert_eq!(result.get_text(0, "value")?, Some("ready")); + db.close()?; + } + println!("OLIPHAUNT_PUBLIC_DATABASE_PASS"); + Ok(()) +} +`; +} + async function runCargoSurface({ lock, surface, root, deadlineMilliseconds, signal }) { const directory = path.join(root, "cargo"); const home = path.join(root, "cargo-home"); @@ -861,12 +884,29 @@ async function runCargoSurface({ lock, surface, root, deadlineMilliseconds, sign rows.push(resolved); entries.push({ entryCarrierId, resolvedCarrierIds: resolved.map(({ id }) => id) }); } + // Compile only the native SDK with its public defaults. The all-feature, + // all-target lock probes above also cover carriers that cannot run on this host. + const executed = []; + for (const carrier of carriers.filter(row => row.name === "oliphaunt")) { + const consumer = path.join(directory, "native-database"); + mkdirSync(path.join(consumer, "src"), { recursive: true }); + writeFileSync(path.join(consumer, "Cargo.toml"), + `[package]\nname = "oliphaunt-public-database"\nversion = "0.0.0"\nedition = "2021"\npublish = false\n\n[dependencies]\nlocked_entry = { package = "oliphaunt", version = ${tomlString(`=${carrier.version}`)} }\n`); + writeFileSync(path.join(consumer, "src/main.rs"), cargoDatabaseSmokeSource()); + await runBoundedCommand("cargo", ["generate-lockfile"], { cwd: consumer, env, deadlineMilliseconds, signal }); + validateCargoResolution(readFileSync(path.join(consumer, "Cargo.lock"), "utf8"), carriers, [carrier.id]); + const output = await runBoundedCommand("cargo", ["run", "--locked", "--", path.join(consumer, "database")], + { cwd: consumer, env, deadlineMilliseconds, signal }); + if (!output.stdout.includes("OLIPHAUNT_PUBLIC_DATABASE_PASS")) throw error(`${carrier.id} did not complete its installed database probe`); + executed.push(carrier.id); + } return { surface: "cargo", - mode: "anonymous-public-independent-entry-all-feature-resolution-no-compile", + mode: "anonymous-public-independent-entry-resolution-and-native-sdk-execution", registry: "https://crates.io", ...resolvedSurfaceCoverage(surface, entries, rows), - receiptCoveredWithoutPayloadFetchCarrierIds: surface.carrierIds, + receiptCoveredCarrierIds: surface.carrierIds, + executedDatabaseCarrierIds: executed.sort(compareText), }; } @@ -926,6 +966,29 @@ export function validateNpmResolution(packageLock, carriers, requiredEntryIds, n return { resolved, installedCarrierIds }; } +export function npmDatabaseSmokeSource(carrier) { + if (!new Set(["@oliphaunt/ts", "@oliphaunt/wasix-ts"]).has(carrier.name)) return null; + return `import assert from 'node:assert/strict'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import Oliphaunt, { extensions } from ${JSON.stringify(carrier.name)}; +import { directory } from ${JSON.stringify(`${carrier.name}/storage/node`)}; +const root = await mkdtemp(path.join(tmpdir(), 'oliphaunt-public-database-')); +try { + for (let attempt = 0; attempt < 2; attempt += 1) { + const db = await Oliphaunt.open({ storage: directory(root), extensions: [extensions.hstore] }); + try { + if (attempt === 0) await db.exec('CREATE EXTENSION hstore'); + const result = await db.query(\`SELECT 'answer=>ready'::hstore -> 'answer' AS value\`); + assert.equal(result.rows[0]?.value, 'ready'); + } finally { await db.close(); } + } +} finally { await rm(root, { recursive: true, force: true }); } +console.log('OLIPHAUNT_PUBLIC_DATABASE_PASS'); +`; +} + async function runNpmSurface({ lock, surface, root, deadlineMilliseconds, signal }) { const directory = path.join(root, "npm"); const home = path.join(root, "npm-home"); @@ -947,6 +1010,7 @@ async function runNpmSurface({ lock, surface, root, deadlineMilliseconds, signal const entries = []; const rows = []; const installed = new Set(); + const executed = []; for (const [index, entryCarrierId] of surface.entryCarrierIds.entries()) { const carrier = byId.get(entryCarrierId); const consumer = path.join(directory, `entry-${String(index).padStart(3, "0")}`); @@ -967,6 +1031,15 @@ async function runNpmSurface({ lock, surface, root, deadlineMilliseconds, signal ], { cwd: consumer, env, deadlineMilliseconds, signal }); const lockJson = JSON.parse(readFileSync(path.join(consumer, "package-lock.json"), "utf8")); const result = validateNpmResolution(lockJson, carriers, [entryCarrierId], consumer); + for (const installedCarrier of carriers.filter(row => result.installedCarrierIds.includes(row.id) && !executed.includes(row.id))) { + const smoke = npmDatabaseSmokeSource(installedCarrier); + if (smoke === null) continue; + const file = path.join(consumer, "database-smoke.mjs"); + writeFileSync(file, smoke); + const output = await runBoundedCommand("node", [file], { cwd: consumer, env, deadlineMilliseconds, signal }); + if (!output.stdout.includes("OLIPHAUNT_PUBLIC_DATABASE_PASS")) throw error(`${installedCarrier.id} did not complete its installed database probe`); + executed.push(installedCarrier.id); + } rows[index] = result.resolved; entries[index] = { entryCarrierId, resolvedCarrierIds: result.resolved.map(({ id }) => id) }; for (const id of result.installedCarrierIds) installed.add(id); @@ -978,6 +1051,7 @@ async function runNpmSurface({ lock, surface, root, deadlineMilliseconds, signal host: `${process.platform}-${process.arch}`, ...resolvedSurfaceCoverage(surface, entries, rows), installedCarrierIds: [...installed].sort(compareText), + executedDatabaseCarrierIds: executed.sort(compareText), receiptCoveredNotHostInstalledCarrierIds: surface.carrierIds.filter((id) => !installed.has(id)).sort(compareText), }; } @@ -1278,12 +1352,23 @@ export function validatePublicConsumerEvidence(evidence, lock, plan) { throw error(`${surface.ecosystem} public consumer evidence has non-canonical ${field} coverage`); } } + if (surface.ecosystem === "npm" || surface.ecosystem === "cargo") { + const expectedExecuted = lock.carriers.filter(carrier => surface.carrierIds.includes(carrier.id) + && (surface.ecosystem === "npm" ? npmDatabaseSmokeSource(carrier) !== null : carrier.name === "oliphaunt")) + .map(carrier => carrier.id).sort(compareText); + if (!sameStrings(observed?.executedDatabaseCarrierIds ?? [], expectedExecuted)) { + throw error(`${surface.ecosystem} evidence omits required installed database execution`); + } + } if (surface.ecosystem === "npm") { const installed = sortedUniqueStrings(observed?.installedCarrierIds ?? [], "npm installedCarrierIds"); const resolved = new Set(coverage.resolved.map(({ id }) => id)); if (installed.some((id) => !resolved.has(id)) || stableJson(installed) !== stableJson(observed.installedCarrierIds)) { throw error("npm host-installed carriers must be a canonical subset of its exact public resolution"); } + if ((observed.executedDatabaseCarrierIds ?? []).some(id => !installed.includes(id))) { + throw error("npm database execution must use a host-installed carrier"); + } const notInstalled = surface.carrierIds.filter((id) => !installed.includes(id)).sort(compareText); if (stableJson(observed.receiptCoveredNotHostInstalledCarrierIds) !== stableJson(notInstalled)) { throw error("npm evidence must explicitly distinguish exhaustive lock resolution from the publish-host installed subset"); @@ -1291,7 +1376,7 @@ export function validatePublicConsumerEvidence(evidence, lock, plan) { } if ( surface.ecosystem === "cargo" - && stableJson(observed.receiptCoveredWithoutPayloadFetchCarrierIds) !== stableJson(surface.carrierIds) + && stableJson(observed.receiptCoveredCarrierIds) !== stableJson(surface.carrierIds) ) { throw error("Cargo evidence must explicitly distinguish registry resolution from receipt-proved payload bytes"); } diff --git a/tools/release/public-consumer-smoke.test.mjs b/tools/release/public-consumer-smoke.test.mjs index 2e5c454e7..7ab5cc272 100644 --- a/tools/release/public-consumer-smoke.test.mjs +++ b/tools/release/public-consumer-smoke.test.mjs @@ -8,6 +8,7 @@ import test from "node:test"; import { PUBLIC_CONSUMER_EVIDENCE_SCHEMA, cargoEntryFeatureNames, + npmDatabaseSmokeSource, publicCargoEnvironment, publicConsumerEvidence, publicConsumerPlan, @@ -343,6 +344,10 @@ test("public probes discard inherited credentials and package-manager substituti }, { PATH: "/usr/bin", CARGO_SOURCE_CRATES_IO_REPLACE_WITH: "local-mirror", + CARGO_TARGET_DIR: "/workspace/target", + CARGO_ENCODED_RUSTFLAGS: "--cfg=workspace", + RUSTC_WRAPPER: "/workspace/wrapper", + RUSTFLAGS: "--cfg=workspace", DENO_CONFIG: "/workspace/deno.json", GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "url.file:///workspace/.insteadOf", @@ -352,6 +357,13 @@ test("public probes discard inherited credentials and package-manager substituti npm_config_userconfig: "/workspace/.npmrc", ORG_GRADLE_PROJECT_repositoryPassword: "secret", RELEASE_TOKEN: "secret", + OLIPHAUNT_INSTALL_DIR: "/workspace/runtime", + OLIPHAUNT_RUNTIME_CACHE_DIR: "/workspace/cache", + LIBOLIPHAUNT_DIR: "/workspace/lib", + LD_LIBRARY_PATH: "/workspace/lib", + DYLD_LIBRARY_PATH: "/workspace/lib", + NODE_OPTIONS: "--import=/workspace/mock.mjs", + NODE_PATH: "/workspace/node_modules", }); assert.deepEqual(env, { PATH: "/usr/bin", @@ -426,19 +438,20 @@ test("Cargo consumer toolchain context fails closed on unpinned or unavailable i }); test("builds canonical lock/receipt-bound evidence and writes it immutably", () => { - const products = [product("alpha", ["npm"])]; - const frozen = lock(products, [carrier("npm:@example/alpha", "alpha", 0)]); - const plan = publicConsumerPlan(frozen, ["alpha"], graph(products)); + const products = [product("oliphaunt-js", ["npm"])]; + const frozen = lock(products, [carrier("npm:@oliphaunt/ts", "oliphaunt-js", 0)]); + const plan = publicConsumerPlan(frozen, ["oliphaunt-js"], graph(products)); const surfaces = [{ surface: "npm", mode: "anonymous-public-independent-entry-host-install-and-lock-resolution", - carrierIds: ["npm:@example/alpha"], + carrierIds: ["npm:@oliphaunt/ts"], dependencyScopes: ["optional", "peer", "runtime"], - entryCarrierIds: ["npm:@example/alpha"], - plannedEntryClosures: [{ entryCarrierId: "npm:@example/alpha", carrierIds: ["npm:@example/alpha"] }], - entries: [{ entryCarrierId: "npm:@example/alpha", resolvedCarrierIds: ["npm:@example/alpha"] }], - installedCarrierIds: ["npm:@example/alpha"], - resolved: [{ id: "npm:@example/alpha", version: "1.2.3", integrity: "sha512-exact" }], + entryCarrierIds: ["npm:@oliphaunt/ts"], + plannedEntryClosures: [{ entryCarrierId: "npm:@oliphaunt/ts", carrierIds: ["npm:@oliphaunt/ts"] }], + entries: [{ entryCarrierId: "npm:@oliphaunt/ts", resolvedCarrierIds: ["npm:@oliphaunt/ts"] }], + installedCarrierIds: ["npm:@oliphaunt/ts"], + executedDatabaseCarrierIds: ["npm:@oliphaunt/ts"], + resolved: [{ id: "npm:@oliphaunt/ts", version: "1.2.3", integrity: "sha512-exact" }], receiptCoveredNotHostInstalledCarrierIds: [], }, { surface: "github", @@ -455,6 +468,12 @@ test("builds canonical lock/receipt-bound evidence and writes it immutably", () }); assert.equal(evidence.schema, PUBLIC_CONSUMER_EVIDENCE_SCHEMA); assert.equal(validatePublicConsumerEvidence(evidence, frozen, plan), evidence); + const unresolvedRuntime = structuredClone(evidence); + unresolvedRuntime.surfaces.find(row => row.surface === "npm").executedDatabaseCarrierIds = []; + assert.throws(() => validatePublicConsumerEvidence(unresolvedRuntime, frozen, plan), /required installed database execution/u); + const notInstalled = structuredClone(evidence); + notInstalled.surfaces.find(row => row.surface === "npm").installedCarrierIds = []; + assert.throws(() => validatePublicConsumerEvidence(notInstalled, frozen, plan), /must use a host-installed carrier/u); const changed = structuredClone(evidence); changed.surfaces.find(({ surface }) => surface === "github").productTags = []; assert.throws(() => validatePublicConsumerEvidence(changed, frozen, plan), /every exact product tag/u); @@ -561,3 +580,56 @@ test("transient registry visibility failures retry from an empty workspace and c rmSync(root, { recursive: true, force: true }); } }); + +test("installed npm database probe executes descriptors, migrations, query, reopen and close", async () => { + const root = mkdtempSync(path.join(tmpdir(), "oliphaunt-database-probe-")); + try { + for (const name of ["@oliphaunt/ts", "@oliphaunt/wasix-ts"]) { + const directory = path.join(root, "node_modules", name); + mkdirSync(directory, { recursive: true }); + writeFileSync(path.join(directory, "package.json"), JSON.stringify({ name, type: "module", exports: { ".": "./index.js", "./storage/node": "./storage.js" } })); + writeFileSync(path.join(directory, "storage.js"), "export const directory = path => ({kind: 'directory', path});"); + writeFileSync(path.join(directory, "index.js"), `import assert from 'node:assert/strict'; +let opens = 0, closes = 0, migrations = 0; +export const extensions = {hstore: {sqlName: 'hstore'}}; +export default {async open(config) { + assert.equal(config.storage.kind, 'directory'); + assert.deepEqual(config.extensions, [extensions.hstore]); + assert.equal(opens, closes); + opens += 1; + return {async exec(sql) { assert.equal(sql, 'CREATE EXTENSION hstore'); migrations += 1; }, + async query() { assert.equal(migrations, 1); return {rows: [{value: process.env.PROBE_WRONG_RESULT ? 'wrong' : 'ready'}]}; }, + async close() { closes += 1; }}; +}}; +process.on('exit', () => { if (!process.env.PROBE_WRONG_RESULT) { assert.equal(opens, 2); assert.equal(closes, 2); } }); +`); + const file = path.join(root, "consumer.mjs"); + writeFileSync(file, npmDatabaseSmokeSource({ name })); + const options = { cwd: root, deadlineMilliseconds: Date.now() + 30_000 }; + const result = await runBoundedCommand("node", [file], options); + assert.match(result.stdout, /OLIPHAUNT_PUBLIC_DATABASE_PASS/u); + await assert.rejects(runBoundedCommand("node", [file], { ...options, env: { ...process.env, PROBE_WRONG_RESULT: "1" } }), /wrong/u); + } + assert.equal(npmDatabaseSmokeSource({ name: "@oliphaunt/react-native" }), null); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test("Cargo evidence requires installed native SDK execution as well as complete receipt coverage", () => { + const products = [product("oliphaunt-rust", ["crates-io"])]; + const frozen = lock(products, [carrier("cargo:oliphaunt", "oliphaunt-rust", 0)]); + const plan = publicConsumerPlan(frozen, ["oliphaunt-rust"], graph(products)); + const surface = plan.surfaces[0]; + const evidence = publicConsumerEvidence({ + lock: frozen, plan, registryReceiptSha256: "e".repeat(64), githubReceiptDigest: "f".repeat(64), + surfaces: [{ + surface: "cargo", carrierIds: surface.carrierIds, entryCarrierIds: surface.entryCarrierIds, + dependencyScopes: surface.dependencyScopes, plannedEntryClosures: surface.entryClosures, + entries: [{ entryCarrierId: "cargo:oliphaunt", resolvedCarrierIds: ["cargo:oliphaunt"] }], + resolved: [{ id: "cargo:oliphaunt", version: "1.2.3", checksum: "d".repeat(64) }], + receiptCoveredCarrierIds: ["cargo:oliphaunt"], executedDatabaseCarrierIds: ["cargo:oliphaunt"], + }, { surface: "github", productTags: plan.github.productTags, swift: null }], + }); + assert.doesNotThrow(() => validatePublicConsumerEvidence(evidence, frozen, plan)); + evidence.surfaces.find(row => row.surface === "cargo").executedDatabaseCarrierIds = []; + assert.throws(() => validatePublicConsumerEvidence(evidence, frozen, plan), /required installed database execution/u); +}); diff --git a/tools/release/stage-native-extension-lifecycle.mjs b/tools/release/stage-native-extension-lifecycle.mjs index 67c3ae60a..ec4e9aca2 100755 --- a/tools/release/stage-native-extension-lifecycle.mjs +++ b/tools/release/stage-native-extension-lifecycle.mjs @@ -26,6 +26,7 @@ import { NATIVE_EXTENSION_ASSET_INDEX_HEADER, isCanonicalNativeExtensionRuntimeIndexRow, } from "./native-extension-asset-index-contract.mjs"; +import { nativeIcuSeedAsset } from "./native-icu-seeds.mjs"; import { requiredRuntimeMemberPaths, requiredToolsMemberPaths, @@ -371,7 +372,7 @@ export function selectedExtensionDependencies(metadata) { return sorted.join(","); } -function stageBaseRuntime(runtimeAssets, output, extensionRows) { +export function stageBaseRuntime(runtimeAssets, output, extensionRows) { const version = currentProductVersionSync("liboliphaunt-native", PREFIX); const runtimeArchive = oneFile(runtimeAssets, `liboliphaunt-${version}-${TARGET}.tar.gz`); const toolsArchive = oneFile(runtimeAssets, `oliphaunt-tools-${version}-${TARGET}.tar.gz`); @@ -394,7 +395,9 @@ function stageBaseRuntime(runtimeAssets, output, extensionRows) { } extract(runtimeEntries, path.join(output, "resources/native-runtime/liboliphaunt-native")); extract(toolsEntries, path.join(output, "resources/native-tools/oliphaunt-tools")); - assertExactFiles(runtimeAssets, [runtimeArchive, toolsArchive], "Linux runtime artifact download"); + // The producer also ships an optional ICU seed, which this standard-profile proof does not load. + const icuSeedArchive = oneFile(runtimeAssets, nativeIcuSeedAsset(version, TARGET)); + assertExactFiles(runtimeAssets, [runtimeArchive, toolsArchive, icuSeedArchive], "Linux runtime artifact download"); return [ artifactRecord("native-runtime", runtimeArchive), artifactRecord("native-tools", toolsArchive), diff --git a/tools/xtask/moon.yml b/tools/xtask/moon.yml index 563b8f2fe..1452618d5 100644 --- a/tools/xtask/moon.yml +++ b/tools/xtask/moon.yml @@ -27,6 +27,8 @@ tasks: - "@group(cargo-workspace)" - "**/*" - "/src/shared/cluster-seed-contract/**/*" + - "/src/extensions/generated/extensions.catalog.json" + - "/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/generated_extensions.rs" options: cache: true runFromWorkspaceRoot: true diff --git a/tools/xtask/src/asset_checks.rs b/tools/xtask/src/asset_checks.rs index 6a8d65231..c9d2152d1 100644 --- a/tools/xtask/src/asset_checks.rs +++ b/tools/xtask/src/asset_checks.rs @@ -786,42 +786,37 @@ pub(crate) fn verify_generated_extension_surface() -> Result<()> { ); } - for extension in &manifest.extensions { - let rust_constant = supported_constants - .get(&extension.sql_name) - .ok_or_else(|| { - anyhow!( - "extension {} missing from supported catalog", - extension.sql_name - ) - })?; + verify_extension_definitions(&generated, &supported_constants)?; + println!("generated extension API matches asset manifest and catalog"); + Ok(()) +} + +fn verify_extension_definitions( + generated: &str, + constants: &BTreeMap, +) -> Result<()> { + for (sql_name, rust_constant) in constants { let definition_const = format!("DEFINITION_{rust_constant}"); - let cargo_feature = format!("extension-{}", extension.sql_name.replace('_', "-")); for (needle, description) in [ ( - format!( - "#[cfg(feature = {cargo_feature:?})]\nconst {definition_const}: Extension =" - ), - "feature-gated extension definition constant", + format!("const {definition_const}: Extension ="), + "extension definition constant", ), ( - format!( - "#[cfg(feature = {cargo_feature:?})]\n pub const {rust_constant}: Self = {definition_const};" - ), - "feature-gated public extension constant", + format!("pub const {rust_constant}: Self = {definition_const};"), + "public extension constant", ), ( - format!("#[cfg(feature = {cargo_feature:?})]\n Self::{rust_constant},"), - "feature-gated Extension::ALL entry", + format!(" Self::{rust_constant},"), + "Extension::ALL entry", ), - (format!("{:?}", extension.sql_name), "extension SQL name"), + (format!("{sql_name:?}"), "extension SQL name"), ] { if !generated.contains(&needle) { bail!("generated extension API is stale: missing {description} {needle}"); } } } - println!("generated extension API matches asset manifest and catalog"); Ok(()) } @@ -1495,3 +1490,33 @@ fn check_wasix_bridge_abi_harness() -> Result<()> { eprintln!("warning: skipping POSIX WASIX bridge ABI harness on non-Unix host"); Ok(()) } + +#[cfg(test)] +mod generated_extension_tests { + use super::*; + + #[test] + fn current_descriptor_surface_passes_without_runtime_build_outputs() { + let catalog: serde_json::Value = serde_json::from_str(include_str!( + "../../../src/extensions/generated/extensions.catalog.json" + )) + .unwrap(); + let constants = catalog["extensions"] + .as_array() + .unwrap() + .iter() + .map(|entry| { + ( + entry["sql-name"].as_str().unwrap().to_owned(), + entry["rust-constant"].as_str().unwrap().to_owned(), + ) + }) + .collect(); + let generated = include_str!( + "../../../src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/generated_extensions.rs" + ); + verify_extension_definitions(generated, &constants).unwrap(); + let missing = generated.replace("pub const HSTORE:", "pub const STALE_HSTORE:"); + assert!(verify_extension_definitions(&missing, &constants).is_err()); + } +} From 9773a0753d46eeb4c49f2563a2681e914c4a02a3 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 11:27:56 +0000 Subject: [PATCH 10/18] fix(sdk): align native TypeScript restore storage descriptors --- docs/maintainers/sdk-api-surface.md | 1 + docs/maintainers/sdk-parity-policy.md | 8 ++++ src/sdks/js/CHANGELOG.md | 4 ++ src/sdks/js/README.md | 10 +++-- src/sdks/js/src/__tests__/client.test.ts | 38 +++++++++++++++++-- .../src/__tests__/native-direct-contract.mjs | 4 +- src/sdks/js/src/__tests__/public-api.test.ts | 15 ++++++++ src/sdks/js/src/client.ts | 11 ++++-- src/sdks/js/src/index.ts | 1 + src/sdks/js/src/storage/node.ts | 4 +- src/sdks/js/src/types.ts | 9 ++++- 11 files changed, 89 insertions(+), 16 deletions(-) diff --git a/docs/maintainers/sdk-api-surface.md b/docs/maintainers/sdk-api-surface.md index d5e29495e..4da77f514 100644 --- a/docs/maintainers/sdk-api-surface.md +++ b/docs/maintainers/sdk-api-surface.md @@ -1669,6 +1669,7 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `QueryValueDecoder` - `RawQueryResult` - `RawQueryRow` +- `RestoreDestination` - `RestoreOptions` - `ServerListen` - `ServerOpenConfig` diff --git a/docs/maintainers/sdk-parity-policy.md b/docs/maintainers/sdk-parity-policy.md index 18e3931ec..c62ae48c0 100644 --- a/docs/maintainers/sdk-parity-policy.md +++ b/docs/maintainers/sdk-parity-policy.md @@ -171,6 +171,14 @@ again. runtime, archive format, capability profile, initialization mode, or extension set. +TypeScript restore destinations reuse the persistent storage descriptors accepted +by open: `directory(path)` on native hosts and the appropriate persistent adapter +on WASIX hosts and browsers. React Native uses `directory(pathOrFileUri)` for both. +Restore excludes temporary and memory storage and does not open a session. Rust +retains `Into` destinations in both runtimes; Kotlin uses `File` and +Swift uses `URL`. These filesystem-only restore APIs do not need an additional +storage wrapper or the connection options used when opening a database. + Cluster initialization and ICU are separate product concepts. The locked convergence target is that ordinary new roots transparently use a runtime-bound cluster seed, while language-native package/build selection enables optional diff --git a/src/sdks/js/CHANGELOG.md b/src/sdks/js/CHANGELOG.md index d991df1d0..206062e0e 100644 --- a/src/sdks/js/CHANGELOG.md +++ b/src/sdks/js/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- `Oliphaunt.restore` now accepts `directory(path)` instead of a raw path string, + matching open, React Native, and WASIX TypeScript storage descriptors. Import + `directory` from `@oliphaunt/ts/storage/node`; temporary storage is not a restore + destination. - Fail broker database objects permanently after helper or IPC failure. Close and explicitly open a new object for PostgreSQL WAL recovery; the SDK never substitutes a new session or replays uncertain work under the old object. diff --git a/src/sdks/js/README.md b/src/sdks/js/README.md index 6df132f76..34ea9705d 100644 --- a/src/sdks/js/README.md +++ b/src/sdks/js/README.md @@ -44,7 +44,7 @@ The deliberate public vocabulary is: - `execProtocolRawStream` for callback delivery of raw backend protocol chunks, including COPY responses, without buffering the complete response. - `backup()` returning the one physical backup format as `Uint8Array`. -- `Oliphaunt.restore(destination, bytes)` for an absent or empty destination. +- `Oliphaunt.restore(directory(path), bytes)` for an absent or empty destination. - `Oliphaunt.openServer(config)` for the distinct local-server handle. `execute` asserts one command with no rows. `query` accepts command-only or @@ -121,24 +121,26 @@ session when its state is unknown. ```ts const source = await Oliphaunt.open({ - storage: { kind: 'directory', path: '.oliphaunt-source' }, + storage: directory('.oliphaunt-source'), }); const bytes = await source.backup(); await source.close(); -await Oliphaunt.restore('.oliphaunt-restored', bytes); +await Oliphaunt.restore(directory('.oliphaunt-restored'), bytes); ``` Backup bytes are a PostgreSQL physical initialization payload containing PGDATA and backup metadata. They do not contain the outer `.oliphaunt.json` descriptor. Restore stages and validates PGDATA, then creates the receiving root identity. There is no archive selector and no replace-existing option. +Open and restore accept the same `directory(path)` helper, including local +`file:` URLs. Restore requires persistent storage and does not open a database. ## Local server ```ts const server = await Oliphaunt.openServer({ - storage: { kind: 'directory', path: '.oliphaunt-server' }, + storage: directory('.oliphaunt-server'), listen: { transport: 'tcp' }, }); console.log(server.connectionString); diff --git a/src/sdks/js/src/__tests__/client.test.ts b/src/sdks/js/src/__tests__/client.test.ts index 94ac52636..b4b8a91c8 100644 --- a/src/sdks/js/src/__tests__/client.test.ts +++ b/src/sdks/js/src/__tests__/client.test.ts @@ -3,6 +3,9 @@ import { mkdtemp, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'vitest'; +import { pathToFileURL } from 'node:url'; + +import { directory } from '../storage/node.js'; import { createOliphauntClient } from '../client.js'; import { extensions as contrib } from '../extensions.js'; @@ -18,6 +21,7 @@ import type { OliphauntDatabase, OliphauntTransaction, OpenConfig, + RestoreDestination, ServerOpenConfig, } from '../types.js'; import type { RuntimeBinding } from '../runtime/types.js'; @@ -33,7 +37,7 @@ test('exposes the minimal database lifecycle and byte backup contract', async () }); try { const db = await client.open({ - storage: { kind: 'directory', path: root }, + storage: directory(root), startupGUCs: { work_mem: '16MB' }, username: 'app', database: 'appdb', @@ -72,7 +76,7 @@ test('exposes the minimal database lifecycle and byte backup contract', async () assert.equal(binding.detachCalls, 1); await assert.rejects(() => db.execute('SELECT 1'), /closed/); - await client.restore(join(root, 'restored'), new Uint8Array([7, 8]), { + await client.restore(directory(pathToFileURL(join(root, 'restored'))), new Uint8Array([7, 8]), { libraryPath: '/opt/oliphaunt/liboliphaunt.so', }); assert.deepEqual(binding.restoreCalls, [ @@ -273,7 +277,31 @@ test('server open preserves both a missing endpoint and handle cleanup failure', } }); -test('copies restore bytes before asynchronous binding resolution', async () => { +test('rejects invalid restore destinations before loading native code', async () => { + let bindingLoads = 0; + const client = createOliphauntClient(() => { + bindingLoads += 1; + return new FakeBinding(); + }); + for (const destination of [ + undefined, + null, + './restored', + { kind: 'temporaryDirectory' }, + { kind: 'directory' }, + { kind: 'directory', path: 42 }, + { kind: 'directory', path: ' ' }, + { kind: 'directory', path: 'bad\0path' }, + ]) { + await assert.rejects( + client.restore(destination as RestoreDestination, Uint8Array.of(1)), + /restore destination/, + ); + } + assert.equal(bindingLoads, 0); +}); + +test('snapshots restore destination and bytes before asynchronous binding resolution', async () => { const root = await mkdtemp(join(tmpdir(), 'oliphaunt-js-restore-snapshot-')); const binding = new FakeBinding(); const releaseBinding = deferred(); @@ -284,7 +312,9 @@ test('copies restore bytes before asynchronous binding resolution', async () => const backup = new Uint8Array([7, 8]); try { - const restoring = client.restore(join(root, 'restored'), backup); + const destination = { kind: 'directory' as const, path: join(root, 'restored') }; + const restoring = client.restore(destination, backup); + destination.path = join(root, 'changed'); backup.fill(0); releaseBinding.resolve(); await restoring; diff --git a/src/sdks/js/src/__tests__/native-direct-contract.mjs b/src/sdks/js/src/__tests__/native-direct-contract.mjs index 36232e978..781d19626 100644 --- a/src/sdks/js/src/__tests__/native-direct-contract.mjs +++ b/src/sdks/js/src/__tests__/native-direct-contract.mjs @@ -20,7 +20,7 @@ export async function assertNativeDatabaseContract(Oliphaunt, config, label) { database = undefined; const restoredRoot = join(root, 'restored'); - await Oliphaunt.restore(restoredRoot, backup); + await Oliphaunt.restore({ kind: 'directory', path: restoredRoot }, backup); assert.match(await readFile(join(restoredRoot, 'pgdata', 'PG_VERSION'), 'utf8'), /^18\s*$/u); database = await Oliphaunt.open({ ...config, @@ -31,7 +31,7 @@ export async function assertNativeDatabaseContract(Oliphaunt, config, label) { database = undefined; await assert.rejects( - Oliphaunt.restore(join(root, 'invalid'), backup.subarray(0, 8)), + Oliphaunt.restore({ kind: 'directory', path: join(root, 'invalid') }, backup.subarray(0, 8)), (error) => error instanceof Error && error.message.length > 0, ); } finally { diff --git a/src/sdks/js/src/__tests__/public-api.test.ts b/src/sdks/js/src/__tests__/public-api.test.ts index 260518ef6..19a30e6ed 100644 --- a/src/sdks/js/src/__tests__/public-api.test.ts +++ b/src/sdks/js/src/__tests__/public-api.test.ts @@ -1,6 +1,8 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import { directory } from '../storage/node.js'; + import { array, binary, @@ -25,6 +27,7 @@ import { type QueryParam, type QueryValue, type RawQueryResult, + type RestoreDestination, type RestoreOptions, type TextQueryParameter, } from '../index.js'; @@ -133,6 +136,18 @@ function assertPublicDatabaseTypes( void assertPublicDatabaseTypes; +function assertPublicRestoreTypes(): void { + const destination: RestoreDestination = directory(new URL('file:///tmp/restored')); + const opened: Promise = Oliphaunt.open({ storage: destination }); + const restored: Promise = Oliphaunt.restore(destination, Uint8Array.of(1)); + // @ts-expect-error Restore requires persistent storage. + Oliphaunt.restore({ kind: 'temporaryDirectory' }, Uint8Array.of(1)); + // @ts-expect-error Use the same storage descriptor as open, not a raw path. + Oliphaunt.restore('/tmp/restored', Uint8Array.of(1)); + void [opened, restored]; +} +void assertPublicRestoreTypes; + const publicHelperTypes: [TextQueryParameter, BinaryQueryParameter, NullQueryParameter] = [ text('value'), binary(Uint8Array.of(1)), diff --git a/src/sdks/js/src/client.ts b/src/sdks/js/src/client.ts index 35f0fac79..8f5ec17ed 100644 --- a/src/sdks/js/src/client.ts +++ b/src/sdks/js/src/client.ts @@ -54,6 +54,7 @@ import type { ServerListen, ServerOpenConfig, ProtocolChunkCallback, + RestoreDestination, RestoreOptions, } from './types.js'; @@ -1110,15 +1111,19 @@ export function createOliphauntClient( }, async restore( - destination: string, + destination: RestoreDestination, backup: BinaryInput, options: RestoreOptions = {}, ): Promise { - validateDirectoryPath(destination, 'restore destination'); + if (destination?.kind !== 'directory' || typeof destination.path !== 'string') { + throw new TypeError('restore destination must be a directory storage descriptor'); + } + const path = destination.path; + validateDirectoryPath(path, 'restore destination'); const bytes = toUint8Array(backup).slice(); const binding = await bindingFor({ libraryPath: options.libraryPath }); await binding.restore({ - destination, + destination: path, bytes, }); }, diff --git a/src/sdks/js/src/index.ts b/src/sdks/js/src/index.ts index 5e13f2a06..3bbf4c093 100644 --- a/src/sdks/js/src/index.ts +++ b/src/sdks/js/src/index.ts @@ -42,6 +42,7 @@ export type { OliphauntTransaction, OliphauntServer, OpenConfig, + RestoreDestination, RestoreOptions, ServerListen, ServerOpenConfig, diff --git a/src/sdks/js/src/storage/node.ts b/src/sdks/js/src/storage/node.ts index 4f5345db9..f1369e308 100644 --- a/src/sdks/js/src/storage/node.ts +++ b/src/sdks/js/src/storage/node.ts @@ -1,10 +1,10 @@ import { fileURLToPath } from 'node:url'; import { validateDirectoryPath } from '../config.js'; -import type { DatabaseStorage } from '../types.js'; +import type { RestoreDestination } from '../types.js'; /** Persist a database in a host directory. Oliphaunt creates it when opened. */ -export function directory(path: string | URL): DatabaseStorage { +export function directory(path: string | URL): RestoreDestination { const location = typeof path === 'string' ? path : fileURLToPath(path); validateDirectoryPath(location, 'database storage directory'); return Object.freeze({ kind: 'directory', path: location }); diff --git a/src/sdks/js/src/types.ts b/src/sdks/js/src/types.ts index 20a918e87..9a8f0fbe4 100644 --- a/src/sdks/js/src/types.ts +++ b/src/sdks/js/src/types.ts @@ -4,6 +4,9 @@ export type DatabaseStorage = | { readonly kind: 'temporaryDirectory' } | { readonly kind: 'directory'; readonly path: string }; +/** Persistent storage that can receive a physical backup. */ +export type RestoreDestination = Exclude; + export type BinaryInput = ArrayBuffer | ArrayBufferView | Uint8Array | ReadonlyArray; type QueryReadOptions = Omit; @@ -123,5 +126,9 @@ export type RestoreOptions = { export type OliphauntClient = { open(config?: OpenConfig): Promise; openServer(config?: ServerOpenConfig): Promise; - restore(destination: string, backup: BinaryInput, options?: RestoreOptions): Promise; + restore( + destination: RestoreDestination, + backup: BinaryInput, + options?: RestoreOptions, + ): Promise; }; From 6db8e98ae3fc6d6ec9ac552411002355c19f534a Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 11:49:38 +0000 Subject: [PATCH 11/18] fix(sdk): remove stale runtime paths and native adapter references --- docs/maintainers/sdk-api-surface.md | 14 + .../oliphaunt-wasix/src/oliphaunt/aot.rs | 6 - .../wasix-ts/src/__tests__/archive.test.ts | 7 + .../src/__tests__/asset-source.test.ts | 22 -- .../src/__tests__/byte-channel-reader.ts | 11 + .../src/__tests__/byte-channel.test.ts | 24 +- .../wasix-ts/src/__tests__/contrib-carrier.ts | 6 +- .../__tests__/extension-descriptor.test.ts | 130 +++---- .../src/__tests__/physical-archive.test.ts | 49 +-- .../__tests__/tool-worker-lifecycle.test.ts | 8 +- .../wasix-ts/src/__tests__/zstd.test.ts | 35 +- src/bindings/wasix-ts/src/archive.ts | 5 +- src/bindings/wasix-ts/src/asset-source.ts | 19 - src/bindings/wasix-ts/src/byte-channel.ts | 63 ---- src/bindings/wasix-ts/src/database-root.ts | 55 --- src/bindings/wasix-ts/src/database.ts | 18 - .../wasix-ts/src/extension-descriptor.ts | 41 +- src/bindings/wasix-ts/src/physical-archive.ts | 17 - src/bindings/wasix-ts/src/types.ts | 16 +- src/bindings/wasix-ts/src/wasix-runtime.ts | 8 - src/bindings/wasix-ts/src/zstd.ts | 17 - .../js/src/__tests__/asset-resolver.test.ts | 354 ++---------------- .../js/src/__tests__/broker-frames.test.ts | 102 ++--- src/sdks/js/src/__tests__/broker-response.ts | 15 + .../js/src/__tests__/memory-duplex-stream.ts | 36 ++ src/sdks/js/src/__tests__/native-smoke.ts | 2 +- .../js/src/__tests__/runtime-adapters.test.ts | 4 +- src/sdks/js/src/native/assets-deno.ts | 45 --- src/sdks/js/src/native/assets-node.ts | 19 +- src/sdks/js/src/native/common.ts | 10 - src/sdks/js/src/native/tar.ts | 158 -------- src/sdks/js/src/native/zip.ts | 139 ------- src/sdks/js/src/protocol.ts | 1 - src/sdks/js/src/runtime/broker-frames.ts | 61 --- src/sdks/js/src/runtime/broker.ts | 4 +- src/sdks/js/src/runtime/byte-stream.ts | 35 -- src/sdks/js/src/runtime/server.ts | 11 +- src/sdks/js/tsconfig.json | 2 + .../oliphaunt/reactnative/OliphauntModule.kt | 42 +-- .../react-native/ios/OliphauntAdapter.swift | 35 +- src/sdks/react-native/src/protocol.ts | 1 - src/sdks/react-native/tsconfig.json | 2 + 42 files changed, 265 insertions(+), 1384 deletions(-) delete mode 100644 src/bindings/wasix-ts/src/__tests__/asset-source.test.ts create mode 100644 src/bindings/wasix-ts/src/__tests__/byte-channel-reader.ts delete mode 100644 src/bindings/wasix-ts/src/asset-source.ts delete mode 100644 src/bindings/wasix-ts/src/zstd.ts create mode 100644 src/sdks/js/src/__tests__/broker-response.ts create mode 100644 src/sdks/js/src/__tests__/memory-duplex-stream.ts delete mode 100644 src/sdks/js/src/native/tar.ts delete mode 100644 src/sdks/js/src/native/zip.ts delete mode 100644 src/sdks/js/src/protocol.ts delete mode 100644 src/sdks/react-native/src/protocol.ts diff --git a/docs/maintainers/sdk-api-surface.md b/docs/maintainers/sdk-api-surface.md index 4da77f514..7adaa13b7 100644 --- a/docs/maintainers/sdk-api-surface.md +++ b/docs/maintainers/sdk-api-surface.md @@ -2033,6 +2033,13 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `TextQueryParameter.format` - `TextQueryParameter.typeOid` - `TextQueryParameter.value` +- `WasixExtensionDescriptor.carriers` +- `WasixExtensionDescriptor.compatibility` +- `WasixExtensionDescriptor.product` +- `WasixExtensionDescriptor.runtime` +- `WasixExtensionDescriptor.schema` +- `WasixExtensionDescriptor.sqlName` +- `WasixExtensionDescriptor.version` - `WasixStorageError.code` - `WasixStorageError.commitState` - `WasixStorageError.constructor()` @@ -2210,6 +2217,13 @@ This version-locked carrier seam is consumed by generated Swift extension produc - `TextQueryParameter.format` - `TextQueryParameter.typeOid` - `TextQueryParameter.value` +- `WasixExtensionDescriptor.carriers` +- `WasixExtensionDescriptor.compatibility` +- `WasixExtensionDescriptor.product` +- `WasixExtensionDescriptor.runtime` +- `WasixExtensionDescriptor.schema` +- `WasixExtensionDescriptor.sqlName` +- `WasixExtensionDescriptor.version` - `WasixStorageError.code` - `WasixStorageError.commitState` - `WasixStorageError.constructor()` diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/aot.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/aot.rs index db3164df8..6d9960309 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/aot.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/aot.rs @@ -241,12 +241,6 @@ pub(crate) fn load_psql_module(engine: &Engine) -> Result { load_artifact_module(engine, "tool:psql") } -#[cfg(feature = "extensions")] -#[allow(dead_code)] -pub(crate) fn load_initdb_module(engine: &Engine) -> Result { - load_artifact_module(engine, "tool:initdb") -} - fn install_artifact(name: &str) -> Result { if let Some(artifact) = installed_artifact(name) { return Ok(artifact); diff --git a/src/bindings/wasix-ts/src/__tests__/archive.test.ts b/src/bindings/wasix-ts/src/__tests__/archive.test.ts index 7169d4977..6c9596b0c 100644 --- a/src/bindings/wasix-ts/src/__tests__/archive.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/archive.test.ts @@ -5,9 +5,16 @@ import { decompressIfNeeded, extractTar, layoutRuntimeSupport, + loadAsset, } from '../archive.js'; describe('WASIX TypeScript archives', () => { + it('rejects local file assets on the browser host', async () => { + await expect(loadAsset('file:///runtime.tar.zst', 'runtime archive')).rejects.toThrow( + 'cannot read package-relative runtime archive URL', + ); + }); + it('preserves uncompressed archive bytes by identity', () => { const bytes = Uint8Array.of(1, 2, 3); diff --git a/src/bindings/wasix-ts/src/__tests__/asset-source.test.ts b/src/bindings/wasix-ts/src/__tests__/asset-source.test.ts deleted file mode 100644 index f7467c333..000000000 --- a/src/bindings/wasix-ts/src/__tests__/asset-source.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; - -import { installPackageAssetReader, readPackageAsset } from '../asset-source.js'; - -describe('WASIX package asset reader', () => { - it('accepts only file URLs after the active host installs one reader', async () => { - await expect(readPackageAsset('file:///runtime.tar.zst', 'runtime archive')).rejects.toThrow( - 'cannot read package-relative runtime archive URL', - ); - - const reader = vi.fn(async (source: URL) => new TextEncoder().encode(source.pathname)); - installPackageAssetReader(reader); - await expect(readPackageAsset('file:///runtime.tar.zst', 'runtime archive')).resolves.toEqual( - new TextEncoder().encode('/runtime.tar.zst'), - ); - expect(reader).toHaveBeenCalledOnce(); - await expect( - readPackageAsset('https://example.test/runtime', 'runtime archive'), - ).rejects.toThrow('cannot read package-relative runtime archive URL'); - expect(() => installPackageAssetReader(reader)).toThrow('already installed'); - }); -}); diff --git a/src/bindings/wasix-ts/src/__tests__/byte-channel-reader.ts b/src/bindings/wasix-ts/src/__tests__/byte-channel-reader.ts new file mode 100644 index 000000000..5f1bd793a --- /dev/null +++ b/src/bindings/wasix-ts/src/__tests__/byte-channel-reader.ts @@ -0,0 +1,11 @@ +import { readWasixByteChannelSync, type WasixByteChannel } from '../byte-channel.js'; + +// The fake worker and its producer share the test thread. Wait for data or EOF +// before calling the blocking reader used by real dedicated workers. +export async function readWasixByteChannel(channel: WasixByteChannel): Promise { + const control = new Int32Array(channel.control); + while (Atomics.load(control, 0) === Atomics.load(control, 1) && Atomics.load(control, 2) === 0) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + return readWasixByteChannelSync(channel); +} diff --git a/src/bindings/wasix-ts/src/__tests__/byte-channel.test.ts b/src/bindings/wasix-ts/src/__tests__/byte-channel.test.ts index 3670f700f..f087a8d7a 100644 --- a/src/bindings/wasix-ts/src/__tests__/byte-channel.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/byte-channel.test.ts @@ -5,49 +5,41 @@ import { failWasixByteChannel, markWasixByteChannelProtocolComplete, markWasixByteChannelProtocolStarted, - readWasixByteChannel, readWasixByteChannelSync, wasixByteChannelProtocolOutcomeUnknown, - wasixByteChannelProtocolStarted, - writeWasixByteChannel, writeWasixByteChannelSync, } from '../byte-channel.js'; describe('bounded WASIX byte channel', () => { - it('preserves byte order across synchronous and asynchronous endpoints', async () => { + it('preserves byte order across dedicated worker endpoints', () => { const channel = createWasixByteChannel(); writeWasixByteChannelSync(channel, Uint8Array.of(1, 2, 3)); - expect(await readWasixByteChannel(channel, 2)).toEqual(Uint8Array.of(1, 2)); - await writeWasixByteChannel(channel, Uint8Array.of(4, 5)); + expect(readWasixByteChannelSync(channel, 2)).toEqual(Uint8Array.of(1, 2)); + writeWasixByteChannelSync(channel, Uint8Array.of(4, 5)); expect(readWasixByteChannelSync(channel)).toEqual(Uint8Array.of(3, 4, 5)); }); - it('reports EOF only after buffered bytes have been consumed', async () => { + it('reports EOF only after buffered bytes have been consumed', () => { const channel = createWasixByteChannel(); - await writeWasixByteChannel(channel, Uint8Array.of(7)); + writeWasixByteChannelSync(channel, Uint8Array.of(7)); closeWasixByteChannel(channel); expect(readWasixByteChannelSync(channel)).toEqual(Uint8Array.of(7)); - await expect(readWasixByteChannel(channel)).resolves.toEqual(new Uint8Array()); + expect(readWasixByteChannelSync(channel)).toEqual(new Uint8Array()); }); - it('fails readers and writers after transport failure', async () => { + it('fails readers and writers after transport failure', () => { const channel = createWasixByteChannel(); failWasixByteChannel(channel); expect(() => readWasixByteChannelSync(channel)).toThrow(/channel failed/); - await expect(writeWasixByteChannel(channel, Uint8Array.of(1))).rejects.toThrow( - /channel failed/, - ); + expect(() => writeWasixByteChannelSync(channel, Uint8Array.of(1))).toThrow(/channel failed/); }); it('shares tool protocol activity without changing channel flow', () => { const channel = createWasixByteChannel(); - expect(wasixByteChannelProtocolStarted(channel)).toBe(false); expect(wasixByteChannelProtocolOutcomeUnknown(channel)).toBe(false); markWasixByteChannelProtocolStarted(channel); - expect(wasixByteChannelProtocolStarted(channel)).toBe(true); expect(wasixByteChannelProtocolOutcomeUnknown(channel)).toBe(true); markWasixByteChannelProtocolComplete(channel); - expect(wasixByteChannelProtocolStarted(channel)).toBe(true); expect(wasixByteChannelProtocolOutcomeUnknown(channel)).toBe(false); }); }); diff --git a/src/bindings/wasix-ts/src/__tests__/contrib-carrier.ts b/src/bindings/wasix-ts/src/__tests__/contrib-carrier.ts index b25ca4bff..5536dd899 100644 --- a/src/bindings/wasix-ts/src/__tests__/contrib-carrier.ts +++ b/src/bindings/wasix-ts/src/__tests__/contrib-carrier.ts @@ -1,6 +1,6 @@ -import { defineWasixExtension } from '../extension-descriptor.js'; +import type { WasixExtensionDescriptor } from '../types.js'; -export const hstore = defineWasixExtension({ +export const hstore = { schema: 'oliphaunt-wasix-extension-v1', runtime: 'wasix', product: 'oliphaunt-extension-contrib-pg18', @@ -44,4 +44,4 @@ export const hstore = defineWasixExtension({ }, }, ], -}); +} as const satisfies WasixExtensionDescriptor; diff --git a/src/bindings/wasix-ts/src/__tests__/extension-descriptor.test.ts b/src/bindings/wasix-ts/src/__tests__/extension-descriptor.test.ts index aeecbedeb..ae60660c1 100644 --- a/src/bindings/wasix-ts/src/__tests__/extension-descriptor.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/extension-descriptor.test.ts @@ -1,13 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { - defineWasixExtension, - serializeWasixExtensionDescriptors, -} from '../extension-descriptor.js'; +import { serializeWasixExtensionDescriptors } from '../extension-descriptor.js'; import type { WasixExtensionCarrier, WasixExtensionDescriptor, - WasixExtensionDescriptorInput, WasixExtensionInstall, } from '../types.js'; @@ -94,50 +90,62 @@ describe('WASIX extension descriptors', () => { ).toThrow('fields must be exactly'); expect(() => - defineWasixExtension({ - ...descriptorInput('pgtap'), - carriers: [carrier('other')], - }), + serializeWasixExtensionDescriptors([ + { + ...descriptorInput('pgtap'), + carriers: [carrier('other')], + }, + ]), ).toThrow("do not contain root SQL name 'pgtap'"); expect(() => - defineWasixExtension({ - ...descriptorInput('pgtap'), - carriers: [carrier('pgtap'), carrier('pgtap')], - }), + serializeWasixExtensionDescriptors([ + { + ...descriptorInput('pgtap'), + carriers: [carrier('pgtap'), carrier('pgtap')], + }, + ]), ).toThrow("repeats carrier SQL name 'pgtap'"); expect(() => - defineWasixExtension({ - ...descriptorInput('pgtap'), - carriers: [ - carrier('pgtap', { - size: 101, - source: Uint8Array.from({ length: 100 }), - }), - ], - }), + serializeWasixExtensionDescriptors([ + { + ...descriptorInput('pgtap'), + carriers: [ + carrier('pgtap', { + size: 101, + source: Uint8Array.from({ length: 100 }), + }), + ], + }, + ]), ).toThrow('byte length must match declared carrier size 101'); expect(() => - defineWasixExtension({ - ...descriptorInput('pgtap'), - carriers: [carrier('pgtap', { archive: 'extensions/renamed.tar.zst' })], - }), + serializeWasixExtensionDescriptors([ + { + ...descriptorInput('pgtap'), + carriers: [carrier('pgtap', { archive: 'extensions/renamed.tar.zst' })], + }, + ]), ).toThrow('archive must be extensions/pgtap.tar.zst'); expect(() => - defineWasixExtension({ - ...descriptorInput('pgtap'), - compatibility: { ...compatibility(), postgresMajor: 'not-a-major' }, - }), + serializeWasixExtensionDescriptors([ + { + ...descriptorInput('pgtap'), + compatibility: { ...compatibility(), postgresMajor: 'not-a-major' }, + }, + ]), ).toThrow('PostgreSQL major must be a positive integer string'); expect(() => - defineWasixExtension({ - ...descriptorInput('pgtap'), - carriers: [carrier('pgtap'), carrier('unused')], - }), + serializeWasixExtensionDescriptors([ + { + ...descriptorInput('pgtap'), + carriers: [carrier('pgtap'), carrier('unused')], + }, + ]), ).toThrow("exact dependency closure for 'pgtap'; unexpected unused"); const missingCreateSchema = install('pgtap') as unknown as { @@ -145,23 +153,27 @@ describe('WASIX extension descriptors', () => { }; delete missingCreateSchema.lifecycle.createSchema; expect(() => - defineWasixExtension({ - ...descriptorInput('pgtap'), - carriers: [ - carrier('pgtap', { - install: missingCreateSchema as unknown as WasixExtensionInstall, - }), - ], - }), + serializeWasixExtensionDescriptors([ + { + ...descriptorInput('pgtap'), + carriers: [ + carrier('pgtap', { + install: missingCreateSchema as unknown as WasixExtensionInstall, + }), + ], + }, + ]), ).toThrow('lifecycle fields must be exactly'); const duplicateStartupConfig = install('pgtap'); duplicateStartupConfig.lifecycle.startupConfig = ['work_mem=4MB', 'work_mem=4MB']; expect(() => - defineWasixExtension({ - ...descriptorInput('pgtap'), - carriers: [carrier('pgtap', { install: duplicateStartupConfig })], - }), + serializeWasixExtensionDescriptors([ + { + ...descriptorInput('pgtap'), + carriers: [carrier('pgtap', { install: duplicateStartupConfig })], + }, + ]), ).toThrow('startupConfig must not repeat values'); const missingLoadOrderFile = { @@ -169,22 +181,14 @@ describe('WASIX extension descriptors', () => { loadOrder: ['lib/postgresql/pgtap.so'], }; expect(() => - defineWasixExtension({ - ...descriptorInput('pgtap'), - carriers: [carrier('pgtap', { install: missingLoadOrderFile })], - }), + serializeWasixExtensionDescriptors([ + { + ...descriptorInput('pgtap'), + carriers: [carrier('pgtap', { install: missingLoadOrderFile })], + }, + ]), ).toThrow('load-order path is absent from installedFiles'); }); - - it('freezes package-authored descriptors and their carrier rows', () => { - const descriptor = extension('pgtap'); - expect(Object.isFrozen(descriptor)).toBe(true); - expect(Object.isFrozen(descriptor.carriers)).toBe(true); - expect(Object.isFrozen(descriptor.carriers[0])).toBe(true); - expect(Object.isFrozen(descriptor.compatibility)).toBe(true); - expect(Object.isFrozen(descriptor.carriers[0]?.install)).toBe(true); - expect(Object.isFrozen(descriptor.carriers[0]?.install.lifecycle.loadSql)).toBe(true); - }); }); type CarrierOverrides = Partial> & { @@ -230,7 +234,7 @@ function install(sqlName: string, dependencies: readonly string[] = []): WasixEx }; } -function compatibility(): WasixExtensionDescriptorInput['compatibility'] { +function compatibility(): WasixExtensionDescriptor['compatibility'] { return { extensionRuntimeContract: 'oliphaunt-extension-runtime-contract-v1', postgresMajor: '18', @@ -242,7 +246,7 @@ function compatibility(): WasixExtensionDescriptorInput['compatibility'] { function descriptorInput( sqlName: string, carriers: readonly WasixExtensionCarrier[] = [carrier(sqlName)], -): WasixExtensionDescriptorInput { +): WasixExtensionDescriptor { const root = carriers.find((candidate) => candidate.sqlName === sqlName) ?? carrier(sqlName); return { schema: 'oliphaunt-wasix-extension-v1', @@ -259,5 +263,5 @@ function extension( sqlName: string, carriers: readonly WasixExtensionCarrier[] = [carrier(sqlName)], ): WasixExtensionDescriptor { - return defineWasixExtension(descriptorInput(sqlName, carriers)); + return descriptorInput(sqlName, carriers); } diff --git a/src/bindings/wasix-ts/src/__tests__/physical-archive.test.ts b/src/bindings/wasix-ts/src/__tests__/physical-archive.test.ts index 72c783de6..1676ce1e6 100644 --- a/src/bindings/wasix-ts/src/__tests__/physical-archive.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/physical-archive.test.ts @@ -13,7 +13,6 @@ import { refreshBackupPgControl, requiredBackupWalNames, snapshotPhysicalBackupBulk, - validateBackupWalRange, withoutPostStopState, } from '../physical-archive.js'; import { PostgresError } from '../query.js'; @@ -92,13 +91,19 @@ describe('WASIX physical archives', () => { expect(calls.map(backupFunction)).toEqual(['pg_backup_start', 'pg_backup_stop']); }); - it('does not stop twice when archive assembly fails after a confirmed stop', async () => { + it.each([ + 'short', + 'missing', + ])('rejects %s WAL without stopping backup twice after a confirmed stop', async (failure) => { const calls: string[] = []; const responses = [startResponse(), stopResponse()]; const directory = backupDirectory(); const readFile = directory.readFile.bind(directory); directory.readFile = async (path) => { - if (path.startsWith('pg_wal/')) return new Uint8Array(1024); + if (path.startsWith('pg_wal/')) { + if (failure === 'missing') throw new Error('WAL file does not exist'); + return new Uint8Array(1024); + } return readFile(path); }; @@ -107,7 +112,7 @@ describe('WASIX physical archives', () => { calls.push(querySql(request)); return nextResponse(responses); }, directory), - ).rejects.toThrow('has the wrong size'); + ).rejects.toThrow(failure === 'missing' ? 'WAL file does not exist' : 'has the wrong size'); expect(calls.map(backupFunction)).toEqual(['pg_backup_start', 'pg_backup_stop']); }); @@ -412,24 +417,6 @@ describe('WASIX physical archives', () => { ); }); - it('validates same-segment and multi-segment WAL ranges', () => { - const size = 1024 * 1024; - const same = '00000001000000000000000A'; - validateBackupWalRange(walSnapshot([[same, size]]), same, same, size); - - const names = [ - '000000010000000000000FFE', - '000000010000000000000FFF', - '000000010000000100000000', - ] as const; - validateBackupWalRange( - walSnapshot(names.map((name) => [name, size])), - names[0], - names[2], - size, - ); - }); - it('matches the shared WAL-range vectors', () => { const text = readFileSync( fileURLToPath( @@ -478,24 +465,6 @@ describe('WASIX physical archives', () => { } }); - it('rejects incomplete or malformed WAL ranges', () => { - const size = 1024 * 1024; - const start = '00000001000000000000000A'; - const stop = '00000001000000000000000B'; - expect(() => validateBackupWalRange(walSnapshot([[start, size]]), start, stop, size)).toThrow( - 'missing WAL segment', - ); - expect(() => - validateBackupWalRange(walSnapshot([[start, size - 1]]), start, start, size), - ).toThrow('wrong size'); - expect(() => validateBackupWalRange(walSnapshot([]), start.toLowerCase(), stop, size)).toThrow( - 'invalid WAL filename', - ); - expect(() => - validateBackupWalRange(walSnapshot([]), start, '00000002000000000000000B', size), - ).toThrow('crosses timelines'); - }); - it('retains only the required post-stop WAL files', () => { const required = '00000001000000000000000A'; const unrelated = '00000001000000000000000B'; diff --git a/src/bindings/wasix-ts/src/__tests__/tool-worker-lifecycle.test.ts b/src/bindings/wasix-ts/src/__tests__/tool-worker-lifecycle.test.ts index 169196bcd..48942b238 100644 --- a/src/bindings/wasix-ts/src/__tests__/tool-worker-lifecycle.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/tool-worker-lifecycle.test.ts @@ -5,10 +5,9 @@ import { createWasixByteChannel, markWasixByteChannelProtocolComplete, markWasixByteChannelProtocolStarted, - readWasixByteChannel, wasixByteChannelProtocolOutcomeUnknown, - wasixByteChannelProtocolStarted, } from '../byte-channel.js'; +import { readWasixByteChannel } from './byte-channel-reader.js'; import { WasixDatabaseImpl, type WasixDatabaseSession } from '../database.js'; import { runWasixToolProcess, @@ -62,7 +61,7 @@ describe('persistent WASIX tool worker lifecycle', () => { backend: beforeBackend, }), ).resolves.toMatchObject({ ok: false, message: 'pre-protocol failure' }); - expect(wasixByteChannelProtocolStarted(beforeFrontend)).toBe(false); + expect(wasixByteChannelProtocolOutcomeUnknown(beforeFrontend)).toBe(false); await expect(readWasixByteChannel(beforeFrontend)).resolves.toEqual(new Uint8Array()); enterProtocol = true; @@ -78,7 +77,7 @@ describe('persistent WASIX tool worker lifecycle', () => { backend: afterBackend, }), ).resolves.toMatchObject({ ok: false, message: 'post-protocol failure' }); - expect(wasixByteChannelProtocolStarted(afterFrontend)).toBe(true); + expect(wasixByteChannelProtocolOutcomeUnknown(afterFrontend)).toBe(true); await expect(readWasixByteChannel(afterFrontend)).rejects.toThrow('byte channel failed'); }); @@ -100,7 +99,6 @@ describe('persistent WASIX tool worker lifecycle', () => { await expect( dispatch({ id: 2, kind: 'run', tool: 'psql', args: [], frontend, backend }), ).resolves.toMatchObject({ ok: false, message: 'WASIX tool mount cleanup failed' }); - expect(wasixByteChannelProtocolStarted(frontend)).toBe(true); expect(wasixByteChannelProtocolOutcomeUnknown(frontend)).toBe(false); await expect(readWasixByteChannel(frontend)).resolves.toEqual(Uint8Array.of(7)); await expect(readWasixByteChannel(frontend)).resolves.toEqual(new Uint8Array()); diff --git a/src/bindings/wasix-ts/src/__tests__/zstd.test.ts b/src/bindings/wasix-ts/src/__tests__/zstd.test.ts index 7dcd8e13a..291cd8fe9 100644 --- a/src/bindings/wasix-ts/src/__tests__/zstd.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/zstd.test.ts @@ -1,4 +1,5 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { expect, it } from 'vitest'; +import { decompressIfNeeded } from '../archive.js'; const frame = Uint8Array.of( 40, @@ -31,34 +32,6 @@ const frame = Uint8Array.of( 116, ); -afterEach(() => vi.resetModules()); - -describe('WASIX zstd decompression', () => { - it('uses the portable fallback by default', async () => { - const { decompressZstd } = await import('../zstd.js'); - - expect(new TextDecoder().decode(decompressZstd(frame))).toBe('oliphaunt-zstd-test'); - }); - - it('selects one installed host decompressor', async () => { - const { decompressZstd, installZstdDecompressor } = await import('../zstd.js'); - const output = Uint8Array.of(4, 2); - const host = vi.fn(() => output); - - installZstdDecompressor(host); - - expect(decompressZstd(frame)).toBe(output); - expect(host).toHaveBeenCalledWith(frame); - expect(() => installZstdDecompressor(host)).toThrow('already installed'); - }); - - it('propagates host decoding failures without retrying', async () => { - const { decompressZstd, installZstdDecompressor } = await import('../zstd.js'); - const failure = new Error('invalid native frame'); - installZstdDecompressor(() => { - throw failure; - }); - - expect(() => decompressZstd(frame)).toThrow(failure); - }); +it('decompresses zstd archive bytes with the portable decoder', () => { + expect(new TextDecoder().decode(decompressIfNeeded(frame))).toBe('oliphaunt-zstd-test'); }); diff --git a/src/bindings/wasix-ts/src/archive.ts b/src/bindings/wasix-ts/src/archive.ts index 271a25245..d34d40c6e 100644 --- a/src/bindings/wasix-ts/src/archive.ts +++ b/src/bindings/wasix-ts/src/archive.ts @@ -1,6 +1,5 @@ -import { readPackageAsset } from './asset-source.js'; import type { SerializedAssetSource } from './rpc.js'; -import { decompressZstd } from './zstd.js'; +import { decompress as decompressZstd } from 'fzstd'; export type DirectoryFiles = Record; @@ -252,7 +251,7 @@ export async function loadAsset(source: SerializedAssetSource, label: string): P return source; } if (source.startsWith('file:')) { - return readPackageAsset(source, label); + throw new Error(`cannot read package-relative ${label} URL ${JSON.stringify(source)}`); } let response: Response; try { diff --git a/src/bindings/wasix-ts/src/asset-source.ts b/src/bindings/wasix-ts/src/asset-source.ts deleted file mode 100644 index 6b87249c6..000000000 --- a/src/bindings/wasix-ts/src/asset-source.ts +++ /dev/null @@ -1,19 +0,0 @@ -export type PackageAssetReader = (source: URL) => Promise; - -let packageAssetReader: PackageAssetReader | undefined; - -/** @internal Installed by the active Node host realm for package-relative file URLs. */ -export function installPackageAssetReader(reader: PackageAssetReader): void { - if (packageAssetReader !== undefined) { - throw new Error('Oliphaunt WASIX package asset reader is already installed'); - } - packageAssetReader = reader; -} - -export async function readPackageAsset(source: string, label: string): Promise { - const url = new URL(source); - if (url.protocol !== 'file:' || packageAssetReader === undefined) { - throw new Error(`cannot read package-relative ${label} URL ${JSON.stringify(source)}`); - } - return packageAssetReader(url); -} diff --git a/src/bindings/wasix-ts/src/byte-channel.ts b/src/bindings/wasix-ts/src/byte-channel.ts index 8715028a5..4caa36d6c 100644 --- a/src/bindings/wasix-ts/src/byte-channel.ts +++ b/src/bindings/wasix-ts/src/byte-channel.ts @@ -4,7 +4,6 @@ const CLOSED = 2; const FAILED = 3; const PROTOCOL_STATE = 4; const CONTROL_WORDS = 5; -const PROTOCOL_IDLE = 0; const PROTOCOL_ACTIVE = 1; const PROTOCOL_COMPLETE = 2; @@ -57,11 +56,6 @@ export function markWasixByteChannelProtocolComplete(channel: WasixByteChannel): ); } -/** @internal Observe protocol activity across the tool/database worker boundary. */ -export function wasixByteChannelProtocolStarted(channel: WasixByteChannel): boolean { - return Atomics.load(channelControl(channel), PROTOCOL_STATE) !== PROTOCOL_IDLE; -} - /** @internal Whether a failed tool may have left an unobserved PostgreSQL outcome. */ export function wasixByteChannelProtocolOutcomeUnknown(channel: WasixByteChannel): boolean { return Atomics.load(channelControl(channel), PROTOCOL_STATE) === PROTOCOL_ACTIVE; @@ -103,45 +97,6 @@ export function writeWasixByteChannelSync(channel: WasixByteChannel, input: Uint } } -/** @internal Non-blocking-realm read. Empty bytes mean EOF. */ -export async function readWasixByteChannel( - channel: WasixByteChannel, - maximumBytes = WASIX_BYTE_CHANNEL_CHUNK_BYTES, -): Promise { - const control = channelControl(channel); - const data = new Uint8Array(channel.data); - for (;;) { - assertChannelHealthy(control); - const read = Atomics.load(control, READ_OFFSET); - const write = Atomics.load(control, WRITE_OFFSET); - if (read !== write) return consume(control, data, read, write, maximumBytes); - if (Atomics.load(control, CLOSED) !== 0) return new Uint8Array(); - await waitForChange(control, WRITE_OFFSET, write); - } -} - -/** @internal Non-blocking-realm write with bounded backpressure. */ -export async function writeWasixByteChannel( - channel: WasixByteChannel, - input: Uint8Array, -): Promise { - const control = channelControl(channel); - const data = new Uint8Array(channel.data); - let offset = 0; - while (offset < input.length) { - assertChannelWritable(control); - const read = Atomics.load(control, READ_OFFSET); - const write = Atomics.load(control, WRITE_OFFSET); - const writable = writableBytes(read, write, data.length); - if (writable === 0) { - await waitForChange(control, READ_OFFSET, read); - continue; - } - const copied = produce(control, data, write, input.subarray(offset), writable); - offset += copied; - } -} - function channelControl(channel: WasixByteChannel): Int32Array { if ( !(channel.control instanceof SharedArrayBuffer) || @@ -204,21 +159,3 @@ function assertChannelWritable(control: Int32Array): void { throw new Error('Oliphaunt WASIX byte channel is closed'); } } - -async function waitForChange(control: Int32Array, index: number, expected: number): Promise { - const waitAsync = ( - Atomics as typeof Atomics & { - waitAsync?: ( - typedArray: Int32Array, - index: number, - value: number, - ) => { async: false; value: string } | { async: true; value: Promise }; - } - ).waitAsync; - if (waitAsync !== undefined) { - const waiting = waitAsync(control, index, expected); - if (waiting.async) await waiting.value; - return; - } - await new Promise((resolve) => setTimeout(resolve, 1)); -} diff --git a/src/bindings/wasix-ts/src/database-root.ts b/src/bindings/wasix-ts/src/database-root.ts index 3abff50ae..8bd1da520 100644 --- a/src/bindings/wasix-ts/src/database-root.ts +++ b/src/bindings/wasix-ts/src/database-root.ts @@ -3,63 +3,8 @@ import { POSTGRES_MAJOR as CARRIER_POSTGRES_MAJOR, } from '@oliphaunt/liboliphaunt-wasix'; -export const DATABASE_ROOT_DESCRIPTOR = '.oliphaunt.json'; -export const DATABASE_ROOT_SCHEMA = 'oliphaunt-database-root-v1'; -export const DATABASE_ROOT_PGDATA = 'pgdata'; export const DATABASE_ROOT_POSTGRES_MAJOR = CARRIER_POSTGRES_MAJOR; export const WASIX_PHYSICAL_FORMAT = CARRIER_PHYSICAL_FORMAT; -export const NATIVE_PHYSICAL_FORMAT = 'native-pg18-v1'; - -export type DatabaseRootDescriptor = Readonly<{ - schema: typeof DATABASE_ROOT_SCHEMA; - engineFamily: 'native' | 'wasix'; - pgdata: typeof DATABASE_ROOT_PGDATA; - postgresMajor: number; - physicalFormat: string; -}>; - -export function wasixDatabaseRootDescriptor(): DatabaseRootDescriptor { - return { - schema: DATABASE_ROOT_SCHEMA, - engineFamily: 'wasix', - pgdata: DATABASE_ROOT_PGDATA, - postgresMajor: DATABASE_ROOT_POSTGRES_MAJOR, - physicalFormat: WASIX_PHYSICAL_FORMAT, - }; -} - -export function parseDatabaseRootDescriptor(value: unknown): DatabaseRootDescriptor | undefined { - if (value === null || typeof value !== 'object' || Array.isArray(value)) return undefined; - const descriptor = value as Record; - const keys = Object.keys(descriptor).sort(); - if ( - keys.length !== 5 || - keys[0] !== 'engineFamily' || - keys[1] !== 'pgdata' || - keys[2] !== 'physicalFormat' || - keys[3] !== 'postgresMajor' || - keys[4] !== 'schema' - ) { - return undefined; - } - if ( - descriptor.schema !== DATABASE_ROOT_SCHEMA || - (descriptor.engineFamily !== 'native' && descriptor.engineFamily !== 'wasix') || - descriptor.pgdata !== DATABASE_ROOT_PGDATA || - descriptor.postgresMajor !== DATABASE_ROOT_POSTGRES_MAJOR || - (descriptor.engineFamily === 'native' - ? descriptor.physicalFormat !== NATIVE_PHYSICAL_FORMAT - : descriptor.physicalFormat !== WASIX_PHYSICAL_FORMAT) - ) { - return undefined; - } - return descriptor as DatabaseRootDescriptor; -} - -/** Parse the root descriptor while rejecting duplicate JSON keys. */ -export function parseDatabaseRootDescriptorText(text: string): DatabaseRootDescriptor | undefined { - return parseDatabaseRootDescriptor(parseJsonWithUniqueObjectKeys(text)); -} /** Parse JSON while retaining the duplicate-key rejection lost by JSON.parse(). */ export function parseJsonWithUniqueObjectKeys(text: string): unknown | undefined { diff --git a/src/bindings/wasix-ts/src/database.ts b/src/bindings/wasix-ts/src/database.ts index 259ebcc7b..65a0d8393 100644 --- a/src/bindings/wasix-ts/src/database.ts +++ b/src/bindings/wasix-ts/src/database.ts @@ -404,14 +404,6 @@ export class WasixDatabaseImpl implements OliphauntDatabase { }); } - /** @internal Serialized connection ownership used by optional server/tool packages. */ - runProtocolConnection( - connection: WasixProtocolConnection, - mode: WasixProtocolConnectionMode, - ): Promise { - return this.reserveProtocolConnection(connection, mode).start(); - } - /** @internal Reserve call order without entering the guest before a client is ready. */ reserveProtocolConnection( connection: WasixProtocolConnection, @@ -1039,16 +1031,6 @@ export class WasixDatabaseImpl implements OliphauntDatabase { } } -/** @internal Optional packages acquire exclusive ownership without expanding the public class. */ -export function runWasixProtocolConnection( - database: OliphauntDatabase, - connection: WasixProtocolConnection, - mode: WasixProtocolConnectionMode, -): Promise { - assertWasixProtocolConnectionTarget(database); - return database.runProtocolConnection(connection, mode); -} - /** @internal Execute pg_dump in the realm that owns this database. */ export function runWasixPgDumpProcess( database: OliphauntDatabase, diff --git a/src/bindings/wasix-ts/src/extension-descriptor.ts b/src/bindings/wasix-ts/src/extension-descriptor.ts index 83a912d0d..707a4e33d 100644 --- a/src/bindings/wasix-ts/src/extension-descriptor.ts +++ b/src/bindings/wasix-ts/src/extension-descriptor.ts @@ -13,7 +13,6 @@ import type { WasixExtensionCarrier, WasixExtensionCompatibility, WasixExtensionDescriptor, - WasixExtensionDescriptorInput, WasixExtensionImport, WasixExtensionInstall, WasixExtensionLifecycle, @@ -79,18 +78,6 @@ export type SerializedWasixExtensions = { carriers: Record; }; -/** Optional package-author helper that validates and deeply freezes a descriptor. */ -export function defineWasixExtension( - input: WasixExtensionDescriptorInput, -): WasixExtensionDescriptor { - validateDescriptor(input, 'WASIX extension descriptor'); - return Object.freeze({ - ...input, - compatibility: Object.freeze({ ...input.compatibility }), - carriers: Object.freeze(input.carriers.map(freezeCarrier)), - }); -} - /** * Validates imported package values and converts roots plus carrier closures to * the worker wire shape. Imported carriers own their exact install metadata; @@ -165,7 +152,7 @@ export function serializeWasixExtensionDescriptors( function validateDescriptor( value: unknown, label: string, -): asserts value is WasixExtensionDescriptorInput { +): asserts value is WasixExtensionDescriptor { const descriptor = requireExactObject(value, DESCRIPTOR_FIELDS, label); if (descriptor.schema !== 'oliphaunt-wasix-extension-v1') { throw new Error(`${label} has unsupported schema`); @@ -378,32 +365,6 @@ function serializeCarrier( }; } -function freezeCarrier(carrier: WasixExtensionCarrier): WasixExtensionCarrier { - const lifecycle = carrier.install.lifecycle; - return Object.freeze({ - ...carrier, - install: Object.freeze({ - ...carrier.install, - nativeModules: Object.freeze( - carrier.install.nativeModules.map((module) => Object.freeze({ ...module })), - ), - dependencies: Object.freeze([...carrier.install.dependencies]), - coreExportsRequired: Object.freeze([...carrier.install.coreExportsRequired]), - loadOrder: Object.freeze([...carrier.install.loadOrder]), - lifecycle: Object.freeze({ - ...lifecycle, - loadSql: Object.freeze([...lifecycle.loadSql]), - postCreateSql: Object.freeze([...lifecycle.postCreateSql]), - startupConfig: Object.freeze([...lifecycle.startupConfig]), - }), - installedFiles: Object.freeze([...carrier.install.installedFiles]), - unresolvedImports: Object.freeze( - carrier.install.unresolvedImports.map((entry) => Object.freeze({ ...entry })), - ), - }), - }); -} - function requireProduct(value: unknown, label: string): string { const product = requireString(value, label); if (!PRODUCT.test(product)) { diff --git a/src/bindings/wasix-ts/src/physical-archive.ts b/src/bindings/wasix-ts/src/physical-archive.ts index ee8b7e337..1b6dc3e08 100644 --- a/src/bindings/wasix-ts/src/physical-archive.ts +++ b/src/bindings/wasix-ts/src/physical-archive.ts @@ -252,23 +252,6 @@ export async function snapshotPhysicalBackupBulk( ); } -/** @internal Prove the online backup contains every required start-to-stop WAL segment. */ -export function validateBackupWalRange( - snapshot: StoredSnapshot, - startName: string, - stopName: string, - segmentSize: number, -): void { - const names = requiredBackupWalNames(startName, stopName, segmentSize); - const files = new Map(snapshot.files.map(({ path, bytes }) => [path, bytes.length])); - for (const name of names) { - const length = files.get(`pg_wal/${name}`); - if (length === undefined) throw new Error(`physical backup is missing WAL segment ${name}`); - if (length !== segmentSize) - throw new Error(`physical backup WAL segment ${name} has the wrong size`); - } -} - /** @internal Enumerate PostgreSQL's inclusive start-to-stop WAL range. */ export function requiredBackupWalNames( startName: string, diff --git a/src/bindings/wasix-ts/src/types.ts b/src/bindings/wasix-ts/src/types.ts index 53bb2a99f..ba8be6dfe 100644 --- a/src/bindings/wasix-ts/src/types.ts +++ b/src/bindings/wasix-ts/src/types.ts @@ -122,7 +122,13 @@ export type WasixExtensionNativeModule = Readonly<{ size: number; }>; -export type WasixExtensionDescriptorInput = Readonly<{ +/** + * A package-authored, runtime-validated WASIX extension import. Applications + * obtain these from extension packages instead of constructing SQL strings. + * The schema and runtime literals discriminate it structurally, so generated + * carrier packages do not need a dependency on this binding. + */ +export type WasixExtensionDescriptor = Readonly<{ schema: 'oliphaunt-wasix-extension-v1'; runtime: 'wasix'; /** Product and version of the root carrier selected by `sqlName`. */ @@ -135,14 +141,6 @@ export type WasixExtensionDescriptorInput = Readonly<{ carriers: readonly WasixExtensionCarrier[]; }>; -/** - * A package-authored, runtime-validated WASIX extension import. Applications - * obtain these from extension packages instead of constructing SQL strings. - * The schema and runtime literals discriminate it structurally, so generated - * carrier packages do not need a dependency on this binding. - */ -export type WasixExtensionDescriptor = WasixExtensionDescriptorInput; - /** Lifecycle fields owned by an independently versioned extension carrier. */ export type WasixExtensionLifecycle = { createExtension: boolean; diff --git a/src/bindings/wasix-ts/src/wasix-runtime.ts b/src/bindings/wasix-ts/src/wasix-runtime.ts index a3229858d..571e6db98 100644 --- a/src/bindings/wasix-ts/src/wasix-runtime.ts +++ b/src/bindings/wasix-ts/src/wasix-runtime.ts @@ -50,14 +50,6 @@ export async function materializeWasixMounts( return { mounts, baseDirectory }; } -/** @internal Materialize runtime support mounts for frontend tools. */ -export function materializeWasixSupportMounts( - DirectoryConstructor: typeof Directory, - layout: Pick, -): Promise> { - return materializeMountMap(DirectoryConstructor, layout); -} - async function materializeMountMap( DirectoryConstructor: typeof Directory, layout: Pick, diff --git a/src/bindings/wasix-ts/src/zstd.ts b/src/bindings/wasix-ts/src/zstd.ts deleted file mode 100644 index cf5ceb635..000000000 --- a/src/bindings/wasix-ts/src/zstd.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { decompress as decompressWithFzstd } from 'fzstd'; - -export type ZstdDecompressor = (bytes: Uint8Array) => Uint8Array; - -let hostZstdDecompressor: ZstdDecompressor | undefined; - -/** @internal Installed by a host realm when it has a native implementation. */ -export function installZstdDecompressor(decompressor: ZstdDecompressor): void { - if (hostZstdDecompressor !== undefined) { - throw new Error('Oliphaunt WASIX zstd decompressor is already installed'); - } - hostZstdDecompressor = decompressor; -} - -export function decompressZstd(bytes: Uint8Array): Uint8Array { - return (hostZstdDecompressor ?? decompressWithFzstd)(bytes); -} diff --git a/src/sdks/js/src/__tests__/asset-resolver.test.ts b/src/sdks/js/src/__tests__/asset-resolver.test.ts index f344064f3..87b4ba7c0 100644 --- a/src/sdks/js/src/__tests__/asset-resolver.test.ts +++ b/src/sdks/js/src/__tests__/asset-resolver.test.ts @@ -10,34 +10,25 @@ import { rename, rm, rmdir, - stat as fsStat, symlink, writeFile, } from 'node:fs/promises'; import { createRequire } from 'node:module'; import { arch, platform, tmpdir } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { deflateRawSync, inflateRawSync } from 'node:zlib'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import { test } from 'vitest'; import { GENERATED_EXTENSION_METADATA } from '../generated/extensions.js'; -import { - type DenoRuntime, - resolvePackageRelativeUrl, - validatePreparedDenoRuntimeExtensions, -} from '../native/assets-deno.js'; +import { resolvePackageRelativeUrl } from '../native/assets-deno.js'; import { materializeNodeExtensionInstall, prepareNodeExtensionInstall, type ResolvedNativeInstall, - resolveNodeIcuDataDirectory, resolveNodeNativeInstall, resolvePackageRelativePath, validatePreparedNodeRuntimeExtensions, } from '../native/assets-node.js'; import { liboliphauntPackageTarget } from '../native/common.js'; -import { extractTarArchive } from '../native/tar.js'; -import { extractZipArchive } from '../native/zip.js'; import { packageMetadataVersion, readTypeScriptPackageJson, @@ -110,8 +101,6 @@ function fixtureExtensionContractManifest( async function main(): Promise { packageTargetsMatchLiboliphauntPackages(); - await tarExtractionRejectsTraversal(); - await zipExtractionWritesFilesAndRejectsTraversal(); packageMetadataPathsAreConfinedToPackageRoot(); await nodeResolverUsesInstalledPackages(); await nodeResolverUsesStandardCarrierRuntime(); @@ -119,48 +108,11 @@ async function main(): Promise { await nodeExtensionMaterializationValidatesSelections(); await nodeExtensionMaterializationAcceptsBuiltInPostgresDependency(); await explicitRuntimeExtensionValidationUsesPreparedFiles(); - await denoPreparedRuntimeRequiresSeparateEmbeddedModules(); await nodeExtensionMaterializationCopiesPackagePayloads(); await nodeExtensionMaterializationRejectsIncompletePackagePayloads(); await typeScriptPackageMetadataMatchesRuntimePackages(); } -async function zipExtractionWritesFilesAndRejectsTraversal(): Promise { - const root = await mkdtemp(join(tmpdir(), 'oliphaunt-js-zip-')); - const host = { - join, - dirname, - async mkdir(path: string) { - await mkdir(path, { recursive: true }); - }, - async writeFile(file: { path: string; bytes: Uint8Array; mode: number }) { - await writeFile(file.path, file.bytes, { mode: file.mode }); - await chmod(file.path, file.mode); - }, - }; - try { - await extractZipArchive( - zipArchive([{ path: 'bin/oliphaunt.dll', mode: 0o755, bytes: utf8('dll') }]), - root, - host, - (bytes) => Uint8Array.from(inflateRawSync(bytes)), - ); - assert.equal(await readFile(join(root, 'bin/oliphaunt.dll'), 'utf8'), 'dll'); - await assert.rejects( - () => - extractZipArchive( - zipArchive([{ path: '../evil', mode: 0o644, bytes: utf8('bad') }]), - root, - host, - (bytes) => Uint8Array.from(inflateRawSync(bytes)), - ), - /unsafe ZIP entry path/, - ); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - function packageTargetsMatchLiboliphauntPackages(): void { const target = liboliphauntPackageTarget('darwin', 'aarch64'); assert.equal(target.id, 'macos-arm64'); @@ -219,33 +171,6 @@ function packageMetadataPathsAreConfinedToPackageRoot(): void { } } -async function tarExtractionRejectsTraversal(): Promise { - const root = await mkdtemp(join(tmpdir(), 'oliphaunt-js-tar-')); - try { - await assert.rejects( - () => - extractTarArchive( - tarArchive([{ path: '../evil', mode: 0o644, bytes: utf8('bad') }]), - root, - { - join, - dirname, - async mkdir(path) { - await mkdir(path, { recursive: true }); - }, - async writeFile(file) { - await writeFile(file.path, file.bytes, { mode: file.mode }); - await chmod(file.path, file.mode); - }, - }, - ), - /escapes/, - ); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - async function nodeResolverUsesInstalledPackages(): Promise { const previousLibraryPath = process.env.LIBOLIPHAUNT_PATH; const previousRuntimeDir = process.env.OLIPHAUNT_RUNTIME_DIR; @@ -298,13 +223,14 @@ async function nodeResolverUsesStandardCarrierRuntime(): Promise { } async function nodeIcuResolverAcceptsValidPortablePackage(): Promise { + const { icuVersion } = await readTypeScriptPackageVersions(); const root = await mkdtemp(join(tmpdir(), 'oliphaunt-js-icu-')); try { await writeFile( join(root, 'package.json'), JSON.stringify({ - name: root, - version: '9.9.9', + name: '@oliphaunt/icu', + version: icuVersion, oliphaunt: { product: 'oliphaunt-icu', kind: 'icu-data', @@ -324,9 +250,24 @@ async function nodeIcuResolverAcceptsValidPortablePackage(): Promise { `schema=oliphaunt-icu-data-v1\nartifactRole=icu-data\nicuDataVersion=76.1\nicuDataForm=files-le\nicuDataTreeSha256=${'a'.repeat(64)}\n`, 'utf8', ); - assert.equal(await resolveNodeIcuDataDirectory('9.9.9', root), await realpath(dataDirectory)); + const descriptor = { + schema: 'oliphaunt-native-icu-v1' as const, + packageName: '@oliphaunt/icu' as const, + version: icuVersion, + packageJsonUrl: pathToFileURL(join(root, 'package.json')).href, + }; + const install = await resolveNodeNativeInstall('/explicit/liboliphaunt.so', descriptor); + assert.equal(install.icuDataDirectory, await realpath(dataDirectory)); + assert.equal(install.catalogProfile, 'icu'); + await assert.rejects( + () => + resolveNodeNativeInstall('/explicit/liboliphaunt.so', { ...descriptor, version: '9.9.8' }), + /ICU package .* is incompatible with runtime/, + ); + const metadata = JSON.parse(await readFile(join(root, 'package.json'), 'utf8')); + await writeFile(join(root, 'package.json'), JSON.stringify({ ...metadata, version: '9.9.8' })); await assert.rejects( - () => resolveNodeIcuDataDirectory('9.9.8', root), + () => resolveNodeNativeInstall('/explicit/liboliphaunt.so', descriptor), /does not match @oliphaunt\/ts icuVersion/, ); } finally { @@ -490,102 +431,6 @@ async function explicitRuntimeExtensionValidationUsesPreparedFiles(): Promise { - const root = await mkdtemp(join(tmpdir(), 'oliphaunt-js-deno-prepared-runtime-')); - const runtime = join(root, 'runtime'); - const embeddedModules = join(runtime, 'lib/modules'); - const deno = fsBackedDenoValidationRuntime(); - try { - await writePreparedHstoreRuntime(runtime, 'linux-x64-gnu'); - - const preferred = await validatePreparedDenoRuntimeExtensions({ - deno, - runtimeDirectory: runtime, - extensions: ['hstore'], - source: 'Deno test runtime', - }); - assert.equal(preferred.runtimeDirectory, runtime); - assert.equal(preferred.moduleDirectory, embeddedModules); - - await rm(join(embeddedModules, 'hstore.so')); - await assert.rejects( - () => - validatePreparedDenoRuntimeExtensions({ - deno, - runtimeDirectory: runtime, - extensions: ['hstore'], - source: 'Deno test runtime', - }), - /module directory is missing required file hstore[.]so/, - ); - - await writeFile(join(embeddedModules, 'hstore.so'), 'embedded hstore'); - await rm(join(embeddedModules, 'dict_snowball.so')); - await assert.rejects( - () => - validatePreparedDenoRuntimeExtensions({ - deno, - runtimeDirectory: runtime, - extensions: ['hstore'], - source: 'Deno test runtime', - }), - /module directory is missing required file dict_snowball[.]so/, - ); - - await writeFile(join(embeddedModules, 'dict_snowball.so'), 'embedded dict_snowball'); - await rm(join(embeddedModules, 'plpgsql.so')); - await assert.rejects( - () => - validatePreparedDenoRuntimeExtensions({ - deno, - runtimeDirectory: runtime, - extensions: ['hstore'], - source: 'Deno test runtime', - }), - /module directory is missing required file plpgsql[.]so/, - ); - - await rm(embeddedModules, { recursive: true }); - await assert.rejects( - () => - validatePreparedDenoRuntimeExtensions({ - deno, - runtimeDirectory: runtime, - extensions: ['hstore'], - source: 'Deno test runtime', - }), - /module directory is missing required file hstore[.]so/, - ); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -function fsBackedDenoValidationRuntime(): DenoRuntime { - return { - build: { os: 'linux', arch: 'x86_64' }, - async readTextFile(path: string | URL) { - return readFile(path, 'utf8'); - }, - async *readDir(path: string | URL) { - for (const entry of await readdir(path, { withFileTypes: true })) { - yield { - name: entry.name, - isFile: entry.isFile(), - isDirectory: entry.isDirectory(), - }; - } - }, - async stat(path: string | URL) { - const metadata = await fsStat(path); - return { - isFile: metadata.isFile(), - isDirectory: metadata.isDirectory(), - }; - }, - }; -} - async function nodeExtensionMaterializationCopiesPackagePayloads(): Promise { const target = liboliphauntPackageTarget(platform(), arch()); const { liboliphauntVersion } = await readTypeScriptPackageVersions(); @@ -1658,161 +1503,6 @@ async function typeScriptPackageMetadataMatchesRuntimePackages(): Promise ); } -type TarEntry = { - path: string; - mode: number; - bytes?: Uint8Array; - directory?: boolean; -}; - -type ZipEntry = { - path: string; - mode: number; - bytes: Uint8Array; -}; - -function zipArchive(entries: ZipEntry[]): Uint8Array { - const chunks: Uint8Array[] = []; - const central: Uint8Array[] = []; - let offset = 0; - for (const entry of entries) { - const name = utf8(entry.path); - const compressed = Uint8Array.from(deflateRawSync(entry.bytes)); - const crc = crc32(entry.bytes); - const local = new Uint8Array(30 + name.length); - writeUInt32LE(local, 0, 0x04034b50); - writeUInt16LE(local, 4, 20); - writeUInt16LE(local, 8, 8); - writeUInt32LE(local, 14, crc); - writeUInt32LE(local, 18, compressed.length); - writeUInt32LE(local, 22, entry.bytes.length); - writeUInt16LE(local, 26, name.length); - local.set(name, 30); - chunks.push(local, compressed); - - const header = new Uint8Array(46 + name.length); - writeUInt32LE(header, 0, 0x02014b50); - writeUInt16LE(header, 4, 20); - writeUInt16LE(header, 6, 20); - writeUInt16LE(header, 10, 8); - writeUInt32LE(header, 16, crc); - writeUInt32LE(header, 20, compressed.length); - writeUInt32LE(header, 24, entry.bytes.length); - writeUInt16LE(header, 28, name.length); - writeUInt32LE(header, 38, (entry.mode & 0o777) << 16); - writeUInt32LE(header, 42, offset); - header.set(name, 46); - central.push(header); - offset += local.length + compressed.length; - } - const centralOffset = offset; - const centralSize = central.reduce((total, chunk) => total + chunk.length, 0); - const eocd = new Uint8Array(22); - writeUInt32LE(eocd, 0, 0x06054b50); - writeUInt16LE(eocd, 8, entries.length); - writeUInt16LE(eocd, 10, entries.length); - writeUInt32LE(eocd, 12, centralSize); - writeUInt32LE(eocd, 16, centralOffset); - return concatBytes([...chunks, ...central, eocd]); -} - -function tarArchive(entries: TarEntry[]): Uint8Array { - const blocks: Uint8Array[] = []; - for (const entry of entries) { - const bytes = entry.bytes ?? new Uint8Array(); - blocks.push( - tarHeader(entry.path, entry.directory === true ? '5' : '0', entry.mode, bytes.length), - ); - if (entry.directory !== true) { - blocks.push(bytes); - const padding = (512 - (bytes.length % 512)) % 512; - if (padding > 0) { - blocks.push(new Uint8Array(padding)); - } - } - } - blocks.push(new Uint8Array(1024)); - const length = blocks.reduce((total, block) => total + block.byteLength, 0); - const archive = new Uint8Array(length); - let offset = 0; - for (const block of blocks) { - archive.set(block, offset); - offset += block.byteLength; - } - return archive; -} - -function tarHeader(path: string, type: '0' | '5', mode: number, size: number): Uint8Array { - const header = new Uint8Array(512); - writeAscii(header, 0, 100, path); - writeOctal(header, 100, 8, mode); - writeOctal(header, 108, 8, 0); - writeOctal(header, 116, 8, 0); - writeOctal(header, 124, 12, size); - writeOctal(header, 136, 12, 0); - header.fill(0x20, 148, 156); - writeAscii(header, 156, 1, type); - writeAscii(header, 257, 6, 'ustar'); - writeAscii(header, 263, 2, '00'); - let checksum = 0; - for (const byte of header) { - checksum += byte; - } - const encoded = checksum.toString(8).padStart(6, '0'); - writeAscii(header, 148, 8, `${encoded}\0 `); - return header; -} - -function writeAscii(buffer: Uint8Array, offset: number, length: number, value: string): void { - const encoded = utf8(value); - if (encoded.byteLength > length) { - throw new Error(`tar test value is too long: ${value}`); - } - buffer.set(encoded, offset); -} - -function writeOctal(buffer: Uint8Array, offset: number, length: number, value: number): void { - writeAscii(buffer, offset, length, `${value.toString(8).padStart(length - 1, '0')}\0`); -} - -function writeUInt16LE(buffer: Uint8Array, offset: number, value: number): void { - buffer[offset] = value & 0xff; - buffer[offset + 1] = (value >>> 8) & 0xff; -} - -function writeUInt32LE(buffer: Uint8Array, offset: number, value: number): void { - buffer[offset] = value & 0xff; - buffer[offset + 1] = (value >>> 8) & 0xff; - buffer[offset + 2] = (value >>> 16) & 0xff; - buffer[offset + 3] = (value >>> 24) & 0xff; -} - -function concatBytes(chunks: Uint8Array[]): Uint8Array { - const length = chunks.reduce((total, chunk) => total + chunk.length, 0); - const out = new Uint8Array(length); - let offset = 0; - for (const chunk of chunks) { - out.set(chunk, offset); - offset += chunk.length; - } - return out; -} - -function crc32(bytes: Uint8Array): number { - let crc = 0xffffffff; - for (const byte of bytes) { - crc ^= byte; - for (let bit = 0; bit < 8; bit += 1) { - crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); - } - } - return (crc ^ 0xffffffff) >>> 0; -} - -function utf8(value: string): Uint8Array { - return new TextEncoder().encode(value); -} - function restoreEnv(name: string, value: string | undefined): void { if (value === undefined) { delete process.env[name]; diff --git a/src/sdks/js/src/__tests__/broker-frames.test.ts b/src/sdks/js/src/__tests__/broker-frames.test.ts index 07ee1ba26..4d72e0ed9 100644 --- a/src/sdks/js/src/__tests__/broker-frames.test.ts +++ b/src/sdks/js/src/__tests__/broker-frames.test.ts @@ -2,16 +2,13 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { - decodeBrokerRequest, decodeBrokerResponse, encodeBrokerRequest, - encodeBrokerResponse, - readBrokerRequest, readBrokerResponse, writeBrokerRequest, - writeBrokerResponse, } from '../runtime/broker-frames.js'; -import { MemoryDuplexStream } from '../runtime/byte-stream.js'; +import { encodeBrokerResponse } from './broker-response.js'; +import { MemoryDuplexStream } from './memory-duplex-stream.js'; import { resolveBrokerStreamCompletion } from '../runtime/broker.js'; async function main(): Promise { @@ -19,33 +16,40 @@ async function main(): Promise { await responseFramesRoundTrip(); rejectsMalformedFrames(); streamCompletionUsesRecoveryAwareErrorPrecedence(); - await streamHelpersUseBinaryFrames(); } async function requestFramesRoundTrip(): Promise { - assert.deepEqual(decodeBrokerRequest(6, new TextEncoder().encode('secret')), { - kind: 'authenticate', - token: 'secret', - }); - assert.deepEqual(decodeBrokerRequest(1, new Uint8Array([1, 2])), { - kind: 'execProtocol', - bytes: new Uint8Array([1, 2]), - }); - assert.deepEqual(decodeBrokerRequest(4, new Uint8Array([3, 4])), { - kind: 'execProtocolStream', - bytes: new Uint8Array([3, 4]), - }); - assert.deepEqual(decodeBrokerRequest(8, new TextEncoder().encode('SELECT 1')), { - kind: 'execSimpleQuery', - sql: 'SELECT 1', - }); - assert.deepEqual(decodeBrokerRequest(3, new Uint8Array()), { kind: 'close' }); - assert.deepEqual(decodeBrokerRequest(5, new Uint8Array()), { - kind: 'backup', - }); - assert.deepEqual(decodeBrokerRequest(7, new Uint8Array()), { - kind: 'cancel', - }); + const requests = [ + [{ kind: 'authenticate', token: 'secret' }, 6, [...new TextEncoder().encode('secret')]], + [{ kind: 'execProtocol', bytes: Uint8Array.of(1, 2) }, 1, [1, 2]], + [{ kind: 'execProtocolStream', bytes: Uint8Array.of(3, 4) }, 4, [3, 4]], + [{ kind: 'execSimpleQuery', sql: 'SELECT 1' }, 8, [...new TextEncoder().encode('SELECT 1')]], + [{ kind: 'close' }, 3, []], + [{ kind: 'backup' }, 5, []], + [{ kind: 'cancel' }, 7, []], + ] as const; + for (const [request, kind, payload] of requests) { + const expected = Uint8Array.from([ + 0x50, + 0x47, + 0x4f, + 0x42, + kind, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + payload.length, + ...payload, + ]); + assert.deepEqual(encodeBrokerRequest(request), expected); + const stream = new MemoryDuplexStream(); + await writeBrokerRequest(stream, request); + assert.deepEqual(stream.output, [expected]); + } } async function responseFramesRoundTrip(): Promise { @@ -78,9 +82,7 @@ async function responseFramesRoundTrip(): Promise { } function rejectsMalformedFrames(): void { - assert.throws(() => decodeBrokerRequest(999, new Uint8Array()), /unknown broker request/); assert.throws(() => decodeBrokerResponse(999, new Uint8Array()), /unknown broker response/); - assert.throws(() => decodeBrokerRequest(5, new Uint8Array([99])), /unexpectedly had a payload/); assert.throws( () => decodeBrokerResponse(104, new Uint8Array([0xff])), /stream callback-aborted frame is not UTF-8/, @@ -126,44 +128,6 @@ function streamCompletionUsesRecoveryAwareErrorPrecedence(): void { ); } -async function streamHelpersUseBinaryFrames(): Promise { - const requestStream = new MemoryDuplexStream(); - await writeBrokerRequest(requestStream, { - kind: 'execProtocol', - bytes: new Uint8Array([0x51, 0, 0, 0, 4]), - }); - assert.deepEqual(await readBrokerRequest(new MemoryDuplexStream(requestStream.output)), { - kind: 'execProtocol', - bytes: new Uint8Array([0x51, 0, 0, 0, 4]), - }); - - const streamingRequest = new MemoryDuplexStream(); - await writeBrokerRequest(streamingRequest, { - kind: 'execProtocolStream', - bytes: new Uint8Array([0x51]), - }); - assert.deepEqual(await readBrokerRequest(new MemoryDuplexStream(streamingRequest.output)), { - kind: 'execProtocolStream', - bytes: new Uint8Array([0x51]), - }); - - const responseStream = new MemoryDuplexStream(); - await writeBrokerResponse(responseStream, { - kind: 'ok', - bytes: new Uint8Array([0x5a]), - }); - assert.deepEqual(await readBrokerResponse(new MemoryDuplexStream(responseStream.output)), { - kind: 'ok', - bytes: new Uint8Array([0x5a]), - }); - - const raw = encodeBrokerRequest({ kind: 'backup' }); - assert.equal(raw[0], 0x50); - assert.equal(raw[1], 0x47); - assert.equal(raw[2], 0x4f); - assert.equal(raw[3], 0x42); -} - test('broker frames', async () => { await main(); }); diff --git a/src/sdks/js/src/__tests__/broker-response.ts b/src/sdks/js/src/__tests__/broker-response.ts new file mode 100644 index 000000000..7af639d5a --- /dev/null +++ b/src/sdks/js/src/__tests__/broker-response.ts @@ -0,0 +1,15 @@ +import type { BrokerResponseFrame } from '../runtime/broker-frames.js'; + +// Simulate the Rust broker's reply; the SDK only encodes requests and reads replies. +export function encodeBrokerResponse(frame: BrokerResponseFrame): Uint8Array { + const kind = { ok: 101, error: 102, chunk: 103, streamCallbackAborted: 104 }[frame.kind]; + const payload = + frame.kind === 'ok' || frame.kind === 'chunk' + ? frame.bytes + : new TextEncoder().encode(frame.message); + const bytes = new Uint8Array(13 + payload.length); + bytes.set([0x50, 0x47, 0x4f, 0x42, kind]); + new DataView(bytes.buffer).setBigUint64(5, BigInt(payload.length)); + bytes.set(payload, 13); + return bytes; +} diff --git a/src/sdks/js/src/__tests__/memory-duplex-stream.ts b/src/sdks/js/src/__tests__/memory-duplex-stream.ts new file mode 100644 index 000000000..a16f901cf --- /dev/null +++ b/src/sdks/js/src/__tests__/memory-duplex-stream.ts @@ -0,0 +1,36 @@ +import type { ByteStream } from '../runtime/byte-stream.js'; + +export class MemoryDuplexStream implements ByteStream { + readonly #input: Uint8Array[]; + readonly output: Uint8Array[] = []; + + constructor(input: ReadonlyArray = []) { + this.#input = [...input]; + } + + async readExactly(length: number): Promise { + const out = new Uint8Array(length); + let offset = 0; + while (offset < length) { + const chunk = this.#input[0]; + if (chunk === undefined) { + throw new Error(`read stream ended before ${length} byte(s) were available`); + } + const take = Math.min(chunk.length, length - offset); + out.set(chunk.subarray(0, take), offset); + offset += take; + if (take === chunk.length) { + this.#input.shift(); + } else { + this.#input[0] = chunk.subarray(take); + } + } + return out; + } + + async writeAll(bytes: Uint8Array): Promise { + this.output.push(bytes.slice()); + } + + async close(): Promise {} +} diff --git a/src/sdks/js/src/__tests__/native-smoke.ts b/src/sdks/js/src/__tests__/native-smoke.ts index 48d4fe77f..618a3e708 100644 --- a/src/sdks/js/src/__tests__/native-smoke.ts +++ b/src/sdks/js/src/__tests__/native-smoke.ts @@ -4,7 +4,7 @@ import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { Oliphaunt } from '../index.js'; -import { simpleQuery } from '../protocol.js'; +import { simpleQuery } from '@oliphaunt/js-core/protocol'; import { parseSimpleQueryRawResponse } from '../query.js'; import { PostgresWireClient } from '../runtime/pgwire.js'; import { assertNativeDatabaseContract } from './native-direct-contract.mjs'; diff --git a/src/sdks/js/src/__tests__/runtime-adapters.test.ts b/src/sdks/js/src/__tests__/runtime-adapters.test.ts index 26c9c3647..7894f82c7 100644 --- a/src/sdks/js/src/__tests__/runtime-adapters.test.ts +++ b/src/sdks/js/src/__tests__/runtime-adapters.test.ts @@ -6,9 +6,9 @@ import { Readable } from 'node:stream'; import { test } from 'vitest'; import { normalizeOpenConfig } from '../config.js'; -import { MemoryDuplexStream } from '../runtime/byte-stream.js'; +import { MemoryDuplexStream } from './memory-duplex-stream.js'; import { BrokerHandle, cancelBrokerStream, createBrokerRuntimeBinding } from '../runtime/broker.js'; -import { encodeBrokerResponse } from '../runtime/broker-frames.js'; +import { encodeBrokerResponse } from './broker-response.js'; import { createForgottenRuntimeHandleCleanup } from '../runtime/forgotten-handle.js'; import { cleanupFailedManagedLaunch, diff --git a/src/sdks/js/src/native/assets-deno.ts b/src/sdks/js/src/native/assets-deno.ts index fe64478c3..748758d59 100644 --- a/src/sdks/js/src/native/assets-deno.ts +++ b/src/sdks/js/src/native/assets-deno.ts @@ -1,6 +1,5 @@ import type { NativeIcuDescriptor } from '@oliphaunt/js-core/resources'; import { createRequire } from 'node:module'; -import { join } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { @@ -9,7 +8,6 @@ import { resolveExplicitLibraryPath, resolveExplicitRuntimeDirectory, } from './common.js'; -import { type RuntimeFileHost, validatePreparedRuntimeExtensions } from './extension-runtime.js'; import { requireIcuDataTreeSha256, requireIcuManifestRelativePath, @@ -124,22 +122,6 @@ export async function resolveDenoNativeInstall( return resolvePackageNativeInstall(deno, target, versions.liboliphauntVersion, icu); } -export async function validatePreparedDenoRuntimeExtensions(config: { - deno: DenoRuntime; - runtimeDirectory?: string; - extensions: ReadonlyArray; - source: string; -}): Promise<{ runtimeDirectory: string; moduleDirectory?: string }> { - const target = liboliphauntPackageTarget(config.deno.build.os, config.deno.build.arch); - return validatePreparedRuntimeExtensions({ - runtimeDirectory: config.runtimeDirectory, - extensions: config.extensions, - target: target.id, - source: config.source, - host: denoRuntimeFileHost(config.deno), - }); -} - async function packageVersions(deno: DenoRuntime): Promise<{ liboliphauntVersion: string; icuPackage: string; @@ -504,30 +486,3 @@ function optionalDenoRuntime(): DenoRuntime | undefined { const deno = (globalThis as { Deno?: DenoRuntime }).Deno; return deno; } - -function denoRuntimeFileHost(deno: DenoRuntime): RuntimeFileHost { - return { - join, - async readDir(path: string) { - const entries: Array<{ name: string; isFile?: boolean }> = []; - for await (const entry of deno.readDir(path)) { - entries.push({ name: entry.name, isFile: entry.isFile }); - } - return entries; - }, - async isDirectory(path: string) { - try { - return (await deno.stat(path)).isDirectory === true; - } catch { - return false; - } - }, - async isFile(path: string) { - try { - return (await deno.stat(path)).isFile === true; - } catch { - return false; - } - }, - }; -} diff --git a/src/sdks/js/src/native/assets-node.ts b/src/sdks/js/src/native/assets-node.ts index 05c10d72a..c546e4f56 100644 --- a/src/sdks/js/src/native/assets-node.ts +++ b/src/sdks/js/src/native/assets-node.ts @@ -348,24 +348,11 @@ async function resolveSelectedIcu( return resources; } -export async function resolveNodeIcuDataDirectory( - expectedVersion?: string, - packageName?: string, -): Promise { - return (await resolveNodeIcuResources(expectedVersion, packageName))?.dataDirectory; -} - async function resolveNodeIcuResources( - expectedVersion?: string, - packageName?: string, + expected: string, + name: string, packageJsonUrl?: string, ): Promise { - const versions = - expectedVersion === undefined || packageName === undefined - ? await packageVersions() - : undefined; - const expected = expectedVersion ?? versions?.icuVersion; - const name = packageName ?? versions?.icuPackage ?? '@oliphaunt/icu'; const packageJsonPath = packageJsonUrl === undefined ? optionalResolvePackageJson(name) : fileURLToPath(packageJsonUrl); if (packageJsonPath === undefined) { @@ -376,7 +363,7 @@ async function resolveNodeIcuResources( if (packageJson.name !== name) { throw new Error(`${name} package metadata has name ${packageJson.name ?? ''}`); } - if (expected !== undefined && packageJson.version !== expected) { + if (packageJson.version !== expected) { throw new Error( `${name} version ${packageJson.version ?? ''} does not match @oliphaunt/ts icuVersion ${expected}`, ); diff --git a/src/sdks/js/src/native/common.ts b/src/sdks/js/src/native/common.ts index 9b25c5e3f..ea69b7d94 100644 --- a/src/sdks/js/src/native/common.ts +++ b/src/sdks/js/src/native/common.ts @@ -16,16 +16,6 @@ export type NativePackageTarget = { runtimeRelativePath: string; }; -export function resolveLibraryPath(libraryPath?: string): string { - const resolved = resolveExplicitLibraryPath(libraryPath); - if (resolved === undefined || resolved.trim().length === 0) { - throw new Error( - 'no liboliphaunt native asset is available; pass libraryPath, set LIBOLIPHAUNT_PATH, or install the compatible @oliphaunt/liboliphaunt-* package', - ); - } - return resolved; -} - export function resolveExplicitLibraryPath(libraryPath?: string): string | undefined { const resolved = libraryPath ?? envVar('LIBOLIPHAUNT_PATH'); if (resolved === undefined || resolved.trim().length === 0) { diff --git a/src/sdks/js/src/native/tar.ts b/src/sdks/js/src/native/tar.ts deleted file mode 100644 index 3ef476536..000000000 --- a/src/sdks/js/src/native/tar.ts +++ /dev/null @@ -1,158 +0,0 @@ -type TarWriteFile = { - path: string; - bytes: Uint8Array; - mode: number; -}; - -export type TarExtractHost = { - join(base: string, relative: string): string; - dirname(path: string): string; - mkdir(path: string): Promise; - writeFile(file: TarWriteFile): Promise; -}; - -const BLOCK_SIZE = 512; -const textDecoder = new TextDecoder(); - -export async function extractTarArchive( - archive: Uint8Array, - destination: string, - host: TarExtractHost, -): Promise { - let offset = 0; - let nextPax: Record | undefined; - let globalPax: Record = {}; - let nextLongName: string | undefined; - - while (offset + BLOCK_SIZE <= archive.byteLength) { - const header = archive.subarray(offset, offset + BLOCK_SIZE); - offset += BLOCK_SIZE; - if (isZeroBlock(header)) { - break; - } - - const type = String.fromCharCode(header[156] ?? 0).replace('\0', '') || '0'; - const size = parseOctal(header.subarray(124, 136), 'tar entry size'); - const mode = parseOctal(header.subarray(100, 108), 'tar entry mode') || 0o644; - const payloadStart = offset; - const payloadEnd = payloadStart + size; - if (payloadEnd > archive.byteLength) { - throw new Error('tar archive ended in the middle of an entry payload'); - } - const payload = archive.subarray(payloadStart, payloadEnd); - offset = payloadStart + roundUpToBlock(size); - - if (type === 'x') { - nextPax = parsePaxPayload(payload); - continue; - } - if (type === 'g') { - globalPax = { ...globalPax, ...parsePaxPayload(payload) }; - continue; - } - if (type === 'L') { - nextLongName = decodeTarString(payload).replace(/\0+$/, ''); - continue; - } - - const pax = { ...globalPax, ...(nextPax ?? {}) }; - nextPax = undefined; - const relativePath = sanitizeTarPath(pax.path ?? nextLongName ?? tarHeaderPath(header)); - nextLongName = undefined; - if (relativePath === undefined) { - continue; - } - - const outputPath = host.join(destination, relativePath); - if (type === '5') { - await host.mkdir(outputPath); - continue; - } - if (type !== '0') { - throw new Error(`unsupported tar entry type '${type}' for ${relativePath}`); - } - await host.mkdir(host.dirname(outputPath)); - await host.writeFile({ - path: outputPath, - bytes: payload.slice(), - mode: mode & 0o777, - }); - } -} - -function isZeroBlock(block: Uint8Array): boolean { - for (const byte of block) { - if (byte !== 0) { - return false; - } - } - return true; -} - -function roundUpToBlock(size: number): number { - return Math.ceil(size / BLOCK_SIZE) * BLOCK_SIZE; -} - -function parseOctal(bytes: Uint8Array, label: string): number { - const text = decodeTarString(bytes).replace(/\0.*$/, '').trim(); - if (text.length === 0) { - return 0; - } - if (!/^[0-7]+$/.test(text)) { - throw new Error(`${label} is not an octal tar field`); - } - const value = Number.parseInt(text, 8); - if (!Number.isSafeInteger(value) || value < 0) { - throw new Error(`${label} is outside the safe integer range`); - } - return value; -} - -function tarHeaderPath(header: Uint8Array): string { - const name = decodeTarString(header.subarray(0, 100)).replace(/\0.*$/, ''); - const prefix = decodeTarString(header.subarray(345, 500)).replace(/\0.*$/, ''); - return prefix.length > 0 ? `${prefix}/${name}` : name; -} - -function sanitizeTarPath(path: string): string | undefined { - const normalized = path.replace(/\\/g, '/').replace(/^\.\/+/, ''); - if (normalized.length === 0 || normalized === '.') { - return undefined; - } - if (normalized.startsWith('/')) { - throw new Error(`tar entry path must be relative: ${path}`); - } - const segments = normalized.split('/').filter((segment) => segment.length > 0); - if (segments.some((segment) => segment === '.' || segment === '..')) { - throw new Error(`tar entry path escapes the install root: ${path}`); - } - return segments.join('/'); -} - -function parsePaxPayload(payload: Uint8Array): Record { - const text = decodeTarString(payload); - const values: Record = {}; - let offset = 0; - while (offset < text.length) { - const space = text.indexOf(' ', offset); - if (space < 0) { - throw new Error('malformed pax header record'); - } - const length = Number.parseInt(text.slice(offset, space), 10); - if (!Number.isSafeInteger(length) || length <= 0) { - throw new Error('invalid pax header record length'); - } - const record = text.slice(space + 1, offset + length); - const equals = record.indexOf('='); - if (equals <= 0 || !record.endsWith('\n')) { - throw new Error('malformed pax header key/value record'); - } - values[record.slice(0, equals)] = record.slice(equals + 1, -1); - offset += length; - } - return values; -} - -function decodeTarString(bytes: Uint8Array): string { - return textDecoder.decode(bytes); -} diff --git a/src/sdks/js/src/native/zip.ts b/src/sdks/js/src/native/zip.ts deleted file mode 100644 index 73b2da515..000000000 --- a/src/sdks/js/src/native/zip.ts +++ /dev/null @@ -1,139 +0,0 @@ -export type ZipFile = { - path: string; - bytes: Uint8Array; - mode: number; -}; - -export type ZipExtractionHost = { - join(root: string, path: string): string; - dirname(path: string): string; - mkdir(path: string): Promise; - writeFile(file: ZipFile): Promise; -}; - -export async function extractZipArchive( - bytes: Uint8Array, - root: string, - host: ZipExtractionHost, - inflateRaw: (bytes: Uint8Array) => Uint8Array | Promise, -): Promise { - const eocdOffset = findEndOfCentralDirectory(bytes); - const entries = readUInt16LE(bytes, eocdOffset + 10); - const centralDirectorySize = readUInt32LE(bytes, eocdOffset + 12); - const centralDirectoryOffset = readUInt32LE(bytes, eocdOffset + 16); - if (centralDirectoryOffset + centralDirectorySize > bytes.length) { - throw new Error('ZIP central directory is outside archive bounds'); - } - - let offset = centralDirectoryOffset; - for (let index = 0; index < entries; index += 1) { - requireSignature(bytes, offset, 0x02014b50, 'central directory header'); - const method = readUInt16LE(bytes, offset + 10); - const compressedSize = readUInt32LE(bytes, offset + 20); - const uncompressedSize = readUInt32LE(bytes, offset + 24); - const nameLength = readUInt16LE(bytes, offset + 28); - const extraLength = readUInt16LE(bytes, offset + 30); - const commentLength = readUInt16LE(bytes, offset + 32); - const externalAttributes = readUInt32LE(bytes, offset + 38); - const localOffset = readUInt32LE(bytes, offset + 42); - const nameStart = offset + 46; - const nameEnd = nameStart + nameLength; - if (nameEnd > bytes.length) { - throw new Error('ZIP entry name is outside archive bounds'); - } - const path = new TextDecoder().decode(bytes.subarray(nameStart, nameEnd)); - const mode = (externalAttributes >>> 16) & 0o777 || 0o644; - offset = nameEnd + extraLength + commentLength; - - const safePath = validateZipPath(path); - if (safePath === undefined || safePath === '.') { - continue; - } - if (safePath.endsWith('/')) { - await host.mkdir(host.join(root, safePath.slice(0, -1))); - continue; - } - - requireSignature(bytes, localOffset, 0x04034b50, 'local file header'); - const localNameLength = readUInt16LE(bytes, localOffset + 26); - const localExtraLength = readUInt16LE(bytes, localOffset + 28); - const dataStart = localOffset + 30 + localNameLength + localExtraLength; - const dataEnd = dataStart + compressedSize; - if (dataEnd > bytes.length) { - throw new Error(`ZIP entry ${safePath} data is outside archive bounds`); - } - const compressed = bytes.subarray(dataStart, dataEnd); - const content = - method === 0 ? compressed : method === 8 ? await inflateRaw(compressed) : undefined; - if (content === undefined) { - throw new Error(`ZIP entry ${safePath} uses unsupported compression method ${method}`); - } - if (content.length !== uncompressedSize) { - throw new Error(`ZIP entry ${safePath} has invalid uncompressed size`); - } - const output = host.join(root, safePath); - await host.mkdir(host.dirname(output)); - await host.writeFile({ path: output, bytes: content, mode }); - } -} - -function findEndOfCentralDirectory(bytes: Uint8Array): number { - const minimumOffset = Math.max(0, bytes.length - 65_557); - for (let offset = bytes.length - 22; offset >= minimumOffset; offset -= 1) { - if (readUInt32LE(bytes, offset) === 0x06054b50) { - return offset; - } - } - throw new Error('ZIP end of central directory was not found'); -} - -function validateZipPath(path: string): string | undefined { - if (path.length === 0 || path.includes('\0') || path.startsWith('/') || path.includes('\\')) { - throw new Error(`unsafe ZIP entry path: ${path}`); - } - const parts: string[] = []; - for (const rawPart of path.split('/')) { - if (rawPart.length === 0 || rawPart === '.') { - continue; - } - if (rawPart === '..') { - throw new Error(`unsafe ZIP entry path: ${path}`); - } - parts.push(rawPart); - } - if (parts.length === 0) { - return undefined; - } - return `${parts.join('/')}${path.endsWith('/') ? '/' : ''}`; -} - -function requireSignature( - bytes: Uint8Array, - offset: number, - signature: number, - label: string, -): void { - if (offset < 0 || offset + 4 > bytes.length || readUInt32LE(bytes, offset) !== signature) { - throw new Error(`invalid ZIP ${label}`); - } -} - -function readUInt16LE(bytes: Uint8Array, offset: number): number { - if (offset < 0 || offset + 2 > bytes.length) { - throw new Error('truncated ZIP archive'); - } - return bytes[offset]! | (bytes[offset + 1]! << 8); -} - -function readUInt32LE(bytes: Uint8Array, offset: number): number { - if (offset < 0 || offset + 4 > bytes.length) { - throw new Error('truncated ZIP archive'); - } - return ( - (bytes[offset]! | - (bytes[offset + 1]! << 8) | - (bytes[offset + 2]! << 16) | - (bytes[offset + 3]! << 24)) >>> - 0 - ); -} diff --git a/src/sdks/js/src/protocol.ts b/src/sdks/js/src/protocol.ts deleted file mode 100644 index f23f4bdb1..000000000 --- a/src/sdks/js/src/protocol.ts +++ /dev/null @@ -1 +0,0 @@ -export * from '@oliphaunt/js-core/protocol'; diff --git a/src/sdks/js/src/runtime/broker-frames.ts b/src/sdks/js/src/runtime/broker-frames.ts index 2ed918e07..761669a24 100644 --- a/src/sdks/js/src/runtime/broker-frames.ts +++ b/src/sdks/js/src/runtime/broker-frames.ts @@ -26,18 +26,6 @@ export async function writeBrokerRequest( await stream.writeAll(encodeBrokerRequest(frame)); } -export async function readBrokerRequest(stream: ByteStream): Promise { - const { kind, payload } = await readFrame(stream); - return decodeBrokerRequest(kind, payload); -} - -export async function writeBrokerResponse( - stream: ByteStream, - frame: BrokerResponseFrame, -): Promise { - await stream.writeAll(encodeBrokerResponse(frame)); -} - export async function readBrokerResponse(stream: ByteStream): Promise { const { kind, payload } = await readFrame(stream); return decodeBrokerResponse(kind, payload); @@ -62,49 +50,6 @@ export function encodeBrokerRequest(frame: BrokerRequestFrame): Uint8Array { } } -export function encodeBrokerResponse(frame: BrokerResponseFrame): Uint8Array { - switch (frame.kind) { - case 'ok': - return encodeFrame(101, frame.bytes); - case 'chunk': - return encodeFrame(103, frame.bytes); - case 'error': - return encodeFrame(102, encodeUtf8(frame.message)); - case 'streamCallbackAborted': - return encodeFrame(104, encodeUtf8(frame.message)); - } -} - -export function decodeBrokerRequest(kind: number, payload: Uint8Array): BrokerRequestFrame { - switch (kind) { - case 6: - return { - kind: 'authenticate', - token: decodeUtf8(payload, 'broker auth frame'), - }; - case 1: - return { kind: 'execProtocol', bytes: payload }; - case 4: - return { kind: 'execProtocolStream', bytes: payload }; - case 8: - return { - kind: 'execSimpleQuery', - sql: decodeUtf8(payload, 'broker simple-query frame'), - }; - case 3: - assertEmptyPayload(payload); - return { kind: 'close' }; - case 5: - assertEmptyPayload(payload); - return { kind: 'backup' }; - case 7: - assertEmptyPayload(payload); - return { kind: 'cancel' }; - default: - throw new Error(`unknown broker request frame ${kind}`); - } -} - export function decodeBrokerResponse(kind: number, payload: Uint8Array): BrokerResponseFrame { switch (kind) { case 101: @@ -156,12 +101,6 @@ function encodeFrame(kind: number, payload: Uint8Array): Uint8Array { return out; } -function assertEmptyPayload(payload: Uint8Array): void { - if (payload.length > 0) { - throw new Error('broker control frame unexpectedly had a payload'); - } -} - const emptyPayload = new Uint8Array(); const utf8 = new TextEncoder(); const strictUtf8 = new TextDecoder('utf-8', { fatal: true }); diff --git a/src/sdks/js/src/runtime/broker.ts b/src/sdks/js/src/runtime/broker.ts index f7e074a69..36ce5c192 100644 --- a/src/sdks/js/src/runtime/broker.ts +++ b/src/sdks/js/src/runtime/broker.ts @@ -455,7 +455,7 @@ async function launchBroker( try { const startupTimeoutMs = brokerStartupTimeoutMs(); const resolvedExecutable = await resolveBrokerExecutable(executable); - const endpoint = await allocateBrokerEndpoint(config); + const endpoint = await allocateBrokerEndpoint(); failedLaunch.paths[0] = endpoint.ipcDir; const nativeInstall = await resolveBrokerNativeInstall(config); const child = spawnManagedChild({ @@ -641,7 +641,7 @@ type BrokerEndpointPlan = | { kind: 'unix'; socket: string; cancelSocket: string; ipcDir: string } | { kind: 'tcp'; listen: string; cancelListen: string; ipcDir?: undefined }; -async function allocateBrokerEndpoint(config: NormalizedOpenConfig): Promise { +async function allocateBrokerEndpoint(): Promise { const canUseUnix = process.platform !== 'win32'; if (canUseUnix) { const ipcDir = await createTempDir('lpgo-'); diff --git a/src/sdks/js/src/runtime/byte-stream.ts b/src/sdks/js/src/runtime/byte-stream.ts index e79089142..242069e5e 100644 --- a/src/sdks/js/src/runtime/byte-stream.ts +++ b/src/sdks/js/src/runtime/byte-stream.ts @@ -3,38 +3,3 @@ export type ByteStream = { writeAll(bytes: Uint8Array): Promise; close(): Promise; }; - -export class MemoryDuplexStream implements ByteStream { - readonly #input: Uint8Array[]; - readonly output: Uint8Array[] = []; - - constructor(input: ReadonlyArray = []) { - this.#input = [...input]; - } - - async readExactly(length: number): Promise { - const out = new Uint8Array(length); - let offset = 0; - while (offset < length) { - const chunk = this.#input[0]; - if (chunk === undefined) { - throw new Error(`read stream ended before ${length} byte(s) were available`); - } - const take = Math.min(chunk.length, length - offset); - out.set(chunk.subarray(0, take), offset); - offset += take; - if (take === chunk.length) { - this.#input.shift(); - } else { - this.#input[0] = chunk.subarray(take); - } - } - return out; - } - - async writeAll(bytes: Uint8Array): Promise { - this.output.push(bytes.slice()); - } - - async close(): Promise {} -} diff --git a/src/sdks/js/src/runtime/server.ts b/src/sdks/js/src/runtime/server.ts index 5080c9f7d..916420cc7 100644 --- a/src/sdks/js/src/runtime/server.ts +++ b/src/sdks/js/src/runtime/server.ts @@ -1,6 +1,6 @@ import type { NativeExtensionDescriptor, NativeIcuDescriptor } from '@oliphaunt/js-core/resources'; import { spawn } from 'node:child_process'; -import { chmod, lstat, mkdir, mkdtemp, readdir, stat } from 'node:fs/promises'; +import { chmod, lstat, mkdir, mkdtemp, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { delimiter, dirname, join, resolve } from 'node:path'; import { createServer } from 'node:net'; @@ -9,14 +9,12 @@ import type { NormalizedOpenConfig } from '../config.js'; import type { ServerListen } from '../types.js'; import { envVar } from '../native/common.js'; import { - connectEndpoint, cleanupFailedManagedLaunch, removeTree, spawnManagedChild, unixSocketPathsFit, waitForManagedChild, type LocalEndpoint, - type FailedManagedLaunch, type ManagedChild, } from './node-adapter.js'; import { PostgresWireClient } from './pgwire.js'; @@ -652,7 +650,6 @@ export async function nativeServerRuntimeEnv( const runtimeDirectory = dirname(toolDirectory); const dynamicLibraryDirs = await nativeDynamicLibraryDirs(runtimeDirectory); const dynamicLibraryEnv = prependEnvPaths( - nativeDynamicLibraryEnvName(), dynamicLibraryDirs, envVar(nativeDynamicLibraryEnvName()), ); @@ -701,11 +698,7 @@ async function nativeDynamicLibraryDirs(runtimeDirectory: string): Promise path.length > 0); if (existing !== undefined && existing.length > 0) { entries.push(existing); diff --git a/src/sdks/js/tsconfig.json b/src/sdks/js/tsconfig.json index d105afcb4..f1a8936fd 100644 --- a/src/sdks/js/tsconfig.json +++ b/src/sdks/js/tsconfig.json @@ -7,6 +7,8 @@ "moduleResolution": "NodeNext", "noEmit": true, "noUncheckedIndexedAccess": true, + "noUnusedLocals": true, + "noUnusedParameters": true, "outDir": "lib", "rootDir": "src", "skipLibCheck": true, diff --git a/src/sdks/react-native/android/src/main/java/dev/oliphaunt/reactnative/OliphauntModule.kt b/src/sdks/react-native/android/src/main/java/dev/oliphaunt/reactnative/OliphauntModule.kt index 655b7a2c1..7ce0934c9 100644 --- a/src/sdks/react-native/android/src/main/java/dev/oliphaunt/reactnative/OliphauntModule.kt +++ b/src/sdks/react-native/android/src/main/java/dev/oliphaunt/reactnative/OliphauntModule.kt @@ -13,7 +13,6 @@ import dev.oliphaunt.DatabaseStorage import dev.oliphaunt.Oliphaunt import dev.oliphaunt.OliphauntConfig import dev.oliphaunt.OliphauntDatabase -import dev.oliphaunt.PostgresStartupGuc import java.io.File import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicBoolean @@ -396,7 +395,7 @@ class OliphauntModule( return ReactNativeAndroidOpenConfig( config = OliphauntConfig( storage = storage, - startupGucs = config.startupGucs("startupGUCs").associate { it.name to it.value }, + startupGucs = config.startupGucs("startupGUCs"), username = username, database = database, extensions = config.extensionDescriptors(), @@ -470,16 +469,13 @@ class OliphauntModule( return value } - private fun ReadableMap.startupGucs(name: String): List = - stringList(name).map { assignment -> + private fun ReadableMap.startupGucs(name: String): Map = + stringList(name).associate { assignment -> val separator = assignment.indexOf('=') if (separator < 0) { throw IllegalArgumentException("PostgreSQL startup GUC string must use name=value") } - PostgresStartupGuc( - name = assignment.substring(0, separator), - value = assignment.substring(separator + 1), - ) + assignment.substring(0, separator) to assignment.substring(separator + 1) } private fun validatePath(value: String?, label: String): String { @@ -492,36 +488,6 @@ class OliphauntModule( return value } - private fun validatePathOverride(value: String?, name: String): String? { - if (value == null) { - return null - } - if (value.isBlank()) { - throw IllegalArgumentException(pathOverrideMessage(name, PathOverrideError.Empty)) - } - if (value.any { it.code == 0 }) { - throw IllegalArgumentException(pathOverrideMessage(name, PathOverrideError.Nul)) - } - return value - } - - private enum class PathOverrideError { - Empty, - Nul, - } - - private fun pathOverrideMessage(name: String, error: PathOverrideError): String = - when (name to error) { - "runtimeDirectory" to PathOverrideError.Empty -> "runtimeDirectory must not be empty" - "runtimeDirectory" to PathOverrideError.Nul -> "runtimeDirectory must not contain NUL bytes" - "resourceRoot" to PathOverrideError.Empty -> "resourceRoot must not be empty" - "resourceRoot" to PathOverrideError.Nul -> "resourceRoot must not contain NUL bytes" - else -> when (error) { - PathOverrideError.Empty -> "$name must not be empty" - PathOverrideError.Nul -> "$name must not contain NUL bytes" - } - } - private enum class StartupIdentityError { Empty, Nul, diff --git a/src/sdks/react-native/ios/OliphauntAdapter.swift b/src/sdks/react-native/ios/OliphauntAdapter.swift index c26174fc9..53b4c49c0 100644 --- a/src/sdks/react-native/ios/OliphauntAdapter.swift +++ b/src/sdks/react-native/ios/OliphauntAdapter.swift @@ -212,7 +212,7 @@ public final class OliphauntAdapterDatabase: NSObject, @unchecked Sendable { let icu = try string(config, "icuVersion").map { OliphauntIcuData(version: $0) } let configuration = OliphauntConfiguration( storage: storage, - startupGUCs: Dictionary(try startupGUCs(config, "startupGUCs").map { ($0.name, $0.value) }, uniquingKeysWith: { _, last in last }), + startupGUCs: try startupGUCs(config, "startupGUCs"), username: username, database: database, extensions: extensions, @@ -330,15 +330,15 @@ public final class OliphauntAdapterDatabase: NSObject, @unchecked Sendable { } } - private static func startupGUCs(_ dictionary: NSDictionary, _ key: String) throws -> [OliphauntStartupGUC] { - try stringArray(dictionary, key).map { assignment in + private static func startupGUCs(_ dictionary: NSDictionary, _ key: String) throws -> [String: String] { + try Dictionary(stringArray(dictionary, key).map { assignment in guard let separator = assignment.firstIndex(of: "=") else { throw adapterError("PostgreSQL startup GUC string must use name=value") } let name = String(assignment[.. String { @@ -351,22 +351,6 @@ public final class OliphauntAdapterDatabase: NSObject, @unchecked Sendable { return "\(key) must be an array of strings" } - private static func env(_ key: String) -> String? { - guard let value = ProcessInfo.processInfo.environment[key], - !value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - else { - return nil - } - return value - } - - private static func urlFromPath(_ path: String?) -> URL? { - guard let path, !path.isEmpty else { - return nil - } - return URL(fileURLWithPath: path) - } - private static func adapterError(_ message: String) -> NSError { NSError( domain: errorDomain, @@ -408,12 +392,3 @@ public final class OliphauntAdapterDatabase: NSObject, @unchecked Sendable { } } } - -private extension String { - func removingPrefix(_ prefix: String) -> String? { - guard hasPrefix(prefix) else { - return nil - } - return String(dropFirst(prefix.count)) - } -} diff --git a/src/sdks/react-native/src/protocol.ts b/src/sdks/react-native/src/protocol.ts deleted file mode 100644 index f23f4bdb1..000000000 --- a/src/sdks/react-native/src/protocol.ts +++ /dev/null @@ -1 +0,0 @@ -export * from '@oliphaunt/js-core/protocol'; diff --git a/src/sdks/react-native/tsconfig.json b/src/sdks/react-native/tsconfig.json index e55f8e213..5a6f908af 100644 --- a/src/sdks/react-native/tsconfig.json +++ b/src/sdks/react-native/tsconfig.json @@ -6,6 +6,8 @@ "moduleResolution": "Bundler", "noEmit": true, "noUncheckedIndexedAccess": true, + "noUnusedLocals": true, + "noUnusedParameters": true, "outDir": "lib/typescript", "rootDir": "src", "skipLibCheck": true, From d1dd97ba07beec8b8a8b12a98ba73ea2fa350461 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 12:09:35 +0000 Subject: [PATCH 12/18] fix(ci): align coverage inventory with SDK cleanup --- coverage/baseline.toml | 9 --------- src/sdks/js/src/__tests__/memory-duplex-stream.ts | 2 +- src/sdks/js/src/runtime/broker-frames.ts | 2 +- src/sdks/js/src/runtime/broker.ts | 3 +-- src/sdks/js/src/runtime/byte-stream.ts | 5 ----- src/sdks/js/src/runtime/node-adapter.ts | 2 +- src/sdks/js/src/runtime/pgwire.ts | 2 +- src/sdks/js/src/runtime/types.ts | 5 +++++ 8 files changed, 10 insertions(+), 20 deletions(-) delete mode 100644 src/sdks/js/src/runtime/byte-stream.ts diff --git a/coverage/baseline.toml b/coverage/baseline.toml index cfb8e1e10..50f9422dd 100644 --- a/coverage/baseline.toml +++ b/coverage/baseline.toml @@ -157,7 +157,6 @@ exclude_globs = [ "src/sdks/js/src/__tests__/**", "src/sdks/js/src/**/*.d.ts", "src/sdks/js/src/**/types.ts", - "src/sdks/js/src/protocol.ts", "src/sdks/js/src/query.ts", "src/sdks/js/lib/**", "src/sdks/js/node_modules/**", @@ -192,13 +191,6 @@ evidence = "native-bindings tests, package checks, and TypeScript native smoke" owner = "@oliphaunt/core" expires = "before-0.2.0" -[[products.oliphaunt-js.waivers]] -path = "src/sdks/js/src/native/tar.ts" -reason = "tar extraction helpers are covered through asset resolver tests and release artifact checks" -evidence = "asset-resolver.test.ts and release artifact validation" -owner = "@oliphaunt/core" -expires = "before-0.2.0" - [[products.oliphaunt-js.waivers]] path = "src/sdks/js/src/runtime/broker.ts" reason = "broker helper lifecycle requires Rust helper/runtime artifacts and belongs in smoke/regression evidence" @@ -243,7 +235,6 @@ source_globs = [ exclude_globs = [ "src/sdks/react-native/src/__tests__/**", "src/sdks/react-native/src/generated/**", - "src/sdks/react-native/src/protocol.ts", "src/sdks/react-native/src/query.ts", "src/sdks/react-native/lib/**", "src/sdks/react-native/node_modules/**", diff --git a/src/sdks/js/src/__tests__/memory-duplex-stream.ts b/src/sdks/js/src/__tests__/memory-duplex-stream.ts index a16f901cf..f80de87c5 100644 --- a/src/sdks/js/src/__tests__/memory-duplex-stream.ts +++ b/src/sdks/js/src/__tests__/memory-duplex-stream.ts @@ -1,4 +1,4 @@ -import type { ByteStream } from '../runtime/byte-stream.js'; +import type { ByteStream } from '../runtime/types.js'; export class MemoryDuplexStream implements ByteStream { readonly #input: Uint8Array[]; diff --git a/src/sdks/js/src/runtime/broker-frames.ts b/src/sdks/js/src/runtime/broker-frames.ts index 761669a24..2b4dd6d26 100644 --- a/src/sdks/js/src/runtime/broker-frames.ts +++ b/src/sdks/js/src/runtime/broker-frames.ts @@ -1,4 +1,4 @@ -import type { ByteStream } from './byte-stream.js'; +import type { ByteStream } from './types.js'; const MAGIC = new Uint8Array([0x50, 0x47, 0x4f, 0x42]); const HEADER_LEN = 13; diff --git a/src/sdks/js/src/runtime/broker.ts b/src/sdks/js/src/runtime/broker.ts index 36ce5c192..ced68969b 100644 --- a/src/sdks/js/src/runtime/broker.ts +++ b/src/sdks/js/src/runtime/broker.ts @@ -19,7 +19,6 @@ import { writeBrokerRequest, type BrokerResponseFrame, } from './broker-frames.js'; -import type { ByteStream } from './byte-stream.js'; import { connectEndpoint, cleanupFailedManagedLaunch, @@ -34,7 +33,7 @@ import { type ManagedChild, type FailedManagedLaunch, } from './node-adapter.js'; -import type { RuntimeBinding, RuntimeHandle } from './types.js'; +import type { ByteStream, RuntimeBinding, RuntimeHandle } from './types.js'; import { throwCollectedCloseFailures } from './close.js'; import { createForgottenRuntimeHandleCleanup } from './forgotten-handle.js'; import { resolveExactNativeRuntimeProfile } from '../native/runtime-profile.js'; diff --git a/src/sdks/js/src/runtime/byte-stream.ts b/src/sdks/js/src/runtime/byte-stream.ts deleted file mode 100644 index 242069e5e..000000000 --- a/src/sdks/js/src/runtime/byte-stream.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type ByteStream = { - readExactly(length: number): Promise; - writeAll(bytes: Uint8Array): Promise; - close(): Promise; -}; diff --git a/src/sdks/js/src/runtime/node-adapter.ts b/src/sdks/js/src/runtime/node-adapter.ts index 740a0ae0d..17aa886b5 100644 --- a/src/sdks/js/src/runtime/node-adapter.ts +++ b/src/sdks/js/src/runtime/node-adapter.ts @@ -6,7 +6,7 @@ import { join } from 'node:path'; import { Socket, createConnection } from 'node:net'; import type { Readable } from 'node:stream'; -import type { ByteStream } from './byte-stream.js'; +import type { ByteStream } from './types.js'; export type LocalEndpoint = | { kind: 'unix'; path: string } diff --git a/src/sdks/js/src/runtime/pgwire.ts b/src/sdks/js/src/runtime/pgwire.ts index 5041208fd..2de722139 100644 --- a/src/sdks/js/src/runtime/pgwire.ts +++ b/src/sdks/js/src/runtime/pgwire.ts @@ -1,4 +1,4 @@ -import type { ByteStream } from './byte-stream.js'; +import type { ByteStream } from './types.js'; import { connectEndpoint, type LocalEndpoint } from './node-adapter.js'; import { throwCollectedCloseFailures } from './close.js'; diff --git a/src/sdks/js/src/runtime/types.ts b/src/sdks/js/src/runtime/types.ts index 33aff90f9..6386fef64 100644 --- a/src/sdks/js/src/runtime/types.ts +++ b/src/sdks/js/src/runtime/types.ts @@ -1,5 +1,10 @@ import type { NormalizedOpenConfig } from '../config.js'; +export type ByteStream = { + readExactly(length: number): Promise; + writeAll(bytes: Uint8Array): Promise; + close(): Promise; +}; export type RuntimeHandle = unknown; /** From de8ed9922c3a16edd32aaca64b2c98d15c93e103 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 16:04:13 +0000 Subject: [PATCH 13/18] fix(sdk): enforce host-specific entrypoints and repair delivery checks --- .../resolve-planned-moon-execution.test.mjs | 1 + .github/workflows/ci.yml | 16 +- docs/maintainers/sdk-api-surface.md | 13 +- examples/browser-wasix/benchmark.ts | 2 +- .../browser-wasix/direct-pg-dump-smoke.ts | 2 +- examples/browser-wasix/main.ts | 14 +- .../browser-wasix/opfs-crash-probe-worker.ts | 2 +- examples/browser-wasix/package-smoke.ts | 16 +- .../browser-wasix/structured-api-smoke.ts | 2 +- examples/browser-wasix/vite.config.ts | 47 +++- src/bindings/wasix-ts/README.md | 16 ++ src/bindings/wasix-ts/moon.yml | 2 + src/bindings/wasix-ts/package.json | 69 ++++-- .../src/__tests__/browser-tools.test.ts | 29 +++ .../wasix-ts/src/__tests__/client.test.ts | 31 ++- .../src/__tests__/entrypoint-types.ts | 54 +++++ .../native-entrypoint-boundary.test.ts | 29 +++ .../src/__tests__/native-session.test.ts | 102 +++++---- .../src/__tests__/server.node.test.ts | 2 +- .../src/__tests__/worker-client.test.ts | 9 + .../wasix-ts/src/__tests__/worker-rpc.test.ts | 38 +++- src/bindings/wasix-ts/src/browser-public.ts | 9 + src/bindings/wasix-ts/src/browser.ts | 12 ++ src/bindings/wasix-ts/src/client-common.ts | 30 +-- src/bindings/wasix-ts/src/client.ts | 8 +- .../wasix-ts/src/descriptor-validation.ts | 13 ++ src/bindings/wasix-ts/src/direct-client.ts | 4 +- src/bindings/wasix-ts/src/direct.node.ts | 2 +- src/bindings/wasix-ts/src/index.bun.ts | 2 +- src/bindings/wasix-ts/src/index.deno.ts | 2 +- src/bindings/wasix-ts/src/index.node.ts | 2 +- src/bindings/wasix-ts/src/index.ts | 2 +- src/bindings/wasix-ts/src/internal.ts | 27 ++- src/bindings/wasix-ts/src/native-addon.ts | 8 +- src/bindings/wasix-ts/src/native-only.ts | 4 + src/bindings/wasix-ts/src/native-public.ts | 9 + src/bindings/wasix-ts/src/native-server.ts | 6 +- src/bindings/wasix-ts/src/native-session.ts | 120 +++-------- .../wasix-ts/src/node-client-common.ts | 6 +- src/bindings/wasix-ts/src/node-client.ts | 4 +- src/bindings/wasix-ts/src/open-config.ts | 38 ++++ src/bindings/wasix-ts/src/storage.ts | 32 +-- .../wasix-ts/src/storage/indexed-db.ts | 2 +- src/bindings/wasix-ts/src/storage/node.ts | 2 +- src/bindings/wasix-ts/src/storage/opfs.ts | 2 +- src/bindings/wasix-ts/src/tool-runtime.ts | 15 +- src/bindings/wasix-ts/src/types.ts | 15 +- src/bindings/wasix-ts/src/worker-client.ts | 16 +- src/bindings/wasix-ts/src/worker-entry.bun.ts | 2 +- .../wasix-ts/src/worker-entry.deno.ts | 2 +- .../wasix-ts/src/worker-entry.node.ts | 2 +- src/bindings/wasix-ts/src/worker-entry.ts | 2 +- .../wasix-ts/src/worker-node-client.ts | 27 +-- src/bindings/wasix-ts/src/worker-rpc.ts | 5 +- .../tools-package/src/__tests__/api.test.ts | 28 +-- .../src/__tests__/wasix-ts-runtime.ts | 10 +- .../wasix-ts/tools-package/src/index.ts | 33 +-- .../src/wasix-ts-internal-shim.d.ts | 1 + .../wasix-ts/tools/entrypoint-consumer.ts | 29 +++ src/docs/content/sdk/typescript/index.mdx | 8 + .../content/sdk/wasix-typescript/index.mdx | 6 + .../wasix/tools/build-aot-target.sh | 2 +- src/sdks/js/README.md | 15 ++ src/sdks/js/moon.yml | 2 + src/sdks/js/package.json | 20 ++ src/sdks/js/src/__tests__/entrypoints.test.ts | 32 +++ src/sdks/js/src/broker.ts | 20 ++ src/sdks/js/src/direct.ts | 23 ++ src/sdks/js/src/native-only.ts | 4 + src/sdks/js/tools/entrypoint-consumer.ts | 17 ++ .../swift/tools/render-extension-products.mjs | 10 +- .../tools/swift-carrier-resolver.test.mjs | 21 ++ .../tools/check-typescript-consumer.mjs | 21 ++ tools/integration/wasix-ts/moon.yml | 3 + .../wasix-ts/packed-node-fixture.mjs | 202 +++++++++--------- .../wasix-ts/stage-resource-packages.mjs | 26 +++ tools/policy/sdk-manifest.toml | 24 +++ tools/release/product-task-model.test.mjs | 1 + .../stage-react-native-resource-packages.mjs | 5 +- ...ge-react-native-resource-packages.test.mjs | 14 +- tools/release/wasix-tools-aot-npm.test.mjs | 30 ++- tools/release/wasix-tools-npm-carrier.mjs | 64 ++++-- tools/release/wasix-typescript-package.mjs | 39 ++-- .../release/wasix-typescript-package.test.mjs | 28 +-- 84 files changed, 1162 insertions(+), 504 deletions(-) create mode 100644 src/bindings/wasix-ts/src/__tests__/browser-tools.test.ts create mode 100644 src/bindings/wasix-ts/src/__tests__/entrypoint-types.ts create mode 100644 src/bindings/wasix-ts/src/__tests__/native-entrypoint-boundary.test.ts create mode 100644 src/bindings/wasix-ts/src/browser-public.ts create mode 100644 src/bindings/wasix-ts/src/browser.ts create mode 100644 src/bindings/wasix-ts/src/native-only.ts create mode 100644 src/bindings/wasix-ts/src/native-public.ts create mode 100644 src/bindings/wasix-ts/src/open-config.ts create mode 100644 src/bindings/wasix-ts/tools/entrypoint-consumer.ts create mode 100644 src/sdks/js/src/__tests__/entrypoints.test.ts create mode 100644 src/sdks/js/src/broker.ts create mode 100644 src/sdks/js/src/direct.ts create mode 100644 src/sdks/js/src/native-only.ts create mode 100644 src/sdks/js/tools/entrypoint-consumer.ts create mode 100644 src/shared/js-core/tools/check-typescript-consumer.mjs create mode 100644 tools/integration/wasix-ts/stage-resource-packages.mjs diff --git a/.github/scripts/resolve-planned-moon-execution.test.mjs b/.github/scripts/resolve-planned-moon-execution.test.mjs index c1f21ea8f..ced193a30 100644 --- a/.github/scripts/resolve-planned-moon-execution.test.mjs +++ b/.github/scripts/resolve-planned-moon-execution.test.mjs @@ -71,6 +71,7 @@ test('resolves a real multi-root job with downloaded dependencies', () => { }), OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: JSON.stringify([ 'liboliphaunt-wasix:runtime-portable', + 'liboliphaunt-wasix:runtime-aot', 'release-tools:wasix-napi-runtime', ]), }, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fb92668e..adcecc4c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1696,9 +1696,23 @@ jobs: name: oliphaunt-wasix-napi-npm-package-linux-x64-gnu path: target/oliphaunt-wasix-napi/npm-packages + - name: Download same-run Linux x64 WASIX tool AOT inputs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: liboliphaunt-wasix-runtime-aot-linux-x64-gnu + path: target/oliphaunt-wasix/sdk-aot-download + + - name: Restore verified Linux x64 tool AOT layout + run: | + artifact_dir=target/oliphaunt-wasix/sdk-aot-download + test "$(tr -d '\r\n' < "$artifact_dir/target-triple.txt")" = x86_64-unknown-linux-gnu + destination=target/oliphaunt-wasix/aot/x86_64-unknown-linux-gnu + mkdir -p "$destination" + cp -R "$artifact_dir/files/." "$destination/" + - name: Build, test, and package the WASIX TypeScript SDK env: - OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-wasix:runtime-portable", "release-tools:wasix-napi-runtime"]' + OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-wasix:runtime-portable", "liboliphaunt-wasix:runtime-aot", "release-tools:wasix-napi-runtime"]' run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh wasix-ts-sdk-package - name: Upload WASIX TypeScript SDK package artifacts diff --git a/docs/maintainers/sdk-api-surface.md b/docs/maintainers/sdk-api-surface.md index 7adaa13b7..0dbcfc0a8 100644 --- a/docs/maintainers/sdk-api-surface.md +++ b/docs/maintainers/sdk-api-surface.md @@ -1629,9 +1629,11 @@ This version-locked carrier seam is consumed by generated Swift extension produc ### Package exports -- `. = {"types":"./lib/index.d.ts","default":"./lib/index.js"}` +- `. = {"types":"./lib/index.d.ts","node":"./lib/index.js","bun":"./lib/index.js","deno":"./lib/index.js","browser":"./lib/native-only.js","default":"./lib/index.js"}` - `./package.json = {"default":"./package.json"}` - `./storage/node = {"types":"./lib/storage/node.d.ts","default":"./lib/storage/node.js"}` +- `./direct = {"types":"./lib/direct.d.ts","node":"./lib/direct.js","bun":"./lib/direct.js","deno":"./lib/direct.js","browser":"./lib/native-only.js","default":"./lib/direct.js"}` +- `./broker = {"types":"./lib/broker.d.ts","node":"./lib/broker.js","bun":"./lib/broker.js","deno":"./lib/broker.js","browser":"./lib/native-only.js","default":"./lib/broker.js"}` ### Types @@ -1851,17 +1853,18 @@ This version-locked carrier seam is consumed by generated Swift extension produc ### Package exports -- `. = {"types":"./lib/index.d.ts","deno":"./lib/index.deno.js","bun":"./lib/index.bun.js","node":"./lib/index.node.js","browser":"./lib/index.js","default":"./lib/index.js"}` -- `./worker = {"types":"./lib/worker-entry.d.ts","deno":"./lib/worker-entry.deno.js","bun":"./lib/worker-entry.bun.js","node":"./lib/worker-entry.node.js","browser":"./lib/worker-entry.js","default":"./lib/worker-entry.js"}` -- `./direct = {"types":"./lib/direct.node.d.ts","deno":"./lib/direct.node.js","bun":"./lib/direct.node.js","node":"./lib/direct.node.js"}` +- `. = {"deno":{"types":"./lib/index.deno.d.ts","default":"./lib/index.deno.js"},"bun":{"types":"./lib/index.bun.d.ts","default":"./lib/index.bun.js"},"node":{"types":"./lib/index.node.d.ts","default":"./lib/index.node.js"},"browser":{"types":"./lib/index.d.ts","default":"./lib/index.js"},"default":{"types":"./lib/index.d.ts","default":"./lib/index.js"}}` +- `./worker = {"deno":{"types":"./lib/worker-entry.deno.d.ts","default":"./lib/worker-entry.deno.js"},"bun":{"types":"./lib/worker-entry.bun.d.ts","default":"./lib/worker-entry.bun.js"},"node":{"types":"./lib/worker-entry.node.d.ts","default":"./lib/worker-entry.node.js"},"browser":{"types":"./lib/worker-entry.d.ts","default":"./lib/worker-entry.js"},"default":{"types":"./lib/worker-entry.d.ts","default":"./lib/worker-entry.js"}}` +- `./direct = {"types":"./lib/direct.node.d.ts","deno":"./lib/direct.node.js","bun":"./lib/direct.node.js","node":"./lib/direct.node.js","browser":"./lib/native-only.js","default":"./lib/native-only.js"}` - `./internal/tools = {"types":"./lib/internal.d.ts","deno":"./lib/internal.node.js","bun":"./lib/internal.node.js","node":"./lib/internal.node.js","browser":"./lib/internal.js","default":"./lib/internal.js"}` -- `./server = {"types":"./lib/server.node.d.ts","deno":"./lib/server.node.js","bun":"./lib/server.node.js","node":"./lib/server.node.js"}` +- `./server = {"types":"./lib/server.node.d.ts","deno":"./lib/server.node.js","bun":"./lib/server.node.js","node":"./lib/server.node.js","browser":"./lib/native-only.js","default":"./lib/native-only.js"}` - `./storage/indexed-db = {"types":"./lib/storage/indexed-db.d.ts","default":"./lib/storage/indexed-db.js"}` - `./storage/opfs = {"types":"./lib/storage/opfs.d.ts","default":"./lib/storage/opfs.js"}` - `./storage/node = {"types":"./lib/storage/node.d.ts","node":"./lib/storage/node.js"}` - `./storage/bun = {"types":"./lib/storage/bun.d.ts","bun":"./lib/storage/bun.js"}` - `./storage/deno = {"types":"./lib/storage/deno.d.ts","deno":"./lib/storage/deno.js"}` - `./package.json = {"default":"./package.json"}` +- `./browser = {"types":"./lib/browser.d.ts","default":"./lib/browser.js"}` ### Types diff --git a/examples/browser-wasix/benchmark.ts b/examples/browser-wasix/benchmark.ts index 9d851927e..d063ad7f1 100644 --- a/examples/browser-wasix/benchmark.ts +++ b/examples/browser-wasix/benchmark.ts @@ -1,6 +1,6 @@ import { PGlite } from '@electric-sql/pglite'; import { PGliteWorker } from '@electric-sql/pglite/worker'; -import Oliphaunt from '@oliphaunt/wasix-ts'; +import Oliphaunt from '@oliphaunt/wasix-ts/browser'; import WorkerOliphaunt from '@oliphaunt/wasix-ts/worker'; import { opfs } from '@oliphaunt/wasix-ts/storage/opfs'; diff --git a/examples/browser-wasix/direct-pg-dump-smoke.ts b/examples/browser-wasix/direct-pg-dump-smoke.ts index d94e089b4..ff64e3590 100644 --- a/examples/browser-wasix/direct-pg-dump-smoke.ts +++ b/examples/browser-wasix/direct-pg-dump-smoke.ts @@ -1,4 +1,4 @@ -import type { OliphauntDatabase } from '@oliphaunt/wasix-ts'; +import type { OliphauntDatabase } from '@oliphaunt/wasix-ts/browser'; import { pgDump } from '@oliphaunt/wasix-tools'; export async function expectDirectPgDump(database: OliphauntDatabase): Promise { diff --git a/examples/browser-wasix/main.ts b/examples/browser-wasix/main.ts index 229b4596b..4bcce76b6 100644 --- a/examples/browser-wasix/main.ts +++ b/examples/browser-wasix/main.ts @@ -1,12 +1,13 @@ import pgtap from '@oliphaunt/extension-pgtap-wasix'; import Oliphaunt, { + extensions as bundledExtensions, type OliphauntDatabase, PostgresError, type QueryParam, type WasixExtensionDescriptor, type WasixStorage, WasixStorageError, -} from '@oliphaunt/wasix-ts'; +} from '@oliphaunt/wasix-ts/browser'; import WorkerOliphaunt from '@oliphaunt/wasix-ts/worker'; import { indexedDB } from '@oliphaunt/wasix-ts/storage/indexed-db'; import { opfs } from '@oliphaunt/wasix-ts/storage/opfs'; @@ -179,8 +180,17 @@ async function expectLargePostgisWorkerModule(): Promise { throw new Error('browser worker canary requires a PostGIS side module larger than 8 MiB'); } - const database = await WorkerOliphaunt.open({ extensions: [postgis] }); + const database = await WorkerOliphaunt.open({ + extensions: [postgis, bundledExtensions.earthdistance], + }); try { + await database.execute('CREATE EXTENSION earthdistance CASCADE'); + const distance = await database.queryRaw( + 'SELECT earth_distance(ll_to_earth(0, 0), ll_to_earth(0, 0)) AS distance', + ); + if (distance.getText(0, 'distance') !== '0') { + throw new Error('bundled contrib dependency closure failed in the browser worker'); + } await database.execute('CREATE EXTENSION postgis'); const version = await readPostgisVersion(database); await database.queryRaw('CREATE TEMP TABLE postgis_nested_error_catch(value integer)'); diff --git a/examples/browser-wasix/opfs-crash-probe-worker.ts b/examples/browser-wasix/opfs-crash-probe-worker.ts index eea427fff..d81cebf3c 100644 --- a/examples/browser-wasix/opfs-crash-probe-worker.ts +++ b/examples/browser-wasix/opfs-crash-probe-worker.ts @@ -1,4 +1,4 @@ -import Oliphaunt, { type OliphauntDatabase } from '@oliphaunt/wasix-ts'; +import Oliphaunt, { type OliphauntDatabase } from '@oliphaunt/wasix-ts/browser'; import { opfs } from '@oliphaunt/wasix-ts/storage/opfs'; type ProbeRequest = Readonly<{ name: string }>; diff --git a/examples/browser-wasix/package-smoke.ts b/examples/browser-wasix/package-smoke.ts index 8e83f16c3..7a7e90ee6 100644 --- a/examples/browser-wasix/package-smoke.ts +++ b/examples/browser-wasix/package-smoke.ts @@ -1,5 +1,5 @@ import pgtap from '@oliphaunt/extension-pgtap-wasix'; -import Oliphaunt, { type OliphauntDatabase } from '@oliphaunt/wasix-ts'; +import Oliphaunt, { extensions, type OliphauntDatabase } from '@oliphaunt/wasix-ts/browser'; import WorkerOliphaunt from '@oliphaunt/wasix-ts/worker'; import { indexedDB } from '@oliphaunt/wasix-ts/storage/indexed-db'; import { pgDump, psql } from '@oliphaunt/wasix-tools'; @@ -28,11 +28,17 @@ try { const storage = indexedDB('packed-browser-smoke'); let database = await Oliphaunt.open({ storage, - extensions: [pgtap], + extensions: [pgtap, extensions.pg_trgm], }); let pgtapVersion: string; try { await database.execute('CREATE EXTENSION pgtap'); + await database.execute('CREATE EXTENSION pg_trgm'); + const similarity = await database.queryRaw( + "SELECT similarity('hello', 'hello')::text AS value", + ); + if (similarity.getText(0, 'value') !== '1') + throw new Error('packed contrib module did not load'); await expectAnswer(database); await expectStructuredApi(database, 'packed browser direct'); pgtapVersion = await readPgtapVersion(database); @@ -48,7 +54,7 @@ try { database = await WorkerOliphaunt.open({ storage, - extensions: [pgtap], + extensions: [pgtap, extensions.pg_trgm], }); try { await expectAnswer(database); @@ -93,7 +99,7 @@ try { } async function expectLogicalTools(): Promise { - const source = await WorkerOliphaunt.open({ extensions: [pgtap] }); + const source = await WorkerOliphaunt.open({ extensions: [pgtap, extensions.pg_trgm] }); let sql: string; try { await psql(source, { script: logicalToolsSeed }); @@ -105,7 +111,7 @@ async function expectLogicalTools(): Promise { await source.close(); } - const target = await WorkerOliphaunt.open({ extensions: [pgtap] }); + const target = await WorkerOliphaunt.open({ extensions: [pgtap, extensions.pg_trgm] }); try { await psql(target, { script: sql }); const result = await target.queryRaw(logicalToolsVerify); diff --git a/examples/browser-wasix/structured-api-smoke.ts b/examples/browser-wasix/structured-api-smoke.ts index 574c7f143..c22de858e 100644 --- a/examples/browser-wasix/structured-api-smoke.ts +++ b/examples/browser-wasix/structured-api-smoke.ts @@ -1,4 +1,4 @@ -import { type OliphauntDatabase, postgresOids } from '@oliphaunt/wasix-ts'; +import { type OliphauntDatabase, postgresOids } from '@oliphaunt/wasix-ts/browser'; type StructuredObjectRow = { answer: number; diff --git a/examples/browser-wasix/vite.config.ts b/examples/browser-wasix/vite.config.ts index 1c51b8695..504cab5e9 100644 --- a/examples/browser-wasix/vite.config.ts +++ b/examples/browser-wasix/vite.config.ts @@ -106,6 +106,7 @@ export default defineConfig({ function packedBrowserPackageExports(consumerRoot: string): Plugin { const expected = new Map([ ['@oliphaunt/wasix-ts', '/@oliphaunt/wasix-ts/lib/index.js'], + ['@oliphaunt/wasix-ts/browser', '/@oliphaunt/wasix-ts/lib/browser.js'], ['@oliphaunt/wasix-ts/worker', '/@oliphaunt/wasix-ts/lib/worker-entry.js'], ['@oliphaunt/wasix-ts/storage/indexed-db', '/@oliphaunt/wasix-ts/lib/storage/indexed-db.js'], ['@oliphaunt/liboliphaunt-wasix', '/@oliphaunt/liboliphaunt-wasix/index.js'], @@ -147,6 +148,7 @@ function wasixAssets(): Plugin { ['@oliphaunt/liboliphaunt-wasix', '\0oliphaunt:liboliphaunt-wasix'], ['@oliphaunt/liboliphaunt-wasix-tools', '\0oliphaunt:liboliphaunt-wasix-tools'], ['@oliphaunt/extension-pgtap-wasix', '\0oliphaunt:extension-pgtap-wasix'], + ['@oliphaunt/extension-contrib-pg18-wasix', '\0oliphaunt:extension-contrib-pg18-wasix'], ['@oliphaunt/extension-pg-uuidv7-wasix', '\0oliphaunt:extension-pg-uuidv7-wasix'], ['@oliphaunt/extension-postgis-wasix', '\0oliphaunt:extension-postgis-wasix'], ]); @@ -182,6 +184,22 @@ function wasixAssets(): Plugin { if (packageName === undefined) { return undefined; } + if (packageName === '@oliphaunt/extension-contrib-pg18-wasix') { + const manifest = JSON.parse(await readFile(resolve(assetRoot, 'manifest.json'), 'utf8')); + const members = requireArray(manifest.extensions, 'extension rows') + .map((row) => requireRecord(row, 'extension row')) + .filter((row) => row['source-kind'] === 'postgres-contrib'); + return ( + await Promise.all( + members.map(async (row) => { + const sqlName = String(row['sql-name']); + routes.set(`/extensions/${sqlName}`, resolve(assetRoot, String(row.archive))); + const descriptor = await developmentDescriptor(packageName, sqlName); + return `export const ${sqlName.replaceAll('-', '_')} = Object.freeze(${JSON.stringify(descriptor)});`; + }), + ) + ).join('\n'); + } let descriptorPromise = descriptorPromises.get(packageName); if (descriptorPromise === undefined) { descriptorPromise = developmentDescriptor(packageName); @@ -252,7 +270,10 @@ async function developmentWasixIdentity(): Promise<{ return { postgresMajor: postgresMajor as number, physicalFormat }; } -async function developmentDescriptor(packageName: string): Promise> { +async function developmentDescriptor( + packageName: string, + contribSqlName?: string, +): Promise> { const manifestBytes = await readFile(resolve(assetRoot, 'manifest.json')); const manifest = JSON.parse(manifestBytes.toString('utf8')) as Record; const versions = JSON.parse( @@ -324,7 +345,14 @@ async function developmentDescriptor(packageName: string): Promise { + const descriptor = await developmentDescriptor(packageName, String(dependency)); + return requireArray(descriptor.carriers, 'dependency carriers'); + }, + ), + ) + ).flat(); return { schema: 'oliphaunt-wasix-extension-v1', runtime: 'wasix', @@ -394,7 +435,7 @@ async function developmentDescriptor(packageName: string): Promise ({ run: vi.fn() })); +vi.mock('../internal-common.js', () => ({ runWasixToolProcess: runtime.run })); +vi.mock('../database.js', () => ({ + getWasixDatabaseIdentity: () => ({ + username: '-application user', + database: '-application database', + }), +})); +import { runWasixToolProcess } from '../internal.js'; +import type { OliphauntDatabase } from '../types.js'; +it('adds browser connection and input arguments once, without splitting user values', () => { + for (const name of ['pg_dump', 'psql'] as const) { + runWasixToolProcess({} as OliphauntDatabase, { + runtimeVersion: '0.1.1', + tool: { name, source: 'file:///tool.wasm', sha256: 'a'.repeat(64), size: 1 }, + args: ['--schema=public'], + ...(name === 'psql' ? { command: 'select 1' } : {}), + }); + const args = runtime.run.mock.calls.at(-1)?.[1].args; + expect(args.slice(0, 1)).toEqual(['--schema=public']); + expect(args.filter((arg: string) => arg.startsWith('--username='))).toEqual([ + '--username=-application user', + ]); + expect(args).toContain('--dbname=-application database'); + expect(args).toContain(name === 'psql' ? '--no-psqlrc' : '--encoding=UTF8'); + if (name === 'psql') expect(args.slice(-2)).toEqual(['--command', 'select 1']); + } +}); diff --git a/src/bindings/wasix-ts/src/__tests__/client.test.ts b/src/bindings/wasix-ts/src/__tests__/client.test.ts index a92be0f33..b81995181 100644 --- a/src/bindings/wasix-ts/src/__tests__/client.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/client.test.ts @@ -23,7 +23,10 @@ vi.mock('../native-session.js', () => ({ restoreNativeWasixDirect: vi.fn(), })); -import { openWasixWithHost } from '../client.js'; +import { openWasixWithHost, Oliphaunt as browser } from '../client.js'; +import { directory } from '../storage/node.js'; +import { indexedDB } from '../storage/indexed-db.js'; +import { restoreNativeWasix, restoreNativeWasixDirect } from '../native-session.js'; import type { OliphauntDatabase } from '../types.js'; let crossOriginDescriptor: PropertyDescriptor | undefined; @@ -96,3 +99,29 @@ function restoreGlobal(name: string, descriptor: PropertyDescriptor | undefined) if (descriptor === undefined) Reflect.deleteProperty(globalThis, name); else Object.defineProperty(globalThis, name, descriptor); } + +it('rejects host-incompatible storage before loading engines or touching restore bytes', async () => { + const load = vi.fn(); + await expect(openWasixWithHost({ storage: directory('/db') } as never, load)).rejects.toThrow( + 'native-only', + ); + await expect(browser.restore(directory('/db') as never, [])).rejects.toThrow('native-only'); + expect(load).not.toHaveBeenCalled(); + const { Oliphaunt: native } = await import('../node-client.js'); + await expect(native.open({ storage: indexedDB('db') } as never)).rejects.toThrow('browser-only'); + await expect(native.restore(indexedDB('db') as never, [])).rejects.toThrow('browser-only'); +}); + +it('passes the exact restore byte view to N-API without another JavaScript copy', async () => { + const bytes = Uint8Array.of(9, 1, 2, 9).subarray(1, 3); + const { Oliphaunt: native } = await import('../node-client.js'); + const { Oliphaunt: direct } = await import('../direct-client.js'); + await native.restore(directory('/db'), bytes); + await direct.restore(directory('/db'), bytes); + expect(vi.mocked(restoreNativeWasix).mock.calls.at(-1)?.[1]).toBe(bytes); + expect(vi.mocked(restoreNativeWasixDirect).mock.calls.at(-1)?.[1]).toBe(bytes); +}); + +it('rejects an explicit browser import in a native host', async () => { + await expect(import('../browser.js')).rejects.toThrow('requires a browser'); +}); diff --git a/src/bindings/wasix-ts/src/__tests__/entrypoint-types.ts b/src/bindings/wasix-ts/src/__tests__/entrypoint-types.ts new file mode 100644 index 000000000..07e26b5e4 --- /dev/null +++ b/src/bindings/wasix-ts/src/__tests__/entrypoint-types.ts @@ -0,0 +1,54 @@ +import type { Oliphaunt as browser } from '../browser.js'; +import type { Oliphaunt as native } from '../index.node.js'; +import type { Oliphaunt as direct } from '../direct.node.js'; +import type { Oliphaunt as browserWorker } from '../worker-entry.js'; +import type { Oliphaunt as nativeWorker } from '../worker-entry.node.js'; +import type { openServer } from '../server.node.js'; +import { memory } from '../storage.js'; +import { directory } from '../storage/node.js'; +import { indexedDB } from '../storage/indexed-db.js'; +import { opfs } from '../storage/opfs.js'; + +// Compiled by the SDK typecheck; never executed against an engine. +export function checkEntrypointTypes( + web: typeof browser, + node: typeof native, + sync: typeof direct, + webWorker: typeof browserWorker, + nodeWorker: typeof nativeWorker, + server: typeof openServer, +): void { + const disk = directory('/database'); + const idb = indexedDB('database'); + const origin = opfs('database'); + const bytes = new Uint8Array(); + for (const client of [web, webWorker]) { + void client.open({ storage: memory() }); + void client.open({ storage: idb }); + void client.open({ storage: origin }); + void client.restore(idb, bytes); + void client.restore(origin, bytes); + // @ts-expect-error A directory descriptor is native-only, including when passed through a variable. + void client.open({ storage: disk }); + // @ts-expect-error Browser restore accepts the same persistent storage kinds as open. + void client.restore(disk, bytes); + // @ts-expect-error Memory cannot receive a persistent restore. + void client.restore(memory(), bytes); + } + for (const client of [node, sync, nodeWorker]) { + void client.open({ storage: memory() }); + void client.open({ storage: disk }); + void client.restore(disk, bytes); + // @ts-expect-error IndexedDB is browser-only. + void client.open({ storage: idb }); + // @ts-expect-error OPFS is browser-only. + void client.open({ storage: origin }); + // @ts-expect-error Native restore cannot write browser storage. + void client.restore(idb, bytes); + // @ts-expect-error Native restore cannot write browser storage. + void client.restore(origin, bytes); + } + void server({ storage: disk }); + // @ts-expect-error The native server cannot use browser storage either. + void server({ storage: origin }); +} diff --git a/src/bindings/wasix-ts/src/__tests__/native-entrypoint-boundary.test.ts b/src/bindings/wasix-ts/src/__tests__/native-entrypoint-boundary.test.ts new file mode 100644 index 000000000..e59cb0d65 --- /dev/null +++ b/src/bindings/wasix-ts/src/__tests__/native-entrypoint-boundary.test.ts @@ -0,0 +1,29 @@ +import { expect, it, vi } from 'vitest'; +import { workerOpenOptions } from './worker-helpers.js'; +const boundary = vi.hoisted(() => ({ serialize: vi.fn(), open: vi.fn() })); +vi.mock('../open-config.js', () => ({ serializeOpenConfig: boundary.serialize })); +vi.mock('../worker-rpc.js', async (importOriginal) => ({ + ...(await importOriginal()), + openWasixWithWorker: boundary.open, +})); +vi.mock('../physical-archive.js', () => { + throw new Error('native entrypoint loaded browser archive code'); +}); +vi.mock('../storage-provider.js', () => { + throw new Error('native entrypoint loaded browser storage providers'); +}); +import { openWasix } from '../worker-node-client.js'; + +it('preserves optional package sources while removing only the embedded core payload sources', async () => { + const options = { + ...workerOpenOptions(), + icu: { dataArchive: { source: 'file:///optional-icu.tar.zst' } }, + extensionCarriers: { vector: { source: 'file:///vector.tar.zst' } }, + }; + boundary.serialize.mockReturnValue(options); + await openWasix(); + const forwarded = boundary.open.mock.calls[0]?.[1]; + expect(forwarded.icu).toBe(options.icu); + expect(forwarded.extensionCarriers).toBe(options.extensionCarriers); + expect(forwarded.runtime.runtimeArchive.source).toBe('oliphaunt:wasix-napi-embedded'); +}); diff --git a/src/bindings/wasix-ts/src/__tests__/native-session.test.ts b/src/bindings/wasix-ts/src/__tests__/native-session.test.ts index fb5ece5b1..2275870d1 100644 --- a/src/bindings/wasix-ts/src/__tests__/native-session.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/native-session.test.ts @@ -116,7 +116,7 @@ describe('WASIX native embedded payload compatibility', () => { session.runTool({ runtimeVersion: '0.1.1', tool: { name: 'pg_dump', sha256: digest, size: 7, source: 'embedded' }, - args: pgDumpArguments(), + args: [], }), ).rejects.toThrow('WASIX pg_dump descriptor does not match the tool in the installed package'); expect(nativeMocks.pgDump).not.toHaveBeenCalled(); @@ -179,7 +179,7 @@ describe('WASIX native embedded payload compatibility', () => { const result = await session.runTool({ runtimeVersion: '0.1.1', tool: { name: 'pg_dump', sha256: digest, size: 7, source: 'embedded' }, - args: pgDumpArguments(), + args: [], }); expect(result).toEqual({ @@ -200,40 +200,42 @@ describe('WASIX native embedded payload compatibility', () => { expect.objectContaining({ profile: 'standard' }), ); expect( - nativeWasixOpenOptions( - { - ...options, - icu: { - schema: 'oliphaunt-wasix-icu-v1', - runtime: 'wasix', - product: 'oliphaunt-icu', - version: '0.1.1', - compatibility: { - runtimeProduct: 'liboliphaunt-wasix', - runtimeVersion: '0.1.1', - postgresMajor: '18', - physicalFormat: 'wasix-pg18-v1', - compatibilityKey: 'wasix-pg18-datum32-v1', - dataVersion: '76.1', - dataForm: 'files-le', - dataTreeSha256: digest, + ( + await nativeWasixOpenOptions( + { + ...options, + icu: { + schema: 'oliphaunt-wasix-icu-v1', + runtime: 'wasix', + product: 'oliphaunt-icu', + version: '0.1.1', + compatibility: { + runtimeProduct: 'liboliphaunt-wasix', + runtimeVersion: '0.1.1', + postgresMajor: '18', + physicalFormat: 'wasix-pg18-v1', + compatibilityKey: 'wasix-pg18-datum32-v1', + dataVersion: '76.1', + dataForm: 'files-le', + dataTreeSha256: digest, + }, + dataArchive: { + archive: 'icu.tar.zst', + sha256: digest, + size: 1, + source: Uint8Array.of(1), + }, + clusterSeedArchive: { + archive: 'seed.tar.zst', + sha256: digest, + size: 1, + source: Uint8Array.of(2), + }, + clusterSeedManifest: { sha256: digest, size: 1, source: Uint8Array.of(3) }, }, - dataArchive: { - archive: 'icu.tar.zst', - sha256: digest, - size: 1, - source: Uint8Array.of(1), - }, - clusterSeedArchive: { - archive: 'seed.tar.zst', - sha256: digest, - size: 1, - source: Uint8Array.of(2), - }, - clusterSeedManifest: { sha256: digest, size: 1, source: Uint8Array.of(3) }, }, - }, - { kind: 'memory' }, + { kind: 'memory' }, + ) ).profile, ).toBe('icu'); }); @@ -262,7 +264,7 @@ describe('WASIX native embedded payload compatibility', () => { const toolOptions = { runtimeVersion: '0.1.1', tool: { name: 'pg_dump' as const, sha256: digest, size: 7, source: 'embedded' }, - args: pgDumpArguments(), + args: [], }; const directTool = await direct.runTool(toolOptions); const actorTool = await actor.runTool(toolOptions); @@ -521,13 +523,23 @@ function extensionCarrier(sqlName: string): SerializedExtensionCarrier { }; } -function pgDumpArguments(): string[] { - return [ - '--encoding=UTF8', - '--no-password', - '--username=postgres', - '--host=127.0.0.1', - '--port=65432', - '--dbname=postgres', - ]; -} +it('registers a shared tools package once and passes user arguments and command directly to Rust', async () => { + const session = await NativeWasixSession.open(workerOpenOptions()); + const tool = { + name: 'pg_dump' as const, + sha256: digest, + size: 7, + source: 'file:///tools/assets/pg_dump.wasix.wasm', + }; + await session.runTool({ runtimeVersion: '0.1.1', tool, args: ['--schema-only'] }); + await session.runTool({ + runtimeVersion: '0.1.1', + tool: { ...tool, name: 'psql', source: 'file:///tools/assets/psql.wasix.wasm' }, + args: ['--quiet'], + command: 'select 1', + }); + expect(nativeMocks.registerTools).toHaveBeenCalledOnce(); + expect(nativeMocks.pgDump).toHaveBeenLastCalledWith(['--schema-only']); + expect(nativeMocks.psql).toHaveBeenLastCalledWith(['--quiet'], 'select 1', undefined); + await session.close(); +}); diff --git a/src/bindings/wasix-ts/src/__tests__/server.node.test.ts b/src/bindings/wasix-ts/src/__tests__/server.node.test.ts index cf5b2f83e..b179308a6 100644 --- a/src/bindings/wasix-ts/src/__tests__/server.node.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/server.node.test.ts @@ -11,7 +11,7 @@ const nativeMocks = vi.hoisted(() => ({ serialize: vi.fn(), })); -vi.mock('../client-common.js', () => ({ +vi.mock('../open-config.js', () => ({ serializeOpenConfig: nativeMocks.serialize, })); vi.mock('../native-session.js', () => ({ diff --git a/src/bindings/wasix-ts/src/__tests__/worker-client.test.ts b/src/bindings/wasix-ts/src/__tests__/worker-client.test.ts index b69b69137..3ec90cbb3 100644 --- a/src/bindings/wasix-ts/src/__tests__/worker-client.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/worker-client.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { openWasix, Oliphaunt } from '../worker-client.js'; import type { WorkerRequest, WorkerResponse } from '../rpc.js'; +import { directory } from '../storage/node.js'; import { indexedDB } from '../storage/indexed-db.js'; let crossOriginDescriptor: PropertyDescriptor | undefined; @@ -103,3 +104,11 @@ function restoreGlobal(name: string, descriptor: PropertyDescriptor | undefined) if (descriptor === undefined) Reflect.deleteProperty(globalThis, name); else Object.defineProperty(globalThis, name, descriptor); } + +it('rejects directory storage before starting a browser worker', async () => { + await expect(Oliphaunt.open({ storage: directory('/db') } as never)).rejects.toThrow( + 'native-only', + ); + await expect(Oliphaunt.restore(directory('/db') as never, [])).rejects.toThrow('native-only'); + expect(FakeBrowserWorker.instances).toHaveLength(0); +}); diff --git a/src/bindings/wasix-ts/src/__tests__/worker-rpc.test.ts b/src/bindings/wasix-ts/src/__tests__/worker-rpc.test.ts index a5732d7f9..79fa9c987 100644 --- a/src/bindings/wasix-ts/src/__tests__/worker-rpc.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/worker-rpc.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { runWasixPgDumpProcess, WASIX_PROTOCOL_CALLBACK_CHUNK_BYTES } from '../database.js'; +import { + tryRunWasixNativeToolProcess, + runWasixPgDumpProcess, + WASIX_PROTOCOL_CALLBACK_CHUNK_BYTES, +} from '../database.js'; import type { WorkerResponse } from '../rpc.js'; import { createWorkerSessionDispatcher } from '../worker-dispatch.js'; import { openWorkerDatabase, WorkerRpc } from '../worker-rpc.js'; @@ -858,3 +862,35 @@ async function postedRequest(port: FakeWorkerPort, index: number) { } return request; } + +it('preserves the installed optional tool location and structured input across the native worker boundary', async () => { + const port = new FakeWorkerPort(); + const opening = openWorkerDatabase(port, workerOpenOptions()); + const open = await postedRequest(port, 0); + port.respond({ id: open.id, ok: true }); + const database = await opening; + const options = { + runtimeVersion: '0.1.1', + tool: { + name: 'psql' as const, + sha256: '4'.repeat(64), + size: 1, + source: 'file:///optional-tools/assets/psql.wasix.wasm', + }, + args: ['--quiet'], + command: 'select 1', + }; + const running = tryRunWasixNativeToolProcess(database, options); + const request = await postedRequest(port, 1); + expect(request).toMatchObject({ method: 'runTool', options }); + port.respond({ + id: request.id, + ok: true, + value: { exitCode: 0, stdout: new Uint8Array(), stderr: new Uint8Array() }, + }); + await running; + const closing = database.close(); + const close = await postedRequest(port, 2); + port.respond({ id: close.id, ok: true }); + await closing; +}); diff --git a/src/bindings/wasix-ts/src/browser-public.ts b/src/bindings/wasix-ts/src/browser-public.ts new file mode 100644 index 000000000..74ea5416e --- /dev/null +++ b/src/bindings/wasix-ts/src/browser-public.ts @@ -0,0 +1,9 @@ +export * from './public.js'; +import type * as Types from './types.js'; +import type * as Storage from './storage.js'; + +type StorageKind = 'memory' | 'indexed-db' | 'opfs'; +export type OpenConfig = Types.OpenConfig; +export type OliphauntClient = Types.OliphauntClient; +export type WasixStorage = Storage.WasixStorage; +export type PersistentWasixStorage = Storage.PersistentWasixStorage>; diff --git a/src/bindings/wasix-ts/src/browser.ts b/src/bindings/wasix-ts/src/browser.ts new file mode 100644 index 000000000..721a01503 --- /dev/null +++ b/src/bindings/wasix-ts/src/browser.ts @@ -0,0 +1,12 @@ +const host = globalThis as typeof globalThis & { + process?: { versions?: { node?: string } }; + Bun?: unknown; + Deno?: unknown; +}; +if (host.process?.versions?.node || host.Bun !== undefined || host.Deno !== undefined) { + throw new Error( + '@oliphaunt/wasix-ts/browser requires a browser or browser worker; use @oliphaunt/wasix-ts on Node.js, Bun, or Deno', + ); +} +export { Oliphaunt, Oliphaunt as default } from './client.js'; +export * from './browser-public.js'; diff --git a/src/bindings/wasix-ts/src/client-common.ts b/src/bindings/wasix-ts/src/client-common.ts index cd3506558..46b2a436b 100644 --- a/src/bindings/wasix-ts/src/client-common.ts +++ b/src/bindings/wasix-ts/src/client-common.ts @@ -1,34 +1,10 @@ -import defaultWasixRuntime from '@oliphaunt/liboliphaunt-wasix'; - -import { serializeWasixExtensionDescriptors } from './extension-descriptor.js'; -import { serializeWasixIcuDescriptor } from './icu-descriptor.js'; import { decodePhysicalArchive } from './physical-archive.js'; import { toUint8Array } from './query.js'; import type { SerializedOpenOptions } from './rpc.js'; -import { serializeWasixRuntimeDescriptor } from './runtime-descriptor.js'; -import { serializeWasixStorage } from './storage.js'; import { restoreWasixStorage, WASIX_PHYSICAL_IDENTITY } from './storage-provider.js'; -import { normalizeWasixStartupGUCs } from './startup-config.js'; -import type { BinaryInput, OpenConfig, WasixRuntimeDescriptor } from './types.js'; - -export function serializeOpenConfig( - config: OpenConfig = {}, - runtimeDescriptor: WasixRuntimeDescriptor = defaultWasixRuntime, -): SerializedOpenOptions { - const extensions = serializeWasixExtensionDescriptors(config.extensions ?? []); - const runtime = serializeWasixRuntimeDescriptor(runtimeDescriptor); - const storage = serializeWasixStorage(config.storage); - return { - runtime, - ...(config.icu === undefined ? {} : { icu: serializeWasixIcuDescriptor(config.icu) }), - extensionCarriers: extensions.carriers, - extensions: extensions.selectedSqlNames, - username: config.username ?? 'postgres', - database: config.database ?? 'postgres', - startupGUCs: normalizeWasixStartupGUCs(config.startupGUCs ?? {}), - storage, - }; -} +import type { BinaryInput, OpenConfig } from './types.js'; +import { serializeOpenConfig } from './open-config.js'; +export { serializeOpenConfig } from './open-config.js'; export async function restoreWasix( storage: OpenConfig['storage'], diff --git a/src/bindings/wasix-ts/src/client.ts b/src/bindings/wasix-ts/src/client.ts index 763cf26da..b3d21a478 100644 --- a/src/bindings/wasix-ts/src/client.ts +++ b/src/bindings/wasix-ts/src/client.ts @@ -1,10 +1,11 @@ -import { restoreWasix, serializeOpenConfig } from './client-common.js'; +import { restoreWasix } from './client-common.js'; +import { requireBrowserStorage, serializeOpenConfig } from './open-config.js'; import { openWasixDirect, type DirectWasixEnvironment, type DirectWasixHost, } from './direct-client-common.js'; -import type { OliphauntClient, OliphauntDatabase, OpenConfig } from './types.js'; +import type { OliphauntClient, OliphauntDatabase, OpenConfig } from './browser-public.js'; /** Open PostgreSQL in the importing browser realm. Guest execution may block that realm. */ export async function openWasix(config: OpenConfig = {}): Promise { @@ -17,6 +18,7 @@ export async function openWasixWithHost( loadHost: () => Promise, ): Promise { const openOptions = serializeOpenConfig(config); + requireBrowserStorage(openOptions); if (globalThis.crossOriginIsolated !== true) { throw new Error( '@oliphaunt/wasix-ts requires COOP: same-origin and COEP: require-corp response headers', @@ -28,7 +30,7 @@ export async function openWasixWithHost( export const Oliphaunt: OliphauntClient = { open: openWasix, - restore: restoreWasix, + restore: (storage, bytes) => restoreWasix(storage, bytes, requireBrowserStorage), }; function browserRealm(): DirectWasixEnvironment { diff --git a/src/bindings/wasix-ts/src/descriptor-validation.ts b/src/bindings/wasix-ts/src/descriptor-validation.ts index 2080aabe5..9bb7aa267 100644 --- a/src/bindings/wasix-ts/src/descriptor-validation.ts +++ b/src/bindings/wasix-ts/src/descriptor-validation.ts @@ -1,3 +1,4 @@ +import type { WasixToolDescriptor } from './tool-runtime.js'; import type { SerializedAssetSource } from './rpc.js'; import type { WasixAssetSource } from './types.js'; @@ -96,3 +97,15 @@ export function requireSafeRelativeAssetPath(value: unknown, label: string): str } return path; } + +export function validateWasixToolDescriptor(tool: WasixToolDescriptor): void { + if (tool.name !== 'pg_dump' && tool.name !== 'psql') { + throw new TypeError('unsupported Oliphaunt WASIX tool'); + } + if (!/^[0-9a-f]{64}$/u.test(tool.sha256)) { + throw new TypeError(`WASIX ${tool.name} SHA-256 is invalid`); + } + if (!Number.isSafeInteger(tool.size) || tool.size <= 0) { + throw new TypeError(`WASIX ${tool.name} size is invalid`); + } +} diff --git a/src/bindings/wasix-ts/src/direct-client.ts b/src/bindings/wasix-ts/src/direct-client.ts index d46c9ed4a..c1a0d9499 100644 --- a/src/bindings/wasix-ts/src/direct-client.ts +++ b/src/bindings/wasix-ts/src/direct-client.ts @@ -1,7 +1,7 @@ -import { serializeOpenConfig } from './client-common.js'; +import { serializeOpenConfig } from './open-config.js'; import { requireNodeStorage, restoreNodeWasixDirect } from './node-client-common.js'; import { openNodeDirect } from './node-direct.js'; -import type { OliphauntClient, OliphauntDatabase, OpenConfig } from './types.js'; +import type { OliphauntClient, OliphauntDatabase, OpenConfig } from './native-public.js'; /** Open PostgreSQL in the importing realm, where native work blocks its event loop. */ export async function openWasix(config: OpenConfig = {}): Promise { diff --git a/src/bindings/wasix-ts/src/direct.node.ts b/src/bindings/wasix-ts/src/direct.node.ts index f242cb4f2..354e8ae25 100644 --- a/src/bindings/wasix-ts/src/direct.node.ts +++ b/src/bindings/wasix-ts/src/direct.node.ts @@ -1,2 +1,2 @@ export { Oliphaunt, Oliphaunt as default } from './direct-client.js'; -export * from './public.js'; +export * from './native-public.js'; diff --git a/src/bindings/wasix-ts/src/index.bun.ts b/src/bindings/wasix-ts/src/index.bun.ts index a99646a35..cf96e1e49 100644 --- a/src/bindings/wasix-ts/src/index.bun.ts +++ b/src/bindings/wasix-ts/src/index.bun.ts @@ -1,2 +1,2 @@ export { Oliphaunt, Oliphaunt as default } from './node-client.js'; -export * from './public.js'; +export * from './native-public.js'; diff --git a/src/bindings/wasix-ts/src/index.deno.ts b/src/bindings/wasix-ts/src/index.deno.ts index a99646a35..cf96e1e49 100644 --- a/src/bindings/wasix-ts/src/index.deno.ts +++ b/src/bindings/wasix-ts/src/index.deno.ts @@ -1,2 +1,2 @@ export { Oliphaunt, Oliphaunt as default } from './node-client.js'; -export * from './public.js'; +export * from './native-public.js'; diff --git a/src/bindings/wasix-ts/src/index.node.ts b/src/bindings/wasix-ts/src/index.node.ts index a99646a35..cf96e1e49 100644 --- a/src/bindings/wasix-ts/src/index.node.ts +++ b/src/bindings/wasix-ts/src/index.node.ts @@ -1,2 +1,2 @@ export { Oliphaunt, Oliphaunt as default } from './node-client.js'; -export * from './public.js'; +export * from './native-public.js'; diff --git a/src/bindings/wasix-ts/src/index.ts b/src/bindings/wasix-ts/src/index.ts index a1bc19102..7d46b25cf 100644 --- a/src/bindings/wasix-ts/src/index.ts +++ b/src/bindings/wasix-ts/src/index.ts @@ -1,2 +1,2 @@ export { Oliphaunt, Oliphaunt as default } from './client.js'; -export * from './public.js'; +export * from './browser-public.js'; diff --git a/src/bindings/wasix-ts/src/internal.ts b/src/bindings/wasix-ts/src/internal.ts index 1327db88d..9aee650f4 100644 --- a/src/bindings/wasix-ts/src/internal.ts +++ b/src/bindings/wasix-ts/src/internal.ts @@ -7,6 +7,7 @@ import { } from './internal-common.js'; import type { WasixToolWorkerRequest, WasixToolWorkerResponse } from './tool-worker-common.js'; +import { getWasixDatabaseIdentity } from './database.js'; export { getWasixDatabaseIdentity } from './database.js'; export type { @@ -19,7 +20,31 @@ export function runWasixToolProcess( database: OliphauntDatabase, options: WasixToolProcessOptions, ): Promise { - return runTool(database, options, createBrowserToolWorker); + const identity = getWasixDatabaseIdentity(database); + const managed = + options.tool.name === 'pg_dump' + ? ['--encoding=UTF8', '--no-password'] + : ['--no-psqlrc', '--no-password', '--set=ON_ERROR_STOP=1']; + return runTool( + database, + { + ...options, + args: [ + ...options.args, + ...managed, + `--username=${identity.username}`, + '--host=127.0.0.1', + '--port=65432', + `--dbname=${identity.database}`, + ...(options.command !== undefined + ? ['--command', options.command] + : options.stdin !== undefined + ? ['--file=-'] + : []), + ], + }, + createBrowserToolWorker, + ); } function createBrowserToolWorker(): WasixToolWorkerPort { diff --git a/src/bindings/wasix-ts/src/native-addon.ts b/src/bindings/wasix-ts/src/native-addon.ts index 7d74580bf..cdc4ddfcd 100644 --- a/src/bindings/wasix-ts/src/native-addon.ts +++ b/src/bindings/wasix-ts/src/native-addon.ts @@ -54,8 +54,8 @@ export type NativeWasixDatabaseHandle = { onChunk: (chunk: Uint8Array) => void, ): 'complete' | 'callbackAborted'; backup(): Uint8Array; - pgDump(args: string[]): NativeWasixToolResult; - psql(args: string[], command?: string, script?: string): NativeWasixToolResult; + pgDump(args: readonly string[]): NativeWasixToolResult; + psql(args: readonly string[], command?: string, script?: string): NativeWasixToolResult; close(): void; }; @@ -67,8 +67,8 @@ export type NativeWasixActorDatabaseHandle = { onChunk: (chunk: Uint8Array) => void, ): Promise<'complete' | 'callbackAborted'>; backup(): Promise; - pgDump(args: string[]): Promise; - psql(args: string[], command?: string, script?: string): Promise; + pgDump(args: readonly string[]): Promise; + psql(args: readonly string[], command?: string, script?: string): Promise; close(): Promise; }; diff --git a/src/bindings/wasix-ts/src/native-only.ts b/src/bindings/wasix-ts/src/native-only.ts new file mode 100644 index 000000000..1999c48b5 --- /dev/null +++ b/src/bindings/wasix-ts/src/native-only.ts @@ -0,0 +1,4 @@ +throw new Error( + 'This @oliphaunt/wasix-ts entrypoint requires Node.js, Bun, or Deno; use @oliphaunt/wasix-ts/browser in a browser', +); +export {}; diff --git a/src/bindings/wasix-ts/src/native-public.ts b/src/bindings/wasix-ts/src/native-public.ts new file mode 100644 index 000000000..1347acd3c --- /dev/null +++ b/src/bindings/wasix-ts/src/native-public.ts @@ -0,0 +1,9 @@ +export * from './public.js'; +import type * as Types from './types.js'; +import type * as Storage from './storage.js'; + +type StorageKind = 'memory' | 'directory'; +export type OpenConfig = Types.OpenConfig; +export type OliphauntClient = Types.OliphauntClient; +export type WasixStorage = Storage.WasixStorage; +export type PersistentWasixStorage = Storage.PersistentWasixStorage>; diff --git a/src/bindings/wasix-ts/src/native-server.ts b/src/bindings/wasix-ts/src/native-server.ts index 03cc24d27..8f6e5693e 100644 --- a/src/bindings/wasix-ts/src/native-server.ts +++ b/src/bindings/wasix-ts/src/native-server.ts @@ -1,6 +1,6 @@ import { resolve } from 'node:path'; -import { serializeOpenConfig } from './client-common.js'; +import { serializeOpenConfig } from './open-config.js'; import type { NativeWasixServerHandle, NativeWasixServerListen } from './native-addon.js'; import { mapNativeError, @@ -8,7 +8,7 @@ import { requireCompatibleNativeWasixAddon, } from './native-session.js'; import { requireNodeStorage } from './node-client-common.js'; -import type { OpenConfig } from './types.js'; +import type { OpenConfig } from './native-public.js'; export type ServerListen = | Readonly<{ transport: 'tcp'; port?: number }> @@ -45,7 +45,7 @@ export async function openServer(config: ServerOpenConfig = {}): Promise { const identity = normalizeWasixDatabaseIdentity(options.username, options.database); return { profile: options.icu === undefined ? 'standard' : 'icu', @@ -503,12 +505,12 @@ export function nativeWasixOpenOptions( icu: { version: options.icu.version, runtimeVersion: options.icu.compatibility.runtimeVersion, - archive: nativeIcuBytes(options.icu.dataArchive.source), + archive: await nativeIcuBytes(options.icu.dataArchive.source), archiveSha256: options.icu.dataArchive.sha256, dataTreeSha256: options.icu.compatibility.dataTreeSha256, - seedArchive: nativeIcuBytes(options.icu.clusterSeedArchive.source), + seedArchive: await nativeIcuBytes(options.icu.clusterSeedArchive.source), seedArchiveSha256: options.icu.clusterSeedArchive.sha256, - seedManifest: nativeIcuBytes(options.icu.clusterSeedManifest.source), + seedManifest: await nativeIcuBytes(options.icu.clusterSeedManifest.source), seedManifestSha256: options.icu.clusterSeedManifest.sha256, }, }), @@ -525,11 +527,12 @@ export function nativeWasixOpenOptions( }; } -function nativeIcuBytes(source: string | Uint8Array): Uint8Array { - if (source instanceof Uint8Array) return Buffer.from(source); +async function nativeIcuBytes(source: string | Uint8Array): Promise { + if (source instanceof Uint8Array) + return Buffer.from(source.buffer, source.byteOffset, source.byteLength); if (!source.startsWith('file:')) throw new TypeError('WASIX native ICU data requires an installed file URL or bytes'); - return readFileSync(fileURLToPath(source)); + return readFile(fileURLToPath(source)); } function nativeStorage(options: SerializedOpenOptions): NativeWasixOpenOptions['storage'] { @@ -557,7 +560,12 @@ function validateNativeToolCall( const key = `${options.tool.name}:${options.tool.sha256}:${options.tool.source}`; const registered = registeredTools.get(addon) ?? new Set(); if (!registered.has(key)) { - addon.registerTools(nativeToolPackage(options.tool)); + const packageDescriptor = nativeToolPackage(options.tool); + const packageKey = JSON.stringify(packageDescriptor); + if (!registered.has(packageKey)) { + addon.registerTools(packageDescriptor); + registered.add(packageKey); + } registered.add(key); registeredTools.set(addon, registered); } @@ -623,70 +631,6 @@ function toolProcessResult(result: NativeWasixToolResult): WasixToolProcessResul }; } -function userPgDumpArguments(args: readonly string[], identity: WasixDatabaseIdentity): string[] { - const suffix = [ - '--encoding=UTF8', - '--no-password', - `--username=${identity.username}`, - '--host=127.0.0.1', - '--port=65432', - `--dbname=${identity.database}`, - ]; - return stripManagedSuffix('pg_dump', args, suffix); -} - -function userPsqlArguments( - args: readonly string[], - stdin: Uint8Array | undefined, - identity: WasixDatabaseIdentity, -): Readonly<{ args: string[]; command?: string; script?: string }> { - const managed = [ - '--no-psqlrc', - '--no-password', - '--set=ON_ERROR_STOP=1', - `--username=${identity.username}`, - '--host=127.0.0.1', - '--port=65432', - `--dbname=${identity.database}`, - ]; - const start = findExactSequence(args, managed); - if (start < 0) throw new Error('Oliphaunt WASIX psql call has an invalid managed argument set'); - const user = args.slice(0, start); - const input = args.slice(start + managed.length); - if (input.length === 0) return { args: user }; - if (input.length === 2 && input[0] === '--command' && input[1] !== undefined) { - return { args: user, command: input[1] }; - } - if (input.length === 1 && input[0] === '--file=-' && stdin !== undefined) { - return { - args: user, - script: new TextDecoder('utf-8', { fatal: true }).decode(stdin), - }; - } - throw new Error('Oliphaunt WASIX psql call has invalid managed input arguments'); -} - -function stripManagedSuffix( - tool: string, - args: readonly string[], - suffix: readonly string[], -): string[] { - if ( - args.length < suffix.length || - !suffix.every((argument, index) => args[args.length - suffix.length + index] === argument) - ) { - throw new Error(`Oliphaunt WASIX ${tool} call has an invalid managed argument set`); - } - return args.slice(0, -suffix.length); -} - -function findExactSequence(values: readonly string[], expected: readonly string[]): number { - for (let start = values.length - expected.length; start >= 0; start -= 1) { - if (expected.every((value, offset) => values[start + offset] === value)) return start; - } - return -1; -} - /** @internal Translate only the exact tagged native storage contract. */ export function mapNativeError(error: unknown): unknown { if (error instanceof WasixStorageError) return error; diff --git a/src/bindings/wasix-ts/src/node-client-common.ts b/src/bindings/wasix-ts/src/node-client-common.ts index 03c2a9fb4..e5da6dc8b 100644 --- a/src/bindings/wasix-ts/src/node-client-common.ts +++ b/src/bindings/wasix-ts/src/node-client-common.ts @@ -1,7 +1,7 @@ import { isAbsolute, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { serializeOpenConfig } from './client-common.js'; +import { serializeOpenConfig } from './open-config.js'; import { hostRuntime } from './host-runtime.js'; import { restoreNativeWasix, restoreNativeWasixDirect } from './native-session.js'; import { toUint8Array } from './query.js'; @@ -16,7 +16,7 @@ export async function restoreNodeWasix( ): Promise { const options = serializeOpenConfig({ storage }); requireNodeStorage(options); - return restoreNativeWasix(options, toUint8Array(bytes).slice()); + return restoreNativeWasix(options, toUint8Array(bytes)); } /** @internal Restore synchronously in the importing realm for `/direct`. */ @@ -26,7 +26,7 @@ export async function restoreNodeWasixDirect( ): Promise { const options = serializeOpenConfig({ storage }); requireNodeStorage(options); - return restoreNativeWasixDirect(options, toUint8Array(bytes).slice()); + return restoreNativeWasixDirect(options, toUint8Array(bytes)); } /** @internal Validate and normalize storage shared by direct and Worker entrypoints. */ diff --git a/src/bindings/wasix-ts/src/node-client.ts b/src/bindings/wasix-ts/src/node-client.ts index 443092ac9..435987b3a 100644 --- a/src/bindings/wasix-ts/src/node-client.ts +++ b/src/bindings/wasix-ts/src/node-client.ts @@ -1,7 +1,7 @@ -import { serializeOpenConfig } from './client-common.js'; +import { serializeOpenConfig } from './open-config.js'; import { requireNodeStorage, restoreNodeWasix } from './node-client-common.js'; import { openNodeActor } from './node-actor.js'; -import type { OliphauntClient, OliphauntDatabase, OpenConfig } from './types.js'; +import type { OliphauntClient, OliphauntDatabase, OpenConfig } from './native-public.js'; /** Open PostgreSQL on a dedicated Rust owner while keeping the caller event loop responsive. */ export async function openWasix(config: OpenConfig = {}): Promise { diff --git a/src/bindings/wasix-ts/src/open-config.ts b/src/bindings/wasix-ts/src/open-config.ts new file mode 100644 index 000000000..d94694017 --- /dev/null +++ b/src/bindings/wasix-ts/src/open-config.ts @@ -0,0 +1,38 @@ +import defaultWasixRuntime from '@oliphaunt/liboliphaunt-wasix'; + +import { serializeWasixExtensionDescriptors } from './extension-descriptor.js'; +import { serializeWasixIcuDescriptor } from './icu-descriptor.js'; +import type { SerializedOpenOptions } from './rpc.js'; +import { serializeWasixRuntimeDescriptor } from './runtime-descriptor.js'; +import { serializeWasixStorage } from './storage.js'; +import { normalizeWasixStartupGUCs } from './startup-config.js'; + +import type { OpenConfig, WasixRuntimeDescriptor } from './types.js'; + +export function serializeOpenConfig( + config: OpenConfig = {}, + runtimeDescriptor: WasixRuntimeDescriptor = defaultWasixRuntime, +): SerializedOpenOptions { + const extensions = serializeWasixExtensionDescriptors(config.extensions ?? []); + const runtime = serializeWasixRuntimeDescriptor(runtimeDescriptor); + const storage = serializeWasixStorage(config.storage); + return { + runtime, + ...(config.icu === undefined ? {} : { icu: serializeWasixIcuDescriptor(config.icu) }), + extensionCarriers: extensions.carriers, + extensions: extensions.selectedSqlNames, + username: config.username ?? 'postgres', + database: config.database ?? 'postgres', + startupGUCs: normalizeWasixStartupGUCs(config.startupGUCs ?? {}), + storage, + }; +} + +/** Reject unsupported browser storage before loading an engine or starting a worker. */ +export function requireBrowserStorage(options: SerializedOpenOptions): void { + if (options.storage.kind === 'directory') { + throw new TypeError( + '@oliphaunt/wasix-ts/browser directory storage is native-only; use memory, IndexedDB, or OPFS', + ); + } +} diff --git a/src/bindings/wasix-ts/src/storage.ts b/src/bindings/wasix-ts/src/storage.ts index 5634a9476..8a5569de2 100644 --- a/src/bindings/wasix-ts/src/storage.ts +++ b/src/bindings/wasix-ts/src/storage.ts @@ -5,12 +5,16 @@ declare const persistentStorageDescriptorBrand: unique symbol; * An opaque storage selection created by this package's storage factories. * The descriptor is deliberately not a bag of user-authored paths or assets. */ -export type WasixStorage = Readonly<{ - [storageDescriptorBrand]: 'oliphaunt-wasix-storage'; +export type WasixStorageKind = 'memory' | 'directory' | 'indexed-db' | 'opfs'; + +export type WasixStorage = Readonly<{ + [storageDescriptorBrand]: Kind; }>; /** Opaque persistent storage accepted by static physical restore. */ -export type PersistentWasixStorage = WasixStorage & +export type PersistentWasixStorage< + Kind extends Exclude = Exclude, +> = WasixStorage & Readonly<{ [persistentStorageDescriptorBrand]: 'oliphaunt-wasix-persistent-storage'; }>; @@ -42,7 +46,7 @@ const descriptorValues = new WeakMap(); * Select a fresh in-memory database. This is also the default when `storage` * is omitted. Reusing the descriptor does not preserve data. */ -export function memory(): WasixStorage { +export function memory(): WasixStorage<'memory'> { return defineStorage({ schema: 'oliphaunt-wasix-storage-v1', kind: 'memory', @@ -50,33 +54,33 @@ export function memory(): WasixStorage { } /** @internal Used by the selectively imported IndexedDB adapter. */ -export function defineIndexedDbStorage(name: string): PersistentWasixStorage { +export function defineIndexedDbStorage(name: string): PersistentWasixStorage<'indexed-db'> { validateIndexedDbDatabaseName(name); return defineStorage({ schema: 'oliphaunt-wasix-storage-v1', kind: 'indexed-db', name, - }) as PersistentWasixStorage; + }) as PersistentWasixStorage<'indexed-db'>; } /** @internal Used by the selectively imported OPFS adapter. */ -export function defineOpfsStorage(name: string): PersistentWasixStorage { +export function defineOpfsStorage(name: string): PersistentWasixStorage<'opfs'> { validateOpfsDatabaseName(name); return defineStorage({ schema: 'oliphaunt-wasix-storage-v1', kind: 'opfs', name, - }) as PersistentWasixStorage; + }) as PersistentWasixStorage<'opfs'>; } /** @internal Used by the selectively imported Node directory adapter. */ -export function defineDirectoryStorage(path: string): PersistentWasixStorage { +export function defineDirectoryStorage(path: string): PersistentWasixStorage<'directory'> { validateHostDirectoryPath(path); return defineStorage({ schema: 'oliphaunt-wasix-storage-v1', kind: 'directory', path, - }) as PersistentWasixStorage; + }) as PersistentWasixStorage<'directory'>; } /** @internal Validate and project the opaque main-thread value for the worker. */ @@ -129,8 +133,10 @@ export function validateHostDirectoryPath(path: unknown): asserts path is string } } -function defineStorage(value: SerializedWasixStorage): WasixStorage { +function defineStorage( + value: Value, +): WasixStorage { const descriptor = Object.freeze({}); - descriptorValues.set(descriptor, Object.freeze(value)); - return descriptor as WasixStorage; + descriptorValues.set(descriptor, Object.freeze(value)); + return descriptor as WasixStorage; } diff --git a/src/bindings/wasix-ts/src/storage/indexed-db.ts b/src/bindings/wasix-ts/src/storage/indexed-db.ts index b6ce2a195..928796c65 100644 --- a/src/bindings/wasix-ts/src/storage/indexed-db.ts +++ b/src/bindings/wasix-ts/src/storage/indexed-db.ts @@ -6,7 +6,7 @@ import { defineIndexedDbStorage, type PersistentWasixStorage } from '../storage. * Every completed protocol operation commits only journaled PGDATA path * changes in one atomic read-write IndexedDB transaction before its Promise resolves. */ -export function indexedDB(name: string): PersistentWasixStorage { +export function indexedDB(name: string): PersistentWasixStorage<'indexed-db'> { return defineIndexedDbStorage(name); } diff --git a/src/bindings/wasix-ts/src/storage/node.ts b/src/bindings/wasix-ts/src/storage/node.ts index 020d54060..8d27ebbd4 100644 --- a/src/bindings/wasix-ts/src/storage/node.ts +++ b/src/bindings/wasix-ts/src/storage/node.ts @@ -9,7 +9,7 @@ import { defineDirectoryStorage, type PersistentWasixStorage } from '../storage. * and performs PostgreSQL-safe durable writes at each native operation * boundary. Network and cross-host shared filesystems are unsupported. */ -export function directory(path: string | URL): PersistentWasixStorage { +export function directory(path: string | URL): PersistentWasixStorage<'directory'> { return defineDirectoryStorage(typeof path === 'string' ? path : fileURLToPath(path)); } diff --git a/src/bindings/wasix-ts/src/storage/opfs.ts b/src/bindings/wasix-ts/src/storage/opfs.ts index e86f23823..c52548fc3 100644 --- a/src/bindings/wasix-ts/src/storage/opfs.ts +++ b/src/bindings/wasix-ts/src/storage/opfs.ts @@ -5,7 +5,7 @@ import { defineOpfsStorage, type PersistentWasixStorage } from '../storage.js'; * same-realm synchronous exact-range I/O; other placements publish to the * same format through the portable journaled path. */ -export function opfs(name: string): PersistentWasixStorage { +export function opfs(name: string): PersistentWasixStorage<'opfs'> { return defineOpfsStorage(name); } diff --git a/src/bindings/wasix-ts/src/tool-runtime.ts b/src/bindings/wasix-ts/src/tool-runtime.ts index 8cecc1ac2..fc367b916 100644 --- a/src/bindings/wasix-ts/src/tool-runtime.ts +++ b/src/bindings/wasix-ts/src/tool-runtime.ts @@ -1,3 +1,5 @@ +import { validateWasixToolDescriptor } from './descriptor-validation.js'; +export { validateWasixToolDescriptor } from './descriptor-validation.js'; import { assertSha256 } from './extensions.js'; import type { Directory, RunWasixOptions } from './host/index.mjs'; import { loadAsset } from './archive.js'; @@ -16,6 +18,7 @@ export type WasixToolProcessOptions = Readonly<{ args: readonly string[]; /** @internal An exact ArrayBuffer-backed view is transferred and consumed. */ stdin?: Uint8Array; + command?: string; }>; export type WasixToolProcessResult = Readonly<{ @@ -130,18 +133,6 @@ export function wasixToolRunOptions( }; } -export function validateWasixToolDescriptor(tool: WasixToolDescriptor): void { - if (tool.name !== 'pg_dump' && tool.name !== 'psql') { - throw new TypeError('unsupported Oliphaunt WASIX tool'); - } - if (!/^[0-9a-f]{64}$/u.test(tool.sha256)) { - throw new TypeError(`WASIX ${tool.name} SHA-256 is invalid`); - } - if (!Number.isSafeInteger(tool.size) || tool.size <= 0) { - throw new TypeError(`WASIX ${tool.name} size is invalid`); - } -} - export function wasixToolAssetIdentity(descriptor: WasixToolDescriptor): string { return `${descriptor.name}:${descriptor.sha256}:${descriptor.size}`; } diff --git a/src/bindings/wasix-ts/src/types.ts b/src/bindings/wasix-ts/src/types.ts index ba8be6dfe..fa657eaad 100644 --- a/src/bindings/wasix-ts/src/types.ts +++ b/src/bindings/wasix-ts/src/types.ts @@ -9,7 +9,7 @@ import type { QueryResult, RawQueryResult, } from './query.js'; -import type { PersistentWasixStorage, WasixStorage } from './storage.js'; +import type { PersistentWasixStorage, WasixStorage, WasixStorageKind } from './storage.js'; type QueryReadOptions = Omit; @@ -199,7 +199,7 @@ export type WasixAssetManifest = { extensions: readonly []; }; -export type OpenConfig = { +export type OpenConfig = { /** Existing PostgreSQL role selected after the fixed superuser bootstrap. */ username?: string; database?: string; @@ -210,7 +210,7 @@ export type OpenConfig = { /** Selectively imported WASIX carriers. SQL strings are intentionally not accepted. */ extensions?: readonly WasixExtensionDescriptor[]; /** Fresh memory by default, or an explicitly imported host storage adapter. */ - storage?: WasixStorage; + storage?: WasixStorage; }; export type OliphauntDatabase = { @@ -286,7 +286,10 @@ export type OliphauntTransaction = { rollback(): Promise; }; -export type OliphauntClient = { - open(config?: OpenConfig): Promise; - restore(storage: PersistentWasixStorage, bytes: BinaryInput): Promise; +export type OliphauntClient = { + open(config?: OpenConfig): Promise; + restore( + storage: PersistentWasixStorage>, + bytes: BinaryInput, + ): Promise; }; diff --git a/src/bindings/wasix-ts/src/worker-client.ts b/src/bindings/wasix-ts/src/worker-client.ts index 839965826..1c3927d76 100644 --- a/src/bindings/wasix-ts/src/worker-client.ts +++ b/src/bindings/wasix-ts/src/worker-client.ts @@ -1,13 +1,19 @@ -import { serializeOpenConfig } from './client-common.js'; -import type { PersistentWasixStorage } from './storage.js'; -import type { BinaryInput, OliphauntClient, OliphauntDatabase, OpenConfig } from './types.js'; +import { requireBrowserStorage, serializeOpenConfig } from './open-config.js'; +import type { PersistentWasixStorage } from './browser-public.js'; +import type { + BinaryInput, + OliphauntClient, + OliphauntDatabase, + OpenConfig, +} from './browser-public.js'; import { openWasixWithWorker, restoreWasixWithWorker, type WasixWorkerPort } from './worker-rpc.js'; /** Open PostgreSQL in a package-owned browser Worker. */ export async function openWasix(config: OpenConfig = {}): Promise { const openOptions = serializeOpenConfig(config); + requireBrowserStorage(openOptions); assertBrowserWorkerEnvironment(); - return openWasixWithWorker(createBrowserWorker, openOptions); + return openWasixWithWorker(createBrowserWorker, openOptions, requireBrowserStorage); } async function restoreBrowserWasix( @@ -15,7 +21,7 @@ async function restoreBrowserWasix( bytes: BinaryInput, ): Promise { assertBrowserWorkerEnvironment(); - return restoreWasixWithWorker(createBrowserWorker, storage, bytes); + return restoreWasixWithWorker(createBrowserWorker, storage, bytes, requireBrowserStorage); } function assertBrowserWorkerEnvironment(): void { diff --git a/src/bindings/wasix-ts/src/worker-entry.bun.ts b/src/bindings/wasix-ts/src/worker-entry.bun.ts index e8cee532b..7c4c9bd8b 100644 --- a/src/bindings/wasix-ts/src/worker-entry.bun.ts +++ b/src/bindings/wasix-ts/src/worker-entry.bun.ts @@ -1,2 +1,2 @@ export { Oliphaunt, Oliphaunt as default } from './worker-node-client.js'; -export * from './public.js'; +export * from './native-public.js'; diff --git a/src/bindings/wasix-ts/src/worker-entry.deno.ts b/src/bindings/wasix-ts/src/worker-entry.deno.ts index e8cee532b..7c4c9bd8b 100644 --- a/src/bindings/wasix-ts/src/worker-entry.deno.ts +++ b/src/bindings/wasix-ts/src/worker-entry.deno.ts @@ -1,2 +1,2 @@ export { Oliphaunt, Oliphaunt as default } from './worker-node-client.js'; -export * from './public.js'; +export * from './native-public.js'; diff --git a/src/bindings/wasix-ts/src/worker-entry.node.ts b/src/bindings/wasix-ts/src/worker-entry.node.ts index e8cee532b..7c4c9bd8b 100644 --- a/src/bindings/wasix-ts/src/worker-entry.node.ts +++ b/src/bindings/wasix-ts/src/worker-entry.node.ts @@ -1,2 +1,2 @@ export { Oliphaunt, Oliphaunt as default } from './worker-node-client.js'; -export * from './public.js'; +export * from './native-public.js'; diff --git a/src/bindings/wasix-ts/src/worker-entry.ts b/src/bindings/wasix-ts/src/worker-entry.ts index 2b4be042c..95578ade4 100644 --- a/src/bindings/wasix-ts/src/worker-entry.ts +++ b/src/bindings/wasix-ts/src/worker-entry.ts @@ -1,2 +1,2 @@ export { Oliphaunt, Oliphaunt as default } from './worker-client.js'; -export * from './public.js'; +export * from './browser-public.js'; diff --git a/src/bindings/wasix-ts/src/worker-node-client.ts b/src/bindings/wasix-ts/src/worker-node-client.ts index 45d3956b3..b8db55eef 100644 --- a/src/bindings/wasix-ts/src/worker-node-client.ts +++ b/src/bindings/wasix-ts/src/worker-node-client.ts @@ -1,13 +1,18 @@ import { Worker } from 'node:worker_threads'; -import { serializeOpenConfig } from './client-common.js'; +import { serializeOpenConfig } from './open-config.js'; import { hostRuntimeName } from './host-runtime.js'; import { requireNodeStorage } from './node-client-common.js'; import { nodeWorkerExecArgv } from './node-worker-options.js'; import { nodeWorkerPort } from './node-worker-port.js'; import type { SerializedOpenOptions } from './rpc.js'; -import type { PersistentWasixStorage } from './storage.js'; -import type { BinaryInput, OliphauntClient, OliphauntDatabase, OpenConfig } from './types.js'; +import type { PersistentWasixStorage } from './native-public.js'; +import type { + BinaryInput, + OliphauntClient, + OliphauntDatabase, + OpenConfig, +} from './native-public.js'; import { openWasixWithWorker, restoreWasixWithWorker, type WasixWorkerPort } from './worker-rpc.js'; /** Open PostgreSQL in a package-owned Node-compatible Worker realm. */ @@ -52,21 +57,5 @@ function withoutNativeAssetPayloads(options: SerializedOpenOptions): SerializedO standardSeedManifest: { ...options.runtime.standardSeedManifest, source }, manifest: { ...options.runtime.manifest, source }, }, - ...(options.icu === undefined - ? {} - : { - icu: { - ...options.icu, - dataArchive: { ...options.icu.dataArchive, source }, - clusterSeedArchive: { ...options.icu.clusterSeedArchive, source }, - clusterSeedManifest: { ...options.icu.clusterSeedManifest, source }, - }, - }), - extensionCarriers: Object.fromEntries( - Object.entries(options.extensionCarriers).map(([sqlName, carrier]) => [ - sqlName, - { ...carrier, source }, - ]), - ), }; } diff --git a/src/bindings/wasix-ts/src/worker-rpc.ts b/src/bindings/wasix-ts/src/worker-rpc.ts index 8c607bb99..7f2de4ef9 100644 --- a/src/bindings/wasix-ts/src/worker-rpc.ts +++ b/src/bindings/wasix-ts/src/worker-rpc.ts @@ -10,7 +10,7 @@ import { type WasixProtocolConnectionMode, type WasixProtocolStreamOutcome, } from './database.js'; -import { serializeOpenConfig } from './client-common.js'; +import { serializeOpenConfig } from './open-config.js'; import { toUint8Array } from './query.js'; import type { SerializedAssetSource, @@ -442,9 +442,6 @@ class WorkerDatabaseSession implements WasixDatabaseSession { method: 'runTool', options: { ...options, - // The release addon owns the verified tool payload. Preserve only - // the descriptor identity across the Worker boundary. - tool: { ...options.tool, source: 'oliphaunt:wasix-napi-embedded' }, args: [...options.args], ...(stdin === undefined ? {} : { stdin }), }, diff --git a/src/bindings/wasix-ts/tools-package/src/__tests__/api.test.ts b/src/bindings/wasix-ts/tools-package/src/__tests__/api.test.ts index b80b8f1dd..3ebf4ea2d 100644 --- a/src/bindings/wasix-ts/tools-package/src/__tests__/api.test.ts +++ b/src/bindings/wasix-ts/tools-package/src/__tests__/api.test.ts @@ -97,28 +97,16 @@ describe('WASIX tools public validation', () => { } }); - it('passes the database startup identity through unambiguous managed long options', async () => { + it('passes user arguments and input without manufacturing native connection arguments', async () => { toolRuntimeCalls.length = 0; - await pgDump(database).catch(() => undefined); + await pgDump(database, { args: ['--schema-only'] }).catch(() => undefined); await psql(database, { command: 'select 1' }).catch(() => undefined); - - expect(toolRuntimeCalls).toHaveLength(2); - for (const call of toolRuntimeCalls) { - expect(call.args).toContain('--username=-application user'); - expect(call.args).toContain('--dbname=-application database'); - expect(call.args).not.toContain('-application database'); - } - }); - - it('marks stdin scripts as non-interactive psql files', async () => { - toolRuntimeCalls.length = 0; - await psql(database, { script: 'select 1' }).catch(() => undefined); - await psql(database, { command: 'select 1' }).catch(() => undefined); - - expect(toolRuntimeCalls[0]?.args).toContain('--file=-'); - expect(toolRuntimeCalls[0]?.args).not.toContain('--command'); - expect(toolRuntimeCalls[1]?.args).not.toContain('--file=-'); - expect(toolRuntimeCalls[1]?.args).toContain('--command'); + await psql(database, { script: 'select 2' }).catch(() => undefined); + expect(toolRuntimeCalls[0]?.args).toEqual(['--schema-only']); + expect(toolRuntimeCalls[1]?.args).toEqual([]); + expect(toolRuntimeCalls[1]?.command).toBe('select 1'); + expect(toolRuntimeCalls[2]?.args).toEqual([]); + expect(new TextDecoder().decode(toolRuntimeCalls[2]?.stdin)).toBe('select 2'); }); it('strictly decodes successful output once at the public boundary', async () => { diff --git a/src/bindings/wasix-ts/tools-package/src/__tests__/wasix-ts-runtime.ts b/src/bindings/wasix-ts/tools-package/src/__tests__/wasix-ts-runtime.ts index 15bc47f25..f8dabacdc 100644 --- a/src/bindings/wasix-ts/tools-package/src/__tests__/wasix-ts-runtime.ts +++ b/src/bindings/wasix-ts/tools-package/src/__tests__/wasix-ts-runtime.ts @@ -1,15 +1,13 @@ -export const toolRuntimeCalls: Array> = []; +export const toolRuntimeCalls: Array< + Readonly<{ args: readonly string[]; command?: string; stdin?: Uint8Array }> +> = []; export const toolRuntimeResponses: Array< Readonly<{ exitCode: number; stdout: Uint8Array; stderr: Uint8Array }> > = []; -export function getWasixDatabaseIdentity(): Readonly<{ username: string; database: string }> { - return { username: '-application user', database: '-application database' }; -} - export async function runWasixToolProcess( _database: unknown, - options: Readonly<{ args: readonly string[] }>, + options: Readonly<{ args: readonly string[]; command?: string; stdin?: Uint8Array }>, ): Promise> { toolRuntimeCalls.push(options); const response = toolRuntimeResponses.shift(); diff --git a/src/bindings/wasix-ts/tools-package/src/index.ts b/src/bindings/wasix-ts/tools-package/src/index.ts index 5bda6b52c..0c42a3866 100644 --- a/src/bindings/wasix-ts/tools-package/src/index.ts +++ b/src/bindings/wasix-ts/tools-package/src/index.ts @@ -1,11 +1,9 @@ import tools from '@oliphaunt/liboliphaunt-wasix-tools'; import type { OliphauntDatabase } from '@oliphaunt/wasix-ts'; -import { getWasixDatabaseIdentity, runWasixToolProcess } from '@oliphaunt/wasix-ts/internal/tools'; +import { runWasixToolProcess } from '@oliphaunt/wasix-ts/internal/tools'; assertToolsCarrier(); -const VIRTUAL_TOOL_HOST = '127.0.0.1'; -const VIRTUAL_TOOL_PORT = '65432'; // PostgreSQL 18 getopt_long optstrings. A value-taking option owns the rest // of its token, so a managed-looking character inside that value stays data. const PG_DUMP_SHORT_OPTIONS = 'abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxXZ:'; @@ -90,16 +88,7 @@ export async function pgDump( PG_DUMP_SHORT_OPTIONS, PG_DUMP_VALUE_OPTIONS, ); - const identity = getWasixDatabaseIdentity(database); - return runTool('pg_dump', database, [ - ...args, - '--encoding=UTF8', - '--no-password', - `--username=${identity.username}`, - `--host=${VIRTUAL_TOOL_HOST}`, - `--port=${VIRTUAL_TOOL_PORT}`, - `--dbname=${identity.database}`, - ]); + return runTool('pg_dump', database, args); } /** @@ -125,24 +114,12 @@ export async function psql( if (command === undefined && script === undefined && args.length === 0) { throw new TypeError('psql requires non-interactive input through command, script, or args'); } - const inputArgs = - command !== undefined ? ['--command', command] : script !== undefined ? ['--file=-'] : []; - const identity = getWasixDatabaseIdentity(database); return runTool( 'psql', database, - [ - ...args, - '--no-psqlrc', - '--no-password', - '--set=ON_ERROR_STOP=1', - `--username=${identity.username}`, - `--host=${VIRTUAL_TOOL_HOST}`, - `--port=${VIRTUAL_TOOL_PORT}`, - `--dbname=${identity.database}`, - ...inputArgs, - ], + args, script === undefined ? undefined : new TextEncoder().encode(script), + command, ); } @@ -151,6 +128,7 @@ async function runTool( database: OliphauntDatabase, args: string[], stdin?: Uint8Array, + command?: string, ): Promise { const descriptor = name === 'pg_dump' ? tools.pgDump : tools.psql; let result: Awaited>; @@ -160,6 +138,7 @@ async function runTool( tool: descriptor, args, stdin, + command, }); } catch (cause) { const detail = cause instanceof Error ? cause.message : String(cause); diff --git a/src/bindings/wasix-ts/tools-package/src/wasix-ts-internal-shim.d.ts b/src/bindings/wasix-ts/tools-package/src/wasix-ts-internal-shim.d.ts index 855b58756..e2426b6eb 100644 --- a/src/bindings/wasix-ts/tools-package/src/wasix-ts-internal-shim.d.ts +++ b/src/bindings/wasix-ts/tools-package/src/wasix-ts-internal-shim.d.ts @@ -23,5 +23,6 @@ export function runWasixToolProcess( }>; args: readonly string[]; stdin?: Uint8Array; + command?: string; }>, ): Promise; diff --git a/src/bindings/wasix-ts/tools/entrypoint-consumer.ts b/src/bindings/wasix-ts/tools/entrypoint-consumer.ts new file mode 100644 index 000000000..61ccf4805 --- /dev/null +++ b/src/bindings/wasix-ts/tools/entrypoint-consumer.ts @@ -0,0 +1,29 @@ +import Browser from '@oliphaunt/wasix-ts/browser'; +import Native from '@oliphaunt/wasix-ts'; +import Direct from '@oliphaunt/wasix-ts/direct'; +import Worker from '@oliphaunt/wasix-ts/worker'; +import { openServer } from '@oliphaunt/wasix-ts/server'; +import { directory } from '@oliphaunt/wasix-ts/storage/node'; +import { indexedDB } from '@oliphaunt/wasix-ts/storage/indexed-db'; +import { opfs } from '@oliphaunt/wasix-ts/storage/opfs'; + +const disk = directory('/db'); +const origin = indexedDB('db'); +const bytes = new Uint8Array(); +void Browser.open({ storage: origin }); +void Browser.restore(opfs('restore'), bytes); +// @ts-expect-error Browser open cannot use host directories. +void Browser.open({ storage: disk }); +// @ts-expect-error Browser restore cannot use host directories. +void Browser.restore(disk, bytes); +for (const client of [Native, Direct, Worker]) { + void client.open({ storage: disk }); + void client.restore(disk, bytes); + // @ts-expect-error Node conditions must select native declarations. + void client.open({ storage: origin }); + // @ts-expect-error Native restore cannot use browser persistence. + void client.restore(origin, bytes); +} +void openServer({ storage: disk }); +// @ts-expect-error Servers are native-only. +void openServer({ storage: origin }); diff --git a/src/docs/content/sdk/typescript/index.mdx b/src/docs/content/sdk/typescript/index.mdx index b03a42cfc..ab049f85b 100644 --- a/src/docs/content/sdk/typescript/index.mdx +++ b/src/docs/content/sdk/typescript/index.mdx @@ -85,3 +85,11 @@ Use [Build With TypeScript](/docs/sdk/typescript/guide) for open/query, configuration, helper resolution, lifecycle, exact extensions, backup, restore, and troubleshooting. Use the [API reference](/docs/sdk/typescript/api-reference) for the public API map. + +## Select an execution mode by import + +`@oliphaunt/ts/direct` and `@oliphaunt/ts/broker` come with the same installation +and expose the same database, restore, and server APIs as the default import. +Neither accepts `topology` in `open`; the direct entrypoint also rejects +`brokerExecutable`, including when options arrive through a variable. The +default `@oliphaunt/ts` import continues to support runtime topology selection. diff --git a/src/docs/content/sdk/wasix-typescript/index.mdx b/src/docs/content/sdk/wasix-typescript/index.mdx index f57f09769..767c559c5 100644 --- a/src/docs/content/sdk/wasix-typescript/index.mdx +++ b/src/docs/content/sdk/wasix-typescript/index.mdx @@ -34,6 +34,12 @@ cross-origin isolated with COOP `same-origin` and COEP `require-corp`. ## Runtime Shape +Use `@oliphaunt/wasix-ts/browser` for an explicit browser import. Its `open` and +`restore` types reject directory storage; native entrypoints reject IndexedDB +and OPFS. Runtime validation also protects JavaScript callers. The `/browser` +entrypoint fails early on Node.js, Bun, and Deno. One package installation still +includes both hosts, but native entrypoints do not load the browser engine. + In browsers the root runs the database in the importing JavaScript realm. On Node-compatible hosts the root uses a dedicated Rust owner and keeps the event loop responsive. `@oliphaunt/wasix-ts/direct` opts into the lowest-overhead diff --git a/src/runtimes/liboliphaunt/wasix/tools/build-aot-target.sh b/src/runtimes/liboliphaunt/wasix/tools/build-aot-target.sh index d09c9f1ad..8d22b00c4 100755 --- a/src/runtimes/liboliphaunt/wasix/tools/build-aot-target.sh +++ b/src/runtimes/liboliphaunt/wasix/tools/build-aot-target.sh @@ -52,7 +52,7 @@ aot_test_command=( OLIPHAUNT_WASM_AOT_VERIFY=full OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT="$proof_root/extension-artifacts" cargo test -p oliphaunt-wasix --locked --no-default-features - --features extension-uuid-ossp,tools + --features extensions,tools --lib "$aot_test_filter" ) oliphaunt_assert_cargo_test_filter_count 4 "$aot_test_filter" "${aot_test_command[@]}" diff --git a/src/sdks/js/README.md b/src/sdks/js/README.md index 34ea9705d..98c94131f 100644 --- a/src/sdks/js/README.md +++ b/src/sdks/js/README.md @@ -22,6 +22,21 @@ console.log(result.rows[0]?.value); await db.close(); ``` +Both execution modes come with one installation. Select a mode through its import +path for stricter options: + +```ts +import Direct from '@oliphaunt/ts/direct'; +import Broker from '@oliphaunt/ts/broker'; + +await using direct = await Direct.open(); +await using broker = await Broker.open(); +``` + +These entrypoints share the root client and preserve its database, restore, and +server APIs. Neither accepts `topology`; `/direct` also rejects `brokerExecutable`. +Browser bundlers receive an early diagnostic directing them to the WASIX SDK. + Direct topology is the default. Set `topology: 'broker'` to place the embedded backend in a helper process while keeping the same database API. If that helper fails, the database object fails permanently; close it and explicitly open a new diff --git a/src/sdks/js/moon.yml b/src/sdks/js/moon.yml index 46e8078de..c57de8d61 100644 --- a/src/sdks/js/moon.yml +++ b/src/sdks/js/moon.yml @@ -88,6 +88,7 @@ tasks: rsync -a --exclude node_modules src/sdks/js/ target/liboliphaunt-sdk-check/oliphaunt-js/package-shape/src/sdks/js/ cp LICENSE THIRD_PARTY_NOTICES.md target/liboliphaunt-sdk-check/oliphaunt-js/package-shape/src/sdks/js/ node src/shared/js-core/tools/stage-package.mjs target/liboliphaunt-sdk-check/oliphaunt-js/package-shape/src/sdks/js src/shared/js-core + node src/shared/js-core/tools/check-typescript-consumer.mjs target/liboliphaunt-sdk-check/oliphaunt-js/package-shape/src/sdks/js src/sdks/js/tools/entrypoint-consumer.ts deps: - "oliphaunt-js:compile" inputs: @@ -96,6 +97,7 @@ tasks: group: "sources" - "@group(pnpm-workspace)" - "/src/shared/js-core/tools/stage-package.mjs" + - "/src/shared/js-core/tools/check-typescript-consumer.mjs" - "**/*" outputs: - "/target/liboliphaunt-sdk-check/oliphaunt-js/package-shape/src/sdks/js/**/*" diff --git a/src/sdks/js/package.json b/src/sdks/js/package.json index 12f4e87ca..7fa983425 100644 --- a/src/sdks/js/package.json +++ b/src/sdks/js/package.json @@ -49,6 +49,10 @@ "exports": { ".": { "types": "./lib/index.d.ts", + "node": "./lib/index.js", + "bun": "./lib/index.js", + "deno": "./lib/index.js", + "browser": "./lib/native-only.js", "default": "./lib/index.js" }, "./package.json": { @@ -57,6 +61,22 @@ "./storage/node": { "types": "./lib/storage/node.d.ts", "default": "./lib/storage/node.js" + }, + "./direct": { + "types": "./lib/direct.d.ts", + "node": "./lib/direct.js", + "bun": "./lib/direct.js", + "deno": "./lib/direct.js", + "browser": "./lib/native-only.js", + "default": "./lib/direct.js" + }, + "./broker": { + "types": "./lib/broker.d.ts", + "node": "./lib/broker.js", + "bun": "./lib/broker.js", + "deno": "./lib/broker.js", + "browser": "./lib/native-only.js", + "default": "./lib/broker.js" } }, "main": "lib/index.js", diff --git a/src/sdks/js/src/__tests__/entrypoints.test.ts b/src/sdks/js/src/__tests__/entrypoints.test.ts new file mode 100644 index 000000000..3625fabb8 --- /dev/null +++ b/src/sdks/js/src/__tests__/entrypoints.test.ts @@ -0,0 +1,32 @@ +import { expect, it, vi } from 'vitest'; +const shared = vi.hoisted(() => ({ open: vi.fn(), restore: vi.fn(), openServer: vi.fn() })); +vi.mock('../index.js', () => ({ Oliphaunt: shared })); +import direct, { type OpenConfig as DirectConfig } from '../direct.js'; +import broker, { type OpenConfig as BrokerConfig } from '../broker.js'; + +it('pins topology while sharing the root client and preserving restore and server APIs', async () => { + await direct.open({ username: 'alice' }); + expect(shared.open).toHaveBeenLastCalledWith({ username: 'alice', topology: 'direct' }); + await broker.open({ brokerExecutable: '/broker' }); + expect(shared.open).toHaveBeenLastCalledWith({ brokerExecutable: '/broker', topology: 'broker' }); + expect(direct.restore).toBe(shared.restore); + expect(broker.restore).toBe(shared.restore); + expect(direct.openServer).toBe(shared.openServer); + await expect(direct.open({ topology: 'broker' } as never)).rejects.toThrow('does not accept'); + await expect(direct.open({ brokerExecutable: '/broker' } as never)).rejects.toThrow( + 'does not accept', + ); + await expect(broker.open({ topology: 'direct' } as never)).rejects.toThrow('does not accept'); +}); + +export function checkModeTypes(): void { + const brokerOptions: BrokerConfig = { brokerExecutable: '/broker' }; + void broker.open(brokerOptions); + // @ts-expect-error Direct mode cannot accept broker configuration, even through a variable. + const directOptions: DirectConfig = brokerOptions; + void directOptions; + // @ts-expect-error An import chooses the topology. + void direct.open({ topology: 'broker' }); + // @ts-expect-error An import chooses the topology. + void broker.open({ topology: 'direct' }); +} diff --git a/src/sdks/js/src/broker.ts b/src/sdks/js/src/broker.ts new file mode 100644 index 000000000..735403e99 --- /dev/null +++ b/src/sdks/js/src/broker.ts @@ -0,0 +1,20 @@ +import { Oliphaunt as client } from './index.js'; +import type * as Types from './types.js'; +export * from './index.js'; + +export type OpenConfig = Omit & { topology?: never }; +export type OliphauntClient = Omit & { + open(config?: OpenConfig): Promise; +}; +export const Oliphaunt: OliphauntClient = { + ...client, + async open(config = {}) { + if (config.topology !== undefined) { + throw new TypeError( + '@oliphaunt/ts/broker does not accept topology; select the execution mode through the import path', + ); + } + return client.open({ ...config, topology: 'broker' }); + }, +}; +export default Oliphaunt; diff --git a/src/sdks/js/src/direct.ts b/src/sdks/js/src/direct.ts new file mode 100644 index 000000000..03342ff81 --- /dev/null +++ b/src/sdks/js/src/direct.ts @@ -0,0 +1,23 @@ +import { Oliphaunt as client } from './index.js'; +import type * as Types from './types.js'; +export * from './index.js'; + +export type OpenConfig = Omit & { + topology?: never; + brokerExecutable?: never; +}; +export type OliphauntClient = Omit & { + open(config?: OpenConfig): Promise; +}; +export const Oliphaunt: OliphauntClient = { + ...client, + async open(config = {}) { + if (config.topology !== undefined || config.brokerExecutable !== undefined) { + throw new TypeError( + '@oliphaunt/ts/direct does not accept topology or brokerExecutable; select the execution mode through the import path', + ); + } + return client.open({ ...config, topology: 'direct' }); + }, +}; +export default Oliphaunt; diff --git a/src/sdks/js/src/native-only.ts b/src/sdks/js/src/native-only.ts new file mode 100644 index 000000000..1f1358f03 --- /dev/null +++ b/src/sdks/js/src/native-only.ts @@ -0,0 +1,4 @@ +throw new Error( + '@oliphaunt/ts requires Node.js, Bun, or Deno; use @oliphaunt/wasix-ts/browser in a browser', +); +export {}; diff --git a/src/sdks/js/tools/entrypoint-consumer.ts b/src/sdks/js/tools/entrypoint-consumer.ts new file mode 100644 index 000000000..6daefaa7f --- /dev/null +++ b/src/sdks/js/tools/entrypoint-consumer.ts @@ -0,0 +1,17 @@ +import Default from '@oliphaunt/ts'; +import Direct from '@oliphaunt/ts/direct'; +import Broker from '@oliphaunt/ts/broker'; +import { directory } from '@oliphaunt/ts/storage/node'; +const config = { storage: directory('/db') }; +void Default.open({ ...config, topology: 'broker' }); +void Direct.open(config); +void Broker.open({ ...config, brokerExecutable: '/broker' }); +void Direct.restore(config.storage, new Uint8Array()); +void Broker.openServer(); +// @ts-expect-error The import already selected direct mode. +void Direct.open({ topology: 'broker' }); +// @ts-expect-error The import already selected broker mode. +void Broker.open({ topology: 'direct' }); +const brokerOptions = { ...config, brokerExecutable: '/broker' }; +// @ts-expect-error Broker-only options cannot leak through variables into direct mode. +void Direct.open(brokerOptions); diff --git a/src/sdks/swift/tools/render-extension-products.mjs b/src/sdks/swift/tools/render-extension-products.mjs index 8dba618d8..d72de422d 100755 --- a/src/sdks/swift/tools/render-extension-products.mjs +++ b/src/sdks/swift/tools/render-extension-products.mjs @@ -850,9 +850,6 @@ export async function writeBundledContrib(selection, outputDir) { if (selection.extensions.some(extension => extension.product !== "oliphaunt-extension-contrib-pg18")) { fail("the base Swift SDK can bundle only runtime-owned contrib extensions"); } - if (selection.nativeDependencies.length > 0) { - fail("bundled contrib must not introduce separately owned native dependency products"); - } const prefix = "generated/swiftpm/contrib"; const generated = path.join(outputDir, prefix); const swiftRoot = path.join(outputDir, "src/sdks/swift/Sources/Oliphaunt"); @@ -860,6 +857,11 @@ export async function writeBundledContrib(selection, outputDir) { const dependencies = []; const imports = ["import Foundation", "import COliphaunt"]; const cases = []; + for (const dependency of selection.nativeDependencies) { + await copyLocalBinaryArtifact(dependency.asset, dependency.binaryTarget, generated); + const target = binaryTargetIR(dependency.binaryTarget, dependency.asset, true); + targets.push({ ...target, path: `${prefix}/${target.path}` }); + } for (const extension of selection.extensions) { if (extension.cTarget) { await copyLocalBinaryArtifact(extension.asset, extension.binaryTarget, generated); @@ -881,7 +883,7 @@ export async function writeBundledContrib(selection, outputDir) { ` throw OliphauntError.engine("missing bundled contrib resources for ${extension.sqlName}")\n }\n` + ` try OliphauntStaticExtensionRegistry.register(\n` + ` product: ${swiftString(extension.product)}, sqlName: ${swiftString(extension.sqlName)}, version: ${swiftString(extension.version)},\n` + - ` dependencies: [${extension.dependencies.map(swiftString).join(", ")}], nativeDependencies: [],\n` + + ` dependencies: [${extension.dependencies.map(swiftString).join(", ")}], nativeDependencies: [${extension.nativeDependencies.map(({ name }) => swiftString(name)).join(", ")}],\n` + ` sharedPreloadLibraries: [${extension.sharedPreloadLibraries.map(swiftString).join(", ")}],\n` + ` nativeModuleStem: ${extension.nativeModuleStem === null ? "nil" : swiftString(extension.nativeModuleStem)},\n` + ` resourceRoot: root, descriptor: ${extension.cFunction ? `${extension.cFunction}()` : "nil"}\n )`); diff --git a/src/sdks/swift/tools/swift-carrier-resolver.test.mjs b/src/sdks/swift/tools/swift-carrier-resolver.test.mjs index e50b8538c..9b97ea26b 100755 --- a/src/sdks/swift/tools/swift-carrier-resolver.test.mjs +++ b/src/sdks/swift/tools/swift-carrier-resolver.test.mjs @@ -691,6 +691,27 @@ async function main() { assert.equal((await fs.stat(path.join(bundledOutput, "src/sdks/swift/Sources/Oliphaunt/ContribResources/cube/Resources/extension-artifact/manifest.properties"))).isFile(), true); assert.match(renderSwiftTargets(bundled.targets), /generated\/swiftpm\/contrib\/Artifacts/u); + // Exercise the bundled writer with a native dependency shared by two members. + // Dependency archive resolution is already qualified by the PostGIS carrier above. + const localPostgisInput = await resolveSwiftCarrierSelection({ carrierFile: carrier, cacheDir: cache, + allowFileUrls: true, localBinaryTargets: true, basePackageVersion: "0.1.0", extensions: ["postgis"] }); + const localPostgis = validateSelection(localPostgisInput, root, { allowFileUrls: true, localBinaryTargets: true }); + const nativeDependency = localPostgis.nativeDependencies[0]; + const withNativeDependency = { + ...contrib, nativeDependencies: [nativeDependency], + extensions: contrib.extensions.map(extension => ({ ...extension, nativeDependencies: [nativeDependency] })), + }; + const nativeBundledOutput = path.join(root, "bundled-native-dependency"); + const nativeBundled = await writeBundledContrib(withNativeDependency, nativeBundledOutput); + const dependencyTargets = nativeBundled.targets.filter(target => target.name === nativeDependency.binaryTarget); + assert.equal(dependencyTargets.length, 1); + const nativeBundledSource = await fs.readFile(path.join(nativeBundledOutput, "src/sdks/swift/Sources/Oliphaunt/OliphauntBundledContrib.swift"), "utf8"); + assert.ok(nativeBundledSource.includes(`nativeDependencies: ["${nativeDependency.name}"]`)); + assert.equal((await fs.stat(path.join(nativeBundledOutput, dependencyTargets[0].path, "Info.plist"))).isFile(), true); + for (const extension of withNativeDependency.extensions) { + assert.ok(nativeBundled.targets.find(target => target.name === extension.cTarget).dependencies.includes(nativeDependency.binaryTarget)); + } + const standalone = path.join(root, "standalone-pgtap"); run(process.execPath, [generator, "--carrier", carrier, "--extension-carrier", pgtapCarrier, "--extensions", "pgtap", "--release-product", "oliphaunt-extension-pgtap", diff --git a/src/shared/js-core/tools/check-typescript-consumer.mjs b/src/shared/js-core/tools/check-typescript-consumer.mjs new file mode 100644 index 000000000..f44df722f --- /dev/null +++ b/src/shared/js-core/tools/check-typescript-consumer.mjs @@ -0,0 +1,21 @@ +#!/usr/bin/env node +import { copyFileSync, rmSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; + +// Resolve only the compiler from the checkout. All SDK imports resolve through +// the staged package's own exports and declarations, as they do after installation. +const [packageDir, fixture] = process.argv.slice(2).map(value => path.resolve(value)); +const compiler = createRequire(new URL('../../../sdks/js/package.json', import.meta.url)).resolve('typescript/bin/tsc'); +const consumer = path.join(packageDir, 'consumer.mts'); +copyFileSync(fixture, consumer); +try { + const result = spawnSync(process.execPath, [compiler, '--noEmit', '--strict', '--skipLibCheck', + '--target', 'ES2022', '--module', 'NodeNext', '--moduleResolution', 'NodeNext', + '--lib', 'ES2023,DOM,ESNext.Disposable', consumer], { stdio: 'inherit' }); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(`installed TypeScript consumer failed: ${fixture}`); +} finally { + rmSync(consumer, { force: true }); +} diff --git a/tools/integration/wasix-ts/moon.yml b/tools/integration/wasix-ts/moon.yml index c0fb23bb4..f9c0bfb71 100644 --- a/tools/integration/wasix-ts/moon.yml +++ b/tools/integration/wasix-ts/moon.yml @@ -37,6 +37,7 @@ tasks: node tools/integration/wasix-ts/smoke-browser.mjs --package-only deps: - "liboliphaunt-wasix:runtime-portable" + - "liboliphaunt-wasix:runtime-aot" - "oliphaunt-wasix-tools-ts:package" - "oliphaunt-wasix-ts:package" - "release-tools:wasix-napi-runtime" @@ -54,6 +55,8 @@ tasks: - "/tools/perf/wasix-node/installed-closure.mjs" - "/tools/perf/wasix-node/plan.mjs" - "/tools/release/wasix-*.mjs" + - "/tools/release/build-extension-ci-artifacts.mjs" + - "/tools/release/package-extension-release-carriers.mjs" options: cache: false runFromWorkspaceRoot: true diff --git a/tools/integration/wasix-ts/packed-node-fixture.mjs b/tools/integration/wasix-ts/packed-node-fixture.mjs index 262f8c404..09e3113f5 100644 --- a/tools/integration/wasix-ts/packed-node-fixture.mjs +++ b/tools/integration/wasix-ts/packed-node-fixture.mjs @@ -1,6 +1,6 @@ import { execFile } from 'node:child_process'; import { createHash } from 'node:crypto'; -import { cp, mkdir, readFile, readdir, writeFile } from 'node:fs/promises'; +import { cp, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { arch, platform } from 'node:os'; import { dirname, isAbsolute, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -50,7 +50,6 @@ export async function createPackedWasixConsumer({ await readFile(resolve(repositoryRoot, '.release-please-manifest.json'), 'utf8'), ); const runtimeVersion = releaseVersions['src/runtimes/liboliphaunt/wasix']; - const extensionVersion = releaseVersions['src/extensions/external/pgtap']; const tarballs = resolve(scratch, 'tarballs'); await mkdir(tarballs, { recursive: true }); @@ -64,6 +63,10 @@ export async function createPackedWasixConsumer({ const toolsCarrier = includeTools ? await packToolsCarrier({ scratch, tarballs, runtimeVersion }) : undefined; + const toolsAot = + includeTools && includeNative + ? await packToolsAotCarrier({ scratch, tarballs, runtimeVersion }) + : undefined; const toolsFacade = includeTools ? await packToolsFacade({ scratch, tarballs, bindingVersion: binding.version }) : undefined; @@ -73,14 +76,23 @@ export async function createPackedWasixConsumer({ const runtime = useStubRuntime ? await packStubRuntime({ scratch, tarballs, runtimeVersion }) : await packRuntime({ scratch, tarballs, runtimeVersion }); - const extension = includePgtap - ? await packPgtap({ scratch, tarballs, runtimeVersion, extensionVersion }) - : undefined; + const extensionPackages = await packExtensions({ scratch, includePgtap }); + const contrib = extensionPackages.find( + (row) => row.name === '@oliphaunt/extension-contrib-pg18-wasix', + ); + const extension = extensionPackages.find( + (row) => row.name === '@oliphaunt/extension-pgtap-wasix', + ); + if (contrib === undefined || (includePgtap && extension === undefined)) + throw new Error( + `same-candidate WASIX extension packages are missing; staged: ${extensionPackages.map((row) => row.name).join(', ')}`, + ); const consumer = resolve(scratch, 'consumer'); await mkdir(consumer, { recursive: true }); const dependencies = { [runtime.name]: pathToFileURL(runtime.file).href, [binding.name]: pathToFileURL(binding.file).href, + [contrib.name]: pathToFileURL(contrib.file).href, }; if (nativeCarrier !== undefined) { dependencies[nativeCarrier.name] = pathToFileURL(nativeCarrier.file).href; @@ -91,6 +103,7 @@ export async function createPackedWasixConsumer({ if (toolsCarrier !== undefined) { dependencies[toolsCarrier.name] = pathToFileURL(toolsCarrier.file).href; } + if (toolsAot !== undefined) dependencies[toolsAot.name] = pathToFileURL(toolsAot.file).href; if (toolsFacade !== undefined) { dependencies[toolsFacade.name] = pathToFileURL(toolsFacade.file).href; } @@ -108,6 +121,8 @@ export async function createPackedWasixConsumer({ extension, toolsCarrier, toolsFacade, + toolsAot, + contrib, ].filter(Boolean); await writeFile( resolve(consumer, 'pnpm-workspace.yaml'), @@ -133,6 +148,8 @@ export async function createPackedWasixConsumer({ packages: { binding, runtime, + contrib, + ...(toolsAot === undefined ? {} : { toolsAot }), ...(nativeCarrier === undefined ? {} : { nativeCarrier }), ...(extension === undefined ? {} : { extension }), ...(toolsCarrier === undefined ? {} : { toolsCarrier }), @@ -339,44 +356,88 @@ function nativeCarrierIdentity(currentPlatform, currentArch) { ); } +function stageResources(options) { + return runFixtureCommand( + resolve(repositoryRoot, 'tools/dev/bun.sh'), + ['tools/integration/wasix-ts/stage-resource-packages.mjs', JSON.stringify(options)], + repositoryRoot, + ); +} + async function packToolsCarrier({ scratch, tarballs, runtimeVersion }) { - requireReleaseVersion(runtimeVersion, 'src/runtimes/liboliphaunt/wasix'); const staging = resolve(scratch, 'tools-carrier'); - const assets = resolve(staging, 'assets'); - await mkdir(assets, { recursive: true }); - const manifest = JSON.parse(await readFile(resolve(assetRoot, 'manifest.json'), 'utf8')); - const descriptors = {}; - for (const [field, key, filename] of [ - ['pgDump', 'pg-dump', 'pg_dump.wasix.wasm'], - ['psql', 'psql', 'psql.wasix.wasm'], - ]) { - const row = manifest[key]; - const bytes = await readFile(resolve(assetRoot, row.path)); - requireDigest(bytes, row.sha256, row.path); - if (bytes.length !== row.size) throw new Error(`${row.path} size differs from its manifest`); - await writeFile(resolve(assets, filename), bytes); - descriptors[field] = { - name: row.name, - sha256: row.sha256, - size: row.size, - filename, - }; - } - const tool = ({ name, sha256: digest, size, filename }) => - `Object.freeze({ name: ${JSON.stringify(name)}, sha256: ${JSON.stringify(digest)}, size: ${size}, source: new URL('./assets/${filename}', import.meta.url).href })`; - await writeFile( - resolve(staging, 'index.js'), - `export default Object.freeze({\n schema: 'oliphaunt-wasix-tools-v1',\n product: 'oliphaunt-wasix-tools',\n version: ${JSON.stringify(runtimeVersion)},\n runtimeProduct: 'liboliphaunt-wasix',\n runtimeVersion: ${JSON.stringify(runtimeVersion)},\n pgDump: ${tool(descriptors.pgDump)},\n psql: ${tool(descriptors.psql)},\n});\n`, - ); - await writeJson(resolve(staging, 'package.json'), { - name: '@oliphaunt/liboliphaunt-wasix-tools', + await stageResources({ + kind: 'tools', version: runtimeVersion, - type: 'module', - exports: { '.': './index.js' }, + packageDir: staging, + assetDirectory: assetRoot, + }); + return pack(staging, tarballs); +} + +async function packToolsAotCarrier({ scratch, tarballs, runtimeVersion }) { + const { target } = nativeCarrierIdentity(platform(), arch()); + const staging = resolve(scratch, 'tools-aot'); + await stageResources({ + kind: 'tools-aot', + version: runtimeVersion, + target, + packageDir: staging, + aotArtifactDirectory: resolve(repositoryRoot, 'target/oliphaunt-wasix/aot'), }); return pack(staging, tarballs); } +async function packExtensions({ scratch, includePgtap }) { + // The canonical artifact builder deliberately keeps its output in the checkout. + const work = await mkdtemp(resolve(repositoryRoot, 'target/wasix-consumer-resources-')); + try { + const artifactRoot = resolve(work, 'artifacts'); + await runFixtureCommand( + resolve(repositoryRoot, 'tools/dev/bun.sh'), + [ + 'tools/release/build-extension-ci-artifacts.mjs', + '--output-root', + artifactRoot, + '--family', + 'wasix', + '--require-wasix', + 'oliphaunt-extension-contrib-pg18', + ...(includePgtap ? ['oliphaunt-extension-pgtap'] : []), + ], + repositoryRoot, + 120_000, + { OLIPHAUNT_WASIX_GENERATED_ASSET_ROOT: assetRoot }, + ); + const stagingRoot = resolve(work, 'packages'); + await stageResources({ kind: 'extensions', artifactRoot, stagingRoot }); + const tarballRoot = resolve(stagingRoot, 'tarballs'); + return await Promise.all( + (await readdir(tarballRoot, { recursive: true })) + .filter((name) => name.endsWith('.tgz')) + .map(async (name) => { + const source = resolve(tarballRoot, name); + const file = resolve(scratch, 'tarballs', source.split(/[\\/]/).at(-1)); + await cp(source, file); + const bytes = await readFile(file); + const entries = readPortableArchiveEntries(file); + const manifest = JSON.parse( + Buffer.from(entries.get('package/package.json').data()).toString('utf8'), + ); + return { + file, + name: manifest.name, + version: manifest.version, + sha256: sha256(bytes), + size: bytes.length, + }; + }), + ); + } finally { + await rm(work, { recursive: true, force: true }); + } +} + async function packToolsFacade({ scratch, tarballs, bindingVersion }) { const source = resolve(packageRoot, 'tools-package'); const staging = resolve(scratch, 'tools-facade'); @@ -535,75 +596,6 @@ export function parseBuildProfile(value) { return configuration; } -async function packPgtap({ scratch, tarballs, runtimeVersion, extensionVersion }) { - requireReleaseVersion(extensionVersion, 'src/extensions/external/pgtap'); - const staging = resolve(scratch, 'pgtap'); - const assets = resolve(staging, 'assets'); - await mkdir(assets, { recursive: true }); - const manifest = JSON.parse(await readFile(resolve(assetRoot, 'manifest.json'), 'utf8')); - const row = manifest.extensions.find((candidate) => candidate['sql-name'] === 'pgtap'); - if (row === undefined) throw new Error('WASIX manifest has no pgtap carrier'); - await cp(resolve(assetRoot, row.archive), resolve(assets, 'pgtap.tar.zst')); - const lifecycle = row.lifecycle; - const carrier = { - product: 'oliphaunt-extension-pgtap', - version: extensionVersion, - sqlName: 'pgtap', - archive: row.archive, - sha256: row.sha256, - size: row.size, - install: { - schema: 'oliphaunt-wasix-extension-install-v1', - name: row.name, - nativeModule: null, - nativeModules: [], - dependencies: row.dependencies, - coreExportsRequired: row['core-exports-required'], - loadOrder: row['load-order'], - lifecycle: { - createExtension: lifecycle['create-extension'], - createSchema: lifecycle['create-schema'], - loadSql: lifecycle['load-sql'], - postCreateSql: lifecycle['post-create-sql'], - startupConfig: lifecycle['startup-config'], - preloadRequired: lifecycle['preload-required'], - restartRequired: lifecycle['restart-required'], - sharedMemoryRequired: lifecycle['shared-memory-required'], - }, - installedFiles: row['installed-files'], - unresolvedImports: row['unresolved-imports'], - }, - }; - const descriptor = { - schema: 'oliphaunt-wasix-extension-v1', - runtime: 'wasix', - product: carrier.product, - version: carrier.version, - compatibility: { - extensionRuntimeContract: 'oliphaunt-extension-runtime-contract-v1', - postgresMajor: manifest.runtime['postgres-version'].split('.')[0], - wasixRuntimeProduct: 'liboliphaunt-wasix', - wasixRuntimeVersion: runtimeVersion, - }, - sqlName: 'pgtap', - carriers: [carrier], - }; - await writeFile( - resolve(staging, 'index.js'), - `const descriptor = ${JSON.stringify(descriptor, null, 2)}; -descriptor.carriers[0].source = new URL('./assets/pgtap.tar.zst', import.meta.url); -export default descriptor; -`, - ); - await writeJson(resolve(staging, 'package.json'), { - name: '@oliphaunt/extension-pgtap-wasix', - version: extensionVersion, - type: 'module', - exports: { '.': './index.js' }, - }); - return pack(staging, tarballs); -} - async function pack(directory, tarballs) { const manifest = JSON.parse(await readFile(resolve(directory, 'package.json'), 'utf8')); const { stdout } = await runFixtureCommand( diff --git a/tools/integration/wasix-ts/stage-resource-packages.mjs b/tools/integration/wasix-ts/stage-resource-packages.mjs new file mode 100644 index 000000000..c031ecc87 --- /dev/null +++ b/tools/integration/wasix-ts/stage-resource-packages.mjs @@ -0,0 +1,26 @@ +#!/usr/bin/env bun +import path from 'node:path'; +import { stageExtensionWasixNpmPackages } from '../../release/package-extension-release-carriers.mjs'; +import { + stageWasixToolsNpmCarrier, + stageWasixToolsAotNpmCarrier, +} from '../../release/wasix-tools-npm-carrier.mjs'; +import { AOT_TARGET_TRIPLES } from '../../release/wasix-cargo-artifact-contract.mjs'; + +// Release metadata uses Bun. Keep the Node/Bun/Deno consumer harness host-neutral. +const { kind, ...options } = JSON.parse(Bun.argv[2]); +if (kind === 'tools') stageWasixToolsNpmCarrier(options); +else if (kind === 'tools-aot') + stageWasixToolsAotNpmCarrier({ + ...options, + aotArtifactDirectory: path.join( + options.aotArtifactDirectory, + AOT_TARGET_TRIPLES[options.target], + ), + }); +else if (kind === 'extensions') { + stageExtensionWasixNpmPackages([options.artifactRoot], options.stagingRoot, { + staged: [], + skipped: [], + }); +} else throw new Error(`unknown WASIX resource package kind: ${kind}`); diff --git a/tools/policy/sdk-manifest.toml b/tools/policy/sdk-manifest.toml index 2548d5671..7a2cd9a28 100644 --- a/tools/policy/sdk-manifest.toml +++ b/tools/policy/sdk-manifest.toml @@ -77,6 +77,14 @@ execution_owner = "caller" main_safe = false topologies = ["wasix-direct"] +[[sdks.wasix-typescript.surfaces]] +id = "browser" +entrypoint = "@oliphaunt/wasix-ts/browser" +calling_contract = "async" +execution_owner = "caller" +main_safe = false +topologies = ["wasix-direct"] + [[sdks.wasix-typescript.surfaces]] id = "default" entrypoint = "@oliphaunt/wasix-ts" @@ -182,3 +190,19 @@ calling_contract = "async" execution_owner = "sdk-runtime" main_safe = true topologies = ["native-direct", "native-broker", "native-server"] + +[[sdks.typescript.surfaces]] +id = "direct" +entrypoint = "@oliphaunt/ts/direct" +calling_contract = "async" +execution_owner = "sdk-runtime" +main_safe = true +topologies = ["native-direct", "native-server"] + +[[sdks.typescript.surfaces]] +id = "broker" +entrypoint = "@oliphaunt/ts/broker" +calling_contract = "async" +execution_owner = "sdk-runtime" +main_safe = true +topologies = ["native-broker", "native-server"] diff --git a/tools/release/product-task-model.test.mjs b/tools/release/product-task-model.test.mjs index ebeda7ac5..06c9fc4bd 100644 --- a/tools/release/product-task-model.test.mjs +++ b/tools/release/product-task-model.test.mjs @@ -187,6 +187,7 @@ test("WASIX TypeScript products build packages and root integration consumes the assert.deepEqual( [...dependencies].sort(), [ + "liboliphaunt-wasix:runtime-aot", "liboliphaunt-wasix:runtime-portable", "oliphaunt-wasix-tools-ts:package", "oliphaunt-wasix-ts:package", diff --git a/tools/release/stage-react-native-resource-packages.mjs b/tools/release/stage-react-native-resource-packages.mjs index c5fef17d2..e97147ca9 100644 --- a/tools/release/stage-react-native-resource-packages.mjs +++ b/tools/release/stage-react-native-resource-packages.mjs @@ -3,7 +3,7 @@ import { readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; import { nativeExtensionCarrierLegal, writeExtensionMetaPackage } from "./package-extension-release-carriers.mjs"; -import { currentProductVersionSync, extensionRegistryPackageTargetSets } from "./release-artifact-targets.mjs"; +import { currentProductVersionSync } from "./release-artifact-targets.mjs"; import { stageLiboliphauntIcuNpmPayload } from "./package-release-carriers.mjs"; export function stageReactNativeResourcePackages({ carrier, selected, icu, outputDir, project, workspace, platform = "ios" }) { @@ -40,7 +40,8 @@ export function stageReactNativeResourcePackages({ carrier, selected, icu, outpu const directory = path.join(outputDir, product); writeExtensionMetaPackage(directory, { product, version, members: sqlNames, - targets: extensionRegistryPackageTargetSets(product, "mobile source qualification").npmTargets, + // Mobile payloads come from the same candidate through Gradle or SwiftPM. + targets: [], iosCarrier: platform === "android" ? undefined : { ...carrier, extensions: members, carriers: carrier.carriers.filter(row => row.product === product), diff --git a/tools/release/stage-react-native-resource-packages.test.mjs b/tools/release/stage-react-native-resource-packages.test.mjs index 9228497d0..550227868 100644 --- a/tools/release/stage-react-native-resource-packages.test.mjs +++ b/tools/release/stage-react-native-resource-packages.test.mjs @@ -1,3 +1,5 @@ +import { spawnSync } from "node:child_process"; +import { portableCommand } from "../../src/runtimes/wasix-napi/tools/portable-command.mjs"; import { test, expect } from "bun:test"; import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync } from "node:fs"; import os from "node:os"; @@ -19,7 +21,7 @@ test("mobile source qualification stages installed package ownership independent const project = path.join(root, "app"); mkdirSync(project); const workspace = path.join(root, "pnpm-workspace.yaml"); - writeFileSync(workspace, 'packages: ["app"]\n'); + writeFileSync(workspace, 'packages: ["app"]\nminimumReleaseAge: 1440\n'); writeFileSync(path.join(project, "package.json"), JSON.stringify({ dependencies: { react: "19.0.0", "@oliphaunt/extension-pgtap": "0.1.0" } })); const args = { carrier, selected: ["vector"], icu: false, outputDir: path.join(root, "packages"), project, workspace }; const dependencies = stageReactNativeResourcePackages(args); @@ -29,6 +31,8 @@ test("mobile source qualification stages installed package ownership independent expect(manifest.dependencies["@oliphaunt/extension-pgtap"]).toBeUndefined(); expect(Bun.YAML.parse(readFileSync(workspace, "utf8")).overrides).toEqual(dependencies); const vectorPackage = JSON.parse(readFileSync(path.join(root, "packages", vector, "package.json"), "utf8")); + expect(vectorPackage.optionalDependencies ?? {}).toEqual({}); + expect(Bun.YAML.parse(readFileSync(workspace, "utf8")).minimumReleaseAge).toBe(1440); expect(vectorPackage.oliphaunt.members).toEqual(["vector"]); expect(vectorPackage.exports["."]["react-native"]).toBeDefined(); const installed = JSON.parse(readFileSync(path.join(root, "packages", contrib, "package.json"), "utf8")); @@ -38,6 +42,14 @@ test("mobile source qualification stages installed package ownership independent const androidPackage = JSON.parse(readFileSync(path.join(root, "packages", vector, "package.json"), "utf8")); expect(androidPackage.exports["."]["react-native"]).toBeDefined(); expect(androidPackage.exports["./ios-carriers"]).toBeUndefined(); + const isolatedApp = JSON.parse(readFileSync(path.join(project, "package.json"), "utf8")); + delete isolatedApp.dependencies.react; + writeFileSync(path.join(project, "package.json"), JSON.stringify(isolatedApp)); + const install = portableCommand("pnpm", ["install", "--offline", "--ignore-scripts", "--no-frozen-lockfile"]); + const installedOffline = spawnSync(install.command, install.args, { cwd: root, encoding: "utf8", timeout: 30000 }); + expect(installedOffline.error).toBeUndefined(); + expect(installedOffline.status, installedOffline.stdout + installedOffline.stderr).toBe(0); + expect(() => stageReactNativeResourcePackages({ ...args, selected: ["missing"] })).toThrow("missing selected carrier"); expect(() => stageReactNativeResourcePackages({ ...args, carrier: { ...carrier, extensions: rows(vector) } })).toThrow("missing the base contrib"); } finally { diff --git a/tools/release/wasix-tools-aot-npm.test.mjs b/tools/release/wasix-tools-aot-npm.test.mjs index cec14e31a..3cbdf2c3e 100644 --- a/tools/release/wasix-tools-aot-npm.test.mjs +++ b/tools/release/wasix-tools-aot-npm.test.mjs @@ -6,7 +6,7 @@ import path from 'node:path'; import { zstdCompressSync } from 'node:zlib'; import { createDeterministicTar } from './cargo-source-package.mjs'; import { canonicalWasixAotMetadata } from './wasix-aot-manifest.mjs'; -import { stageWasixToolsAotNpmCarrier } from './wasix-tools-npm-carrier.mjs'; +import { stageWasixToolsAotNpmCarrier, stageWasixToolsNpmCarrier } from './wasix-tools-npm-carrier.mjs'; test('the optional tools host package contains only pg_dump and psql from the verified runtime release', () => { const root = mkdtempSync(path.join(tmpdir(), 'oliphaunt-tools-aot-')); @@ -35,9 +35,37 @@ test('the optional tools host package contains only pg_dump and psql from the ve const packageDir = path.join(root, 'package'); const manifest = stageWasixToolsAotNpmCarrier({ version: '0.2.0', target: 'linux-x64-gnu', packageDir, aotReleaseArchive: archive }); + const fromDirectory = stageWasixToolsAotNpmCarrier({ version: '0.2.0', target: 'linux-x64-gnu', + packageDir: path.join(root, 'from-directory'), aotArtifactDirectory: source }); + expect(fromDirectory).toEqual(manifest); + expect(readFileSync(path.join(root, 'from-directory', 'aot-manifest.json'))).toEqual(readFileSync(path.join(packageDir, 'aot-manifest.json'))); expect(manifest.name).toBe('@oliphaunt/liboliphaunt-wasix-tools-linux-x64-gnu'); expect(existsSync(path.join(packageDir, '0.bin.zst'))).toBe(false); expect(JSON.parse(readFileSync(path.join(packageDir, 'aot-manifest.json'))).artifacts.map(row => row.name)) .toEqual(['tool:pg_dump', 'tool:psql']); } finally { rmSync(root, { recursive: true, force: true }); } }); + +test('portable tools staged from producer files retain the published manifest and reject changed bytes', () => { + const root = mkdtempSync(path.join(tmpdir(), 'oliphaunt-portable-tools-')); + try { + const assets = path.join(root, 'assets'); + mkdirSync(path.join(assets, 'bin'), { recursive: true }); + const manifest = {}; + for (const [key, name] of [['pg-dump', 'pg_dump'], ['psql', 'psql']]) { + const bytes = Buffer.from(`verified-${name}`); + const relative = `bin/${name}.wasix.wasm`; + writeFileSync(path.join(assets, relative), bytes); + manifest[key] = { path: relative, sha256: createHash('sha256').update(bytes).digest('hex'), size: bytes.length }; + } + writeFileSync(path.join(assets, 'manifest.json'), JSON.stringify(manifest)); + const packageDir = path.join(root, 'package'); + stageWasixToolsNpmCarrier({ version: '0.2.0', assetDirectory: assets, packageDir }); + const published = JSON.parse(readFileSync(path.join(packageDir, 'package.json'))); + expect(published.oliphaunt.kind).toBe('wasix-tools'); + expect(published.oliphaunt.tools.pg_dump.sha256).toBe(manifest['pg-dump'].sha256); + expect(published.optionalDependencies['@oliphaunt/liboliphaunt-wasix-tools-linux-x64-gnu']).toBe('0.2.0'); + writeFileSync(path.join(assets, 'bin/psql.wasix.wasm'), 'tampered'); + expect(() => stageWasixToolsNpmCarrier({ version: '0.2.0', assetDirectory: assets, packageDir })).toThrow('differs from its producer manifest'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/release/wasix-tools-npm-carrier.mjs b/tools/release/wasix-tools-npm-carrier.mjs index 34bdc9322..1acbad0f3 100644 --- a/tools/release/wasix-tools-npm-carrier.mjs +++ b/tools/release/wasix-tools-npm-carrier.mjs @@ -3,6 +3,7 @@ import { createHash } from 'node:crypto'; import { lstatSync, + readFileSync, mkdirSync, rmSync, statSync, @@ -67,30 +68,44 @@ function regularArchive(file) { } } -export function wasixToolsNpmInputs({ portableReleaseArchive }) { - const archive = path.resolve(portableReleaseArchive); - regularArchive(archive); - validatePortableReleaseAsset(archive); - const entries = readPortableArchiveEntries(archive); +export function wasixToolsNpmInputs({ portableReleaseArchive, assetDirectory }) { + if ((portableReleaseArchive === undefined) === (assetDirectory === undefined)) { + fail('tools staging requires exactly one release archive or producer asset directory'); + } + let entries; + let manifest; + if (portableReleaseArchive !== undefined) { + const archive = path.resolve(portableReleaseArchive); + regularArchive(archive); + validatePortableReleaseAsset(archive); + entries = readPortableArchiveEntries(archive); + } else { + manifest = JSON.parse(readFileSync(path.join(assetDirectory, 'manifest.json'), 'utf8')); + } const tools = {}; for (const [descriptorName, spec] of Object.entries(RELEASE_TOOLS)) { - const bytes = requiredEntry(entries, spec.member, archive); - tools[descriptorName] = Object.freeze({ - name: spec.name, - sha256: sha256(bytes), - size: bytes.length, - bytes, - }); + const relative = `bin/${spec.name}.wasix.wasm`; + const bytes = entries === undefined + ? readFileSync(path.join(assetDirectory, relative)) + : requiredEntry(entries, spec.member, portableReleaseArchive); + const digest = sha256(bytes); + if (manifest !== undefined) { + const row = manifest[descriptorName === 'pgDump' ? 'pg-dump' : 'psql']; + if (row?.path !== relative || row.sha256 !== digest || row.size !== bytes.length) { + fail(`${spec.name} differs from its producer manifest`); + } + } + tools[descriptorName] = Object.freeze({ name: spec.name, sha256: digest, size: bytes.length, bytes }); } return Object.freeze(tools); } -export function stageWasixToolsNpmCarrier({ version, portableReleaseArchive, packageDir }) { +export function stageWasixToolsNpmCarrier({ version, portableReleaseArchive, assetDirectory, packageDir }) { if (typeof version !== 'string' || !/^\d+\.\d+\.\d+$/u.test(version)) { throw new TypeError(`${TOOL}: version must be an exact semantic version`); } const output = path.resolve(packageDir); - const tools = wasixToolsNpmInputs({ portableReleaseArchive }); + const tools = wasixToolsNpmInputs({ portableReleaseArchive, assetDirectory }); rmSync(output, { recursive: true, force: true }); mkdirSync(path.join(output, 'assets'), { recursive: true }); for (const [name, tool] of Object.entries(tools)) { @@ -177,24 +192,29 @@ export function packWasixToolsNpmCarrier({ return Object.freeze({ ...staged, tarball }); } -export function stageWasixToolsAotNpmCarrier({ version, target, packageDir, aotReleaseArchive }) { - const entries = readPortableArchiveEntries(aotReleaseArchive); - const candidates = [...entries.keys()].filter(member => member.endsWith('/manifest.json')); +export function stageWasixToolsAotNpmCarrier({ version, target, packageDir, aotReleaseArchive, aotArtifactDirectory }) { + if ((aotReleaseArchive === undefined) === (aotArtifactDirectory === undefined)) { + fail('tools AOT staging requires exactly one release archive or producer artifact directory'); + } + const entries = aotReleaseArchive === undefined ? undefined : readPortableArchiveEntries(aotReleaseArchive); + const candidates = entries === undefined ? ['manifest.json'] : [...entries.keys()].filter(member => member.endsWith('/manifest.json')); if (candidates.length !== 1) fail('tools AOT release must contain exactly one manifest'); const manifestPath = candidates[0]; const prefix = path.posix.dirname(manifestPath); - const manifest = JSON.parse(requiredEntry(entries, manifestPath, aotReleaseArchive).toString('utf8')); + const readArtifact = relative => entries === undefined + ? readFileSync(path.join(aotArtifactDirectory, relative)) + : requiredEntry(entries, `${prefix}/${relative}`, aotReleaseArchive); + const manifest = JSON.parse(readArtifact('manifest.json').toString('utf8')); assertCanonicalWasixAotManifest(manifest, { expectedTarget: AOT_TARGET_TRIPLES[target] }); - const artifacts = assertWasixAotArtifactPayloads(manifest, { - readArtifact: relative => requiredEntry(entries, `${prefix}/${relative}`, aotReleaseArchive), - }).filter(artifact => ['tool:pg_dump', 'tool:psql'].includes(artifact.name)); + const artifacts = assertWasixAotArtifactPayloads(manifest, { readArtifact }) + .filter(artifact => ['tool:pg_dump', 'tool:psql'].includes(artifact.name)); if (artifacts.length !== 2) fail('tools AOT release is missing pg_dump or psql'); rmSync(packageDir, { recursive: true, force: true }); mkdirSync(packageDir, { recursive: true }); for (const artifact of artifacts) { const output = path.join(packageDir, artifact.path); mkdirSync(path.dirname(output), { recursive: true }); - writeFileSync(output, requiredEntry(entries, `${prefix}/${artifact.path}`, aotReleaseArchive)); + writeFileSync(output, readArtifact(artifact.path)); } const bytes = Buffer.from(`${JSON.stringify({ ...manifest, artifacts }, null, 2)}\n`); writeFileSync(path.join(packageDir, 'aot-manifest.json'), bytes); diff --git a/tools/release/wasix-typescript-package.mjs b/tools/release/wasix-typescript-package.mjs index 827f15199..6056734c3 100644 --- a/tools/release/wasix-typescript-package.mjs +++ b/tools/release/wasix-typescript-package.mjs @@ -85,6 +85,7 @@ export function assertWasixTypescriptManifest(manifest, label = `${PACKAGE_NAME} const root = manifest.exports?.['.']; const expectedExports = [ '.', + './browser', './direct', './worker', './package.json', @@ -101,37 +102,41 @@ export function assertWasixTypescriptManifest(manifest, label = `${PACKAGE_NAME} } if ( JSON.stringify(Object.keys(root ?? {})) - !== JSON.stringify(['types', 'deno', 'bun', 'node', 'browser', 'default']) - || root?.types !== './lib/index.d.ts' - || root?.deno !== './lib/index.deno.js' - || root?.bun !== './lib/index.bun.js' - || root?.browser !== './lib/index.js' - || root?.node !== './lib/index.node.js' - || root?.default !== './lib/index.js' + !== JSON.stringify(['deno', 'bun', 'node', 'browser', 'default']) + || JSON.stringify(root?.deno) !== JSON.stringify({ types: './lib/index.deno.d.ts', default: './lib/index.deno.js' }) + || JSON.stringify(root?.bun) !== JSON.stringify({ types: './lib/index.bun.d.ts', default: './lib/index.bun.js' }) + || JSON.stringify(root?.browser) !== JSON.stringify({ types: './lib/index.d.ts', default: './lib/index.js' }) + || JSON.stringify(root?.node) !== JSON.stringify({ types: './lib/index.node.d.ts', default: './lib/index.node.js' }) + || JSON.stringify(root?.default) !== JSON.stringify({ types: './lib/index.d.ts', default: './lib/index.js' }) ) { fail(`${label} must expose exact browser, Node, Bun, and Deno conditional entrypoints`); } + if (JSON.stringify(manifest.exports['./browser']) !== JSON.stringify({ types: './lib/browser.d.ts', default: './lib/browser.js' }) + || JSON.stringify(manifest.sideEffects) !== JSON.stringify(['./lib/browser.js', './lib/native-only.js'])) { + fail(`${label} must retain the explicit browser entrypoint and environment guards`); + } const worker = manifest.exports?.['./worker']; if ( JSON.stringify(Object.keys(worker ?? {})) - !== JSON.stringify(['types', 'deno', 'bun', 'node', 'browser', 'default']) - || worker?.types !== './lib/worker-entry.d.ts' - || worker?.deno !== './lib/worker-entry.deno.js' - || worker?.bun !== './lib/worker-entry.bun.js' - || worker?.node !== './lib/worker-entry.node.js' - || worker?.browser !== './lib/worker-entry.js' - || worker?.default !== './lib/worker-entry.js' + !== JSON.stringify(['deno', 'bun', 'node', 'browser', 'default']) + || JSON.stringify(worker?.deno) !== JSON.stringify({ types: './lib/worker-entry.deno.d.ts', default: './lib/worker-entry.deno.js' }) + || JSON.stringify(worker?.bun) !== JSON.stringify({ types: './lib/worker-entry.bun.d.ts', default: './lib/worker-entry.bun.js' }) + || JSON.stringify(worker?.node) !== JSON.stringify({ types: './lib/worker-entry.node.d.ts', default: './lib/worker-entry.node.js' }) + || JSON.stringify(worker?.browser) !== JSON.stringify({ types: './lib/worker-entry.d.ts', default: './lib/worker-entry.js' }) + || JSON.stringify(worker?.default) !== JSON.stringify({ types: './lib/worker-entry.d.ts', default: './lib/worker-entry.js' }) ) { fail(`${label} must expose the exact browser, Node, Bun, and Deno worker entrypoint`); } const direct = manifest.exports?.['./direct']; if ( JSON.stringify(Object.keys(direct ?? {})) - !== JSON.stringify(['types', 'deno', 'bun', 'node']) + !== JSON.stringify(['types', 'deno', 'bun', 'node', 'browser', 'default']) || direct?.types !== './lib/direct.node.d.ts' || direct?.deno !== './lib/direct.node.js' || direct?.bun !== './lib/direct.node.js' || direct?.node !== './lib/direct.node.js' + || direct?.browser !== './lib/native-only.js' + || direct?.default !== './lib/native-only.js' ) { fail(`${label} must expose one exact host-only conditional direct entrypoint`); } @@ -149,11 +154,13 @@ export function assertWasixTypescriptManifest(manifest, label = `${PACKAGE_NAME} const server = manifest.exports?.['./server']; if ( JSON.stringify(Object.keys(server ?? {})) - !== JSON.stringify(['types', 'deno', 'bun', 'node']) + !== JSON.stringify(['types', 'deno', 'bun', 'node', 'browser', 'default']) || server?.types !== './lib/server.node.d.ts' || server?.deno !== './lib/server.node.js' || server?.bun !== './lib/server.node.js' || server?.node !== './lib/server.node.js' + || server?.browser !== './lib/native-only.js' + || server?.default !== './lib/native-only.js' ) { fail(`${label} must expose one exact host-only conditional local-server entrypoint`); } diff --git a/tools/release/wasix-typescript-package.test.mjs b/tools/release/wasix-typescript-package.test.mjs index daca8b53d..33502e3bf 100644 --- a/tools/release/wasix-typescript-package.test.mjs +++ b/tools/release/wasix-typescript-package.test.mjs @@ -22,28 +22,30 @@ function manifest() { '@oliphaunt/wasix-napi-linux-x64-gnu': '1.2.3', '@oliphaunt/wasix-napi-win32-x64-msvc': '1.2.3', }, + sideEffects: ['./lib/browser.js', './lib/native-only.js'], exports: { + './browser': { types: './lib/browser.d.ts', default: './lib/browser.js' }, '.': { - types: './lib/index.d.ts', - deno: './lib/index.deno.js', - bun: './lib/index.bun.js', - node: './lib/index.node.js', - browser: './lib/index.js', - default: './lib/index.js', + deno: { types: './lib/index.deno.d.ts', default: './lib/index.deno.js' }, + bun: { types: './lib/index.bun.d.ts', default: './lib/index.bun.js' }, + node: { types: './lib/index.node.d.ts', default: './lib/index.node.js' }, + browser: { types: './lib/index.d.ts', default: './lib/index.js' }, + default: { types: './lib/index.d.ts', default: './lib/index.js' }, }, './worker': { - types: './lib/worker-entry.d.ts', - deno: './lib/worker-entry.deno.js', - bun: './lib/worker-entry.bun.js', - node: './lib/worker-entry.node.js', - browser: './lib/worker-entry.js', - default: './lib/worker-entry.js', + deno: { types: './lib/worker-entry.deno.d.ts', default: './lib/worker-entry.deno.js' }, + bun: { types: './lib/worker-entry.bun.d.ts', default: './lib/worker-entry.bun.js' }, + node: { types: './lib/worker-entry.node.d.ts', default: './lib/worker-entry.node.js' }, + browser: { types: './lib/worker-entry.d.ts', default: './lib/worker-entry.js' }, + default: { types: './lib/worker-entry.d.ts', default: './lib/worker-entry.js' }, }, './direct': { types: './lib/direct.node.d.ts', deno: './lib/direct.node.js', bun: './lib/direct.node.js', node: './lib/direct.node.js', + browser: './lib/native-only.js', + default: './lib/native-only.js', }, './internal/tools': { types: './lib/internal.d.ts', @@ -58,6 +60,8 @@ function manifest() { deno: './lib/server.node.js', bun: './lib/server.node.js', node: './lib/server.node.js', + browser: './lib/native-only.js', + default: './lib/native-only.js', }, './storage/node': { types: './lib/storage/node.d.ts', From 282efd917b4215d997a0386b79c1d4a5ffd2d8b6 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 9 Sep 2026 17:23:21 +0000 Subject: [PATCH 14/18] fix(wasix): use version compatibility and prepare resources off-thread --- Cargo.lock | 11 +- Cargo.toml | 1 + benchmarks/wasix/node-pglite-memory-v2.json | 2 +- docs/architecture/cluster-seeds-and-icu.md | 2 +- docs/architecture/database-storage.md | 7 +- docs/internal/PG18_WASIX_PERF_STATUS.md | 4 +- docs/internal/PG18_WASIX_POSTGRES.md | 85 ++------- docs/maintainers/assets.md | 8 +- .../consumer-dx-release-blueprint.md | 2 +- docs/maintainers/sdk-api-surface.md | 1 + .../crates/oliphaunt-wasix/README.md | 2 +- .../crates/oliphaunt-wasix/src/async_api.rs | 109 ++++++++--- .../crates/oliphaunt-wasix/src/lib.rs | 4 +- .../oliphaunt-wasix/src/oliphaunt/aot.rs | 53 ++---- .../oliphaunt-wasix/src/oliphaunt/assets.rs | 53 ++---- .../oliphaunt-wasix/src/oliphaunt/base.rs | 13 -- .../oliphaunt-wasix/src/oliphaunt/builder.rs | 2 +- .../oliphaunt-wasix/src/oliphaunt/server.rs | 2 +- .../oliphaunt-wasix/src/oliphaunt/tools.rs | 20 ++ src/bindings/wasix-ts/ARCHITECTURE.md | 4 +- src/bindings/wasix-ts/README.md | 19 +- src/bindings/wasix-ts/package.json | 2 +- .../wasix-ts/src/__tests__/extensions.test.ts | 11 +- .../src/__tests__/native-addon.test.ts | 6 +- .../native-extension-packages.test.ts | 12 +- .../src/__tests__/native-session.test.ts | 4 +- .../src/__tests__/runtime-descriptor.test.ts | 8 + .../wasix-ts/src/__tests__/storage.test.ts | 4 +- .../wasix-ts/src/descriptor-validation.ts | 2 +- src/bindings/wasix-ts/src/extensions.ts | 17 +- src/bindings/wasix-ts/src/native-addon.ts | 10 +- .../wasix-ts/src/native-extension-packages.ts | 108 ++++++----- src/bindings/wasix-ts/src/native-session.ts | 27 ++- src/bindings/wasix-ts/src/rpc.ts | 4 +- src/bindings/wasix-ts/src/types.ts | 2 - src/runtimes/liboliphaunt/icu/Cargo.toml | 17 +- src/runtimes/liboliphaunt/icu/README.md | 2 +- src/runtimes/liboliphaunt/icu/build.rs | 177 +----------------- src/runtimes/liboliphaunt/icu/src/lib.rs | 30 +-- src/runtimes/liboliphaunt/native/release.toml | 9 + .../wasix/crates/assets/src/lib.rs | 14 -- .../liboliphaunt/wasix/crates/icu/Cargo.toml | 19 ++ .../liboliphaunt/wasix/crates/icu/README.md | 5 + .../liboliphaunt/wasix/crates/icu/build.rs | 37 ++++ .../liboliphaunt/wasix/crates/icu/src/lib.rs | 3 + src/runtimes/liboliphaunt/wasix/release.toml | 7 +- src/runtimes/wasix-napi/README.md | 7 +- src/runtimes/wasix-napi/build.rs | 2 +- src/runtimes/wasix-napi/package.json | 2 +- .../packages/darwin-arm64/package.json | 2 +- .../packages/linux-arm64-gnu/package.json | 2 +- .../packages/linux-x64-gnu/package.json | 2 +- .../packages/win32-x64-msvc/package.json | 2 +- .../wasix-napi/src/extension_package.rs | 86 ++++++--- src/runtimes/wasix-napi/src/icu_data.rs | 103 +++++++--- src/runtimes/wasix-napi/src/lib.rs | 80 ++++---- .../wasix-napi/tests/native.integration.mjs | 20 +- .../wasix-napi/tools/check-build-inputs.mjs | 18 +- .../tools/check-build-inputs.test.mjs | 5 +- .../wasix-napi/tools/smoke-packaged-addon.mjs | 4 +- .../fixtures/icu.valid.json | 1 - .../fixtures/profile-mismatch.invalid.json | 1 - .../fixtures/standard.valid.json | 1 - .../wasix-ts/packed-node-fixture.mjs | 4 +- tools/integration/wasix-ts/smoke-browser.mjs | 1 - tools/perf/wasix-node/plan.mjs | 10 +- tools/perf/wasix-node/plan.test.mjs | 4 +- tools/policy/check-cluster-seed-contract.mjs | 2 +- tools/release/check-release-metadata.mjs | 2 +- .../extension-wasix-npm-packages.test.mjs | 2 +- tools/release/moon.yml | 5 +- tools/release/native-icu-seeds.test.mjs | 33 ++++ .../release/normal-publication-plan.test.mjs | 8 +- .../package-liboliphaunt-cargo-artifacts.mjs | 25 +++ ...kage-liboliphaunt-cargo-artifacts.test.mjs | 25 ++- ...age_liboliphaunt_wasix_cargo_artifacts.mjs | 11 +- .../package_oliphaunt_wasix_sdk_crate.mjs | 2 +- tools/release/publication-catalog.test.mjs | 2 +- tools/release/publication-lock.test.mjs | 4 +- .../release/wasix-cargo-artifact-contract.mjs | 2 +- .../wasix-cargo-license-contract.test.mjs | 2 +- tools/release/wasix-icu-npm-carrier.test.mjs | 3 +- .../wasix-napi-package-contract.test.mjs | 4 +- tools/release/wasix-runtime-npm-carrier.mjs | 12 -- .../wasix-runtime-npm-carrier.test.mjs | 9 +- tools/release/wasix-typescript-package.mjs | 2 +- .../release/wasix-typescript-package.test.mjs | 2 +- tools/xtask/src/asset_checks.rs | 2 +- tools/xtask/src/asset_io.rs | 4 - tools/xtask/src/asset_manifest.rs | 8 - tools/xtask/src/asset_pipeline.rs | 171 ++--------------- tools/xtask/src/main.rs | 1 - tools/xtask/src/postgres_guard.rs | 2 +- 93 files changed, 749 insertions(+), 894 deletions(-) create mode 100644 src/runtimes/liboliphaunt/wasix/crates/icu/Cargo.toml create mode 100644 src/runtimes/liboliphaunt/wasix/crates/icu/README.md create mode 100644 src/runtimes/liboliphaunt/wasix/crates/icu/build.rs create mode 100644 src/runtimes/liboliphaunt/wasix/crates/icu/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index aa4f9803c..749d429b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2479,8 +2479,6 @@ version = "0.2.0" dependencies = [ "oliphaunt-resources", "sha2 0.10.9", - "tar", - "zstd", ] [[package]] @@ -2589,6 +2587,15 @@ dependencies = [ "zstd", ] +[[package]] +name = "oliphaunt-wasix-icu" +version = "0.2.0" +dependencies = [ + "oliphaunt-resources", + "serde_json", + "sha2 0.10.9", +] + [[package]] name = "oliphaunt-wasix-napi" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 83d18ef05..84f690edc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "src/runtimes/liboliphaunt/native/crates/tools", "src/runtimes/broker", "src/runtimes/liboliphaunt/icu", + "src/runtimes/liboliphaunt/wasix/crates/icu", "src/runtimes/liboliphaunt/wasix/crates/assets", "src/runtimes/liboliphaunt/wasix/crates/tools", "src/runtimes/liboliphaunt/wasix/crates/aot/aarch64-apple-darwin", diff --git a/benchmarks/wasix/node-pglite-memory-v2.json b/benchmarks/wasix/node-pglite-memory-v2.json index 633b49f08..18704112b 100644 --- a/benchmarks/wasix/node-pglite-memory-v2.json +++ b/benchmarks/wasix/node-pglite-memory-v2.json @@ -10,7 +10,7 @@ "schema": "oliphaunt-wasix-napi-host-v1", "product": "oliphaunt-wasix-napi", "binary": "oliphaunt_wasix_napi.node", - "addonAbiVersion": 1, + "addonAbiVersion": 2, "nodeApiVersion": 8, "profiles": [ "standard", diff --git a/docs/architecture/cluster-seeds-and-icu.md b/docs/architecture/cluster-seeds-and-icu.md index fb8387865..9b5901916 100644 --- a/docs/architecture/cluster-seeds-and-icu.md +++ b/docs/architecture/cluster-seeds-and-icu.md @@ -139,7 +139,7 @@ Other SDKs retain language-native package selection: | Swift | base SwiftPM runtime resources | independent `OliphauntICU` package and `icu: OliphauntICU.descriptor` | | Kotlin / Java | ordinary Maven runtime resources | ICU dependency and explicit `ICU.data` configuration | | React Native | ordinary generated native resources | import `@oliphaunt/icu`, pass `{ icu }` | -| Rust WASIX | portable runtime artifact | `oliphaunt-icu` dependency and `.icu(oliphaunt_icu::ICU)` | +| Rust WASIX | portable runtime artifact | `oliphaunt-wasix-icu` dependency and `.icu(oliphaunt_wasix_icu::ICU)` | | WASIX TypeScript | default runtime descriptor | explicit `@oliphaunt/wasix-icu` descriptor | This is semantic parity, not identical signatures. diff --git a/docs/architecture/database-storage.md b/docs/architecture/database-storage.md index 7863405b4..99444917e 100644 --- a/docs/architecture/database-storage.md +++ b/docs/architecture/database-storage.md @@ -163,10 +163,9 @@ build physically interchangeable. Native roots are shared among compatible native SDKs. Rust and WASIX TypeScript use the same root descriptor and WASIX physical-format value, but cross-binding root handoff is not a supported or qualified workflow. The managed-root descriptor is -written once when the root is created. WASIX source fingerprints remain -asset-graph coherence identities used to reject mixed runtime, cluster-seed, -AOT, and extension build outputs; they -are not a physical-reopen key or binding identity in the root. Both runtime +written once when the root is created. Runtime/resource versions determine +package compatibility, and artifact checksums detect corruption. Neither is +a physical-reopen key or binding identity in the root. Both runtime families validate either exact descriptor shape. Opening another family's root is not a supported transfer path, so the SDKs add no cross-family rejection policy; the underlying PostgreSQL/runtime behavior is authoritative. diff --git a/docs/internal/PG18_WASIX_PERF_STATUS.md b/docs/internal/PG18_WASIX_PERF_STATUS.md index 4f91265ec..2a9239d05 100644 --- a/docs/internal/PG18_WASIX_PERF_STATUS.md +++ b/docs/internal/PG18_WASIX_PERF_STATUS.md @@ -77,8 +77,8 @@ Current active artifact: - release-host reruns can use the stable PG18 generated asset and AOT paths: `OLIPHAUNT_WASM_GENERATED_ASSETS_DIR=.../target/oliphaunt-wasix/assets`, and `OLIPHAUNT_WASM_GENERATED_AOT_DIR=.../target/oliphaunt-wasix/aot`. - Asset, PGDATA-template, and AOT manifests carry source-fingerprint metadata, - so a stale artifact fails with a fingerprint mismatch before measurement. + Historical builds recorded source fingerprints in manifests. Current loaders + use runtime versions for compatibility and checksums for payload integrity. Current 37-patch O2 release-host three-run median against same-host PG17.5 `0.5.0` and the documented PG17.5 release-lane table in diff --git a/docs/internal/PG18_WASIX_POSTGRES.md b/docs/internal/PG18_WASIX_POSTGRES.md index b8605ca0c..a0c9d3b7b 100644 --- a/docs/internal/PG18_WASIX_POSTGRES.md +++ b/docs/internal/PG18_WASIX_POSTGRES.md @@ -419,27 +419,12 @@ the applied runtime, tool, contrib, and hot-path patch markers. The PG18 source-spine command defaults to source-only validation; `--strict-local` also requires the shared non-backend source checkouts to be present, clean, and pinned. -Once those outputs exist, `assets cluster-seeds` and -`assets package` discover the PG18 build tree, -derive manifest PostgreSQL versions from the prepared PG18 source markers, and -write explicit PG18 source-fingerprint and PG18 source pins into -generated asset manifests. PG18 packaged asset discovery rejects manifests -whose fingerprint does not match the current PostgreSQL tarball plus patch -series hash. Contrib extension control files are staged from the active -PostgreSQL source tree, not from the generated PG17 catalog metadata. The -default path remains the released PG17.5 lane unless the source selection is -explicitly selected. - -PG18 build outputs must also carry the same source fingerprint and PostgreSQL -version markers as the prepared source. The backend Docker entrypoint stamps -those markers after configure, companion build stages fail closed if either -marker drifts, and xtask checks the markers again before packaging, template, or -build-output manifest generation can consume an existing build tree. - -Cluster seed manifests produced by `assets cluster-seeds` also carry camelCase -`sourceLane` metadata, and the asset manifest's `cluster-seeds` entries record -the same lane. PG18 cluster seeds also carry the same source fingerprint as the -runtime assets. +Once those outputs exist, `assets cluster-seeds` and `assets package` discover +the PG18 build tree and record PostgreSQL versions and source pins. Runtime and +resource versions govern compatibility. Source hashes remain local build-cache +markers so patch edits invalidate stale prepared sources and build trees; they +are not shipped compatibility keys. Contrib control files come from the selected +PostgreSQL source tree. The source-spine guard also checks the prepared PG18 tree against the runtime assets and promoted contrib build plan. The required `plpgsql`, @@ -477,20 +462,8 @@ local discovery. Portable assets now write to the stable generated directory `target/oliphaunt-wasix/assets`. AOT intermediates remain under `target/oliphaunt-wasix/wasix-build/build/aot`, while packaged -AOT outputs write to the stable generated directory -`target/oliphaunt-wasix/aot`. Packaged AOT manifests carry explicit -`source fingerprint`, source-fingerprint, and `postgres-version` metadata, and -`assets check-aot` verifies those fields before checking module hashes. - -The Rust asset parser preserves the same source-fingerprint metadata that xtask -writes into PG18 asset manifests. Embedded cluster seed manifests must match -the top-level asset manifest fingerprint, and bundled AOT manifests must match -the same fingerprint and PostgreSQL version before their module hashes are -accepted. The `liboliphaunt-wasix-portable` build script probes -`target/oliphaunt-wasix/assets` plus the publishable payload unless -`OLIPHAUNT_WASM_GENERATED_ASSETS_DIR` explicitly overrides the asset directory. -Any selected PG18 manifest must carry a non-empty source-fingerprint plus a -PostgreSQL 18 runtime version before embedding. +AOT outputs write to `target/oliphaunt-wasix/aot`. The loader checks runtime, +PostgreSQL, engine, and target compatibility and verifies artifact checksums. Runtime reuse has the same fail-closed stance. A full-local runtime root is only reused when its saved runtime source key matches the currently embedded @@ -506,28 +479,10 @@ Cargo workspace crate as an unrelated final step. Run `moon run repo:package` for the workspace crate gate; each release carrier also enforces the registry limit while creating the exact package it publishes. -Perf reports now carry WASIX runtime asset provenance when the measured engine -is the bundled WASIX runtime. The JSON field is `wasixRuntimeAssets` and -records the asset source selection, PostgreSQL version, optional PG18 source -fingerprint, and PGDATA-template lane/fingerprint/version. Native PostgreSQL, -SQLite, native liboliphaunt, and Node Oliphaunt controls omit the field. This -keeps future PG18-versus-released-lane benchmark reports self-identifying even -when both lanes can be built from the same xtask binary. - -`assets check` and `assets verify-committed` include a source-fingerprint -isolation guard. It validates lane selection and output paths, rejects legacy -PG17/Oliphaunt inputs, checks the patch manifest/series and prepared-source -fingerprints, syntax-checks build scripts, and compiles the C ABI harnesses. -It deliberately does not grep for exact positive implementation spellings; -patch application, compilation, artifact/export validation, smoke, and -regression tasks own those proofs. -Generated asset manifest -validation also checks the manifest `source fingerprint` field when a lane is selected, -so PG18 packaged assets cannot pass as released-lane assets by version inference -alone. Build-output manifests carry the same PG18 source fingerprint and are -ignored by export-list generation if the fingerprint no longer matches the -current source stack; PG18 build-output manifest module paths must also stay -under the PG18 build root instead of the released build root. +`assets check` and `assets verify-committed` validate source selection and output +paths, the patch series, prepared-source build-cache markers, build-script syntax, +and C ABI harnesses. Runtime manifests use version compatibility, without source +fingerprint fields. Build-output module paths must stay under the PG18 build root. Promoted extension packaging is also fail-closed. PostgreSQL contrib and PGXS style extensions are lane-scoped through the selected build directory. The PG18 @@ -590,20 +545,8 @@ Verified locally: including the PG18 source-prep and backend entrypoint scripts. - `assets verify-committed` additionally validates the source-fingerprint isolation guard and the source-controlled WASIX export list. -- Generated asset manifests now carry explicit `source fingerprint` metadata and a - source fingerprint that must match the current PG18 tarball plus patch stack. -- Existing PG18 build trees are accepted only when their stamped source - fingerprint and PostgreSQL version markers match the prepared source. -- Packaged AOT manifests now carry explicit `source fingerprint`, `postgres-version`, - and source-fingerprint metadata. -- Cluster seed manifests and asset-manifest `cluster-seeds` entries now carry - lane metadata as well, plus PG18 source fingerprints. -- Runtime asset parsing preserves PG18 source fingerprints, and embedded cluster - seed/AOT manifests are checked against the bundled asset manifest before - use. -- WASIX perf reports include bundled runtime asset provenance, so benchmark JSON - identifies the measured source selection, PostgreSQL version, and PG18 source - fingerprint before the numbers are compared. +- Existing PG18 build trees retain producer-local source and version markers. +- Runtime, seed, and AOT manifests use version compatibility and payload checksums. - Unit coverage checks that PG18 extension manifests use packaged control files and reject released-lane path leaks; the legacy PG17 source selection is no longer selectable. diff --git a/docs/maintainers/assets.md b/docs/maintainers/assets.md index 1985b013e..72962013c 100644 --- a/docs/maintainers/assets.md +++ b/docs/maintainers/assets.md @@ -35,7 +35,8 @@ applications pass their exported descriptors alongside the SDK's contrib values. See the [WASIX Rust guide](../../src/docs/content/sdk/wasix-rust/guide.mdx) for setup. The optional `tools` feature selects the split tool and tools-AOT carriers. -ICU data and its matching seed come from `oliphaunt-icu`. Neither ICU nor external +ICU data and its matching seed come from `oliphaunt-icu` for native Rust or +`oliphaunt-wasix-icu` for WASIX Rust. Neither ICU nor external extensions use SDK feature flags. Ordinary applications need no build script. ## Cache Behavior @@ -102,8 +103,9 @@ uses the same provenance path: ICU code is source-pinned in little-endian data archive is independently pinned in `src/sources/third-party/shared/icu-data.toml`. Native and WASIX builders compile target-specific ICU code but expand that one data archive into the shared -files-data identity. ICU data is packaged as a separate `oliphaunt-icu` -payload; standard native and WASIX runtime artifacts do not carry `share/icu`. +files-data identity. Optional `oliphaunt-icu` and `oliphaunt-wasix-icu` Cargo +packages carry the data and matching seeds for their respective runtimes; +standard native and WASIX runtime artifacts do not carry `share/icu`. That payload supplies runtime capability; it is distinct from the per-database `pg_collation` catalog state created during `initdb`. An ICU-enabled new root therefore requires the matching `icu` cluster seed diff --git a/docs/maintainers/consumer-dx-release-blueprint.md b/docs/maintainers/consumer-dx-release-blueprint.md index bfcf2ff4d..6d977dc17 100644 --- a/docs/maintainers/consumer-dx-release-blueprint.md +++ b/docs/maintainers/consumer-dx-release-blueprint.md @@ -86,7 +86,7 @@ filesystem library. It does not define an application-data storage kind. Validate installed package ownership, version, runtime compatibility, target, archive layout, and hashes before loading payloads. Native AOT validation also -checks compiler/runtime identity and the source fingerprint before deserialization. +checks runtime, compiler, and target versions and verifies artifact integrity before deserialization. An arbitrary caller-supplied path and adjacent checksum are not package provenance. Native package installation is a trust boundary equivalent to installing native code; package verification is not a claim of cryptographic publisher signatures. diff --git a/docs/maintainers/sdk-api-surface.md b/docs/maintainers/sdk-api-surface.md index 0dbcfc0a8..51b3ff94a 100644 --- a/docs/maintainers/sdk-api-surface.md +++ b/docs/maintainers/sdk-api-surface.md @@ -859,6 +859,7 @@ These symbols require no optional Cargo feature. Target-gated symbols (for examp - `oliphaunt_wasix::tools::PsqlOptions.command()` - `oliphaunt_wasix::tools::PsqlOptions.new()` - `oliphaunt_wasix::tools::PsqlOptions.script()` +- `oliphaunt_wasix::tools::installed_tool_identity()` - `oliphaunt_wasix::tools::installed_tool_wasm()` - `oliphaunt_wasix::tools::register_installed_package()` diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/README.md b/src/bindings/wasix-rust/crates/oliphaunt-wasix/README.md index fb9fb6fa3..d66a2eda3 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/README.md +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/README.md @@ -141,7 +141,7 @@ build script or SDK `extension-vector` feature is required. The builder accepts selection prepares artifacts and startup requirements; migrations remain application-owned. -For ICU collations, add `oliphaunt-icu` and pass `.icu(oliphaunt_icu::ICU)`. +For ICU collations, add `oliphaunt-wasix-icu` and pass `.icu(oliphaunt_wasix_icu::ICU)`. The optional package supplies ICU data and the matching initialization seed. Omitting it selects standard initialization. diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/async_api.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/async_api.rs index 3129b83df..8e83e7e2c 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/async_api.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/async_api.rs @@ -463,8 +463,9 @@ impl Drop for DatabaseOwnerInner { } impl DatabaseOwner { - fn open_with_completion(builder: DirectOliphauntBuilder, completion: C) + fn open_with_completion(configure: F, completion: C) where + F: FnOnce() -> Result + Send + 'static, C: FnOnce(Result) + Send + 'static, { let completion = SharedCompletion::new(completion); @@ -486,7 +487,7 @@ impl DatabaseOwner { "WASIX database owner stopped before open completed", ); let opened = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| builder.open())); + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| configure()?.open())); let database = match opened { Ok(Ok(database)) => { completion.complete(Ok(Self { @@ -1766,7 +1767,7 @@ impl AsyncOliphauntBuilder { self } - /// Select ICU data from the optional `oliphaunt-icu` package. + /// Select ICU data from the optional `oliphaunt-wasix-icu` package. pub fn icu(mut self, data: oliphaunt_resources::IcuData) -> Self { self.inner = self.inner.icu(data); self @@ -1822,15 +1823,18 @@ impl AsyncOliphauntBuilder { /// Construct the Wasmer runtime and PostgreSQL session on its permanent owner thread. pub async fn open(self) -> Result { let (reply, receiver) = oneshot::channel(); - DatabaseOwner::open_with_completion(self.inner, move |result| { - let _ = reply.send(result.map(|owner| AsyncOliphaunt { owner })); - }); + DatabaseOwner::open_with_completion( + move || Ok(self.inner), + move |result| { + let _ = reply.send(result.map(|owner| AsyncOliphaunt { owner })); + }, + ); receiver .await .map_err(|_| Error::lifecycle("WASIX database owner stopped before open completed"))? } - /// Construct the database on its owner thread and report completion without + /// Configure resources and construct the database on its owner thread, reporting completion without /// creating or polling a Rust future. /// /// Completion runs on the new owner thread after successful construction, @@ -1838,13 +1842,21 @@ impl AsyncOliphauntBuilder { /// invoked exactly once. #[cfg(any(feature = "__internal-napi", test))] #[doc(hidden)] - pub fn open_with_completion(self, completion: C) + pub fn open_with_completion(configure: F, completion: C) where + F: FnOnce() -> std::result::Result + Send + 'static, C: FnOnce(Result) + Send + 'static, { - DatabaseOwner::open_with_completion(self.inner, move |result| { - completion(result.map(|owner| AsyncOliphaunt { owner })); - }); + DatabaseOwner::open_with_completion( + move || { + configure() + .map(|builder| builder.inner) + .map_err(|error| Error::from_anyhow(crate::error::invalid_configuration(error))) + }, + move |result| { + completion(result.map(|owner| AsyncOliphaunt { owner })); + }, + ); } } @@ -2402,7 +2414,7 @@ impl AsyncOliphauntServerBuilder { self } - /// Select ICU data from the optional `oliphaunt-icu` package. + /// Select ICU data from the optional `oliphaunt-wasix-icu` package. pub fn icu(mut self, data: oliphaunt_resources::IcuData) -> Self { self.inner = self.inner.icu(data); self @@ -2465,16 +2477,20 @@ impl AsyncOliphauntServerBuilder { /// Start the server and await its bound endpoint. pub async fn start(self) -> Result { let (reply, receiver) = oneshot::channel(); - self.start_with_reply(move |result| { - let _ = reply.send(result); - }); + Self::start_with_reply( + move || Ok(self), + move |result| { + let _ = reply.send(result); + }, + ); receiver .await .map_err(|_| Error::lifecycle("WASIX server owner stopped before start completed"))? } - fn start_with_reply(self, completion: C) + fn start_with_reply(configure: F, completion: C) where + F: FnOnce() -> Result + Send + 'static, C: FnOnce(Result) + Send + 'static, { let completion = SharedCompletion::new(completion); @@ -2493,8 +2509,9 @@ impl AsyncOliphauntServerBuilder { thread_completion, "WASIX server owner stopped before start completed", ); - let opened = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.inner.start())); + let opened = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + configure()?.inner.start() + })); let server = match opened { Ok(Ok(server)) => server, Ok(Err(error)) => { @@ -2547,16 +2564,23 @@ impl AsyncOliphauntServerBuilder { } } - /// Start the server and report its bound endpoint without creating or + /// Configure resources and start the server on its owner thread without creating or /// polling a Rust future. Completion runs exactly once, including thread /// spawn failure and owner loss during startup. #[cfg(any(feature = "__internal-napi", test))] #[doc(hidden)] - pub fn start_with_completion(self, completion: C) + pub fn start_with_completion(configure: F, completion: C) where + F: FnOnce() -> std::result::Result + Send + 'static, C: FnOnce(Result) + Send + 'static, { - self.start_with_reply(completion); + Self::start_with_reply( + move || { + configure() + .map_err(|error| Error::from_anyhow(crate::error::invalid_configuration(error))) + }, + completion, + ); } } @@ -3862,3 +3886,46 @@ mod close_tests { assert!(harness.server.is_closed()); } } + +#[cfg(test)] +mod configured_owner_tests { + use super::*; + + #[test] + fn resource_configuration_runs_on_owners_and_failure_completes_once() { + let caller = thread::current().id(); + let (sent, received) = mpsc::channel(); + AsyncOliphauntBuilder::open_with_completion( + move || { + assert_ne!(thread::current().id(), caller); + Err("invalid database resources".to_owned()) + }, + move |result| { + sent.send(result.err().unwrap()).unwrap(); + }, + ); + let error = received + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert_eq!(error.kind(), crate::ErrorKind::InvalidConfiguration); + assert!(error.to_string().contains("invalid database resources")); + assert!(received.recv().is_err()); + + let (sent, received) = mpsc::channel(); + AsyncOliphauntServerBuilder::start_with_completion( + move || { + assert_ne!(thread::current().id(), caller); + Err("invalid server resources".to_owned()) + }, + move |result| { + sent.send(result.err().unwrap()).unwrap(); + }, + ); + let error = received + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert_eq!(error.kind(), crate::ErrorKind::InvalidConfiguration); + assert!(error.to_string().contains("invalid server resources")); + assert!(received.recv().is_err()); + } +} diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/lib.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/lib.rs index 11f17f2aa..f187df7d9 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/lib.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/lib.rs @@ -40,5 +40,7 @@ pub mod tools { PgDumpOptions, PostgresToolError, PostgresToolOutput, PsqlOptions, }; #[doc(hidden)] - pub use crate::oliphaunt::tools::{installed_tool_wasm, register_installed_package}; + pub use crate::oliphaunt::tools::{ + installed_tool_identity, installed_tool_wasm, register_installed_package, + }; } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/aot.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/aot.rs index 6d9960309..eb0a59bbb 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/aot.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/aot.rs @@ -110,16 +110,7 @@ fn validate_package_aot_header(manifest: &AotManifest) -> Result<()> { manifest.source_lane.as_deref() == Some("stable"), "package AOT source lane mismatch" ); - ensure!( - manifest - .source_fingerprint - .as_deref() - .is_some_and(|value| value.len() == 64 - && value - .bytes() - .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())), - "package AOT must identify its runtime source fingerprint" - ); + ensure!( manifest.target_triple == target_triple(), "package AOT target mismatch" @@ -148,6 +139,10 @@ pub(crate) fn load_package_module( name: &str, wasm: &[u8], ) -> Result { + ensure!( + package.runtime_version() == liboliphaunt_wasix_portable::PACKAGE_VERSION, + "package AOT runtime version mismatch" + ); let manifest: AotManifest = serde_json::from_str(package.aot_manifest()) .context("parse selected extension AOT manifest")?; validate_package_aot_header(&manifest)?; @@ -156,10 +151,7 @@ pub(crate) fn load_package_module( manifest.source_lane == runtime.source_lane, "package AOT source lane mismatch" ); - ensure!( - manifest.source_fingerprint == runtime.source_fingerprint, - "extension AOT runtime source fingerprint mismatch" - ); + ensure!( manifest.postgres_version.as_deref() == Some(runtime.postgres_version.as_str()), "extension AOT PostgreSQL version mismatch" @@ -188,8 +180,12 @@ pub(crate) fn load_package_module( "extension package must contain exactly one AOT payload {name}" ); let bytes = matches[0].1; - validate_compressed_artifact_manifest(name, artifact, bytes)?; - let key = format!("package:{name}:{}", artifact.sha256); + let key = format!( + "package:{name}:{}:{:p}:{}", + artifact.sha256, + bytes.as_ptr(), + bytes.len() + ); let mut modules = MODULE_CACHE .get_or_init(|| Mutex::new(HashMap::new())) .lock() @@ -197,6 +193,7 @@ pub(crate) fn load_package_module( if let Some(module) = modules.get(&key) { return Ok(module.clone()); } + validate_compressed_artifact_manifest(name, artifact, bytes)?; let size = artifact .raw_size .context("extension AOT is missing raw-size")?; @@ -491,16 +488,6 @@ fn target_manifest_artifact(name: &str) -> Result { manifest.wasmer_wasix_version ); let metadata = assets::asset_manifest_metadata()?; - if let Some(expected) = metadata.source_fingerprint.as_deref() { - ensure!( - manifest.source_fingerprint.as_deref() == Some(expected), - "AOT manifest source fingerprint mismatch: manifest={} assets={expected}", - manifest - .source_fingerprint - .as_deref() - .unwrap_or("") - ); - } let postgres_version = manifest .postgres_version .as_deref() @@ -581,10 +568,7 @@ fn merge_tools_aot_manifest(manifest: &mut AotManifest) -> Result<()> { tools_manifest.wasmer_wasix_version, manifest.wasmer_wasix_version ); - ensure!( - tools_manifest.source_fingerprint == manifest.source_fingerprint, - "tools AOT manifest source fingerprint mismatch" - ); + ensure!( tools_manifest.postgres_version == manifest.postgres_version, "tools AOT manifest postgres version mismatch" @@ -658,10 +642,7 @@ fn merge_extension_aot_manifests(_manifest: &mut AotManifest) -> Result<()> { extension_manifest.wasmer_wasix_version, manifest.wasmer_wasix_version ); - ensure!( - extension_manifest.source_fingerprint == manifest.source_fingerprint, - "extension AOT manifest source fingerprint mismatch for '{sql_name}'" - ); + ensure!( extension_manifest.postgres_version == manifest.postgres_version, "extension AOT manifest postgres version mismatch for '{sql_name}'" @@ -1060,7 +1041,6 @@ struct AotManifest { format_version: Option, #[allow(dead_code)] source_lane: Option, - source_fingerprint: Option, postgres_version: Option, target_triple: String, engine: String, @@ -1218,7 +1198,7 @@ mod tests { #[test] fn package_aot_header_rejects_foreign_or_incomplete_executable_identities() { let valid = serde_json::json!({ - "format-version": 1, "source-lane": "stable", "source-fingerprint": "a".repeat(64), + "format-version": 1, "source-lane": "stable", "postgres-version": "18.4", "target-triple": target_triple(), "engine": EXPECTED_AOT_ENGINE, "wasmer-version": EXPECTED_WASMER_VERSION, "wasmer-wasix-version": EXPECTED_WASMER_WASIX_VERSION, "artifacts": [] @@ -1227,7 +1207,6 @@ fn package_aot_header_rejects_foreign_or_incomplete_executable_identities() { for (field, value) in [ ("format-version", serde_json::json!(2)), ("source-lane", serde_json::json!("other")), - ("source-fingerprint", serde_json::Value::Null), ("target-triple", serde_json::json!("other")), ("engine", serde_json::json!("other")), ("wasmer-version", serde_json::json!("other")), diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/assets.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/assets.rs index 81df2c3c6..1b132a347 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/assets.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/assets.rs @@ -3,11 +3,9 @@ use anyhow::{Context, Result, ensure}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct AssetManifestMetadata { pub source_lane: Option, - pub source_fingerprint: Option, pub postgres_version: String, pub runtime_module_sha256: String, pub cluster_seed_source_lane: Option, - pub cluster_seed_source_fingerprint: Option, pub cluster_seed_postgres_version: Option, pub cluster_seed_profile: String, pub cluster_seed_compatibility_key: String, @@ -61,6 +59,18 @@ pub(crate) fn register_icu(data: oliphaunt_resources::IcuData) -> Result<()> { data.runtime_version == liboliphaunt_wasix_portable::PACKAGE_VERSION, "ICU package is incompatible with the selected WASIX runtime" ); + if let Some(selected) = SELECTED_ICU.get() { + let same_bytes = |left: Option<&[u8]>, right: Option<&[u8]>| matches!((left, right), (Some(left), Some(right)) if std::ptr::eq(left, right)); + if selected.version == data.version + && selected.wasix_archive_sha256 == data.wasix_archive_sha256 + && selected.wasix_data_tree_sha256 == data.wasix_data_tree_sha256 + && same_bytes(selected.wasix_archive, data.wasix_archive) + && same_bytes(selected.wasix_seed_archive, data.wasix_seed_archive) + && same_bytes(selected.wasix_seed_manifest, data.wasix_seed_manifest) + { + return Ok(()); + } + } let archive = data .wasix_archive .context("selected ICU package has no WASIX archive")?; @@ -123,24 +133,12 @@ pub(crate) fn asset_manifest_metadata_for( ) -> Result { let manifest = liboliphaunt_wasix_portable::manifest().context("parse oliphaunt-wasix asset manifest")?; - if cluster_seed_manifest(selected_profile).is_some() { - let seed = manifest - .cluster_seeds - .get(selected_profile.as_str()) - .context("embedded WASIX assets are missing the selected cluster seed entry")?; - validate_embedded_source_fingerprints( - manifest.source_fingerprint.as_deref(), - seed.source_fingerprint.as_deref(), - )?; - } let seed = manifest.cluster_seeds.get(selected_profile.as_str()); Ok(AssetManifestMetadata { source_lane: manifest.source_lane, - source_fingerprint: manifest.source_fingerprint, postgres_version: manifest.runtime.postgres_version, runtime_module_sha256: manifest.runtime.module_sha256, cluster_seed_source_lane: seed.and_then(|seed| seed.source_lane.clone()), - cluster_seed_source_fingerprint: seed.and_then(|seed| seed.source_fingerprint.clone()), cluster_seed_postgres_version: seed.map(|seed| seed.postgres_version.clone()), cluster_seed_profile: selected_profile.as_str().to_owned(), cluster_seed_compatibility_key: seed @@ -149,23 +147,6 @@ pub(crate) fn asset_manifest_metadata_for( }) } -fn validate_embedded_source_fingerprints( - asset_fingerprint: Option<&str>, - seed_fingerprint: Option<&str>, -) -> Result<()> { - let asset_fingerprint = asset_fingerprint - .filter(|value| !value.trim().is_empty()) - .context("embedded WASIX asset manifest is missing source-fingerprint metadata")?; - let seed_fingerprint = seed_fingerprint - .filter(|value| !value.trim().is_empty()) - .context("embedded WASIX cluster seed is missing source-fingerprint metadata")?; - ensure!( - seed_fingerprint == asset_fingerprint, - "embedded WASIX runtime and cluster seed source fingerprints differ" - ); - Ok(()) -} - pub(crate) fn runtime_archive() -> Option<&'static [u8]> { liboliphaunt_wasix_portable::runtime_archive() } @@ -267,7 +248,6 @@ mod tests { CatalogProfile, asset_manifest_metadata, cluster_seed_archive, cluster_seed_manifest, expected_icu_data_archive_sha256, expected_icu_data_tree_sha256, expected_runtime_archive_sha256, icu_data_archive, runtime_archive, - validate_embedded_source_fingerprints, }; #[test] @@ -301,13 +281,4 @@ mod tests { assert_eq!(expected_icu_data_archive_sha256().is_some(), has_icu_assets); assert_eq!(expected_icu_data_tree_sha256().is_some(), has_icu_assets); } - - #[test] - fn embedded_source_fingerprints_are_required_and_equal() { - validate_embedded_source_fingerprints(Some("source-key"), Some("source-key")) - .expect("matching identities"); - assert!(validate_embedded_source_fingerprints(None, Some("source-key")).is_err()); - assert!(validate_embedded_source_fingerprints(Some("source-key"), Some(" ")).is_err()); - assert!(validate_embedded_source_fingerprints(Some("runtime"), Some("seed")).is_err()); - } } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/base.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/base.rs index 95c6f358a..1be44af3a 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/base.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/base.rs @@ -186,7 +186,6 @@ struct ClusterSeedRuntimeIdentity { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct ClusterSeedSourceIdentity { - fingerprint: String, catalog_version: String, lane: String, producer: String, @@ -1231,18 +1230,6 @@ fn validate_cluster_seed_manifest_metadata( ); } - let expected_fingerprint = metadata - .cluster_seed_source_fingerprint - .as_deref() - .or(metadata.source_fingerprint.as_deref()); - if let Some(expected) = expected_fingerprint { - ensure!( - manifest.source.fingerprint == expected, - "embedded cluster seed source fingerprint mismatch: seed={} assets={expected}", - manifest.source.fingerprint - ); - } - Ok(()) } diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs index 6b6fc01f9..e82525274 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/builder.rs @@ -59,7 +59,7 @@ impl OliphauntBuilder { self } - /// Select ICU data from the optional `oliphaunt-icu` package. + /// Select ICU data from the optional `oliphaunt-wasix-icu` package. pub fn icu(mut self, data: oliphaunt_resources::IcuData) -> Self { self.icu = Some(data); self.catalog_profile = CatalogProfile::Icu; diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/server.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/server.rs index 0d822123a..91425079a 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/server.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/server.rs @@ -310,7 +310,7 @@ impl OliphauntServerBuilder { self } - /// Select ICU data from the optional `oliphaunt-icu` package. + /// Select ICU data from the optional `oliphaunt-wasix-icu` package. pub fn icu(mut self, data: oliphaunt_resources::IcuData) -> Self { self.icu = Some(data); self.catalog_profile = CatalogProfile::Icu; diff --git a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/tools.rs b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/tools.rs index 98c21a84b..923c628af 100644 --- a/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/tools.rs +++ b/src/bindings/wasix-rust/crates/oliphaunt-wasix/src/oliphaunt/tools.rs @@ -433,6 +433,16 @@ pub fn register_installed_package(package: oliphaunt_resources::WasixPackage) -> package.runtime_version() == liboliphaunt_wasix_portable::PACKAGE_VERSION, "installed tools package runtime mismatch" ); + if let Some(selected) = INSTALLED_TOOLS.get() { + if selected.product() == package.product() + && selected.version() == package.version() + && std::ptr::eq(selected.archives(), package.archives()) + && std::ptr::eq(selected.aot_artifacts(), package.aot_artifacts()) + && std::ptr::eq(selected.aot_manifest(), package.aot_manifest()) + { + return Ok(()); + } + } for name in ["pg_dump", "psql"] { let matches: Vec<_> = package .archives() @@ -477,6 +487,16 @@ pub fn installed_tool_wasm(name: &str) -> Option<&'static [u8]> { .map(|(_, bytes, _)| *bytes) } +#[doc(hidden)] +pub fn installed_tool_identity(name: &str) -> Option<(&'static str, usize)> { + INSTALLED_TOOLS + .get()? + .archives() + .iter() + .find(|(entry, _, _)| *entry == name) + .map(|(_, bytes, hash)| (*hash, bytes.len())) +} + fn pg_dump_wasm_asset() -> Result<&'static [u8]> { assets::pg_dump_wasm() .filter(|bytes| !bytes.is_empty()) diff --git a/src/bindings/wasix-ts/ARCHITECTURE.md b/src/bindings/wasix-ts/ARCHITECTURE.md index b7ad2809e..8ac9deb80 100644 --- a/src/bindings/wasix-ts/ARCHITECTURE.md +++ b/src/bindings/wasix-ts/ARCHITECTURE.md @@ -320,8 +320,8 @@ initial capacity use the portable path. Its size is an implementation detail, not a public database-capacity limit. Compatibility uses the PostgreSQL major and versioned WASIX physical format. -Runtime hashes and source fingerprints still reject mixed runtime, cluster-seed, -AOT, and extension build outputs, while package and carrier changes do not +Runtime/resource versions govern package compatibility, and checksums verify +artifact integrity, while package and carrier changes do not rewrite the managed-root descriptor or reject an unchanged physical format. Safe extension upgrade or removal remains an explicit migration concern rather than a reason to reject every change in the available carrier set. diff --git a/src/bindings/wasix-ts/README.md b/src/bindings/wasix-ts/README.md index 2ea6eda76..8ca29299b 100644 --- a/src/bindings/wasix-ts/README.md +++ b/src/bindings/wasix-ts/README.md @@ -203,8 +203,8 @@ pgdata/ ``` The descriptor records the shared database-root schema, PostgreSQL major, and -WASIX physical format. Runtime source fingerprints and package hashes validate -the asset graph; they are not physical-reopen identity. Native and WASIX roots +WASIX physical format. Runtime/resource versions determine package compatibility; checksums verify +artifact integrity. Neither is a physical-reopen identity. Native and WASIX roots are not rejected merely because of the originating family. Rust and WASIX TypeScript bindings use the same root and physical-archive @@ -332,14 +332,21 @@ await using database = await WorkerOliphaunt.open(); All imports expose the same PostgreSQL interface and retain the promise-shaped public API. A Promise does not itself imply off-thread execution. In a browser, -the root steps the Wasmer guest in the importing realm. On native hosts, the +`/browser` steps the Wasmer guest in the importing realm. On native hosts, the root uses one Rust owner actor so PostgreSQL does not block the importing event -loop. `/direct` calls the synchronous Rust database on the importing thread and +loop. Resource reads, verification, and startup run on that Rust owner; +caller-owned byte inputs are copied before crossing threads. Optional tool +registration uses Node background work before execution on the database owner. +`/direct` calls the synchronous Rust database on the importing thread and removes that actor hop. `/worker` uses a real package-owned JavaScript Worker on every runtime and loads the direct implementation inside it. -Importing the browser root or `/direct` from an application Worker blocks only -that Worker; importing the browser root in a Window can block the page. Browser +This differs from native `@oliphaunt/ts/direct`: there, “direct” selects an +in-process database topology and calls still run asynchronously. WASIX +`/direct` selects execution on the importing thread. + +Importing `/browser` from an application Worker blocks only that Worker; +importing `/browser` in a Window can block the page. Browser Worker use requires cross-origin isolation. Chromium Window compilation of native side modules larger than 8 MiB requires `/worker`. diff --git a/src/bindings/wasix-ts/package.json b/src/bindings/wasix-ts/package.json index 68df60c9d..d9795267f 100644 --- a/src/bindings/wasix-ts/package.json +++ b/src/bindings/wasix-ts/package.json @@ -22,7 +22,7 @@ "runtimeVersion": "0.2.0", "wasixNapiProduct": "oliphaunt-wasix-napi", "wasixNapiVersion": "0.1.0", - "wasixAddonAbiVersion": 1, + "wasixAddonAbiVersion": 2, "nodeApiVersion": 8, "browserHost": "wasmer-js-patched", "serverHost": "wasix-rust-napi" diff --git a/src/bindings/wasix-ts/src/__tests__/extensions.test.ts b/src/bindings/wasix-ts/src/__tests__/extensions.test.ts index 51af6f87b..98e5a6b95 100644 --- a/src/bindings/wasix-ts/src/__tests__/extensions.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/extensions.test.ts @@ -310,11 +310,6 @@ describe('WASIX TypeScript extensions', () => { expect(() => parseWasixAssetManifest(new TextEncoder().encode(JSON.stringify(invalid))), ).toThrow('format-version 2'); - - const missingFingerprint = { ...expected, 'source-fingerprint': undefined }; - expect(() => - parseWasixAssetManifest(new TextEncoder().encode(JSON.stringify(missingFingerprint))), - ).toThrow('WASIX asset source fingerprint'); }); }); @@ -414,7 +409,7 @@ function carrierMap( function manifest(): WasixAssetManifest { return { 'format-version': 2, - 'source-fingerprint': 'postgres-source-fingerprint', + runtime: { archive: 'oliphaunt.wasix.tar.zst', sha256: '0'.repeat(64), @@ -438,7 +433,7 @@ function manifest(): WasixAssetManifest { sha256: '4'.repeat(64), size: 100, 'runtime-module-sha256': '1'.repeat(64), - 'source-fingerprint': 'postgres-source-fingerprint', + 'postgres-version': '18', 'physical-format': 'wasix-pg18-v1', 'compatibility-key': 'wasix-pg18-datum32-v1', @@ -451,7 +446,7 @@ function manifest(): WasixAssetManifest { sha256: '6'.repeat(64), size: 101, 'runtime-module-sha256': '1'.repeat(64), - 'source-fingerprint': 'postgres-source-fingerprint', + 'postgres-version': '18', 'physical-format': 'wasix-pg18-v1', 'compatibility-key': 'wasix-pg18-datum32-v1', diff --git a/src/bindings/wasix-ts/src/__tests__/native-addon.test.ts b/src/bindings/wasix-ts/src/__tests__/native-addon.test.ts index 62149c39c..6a9e23665 100644 --- a/src/bindings/wasix-ts/src/__tests__/native-addon.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/native-addon.test.ts @@ -160,14 +160,14 @@ function addonFixture(): NativeWasixAddon { NativeWasixServer: Server, async restore() {}, restoreDirect() {}, - addonAbiVersion: () => 1, + addonAbiVersion: () => 2, nodeApiVersion: () => 8, runtimeVersion: () => '0.1.1', supportedProfiles: () => ['standard', 'icu'], payloadIdentity: () => `${'a'.repeat(64)}:1`, extensionIdentity: () => `${'a'.repeat(64)}:1`, toolIdentity: () => `${'a'.repeat(64)}:1`, - registerTools: () => {}, + registerTools: async () => {}, }; } @@ -179,7 +179,7 @@ function metadata() { runtimeVersion: '0.1.1', wasixNapiProduct: 'oliphaunt-wasix-napi', wasixNapiVersion: '0.1.1', - wasixAddonAbiVersion: 1, + wasixAddonAbiVersion: 2, nodeApiVersion: 8, }, }; diff --git a/src/bindings/wasix-ts/src/__tests__/native-extension-packages.test.ts b/src/bindings/wasix-ts/src/__tests__/native-extension-packages.test.ts index b60dc5ac5..d21f2bde3 100644 --- a/src/bindings/wasix-ts/src/__tests__/native-extension-packages.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/native-extension-packages.test.ts @@ -11,7 +11,7 @@ afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); -it('resolves an imported package by its file identity, including npm aliases, and rejects substituted versions', () => { +it('resolves an imported package by its file identity, including npm aliases, and rejects substituted versions', async () => { const root = mkdtempSync(join(tmpdir(), 'oliphaunt-extension-')); roots.push(root); const owner = join(root, 'node_modules', 'my-pgtap-version'); @@ -41,7 +41,7 @@ it('resolves an imported package by its file identity, including npm aliases, an }, }; writeFileSync(join(owner, 'package.json'), JSON.stringify(manifest)); - expect(nativeExtensionPackages(options)).toEqual([ + expect(await nativeExtensionPackages(options)).toEqual([ { sqlName: 'pgtap', product: carrier.product, @@ -51,7 +51,11 @@ it('resolves an imported package by its file identity, including npm aliases, an ]); manifest.version = '1.3.5'; writeFileSync(join(owner, 'package.json'), JSON.stringify(manifest)); - expect(() => nativeExtensionPackages(options)).toThrow('does not match its installed package'); + await expect(nativeExtensionPackages(options)).rejects.toThrow( + 'does not match its installed package', + ); carrier.source = 'https://example.com/untrusted.tar.zst'; - expect(() => nativeExtensionPackages(options)).toThrow('requires an installed package file URL'); + await expect(nativeExtensionPackages(options)).rejects.toThrow( + 'requires an installed package file URL', + ); }); diff --git a/src/bindings/wasix-ts/src/__tests__/native-session.test.ts b/src/bindings/wasix-ts/src/__tests__/native-session.test.ts index 2275870d1..c0e275ec0 100644 --- a/src/bindings/wasix-ts/src/__tests__/native-session.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/native-session.test.ts @@ -135,7 +135,7 @@ describe('WASIX native embedded payload compatibility', () => { nativeMocks.open.mockImplementation(() => { throw Object.assign(new Error('this deliberately says corrupt and available'), { oliphauntWasixError: 'storage', - oliphauntWasixAddonAbi: 1, + oliphauntWasixAddonAbi: 2, code: 'busy', commitState: 'unchanged', phase: 'ownership', @@ -391,7 +391,7 @@ function addon(): NativeWasixAddon { }) as unknown as NativeWasixAddon['NativeWasixServer'], async restore() {}, restoreDirect() {}, - addonAbiVersion: () => 1, + addonAbiVersion: () => 2, nodeApiVersion: () => 8, runtimeVersion: () => '0.1.1', supportedProfiles: () => ['standard', 'icu'], diff --git a/src/bindings/wasix-ts/src/__tests__/runtime-descriptor.test.ts b/src/bindings/wasix-ts/src/__tests__/runtime-descriptor.test.ts index d38eab1fc..0a8345e71 100644 --- a/src/bindings/wasix-ts/src/__tests__/runtime-descriptor.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/runtime-descriptor.test.ts @@ -1,9 +1,17 @@ +import { Buffer } from 'node:buffer'; +import { serializeAssetSource } from '../descriptor-validation.js'; import { describe, expect, it } from 'vitest'; import { serializeWasixRuntimeDescriptor } from '../runtime-descriptor.js'; import type { WasixRuntimeDescriptor } from '../types.js'; describe('WASIX runtime descriptors', () => { + it('owns Buffer snapshots, including views with an offset', () => { + const bytes = Buffer.from([0, 1, 2, 3]); + const snapshot = serializeAssetSource(bytes.subarray(1, 3)); + bytes.fill(9); + expect(snapshot).toEqual(Uint8Array.of(1, 2)); + }); it('serializes one exact runtime identity and preserves package-relative URLs', () => { const value = descriptor(); const serialized = serializeWasixRuntimeDescriptor(value); diff --git a/src/bindings/wasix-ts/src/__tests__/storage.test.ts b/src/bindings/wasix-ts/src/__tests__/storage.test.ts index 3def57cc6..1f48541b6 100644 --- a/src/bindings/wasix-ts/src/__tests__/storage.test.ts +++ b/src/bindings/wasix-ts/src/__tests__/storage.test.ts @@ -126,7 +126,7 @@ describe('WASIX storage descriptors', () => { const original = Object.assign(new Error('pg_dump reported an impossible success error'), { name: 'OliphauntWasixToolError', oliphauntWasixError: 'tool' as const, - oliphauntWasixAddonAbi: 1 as const, + oliphauntWasixAddonAbi: 2 as const, code: 'tool-error' as const, tool: 'pg_dump', exitCode: 0, @@ -139,7 +139,7 @@ describe('WASIX storage descriptors', () => { expect(roundTrip).toMatchObject({ name: 'OliphauntWasixToolError', oliphauntWasixError: 'tool', - oliphauntWasixAddonAbi: 1, + oliphauntWasixAddonAbi: 2, code: 'tool-error', tool: 'pg_dump', exitCode: 0, diff --git a/src/bindings/wasix-ts/src/descriptor-validation.ts b/src/bindings/wasix-ts/src/descriptor-validation.ts index 9bb7aa267..b211eacb7 100644 --- a/src/bindings/wasix-ts/src/descriptor-validation.ts +++ b/src/bindings/wasix-ts/src/descriptor-validation.ts @@ -80,7 +80,7 @@ export function serializeAssetSource(source: WasixAssetSource): SerializedAssetS return source.href; } if (source instanceof Uint8Array) { - return source.slice(); + return new Uint8Array(source); } return new Uint8Array(source.slice(0)); } diff --git a/src/bindings/wasix-ts/src/extensions.ts b/src/bindings/wasix-ts/src/extensions.ts index 1afe2aab7..9a724abbf 100644 --- a/src/bindings/wasix-ts/src/extensions.ts +++ b/src/bindings/wasix-ts/src/extensions.ts @@ -39,7 +39,6 @@ type ClusterSeedManifest = { initdbSha256: string; }; source: { - fingerprint: string; catalogVersion: string; lane: string; producer: string; @@ -412,7 +411,6 @@ export function parseWasixAssetManifest(bytes: Uint8Array): WasixAssetManifest { requireSafeInteger(seed.size, `WASIX ${profile} cluster seed archive size`); requireSha256(seed['runtime-module-sha256'], `WASIX ${profile} seed runtime module`); requireString(seed['postgres-version'], `WASIX ${profile} seed PostgreSQL version`); - requireString(seed['source-fingerprint'], `WASIX ${profile} seed source fingerprint`); if ( seed['physical-format'] !== 'wasix-pg18-v1' || seed['compatibility-key'] !== 'wasix-pg18-datum32-v1' @@ -425,7 +423,6 @@ export function parseWasixAssetManifest(bytes: Uint8Array): WasixAssetManifest { throw new Error('WASIX standard cluster seed must not identify ICU data'); } } - requireString(manifest['source-fingerprint'], 'WASIX asset source fingerprint'); const runtimeSupport = requireArray(manifest['runtime-support'], 'WASIX runtime-support entries'); for (const [index, value] of runtimeSupport.entries()) { @@ -736,12 +733,8 @@ function parseClusterSeedManifest( } const source = requireObject(root.source, `${label} source`); - requireExactKeys( - source, - ['catalogVersion', 'fingerprint', 'lane', 'producer'], - `${label} source`, - ); - for (const field of ['catalogVersion', 'fingerprint', 'lane', 'producer'] as const) { + requireExactKeys(source, ['catalogVersion', 'lane', 'producer'], `${label} source`); + for (const field of ['catalogVersion', 'lane', 'producer'] as const) { requireString(source[field], `${label} source ${field}`); } @@ -889,12 +882,6 @@ function verifyClusterSeedIdentity( ) { throw new Error(`WASIX ${profile} cluster seed was produced by a different runtime module`); } - if ( - seed.source.fingerprint !== outer['source-fingerprint'] || - seed.source.fingerprint !== selected['source-fingerprint'] - ) { - throw new Error(`WASIX ${profile} cluster seed has a different source fingerprint`); - } if ( seed.archive.path !== archive.archive || seed.archive.path !== selected.archive || diff --git a/src/bindings/wasix-ts/src/native-addon.ts b/src/bindings/wasix-ts/src/native-addon.ts index cdc4ddfcd..059715cbf 100644 --- a/src/bindings/wasix-ts/src/native-addon.ts +++ b/src/bindings/wasix-ts/src/native-addon.ts @@ -21,12 +21,12 @@ export type NativeWasixOpenOptions = Readonly<{ icu?: Readonly<{ version: string; runtimeVersion: string; - archive: Uint8Array; + archive: string | Uint8Array; archiveSha256: string; dataTreeSha256: string; - seedArchive: Uint8Array; + seedArchive: string | Uint8Array; seedArchiveSha256: string; - seedManifest: Uint8Array; + seedManifest: string | Uint8Array; seedManifestSha256: string; }>; }>; @@ -108,7 +108,7 @@ export type NativeWasixAddon = { ): string; extensionIdentity(sqlName: string): string; toolIdentity(name: 'pg_dump' | 'psql'): string; - registerTools(options: { packageJson: string; aotPackageJson: string }): void; + registerTools(options: { packageJson: string; aotPackageJson: string }): Promise; }; type WasixPackageMetadata = Readonly<{ @@ -383,7 +383,7 @@ export function validateNativeWasixAddon( throw new Error(`Oliphaunt WASIX native addon ${path} has an invalid export surface`); } const expectedAbi = metadata.oliphaunt?.wasixAddonAbiVersion; - if (expectedAbi !== 1 || addon.addonAbiVersion() !== expectedAbi) { + if (expectedAbi !== 2 || addon.addonAbiVersion() !== expectedAbi) { throw new Error(`Oliphaunt WASIX native addon ${path} has an incompatible addon ABI`); } const expectedNodeApi = metadata.oliphaunt?.nodeApiVersion; diff --git a/src/bindings/wasix-ts/src/native-extension-packages.ts b/src/bindings/wasix-ts/src/native-extension-packages.ts index 8c8f195af..02d4260bc 100644 --- a/src/bindings/wasix-ts/src/native-extension-packages.ts +++ b/src/bindings/wasix-ts/src/native-extension-packages.ts @@ -1,4 +1,4 @@ -import { readFileSync, realpathSync } from 'node:fs'; +import { readFile, realpath } from 'node:fs/promises'; import { createRequire } from 'node:module'; import { arch, platform } from 'node:os'; import { dirname, join } from 'node:path'; @@ -7,16 +7,16 @@ import { nativeTarget, type NativeExtensionPackage } from './native-addon.js'; import type { SerializedOpenOptions } from './rpc.js'; import type { WasixToolProcessOptions } from './tool-runtime.js'; -export function nativeToolPackage(tool: WasixToolProcessOptions['tool']): { +export async function nativeToolPackage(tool: WasixToolProcessOptions['tool']): Promise<{ packageJson: string; aotPackageJson: string; -} { +}> { if (typeof tool.source !== 'string' || !tool.source.startsWith('file:')) { throw new Error('WASIX native tools require an installed package file URL'); } - const modulePath = realpathSync(fileURLToPath(tool.source)); + const modulePath = await realpath(fileURLToPath(tool.source)); const packageJson = join(dirname(dirname(modulePath)), 'package.json'); - const manifest = JSON.parse(readFileSync(packageJson, 'utf8')); + const manifest = JSON.parse(await readFile(packageJson, 'utf8')); const name = '@oliphaunt/liboliphaunt-wasix-tools'; const payload = manifest.oliphaunt?.tools?.[tool.name]; if ( @@ -39,52 +39,56 @@ export function nativeToolPackage(tool: WasixToolProcessOptions['tool']): { } /** Installed native dependencies have the same trust as the application's imports. */ -export function nativeExtensionPackages(options: SerializedOpenOptions): NativeExtensionPackage[] { - return Object.values(options.extensionCarriers) - .filter((carrier) => carrier.product !== 'oliphaunt-extension-contrib-pg18') - .map((carrier) => { - if (typeof carrier.source !== 'string' || !carrier.source.startsWith('file:')) { - throw new Error( - `WASIX native extension ${carrier.sqlName} requires an installed package file URL`, - ); - } - const archive = realpathSync(fileURLToPath(carrier.source)); - const root = dirname(dirname(dirname(archive))); - const packageJson = join(root, 'package.json'); - const manifest = JSON.parse(readFileSync(packageJson, 'utf8')); - const packageName = `@oliphaunt/${carrier.product.slice('oliphaunt-'.length)}-wasix`; - const payload = manifest.oliphaunt?.carriers?.[carrier.sqlName]; - if ( - manifest.name !== packageName || - manifest.version !== carrier.version || - manifest.oliphaunt?.product !== carrier.product || - manifest.oliphaunt?.kind !== 'exact-extension-wasix' || - manifest.oliphaunt?.wasixRuntimeVersion !== options.runtime.version || - payload?.path !== `extensions/${carrier.sqlName}/extension.tar.zst` || - payload.sha256 !== carrier.sha256 || - payload.size !== carrier.size || - typeof payload.requiresAot !== 'boolean' || - archive !== realpathSync(join(root, payload.path)) - ) { - throw new Error( - `WASIX extension ${carrier.sqlName} descriptor does not match its installed package`, - ); - } - let aotPackageJson: string | undefined; - if (payload.requiresAot) { - const target = nativeTarget(platform(), arch()); - const aotName = `${packageName}-${target.id}`; - if (manifest.optionalDependencies?.[aotName] !== carrier.version) { - throw new Error(`WASIX extension ${carrier.sqlName} has no exact host AOT dependency`); +export async function nativeExtensionPackages( + options: SerializedOpenOptions, +): Promise { + return Promise.all( + Object.values(options.extensionCarriers) + .filter((carrier) => carrier.product !== 'oliphaunt-extension-contrib-pg18') + .map(async (carrier) => { + if (typeof carrier.source !== 'string' || !carrier.source.startsWith('file:')) { + throw new Error( + `WASIX native extension ${carrier.sqlName} requires an installed package file URL`, + ); } - aotPackageJson = createRequire(packageJson).resolve(`${aotName}/package.json`); - } - return { - sqlName: carrier.sqlName, - product: carrier.product, - version: carrier.version, - packageJson, - ...(aotPackageJson === undefined ? {} : { aotPackageJson }), - }; - }); + const archive = await realpath(fileURLToPath(carrier.source)); + const root = dirname(dirname(dirname(archive))); + const packageJson = join(root, 'package.json'); + const manifest = JSON.parse(await readFile(packageJson, 'utf8')); + const packageName = `@oliphaunt/${carrier.product.slice('oliphaunt-'.length)}-wasix`; + const payload = manifest.oliphaunt?.carriers?.[carrier.sqlName]; + if ( + manifest.name !== packageName || + manifest.version !== carrier.version || + manifest.oliphaunt?.product !== carrier.product || + manifest.oliphaunt?.kind !== 'exact-extension-wasix' || + manifest.oliphaunt?.wasixRuntimeVersion !== options.runtime.version || + payload?.path !== `extensions/${carrier.sqlName}/extension.tar.zst` || + payload.sha256 !== carrier.sha256 || + payload.size !== carrier.size || + typeof payload.requiresAot !== 'boolean' || + archive !== (await realpath(join(root, payload.path))) + ) { + throw new Error( + `WASIX extension ${carrier.sqlName} descriptor does not match its installed package`, + ); + } + let aotPackageJson: string | undefined; + if (payload.requiresAot) { + const target = nativeTarget(platform(), arch()); + const aotName = `${packageName}-${target.id}`; + if (manifest.optionalDependencies?.[aotName] !== carrier.version) { + throw new Error(`WASIX extension ${carrier.sqlName} has no exact host AOT dependency`); + } + aotPackageJson = createRequire(packageJson).resolve(`${aotName}/package.json`); + } + return { + sqlName: carrier.sqlName, + product: carrier.product, + version: carrier.version, + packageJson, + ...(aotPackageJson === undefined ? {} : { aotPackageJson }), + }; + }), + ); } diff --git a/src/bindings/wasix-ts/src/native-session.ts b/src/bindings/wasix-ts/src/native-session.ts index ab1281b57..dcc42fd37 100644 --- a/src/bindings/wasix-ts/src/native-session.ts +++ b/src/bindings/wasix-ts/src/native-session.ts @@ -1,4 +1,3 @@ -import { readFile } from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; import { WasixStorageError, @@ -138,7 +137,7 @@ export class NativeWasixSession implements WasixDatabaseSession { async runTool(options: WasixToolProcessOptions): Promise { this.#assertOpen(); - validateNativeToolCall(this.#addon, this.#runtimeVersion, options); + await validateNativeToolCall(this.#addon, this.#runtimeVersion, options); if (options.tool.name === 'pg_dump') { try { return toolProcessResult(this.#handle.pgDump(options.args)); @@ -278,7 +277,7 @@ export class NativeWasixActorSession implements WasixDatabaseSession { async runTool(options: WasixToolProcessOptions): Promise { this.#assertOpen(); - validateNativeToolCall(this.#addon, this.#runtimeVersion, options); + await validateNativeToolCall(this.#addon, this.#runtimeVersion, options); try { if (options.tool.name === 'pg_dump') { return toolProcessResult(await this.#handle.pgDump(options.args)); @@ -505,12 +504,12 @@ export async function nativeWasixOpenOptions( icu: { version: options.icu.version, runtimeVersion: options.icu.compatibility.runtimeVersion, - archive: await nativeIcuBytes(options.icu.dataArchive.source), + archive: nativeIcuSource(options.icu.dataArchive.source), archiveSha256: options.icu.dataArchive.sha256, dataTreeSha256: options.icu.compatibility.dataTreeSha256, - seedArchive: await nativeIcuBytes(options.icu.clusterSeedArchive.source), + seedArchive: nativeIcuSource(options.icu.clusterSeedArchive.source), seedArchiveSha256: options.icu.clusterSeedArchive.sha256, - seedManifest: await nativeIcuBytes(options.icu.clusterSeedManifest.source), + seedManifest: nativeIcuSource(options.icu.clusterSeedManifest.source), seedManifestSha256: options.icu.clusterSeedManifest.sha256, }, }), @@ -522,17 +521,17 @@ export async function nativeWasixOpenOptions( ...(Object.values(options.extensionCarriers).some( (carrier) => carrier.product !== 'oliphaunt-extension-contrib-pg18', ) - ? { extensionPackages: nativeExtensionPackages(options) } + ? { extensionPackages: await nativeExtensionPackages(options) } : {}), }; } -async function nativeIcuBytes(source: string | Uint8Array): Promise { +function nativeIcuSource(source: string | Uint8Array): string | Uint8Array { if (source instanceof Uint8Array) return Buffer.from(source.buffer, source.byteOffset, source.byteLength); if (!source.startsWith('file:')) throw new TypeError('WASIX native ICU data requires an installed file URL or bytes'); - return readFile(fileURLToPath(source)); + return fileURLToPath(source); } function nativeStorage(options: SerializedOpenOptions): NativeWasixOpenOptions['storage'] { @@ -546,11 +545,11 @@ function nativeStorage(options: SerializedOpenOptions): NativeWasixOpenOptions[' const registeredTools = new WeakMap>(); -function validateNativeToolCall( +async function validateNativeToolCall( addon: NativeWasixAddon, runtimeVersion: string, options: WasixToolProcessOptions, -): void { +): Promise { if (options.runtimeVersion !== '' && options.runtimeVersion !== runtimeVersion) { throw new Error( `WASIX tools runtime ${options.runtimeVersion} is incompatible with database runtime ${runtimeVersion}`, @@ -560,10 +559,10 @@ function validateNativeToolCall( const key = `${options.tool.name}:${options.tool.sha256}:${options.tool.source}`; const registered = registeredTools.get(addon) ?? new Set(); if (!registered.has(key)) { - const packageDescriptor = nativeToolPackage(options.tool); + const packageDescriptor = await nativeToolPackage(options.tool); const packageKey = JSON.stringify(packageDescriptor); if (!registered.has(packageKey)) { - addon.registerTools(packageDescriptor); + await addon.registerTools(packageDescriptor); registered.add(packageKey); } registered.add(key); @@ -677,7 +676,7 @@ function nativeStorageError(error: unknown): NativeStorageError | undefined { const candidate = error as Record; if ( candidate.oliphauntWasixError !== 'storage' || - candidate.oliphauntWasixAddonAbi !== 1 || + candidate.oliphauntWasixAddonAbi !== 2 || !memberOf(candidate.code, STORAGE_CODES) || !memberOf(candidate.commitState, STORAGE_COMMIT_STATES) || !memberOf(candidate.phase, STORAGE_PHASES) diff --git a/src/bindings/wasix-ts/src/rpc.ts b/src/bindings/wasix-ts/src/rpc.ts index d892c2b48..4945442fc 100644 --- a/src/bindings/wasix-ts/src/rpc.ts +++ b/src/bindings/wasix-ts/src/rpc.ts @@ -249,7 +249,7 @@ export function deserializeWorkerError(error: SerializedWorkerError): Error { restored.name = error.name; return Object.assign(restored, { oliphauntWasixError: 'tool' as const, - oliphauntWasixAddonAbi: 1 as const, + oliphauntWasixAddonAbi: 2 as const, code: error.code, tool: error.tool, exitCode: error.exitCode, @@ -276,7 +276,7 @@ function nativeToolError(error: unknown): NativeToolError | undefined { const candidate = error as Record; if ( candidate.oliphauntWasixError !== 'tool' || - candidate.oliphauntWasixAddonAbi !== 1 || + candidate.oliphauntWasixAddonAbi !== 2 || candidate.code !== 'tool-error' || typeof candidate.message !== 'string' || typeof candidate.tool !== 'string' || diff --git a/src/bindings/wasix-ts/src/types.ts b/src/bindings/wasix-ts/src/types.ts index fa657eaad..b47da86dd 100644 --- a/src/bindings/wasix-ts/src/types.ts +++ b/src/bindings/wasix-ts/src/types.ts @@ -156,7 +156,6 @@ export type WasixExtensionLifecycle = { /** Host-relevant subset of the generated liboliphaunt WASIX asset manifest. */ export type WasixAssetManifest = { 'format-version': 2; - 'source-fingerprint': string; runtime: { archive: string; sha256: string; @@ -187,7 +186,6 @@ export type WasixAssetManifest = { sha256: string; size: number; 'runtime-module-sha256': string; - 'source-fingerprint': string; 'postgres-version': string; 'physical-format': 'wasix-pg18-v1'; 'compatibility-key': 'wasix-pg18-datum32-v1'; diff --git a/src/runtimes/liboliphaunt/icu/Cargo.toml b/src/runtimes/liboliphaunt/icu/Cargo.toml index cba1c30e7..0936c6ff9 100644 --- a/src/runtimes/liboliphaunt/icu/Cargo.toml +++ b/src/runtimes/liboliphaunt/icu/Cargo.toml @@ -3,7 +3,7 @@ name = "oliphaunt-icu" version = "0.2.0" edition = "2024" rust-version = "1.93" -description = "Optional ICU data files for Oliphaunt runtimes." +description = "Optional ICU data and cluster seeds for the Oliphaunt native runtime." readme = "README.md" repository = "https://github.com/f0rr0/oliphaunt" homepage = "https://oliphaunt.dev" @@ -11,18 +11,7 @@ documentation = "https://docs.rs/oliphaunt-icu" license = "MIT AND PostgreSQL AND Unicode-3.0" links = "oliphaunt_artifact_oliphaunt_icu" build = "build.rs" -include = [ - "Cargo.toml", - "README.md", - "build.rs", - "src/**", - "payload/**", - "LICENSE", - "THIRD_PARTY_NOTICES.md", - "THIRD_PARTY_NOTICES.oliphaunt-wasix.md", - "THIRD_PARTY_LICENSES/ICU-LICENSE", - "THIRD_PARTY_LICENSES/PostgreSQL-COPYRIGHT", -] +include = ["Cargo.toml","README.md","build.rs","src/**","payload/**","LICENSE","THIRD_PARTY_NOTICES.md","THIRD_PARTY_NOTICES.liboliphaunt-native.md","THIRD_PARTY_LICENSES/PostgreSQL-COPYRIGHT","THIRD_PARTY_LICENSES/ICU-LICENSE"] [lib] path = "src/lib.rs" @@ -32,5 +21,3 @@ oliphaunt-resources = { version = "0.2.0", path = "../../../sdks/rust/crates/oli [build-dependencies] sha2 = "0.10" -tar = "0.4" -zstd = { version = "0.13", default-features = false } diff --git a/src/runtimes/liboliphaunt/icu/README.md b/src/runtimes/liboliphaunt/icu/README.md index f3d9b1918..7be5ab1f6 100644 --- a/src/runtimes/liboliphaunt/icu/README.md +++ b/src/runtimes/liboliphaunt/icu/README.md @@ -15,5 +15,5 @@ let db = oliphaunt::Oliphaunt::builder() ``` The package embeds the current native target's seed and ICU data. WASIX -consumers use the package's WASIX descriptor with `oliphaunt-wasix` instead. +consumers use `oliphaunt-wasix-icu` with `oliphaunt-wasix` instead. Base runtime carriers include only the standard seed. diff --git a/src/runtimes/liboliphaunt/icu/build.rs b/src/runtimes/liboliphaunt/icu/build.rs index 82da64bf6..ee2b569d7 100644 --- a/src/runtimes/liboliphaunt/icu/build.rs +++ b/src/runtimes/liboliphaunt/icu/build.rs @@ -1,7 +1,7 @@ use std::env; use std::fs; use std::io::{self, Read}; -use std::path::{Component, Path, PathBuf}; +use std::path::{Path, PathBuf}; use sha2::{Digest, Sha256}; @@ -9,52 +9,25 @@ const ARTIFACT_SCHEMA: &str = "oliphaunt-artifact-manifest-v1"; const ARTIFACT_PRODUCT: &str = "oliphaunt-icu"; const ARTIFACT_KIND: &str = "icu-data"; const ARTIFACT_TARGET: &str = "portable"; -const PACKAGED_ICU_ARCHIVE: &str = "payload/icu-data.tar.zst"; fn main() { - let native_version_file = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()) - .join("payload/native-runtime-version"); - println!("cargo:rerun-if-changed={}", native_version_file.display()); - let native_version = fs::read_to_string(&native_version_file) - .unwrap_or_else(|_| env::var("CARGO_PKG_VERSION").unwrap()); - let native_version = native_version.trim(); - assert!( - !native_version.is_empty() - && native_version - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || b".-+".contains(&byte)), - "invalid native runtime version" - ); - println!("cargo:rustc-env=OLIPHAUNT_ICU_NATIVE_RUNTIME_VERSION={native_version}"); println!("cargo:rerun-if-env-changed=OLIPHAUNT_ICU_DATA_DIR"); println!("cargo:rerun-if-env-changed=OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD"); let out_dir = PathBuf::from(env::var_os("OUT_DIR").expect("OUT_DIR is set by Cargo")); - let out = out_dir.join("generated_icu.rs"); - if let Some(archive) = find_packaged_icu_archive() { - println!("cargo:rerun-if-changed={}", archive.display()); - let extracted_root = unpack_icu_archive(&archive, &out_dir.join("icu-data-expanded")); - emit_icu_artifact(&out, &out_dir, &archive, &extracted_root); - } else if let Some(icu_root) = find_icu_data_root() { + if let Some(icu_root) = find_icu_data_root() { emit_rerun_directives(&icu_root); - let archive = out_dir.join("icu-data.tar.zst"); - write_icu_archive(&icu_root, &archive); - emit_icu_artifact(&out, &out_dir, &archive, &icu_root); + emit_icu_artifact(&out_dir, &icu_root); } else { if env::var_os("OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD").is_some() { - panic!( - "release packaging requires package-local ICU data under payload/icu-data.tar.zst or payload/share/icu" - ); + panic!("release packaging requires package-local ICU data under payload/share/icu"); } - write_generated_icu(&out, None); fs::write(out_dir.join("native_icu.rs"), "&[]\n").expect("write empty native ICU index"); } } -fn emit_icu_artifact(out: &Path, out_dir: &Path, archive: &Path, icu_root: &Path) { - let archive_sha256 = sha256_file(archive).expect("digest ICU data archive"); +fn emit_icu_artifact(out_dir: &Path, icu_root: &Path) { let data_tree_sha256 = logical_tree_sha256(icu_root).expect("digest ICU logical data tree"); - write_generated_icu(out, Some((archive, &archive_sha256, &data_tree_sha256))); let receipt = out_dir.join("native-icu.properties"); fs::write(&receipt, format!("schema=oliphaunt-icu-data-v1\nartifactRole=icu-data\nicuDataVersion=76.1\nicuDataForm=files-le\nicuDataTreeSha256={data_tree_sha256}\n")).expect("write native ICU receipt"); emit_artifact_manifest(out_dir, icu_root, &receipt); @@ -133,13 +106,6 @@ fn native_seed_root() -> Option<(&'static str, PathBuf)> { Some((target, root)) } -fn find_packaged_icu_archive() -> Option { - let manifest_dir = - PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set")); - let archive = manifest_dir.join(PACKAGED_ICU_ARCHIVE); - archive.is_file().then_some(archive) -} - fn find_icu_data_root() -> Option { let manifest_dir = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set")); @@ -160,78 +126,6 @@ fn icu_candidates(manifest_dir: &Path) -> Vec { candidates } -fn unpack_icu_archive(archive: &Path, destination: &Path) -> PathBuf { - if destination.exists() { - fs::remove_dir_all(destination).expect("remove previously unpacked ICU data archive"); - } - fs::create_dir_all(destination).expect("create ICU data archive destination"); - let file = fs::File::open(archive).expect("open packaged ICU data archive"); - let decoder = zstd::stream::read::Decoder::new(file).expect("decode packaged ICU data archive"); - let mut archive_reader = tar::Archive::new(decoder); - let entries = archive_reader - .entries() - .expect("read packaged ICU data archive entries"); - let mut entry_count = 0_usize; - for entry in entries { - entry_count += 1; - assert!( - entry_count <= 8192, - "packaged ICU data archive has too many entries" - ); - let mut entry = entry.expect("read packaged ICU data archive entry"); - let path = entry - .path() - .expect("read packaged ICU data archive entry path") - .into_owned(); - let relative = icu_archive_relative_path(&path); - let destination_path = destination.join(&relative); - let entry_type = entry.header().entry_type(); - if entry_type.is_dir() { - fs::create_dir_all(&destination_path).expect("create ICU data archive directory"); - continue; - } - if !entry_type.is_file() { - panic!( - "packaged ICU data archive entry {} has unsupported type {:?}", - path.display(), - entry_type - ); - } - if let Some(parent) = destination_path.parent() { - fs::create_dir_all(parent).expect("create ICU data archive entry parent"); - } - entry - .unpack(&destination_path) - .expect("unpack packaged ICU data archive entry"); - } - let root = destination.join("share/icu"); - canonical_icu_data_root(&root).expect("packaged ICU data archive contains share/icu data") -} - -fn icu_archive_relative_path(path: &Path) -> PathBuf { - let mut relative = PathBuf::new(); - let mut components = Vec::new(); - for component in path.components() { - match component { - Component::CurDir => {} - Component::Normal(part) => { - relative.push(part); - components.push(part.to_owned()); - } - _ => panic!("unsafe packaged ICU data archive entry {}", path.display()), - } - } - let under_share_icu = components.first().and_then(|part| part.to_str()) == Some("share") - && components.get(1).and_then(|part| part.to_str()) == Some("icu"); - if !under_share_icu { - panic!( - "packaged ICU data archive entry {} must stay under share/icu", - path.display() - ); - } - relative -} - fn canonical_icu_data_root(candidate: &Path) -> Option { if icu_root_contains_data(candidate) { return Some(candidate.to_path_buf()); @@ -279,67 +173,6 @@ fn emit_rerun_directives(root: &Path) { } } -fn write_icu_archive(icu_root: &Path, archive: &Path) { - let file = fs::File::create(archive).expect("create ICU data archive"); - let encoder = zstd::stream::write::Encoder::new(file, 19).expect("create zstd encoder"); - let mut builder = tar::Builder::new(encoder); - for source in collect_files(icu_root).expect("collect ICU data files") { - let relative = source - .strip_prefix(icu_root) - .expect("ICU file stays under ICU root"); - let archive_path = Path::new("share/icu").join(relative); - let bytes = fs::read(&source).expect("read ICU data file"); - let mut header = tar::Header::new_gnu(); - header.set_size(bytes.len() as u64); - header.set_mode(0o644); - header.set_uid(0); - header.set_gid(0); - header.set_mtime(0); - header.set_cksum(); - builder - .append_data(&mut header, &archive_path, bytes.as_slice()) - .expect("append ICU data file"); - } - let encoder = builder.into_inner().expect("finish ICU tar archive"); - encoder.finish().expect("finish ICU zstd archive"); -} - -fn write_generated_icu(out: &Path, archive: Option<(&Path, &str, &str)>) { - let text = match archive { - Some((archive, archive_sha256, data_tree_sha256)) => format!( - "pub const HAS_ICU_DATA: bool = true;\n\ - pub const ICU_DATA_ARCHIVE_SHA256: Option<&str> = Some({archive_sha256:?});\n\ - pub const ICU_DATA_TREE_SHA256: Option<&str> = Some({data_tree_sha256:?});\n\ - pub const fn icu_data_archive() -> Option<&'static [u8]> {{ Some(include_bytes!({archive:?})) }}\n", - archive = archive.to_string_lossy(), - ), - None => "pub const HAS_ICU_DATA: bool = false;\n\ - pub const ICU_DATA_ARCHIVE_SHA256: Option<&str> = None;\n\ - pub const ICU_DATA_TREE_SHA256: Option<&str> = None;\n\ - pub const fn icu_data_archive() -> Option<&'static [u8]> { None }\n" - .to_owned(), - }; - let mut text = text; - for (name, file) in [ - ("ICU_SEED_ARCHIVE", "icu.tar.zst"), - ("ICU_SEED_MANIFEST", "icu.json"), - ] { - let seed = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("manifest dir")) - .join("payload/cluster-seeds") - .join(file); - println!("cargo:rerun-if-changed={}", seed.display()); - let body = if seed.is_file() { - format!("Some(include_bytes!({seed:?}))") - } else { - "None".into() - }; - text.push_str(&format!( - "pub const {name}: Option<&'static [u8]> = {body};\n" - )); - } - fs::write(out, text).expect("write generated ICU data module"); -} - fn emit_artifact_manifest(out_dir: &Path, icu_root: &Path, receipt: &Path) { let version = env::var("CARGO_PKG_VERSION").expect("CARGO_PKG_VERSION is set by Cargo"); let manifest_path = out_dir.join("oliphaunt-artifact.toml"); diff --git a/src/runtimes/liboliphaunt/icu/src/lib.rs b/src/runtimes/liboliphaunt/icu/src/lib.rs index d9cc5e262..150b1c5b6 100644 --- a/src/runtimes/liboliphaunt/icu/src/lib.rs +++ b/src/runtimes/liboliphaunt/icu/src/lib.rs @@ -1,28 +1,14 @@ #![deny(unsafe_code)] -include!(concat!(env!("OUT_DIR"), "/generated_icu.rs")); - -/// Select this optional ICU package for a native or WASIX database. +/// Optional ICU data and seed for the matching native runtime. pub const ICU: oliphaunt_resources::IcuData = oliphaunt_resources::IcuData { version: env!("CARGO_PKG_VERSION"), - native_runtime_version: env!("OLIPHAUNT_ICU_NATIVE_RUNTIME_VERSION"), - runtime_version: env!("CARGO_PKG_VERSION"), + native_runtime_version: env!("CARGO_PKG_VERSION"), + runtime_version: "unavailable", resources: include!(concat!(env!("OUT_DIR"), "/native_icu.rs")), - wasix_archive: icu_data_archive(), - wasix_archive_sha256: ICU_DATA_ARCHIVE_SHA256, - wasix_data_tree_sha256: ICU_DATA_TREE_SHA256, - wasix_seed_archive: ICU_SEED_ARCHIVE, - wasix_seed_manifest: ICU_SEED_MANIFEST, + wasix_archive: None, + wasix_archive_sha256: None, + wasix_data_tree_sha256: None, + wasix_seed_archive: None, + wasix_seed_manifest: None, }; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn packaged_identities_track_the_embedded_payload() { - assert_eq!(HAS_ICU_DATA, icu_data_archive().is_some()); - assert_eq!(HAS_ICU_DATA, ICU_DATA_ARCHIVE_SHA256.is_some()); - assert_eq!(HAS_ICU_DATA, ICU_DATA_TREE_SHA256.is_some()); - } -} diff --git a/src/runtimes/liboliphaunt/native/release.toml b/src/runtimes/liboliphaunt/native/release.toml index f4d0bf043..1cef0f188 100644 --- a/src/runtimes/liboliphaunt/native/release.toml +++ b/src/runtimes/liboliphaunt/native/release.toml @@ -3,6 +3,7 @@ owner = "@oliphaunt/core" kind = "native-core" publish_targets = ["github-release-assets", "npm", "maven-central", "crates-io"] registry_packages = [ + "crates:oliphaunt-icu", "crates:liboliphaunt-native-linux-arm64-gnu", "crates:liboliphaunt-native-linux-x64-gnu", "crates:liboliphaunt-native-macos-arm64", @@ -38,3 +39,11 @@ release_artifacts = [ "runtime-resources-android-datum64", "icu-data", ] + +derived_version_files = ["src/runtimes/liboliphaunt/icu/Cargo.toml"] +shared_source_paths = ["src/runtimes/liboliphaunt/icu"] + +[compatibility_versions.liboliphaunt-native-icu] +source_product = "liboliphaunt-native" +path = "src/runtimes/liboliphaunt/icu/Cargo.toml" +parser = "toml:package.version" diff --git a/src/runtimes/liboliphaunt/wasix/crates/assets/src/lib.rs b/src/runtimes/liboliphaunt/wasix/crates/assets/src/lib.rs index f57d08c53..a50d4c251 100644 --- a/src/runtimes/liboliphaunt/wasix/crates/assets/src/lib.rs +++ b/src/runtimes/liboliphaunt/wasix/crates/assets/src/lib.rs @@ -18,8 +18,6 @@ pub struct AssetManifest { pub format_version: u32, #[serde(default)] pub source_lane: Option, - #[serde(default)] - pub source_fingerprint: Option, pub runtime: RuntimeAsset, #[serde(default)] pub runtime_support: Vec, @@ -75,8 +73,6 @@ pub struct ClusterSeedAsset { pub source_pins_sha256: String, #[serde(default)] pub source_lane: Option, - #[serde(default)] - pub source_fingerprint: Option, pub postgres_version: String, pub catalog_version: String, pub init_profile: String, @@ -245,7 +241,6 @@ mod tests { r#"{ "format-version": 2, "source-lane": "stable", - "source-fingerprint": "postgresql-18.4:patch-stack", "runtime": { "archive": "oliphaunt.wasix.tar.zst", "sha256": "runtime-archive", @@ -266,7 +261,6 @@ mod tests { "initdb-module-sha256": "initdb-module", "source-pins-sha256": "source-pins", "source-lane": "stable", - "source-fingerprint": "postgresql-18.4:patch-stack", "postgres-version": "18", "catalog-version": "202505281", "init-profile": "default", @@ -282,19 +276,11 @@ mod tests { .expect("PG18 asset manifest metadata should parse"); assert_eq!(manifest.source_lane.as_deref(), Some("stable")); - assert_eq!( - manifest.source_fingerprint.as_deref(), - Some("postgresql-18.4:patch-stack") - ); let seed = manifest .cluster_seeds .get("standard") .expect("standard cluster seed asset"); assert_eq!(seed.catalog_profile, "standard"); assert_eq!(seed.source_lane.as_deref(), Some("stable")); - assert_eq!( - seed.source_fingerprint.as_deref(), - Some("postgresql-18.4:patch-stack") - ); } } diff --git a/src/runtimes/liboliphaunt/wasix/crates/icu/Cargo.toml b/src/runtimes/liboliphaunt/wasix/crates/icu/Cargo.toml new file mode 100644 index 000000000..74a52234c --- /dev/null +++ b/src/runtimes/liboliphaunt/wasix/crates/icu/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "oliphaunt-wasix-icu" +version = "0.2.0" +edition = "2024" +rust-version = "1.93" +description = "Optional ICU data and cluster seed for the Oliphaunt WASIX runtime." +readme = "README.md" +repository = "https://github.com/f0rr0/oliphaunt" +homepage = "https://oliphaunt.dev" +license = "MIT AND PostgreSQL AND Unicode-3.0" +build = "build.rs" +include = ["Cargo.toml","README.md","build.rs","src/**","payload/**","LICENSE","THIRD_PARTY_NOTICES.md","THIRD_PARTY_NOTICES.oliphaunt-wasix.md","THIRD_PARTY_LICENSES/PostgreSQL-COPYRIGHT","THIRD_PARTY_LICENSES/ICU-LICENSE"] + +[dependencies] +oliphaunt-resources = { version = "0.2.0", path = "../../../../../sdks/rust/crates/oliphaunt-resources" } + +[build-dependencies] +sha2 = "0.10" +serde_json = "1" diff --git a/src/runtimes/liboliphaunt/wasix/crates/icu/README.md b/src/runtimes/liboliphaunt/wasix/crates/icu/README.md new file mode 100644 index 000000000..fcc99312a --- /dev/null +++ b/src/runtimes/liboliphaunt/wasix/crates/icu/README.md @@ -0,0 +1,5 @@ +# oliphaunt-wasix-icu + +Optional ICU data and cluster seed for the matching WASIX runtime version. +Add this crate explicitly and pass `.icu(oliphaunt_wasix_icu::ICU)` to your builder. +For native Rust use `oliphaunt-icu` instead. diff --git a/src/runtimes/liboliphaunt/wasix/crates/icu/build.rs b/src/runtimes/liboliphaunt/wasix/crates/icu/build.rs new file mode 100644 index 000000000..cc35eb04d --- /dev/null +++ b/src/runtimes/liboliphaunt/wasix/crates/icu/build.rs @@ -0,0 +1,37 @@ +use sha2::{Digest, Sha256}; +use std::{env, fs, path::PathBuf}; + +fn main() { + let payload = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("payload"); + println!("cargo:rerun-if-changed={}", payload.display()); + println!("cargo:rerun-if-env-changed=OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD"); + let archive = payload.join("icu-data.tar.zst"); + let seed = payload.join("cluster-seeds/icu.tar.zst"); + let manifest = payload.join("cluster-seeds/icu.json"); + let fields = if archive.is_file() { + let bytes = fs::read(&archive).expect("read ICU archive"); + let hash = format!("{:x}", Sha256::digest(&bytes)); + let metadata: serde_json::Value = + serde_json::from_slice(&fs::read(&manifest).expect("read ICU seed manifest")) + .expect("parse ICU seed manifest"); + assert_eq!( + metadata["runtime"]["version"], + env::var("CARGO_PKG_VERSION").unwrap() + ); + assert_eq!(metadata["catalogProfile"], "icu"); + let tree = metadata["icu"]["dataTreeSha256"] + .as_str() + .expect("ICU tree digest"); + assert!(seed.is_file(), "missing ICU cluster seed"); + format!( + "wasix_archive: Some(include_bytes!({archive:?})), wasix_archive_sha256: Some({hash:?}), wasix_data_tree_sha256: Some({tree:?}), wasix_seed_archive: Some(include_bytes!({seed:?})), wasix_seed_manifest: Some(include_bytes!({manifest:?}))," + ) + } else { + assert!( + env::var_os("OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD").is_none(), + "release package requires ICU payload" + ); + "wasix_archive: None, wasix_archive_sha256: None, wasix_data_tree_sha256: None, wasix_seed_archive: None, wasix_seed_manifest: None,".into() + }; + fs::write(PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("icu.rs"), format!("/// Optional ICU data for the matching WASIX runtime.\npub const ICU: oliphaunt_resources::IcuData = oliphaunt_resources::IcuData {{ version: env!(\"CARGO_PKG_VERSION\"), runtime_version: env!(\"CARGO_PKG_VERSION\"), native_runtime_version: \"unavailable\", resources: &[], {fields} }};\n")).expect("write ICU descriptor"); +} diff --git a/src/runtimes/liboliphaunt/wasix/crates/icu/src/lib.rs b/src/runtimes/liboliphaunt/wasix/crates/icu/src/lib.rs new file mode 100644 index 000000000..5095b7e4c --- /dev/null +++ b/src/runtimes/liboliphaunt/wasix/crates/icu/src/lib.rs @@ -0,0 +1,3 @@ +#![deny(unsafe_code)] + +include!(concat!(env!("OUT_DIR"), "/icu.rs")); diff --git a/src/runtimes/liboliphaunt/wasix/release.toml b/src/runtimes/liboliphaunt/wasix/release.toml index ff73aa9d1..b411a8569 100644 --- a/src/runtimes/liboliphaunt/wasix/release.toml +++ b/src/runtimes/liboliphaunt/wasix/release.toml @@ -3,7 +3,7 @@ owner = "@oliphaunt/wasix" kind = "wasm-runtime" publish_targets = ["github-release-assets", "crates-io", "npm"] registry_packages = [ - "crates:oliphaunt-icu", + "crates:oliphaunt-wasix-icu", "crates:liboliphaunt-wasix-portable", "crates:oliphaunt-wasix-tools", "crates:liboliphaunt-wasix-aot-aarch64-apple-darwin", @@ -25,10 +25,9 @@ registry_packages = [ release_artifacts = [ "release-assets", ] -derived_version_files = ["src/runtimes/liboliphaunt/icu/Cargo.toml"] -shared_source_paths = ["src/runtimes/liboliphaunt/icu"] +derived_version_files = ["src/runtimes/liboliphaunt/wasix/crates/icu/Cargo.toml"] [compatibility_versions.liboliphaunt-wasix-icu] source_product = "liboliphaunt-wasix" -path = "src/runtimes/liboliphaunt/icu/Cargo.toml" +path = "src/runtimes/liboliphaunt/wasix/crates/icu/Cargo.toml" parser = "toml:package.version" diff --git a/src/runtimes/wasix-napi/README.md b/src/runtimes/wasix-napi/README.md index dcbd12343..1d21fedff 100644 --- a/src/runtimes/wasix-napi/README.md +++ b/src/runtimes/wasix-napi/README.md @@ -9,7 +9,7 @@ The addon supports four purpose-specific TypeScript placement paths: - the direct TypeScript entry point opens and runs the database on its caller's JavaScript thread; - the default native-host entry point uses one Rust database-owner actor so - synchronous guest work does not block the importing event loop; + resource preparation, verification, and guest work do not block the importing event loop; - the `/worker` entry point loads the direct class inside a real package-owned JavaScript Worker; and - `/server` wraps the Rust listener owner directly. @@ -51,6 +51,11 @@ reports exit code zero; unrelated runtime failures remain thrown errors. installed tools package. The TypeScript adapter compares these identities with its validated public descriptors before use. +`registerTools` returns a Promise and prepares the optional package in Node +background work. It reuses verified immutable payloads on repeated registration. +ICU file paths go to Rust directly; caller-owned byte inputs are detached before +crossing to an owner. + `payloadIdentity(component)` identifies the embedded runtime archive and standard seed archive/manifest. ICU payloads come from the selected ICU package. diff --git a/src/runtimes/wasix-napi/build.rs b/src/runtimes/wasix-napi/build.rs index 811075088..432aa63cb 100644 --- a/src/runtimes/wasix-napi/build.rs +++ b/src/runtimes/wasix-napi/build.rs @@ -13,7 +13,7 @@ fn main() { for name in RELEASE_INPUT_ENVS { println!("cargo::rerun-if-env-changed={name}"); } - println!("cargo::rustc-env=OLIPHAUNT_WASIX_NAPI_ABI_VERSION=1"); + println!("cargo::rustc-env=OLIPHAUNT_WASIX_NAPI_ABI_VERSION=2"); validate_release_inputs(); } diff --git a/src/runtimes/wasix-napi/package.json b/src/runtimes/wasix-napi/package.json index 6507473dc..aac48db3e 100644 --- a/src/runtimes/wasix-napi/package.json +++ b/src/runtimes/wasix-napi/package.json @@ -15,7 +15,7 @@ "runtimeVersion": "0.2.0", "rustBindingProduct": "oliphaunt-wasix-rust", "rustBindingVersion": "0.2.0", - "addonAbiVersion": 1, + "addonAbiVersion": 2, "nodeApiVersion": 8, "profiles": [ "standard", diff --git a/src/runtimes/wasix-napi/packages/darwin-arm64/package.json b/src/runtimes/wasix-napi/packages/darwin-arm64/package.json index 016dbcb32..61ac4168a 100644 --- a/src/runtimes/wasix-napi/packages/darwin-arm64/package.json +++ b/src/runtimes/wasix-napi/packages/darwin-arm64/package.json @@ -20,7 +20,7 @@ "target": "macos-arm64", "runtimeProduct": "liboliphaunt-wasix", "runtimeVersion": "0.2.0", - "addonAbiVersion": 1, + "addonAbiVersion": 2, "nodeApiVersion": 8, "profiles": [ "standard", diff --git a/src/runtimes/wasix-napi/packages/linux-arm64-gnu/package.json b/src/runtimes/wasix-napi/packages/linux-arm64-gnu/package.json index cc4c13f41..b9cb5010c 100644 --- a/src/runtimes/wasix-napi/packages/linux-arm64-gnu/package.json +++ b/src/runtimes/wasix-napi/packages/linux-arm64-gnu/package.json @@ -23,7 +23,7 @@ "target": "linux-arm64-gnu", "runtimeProduct": "liboliphaunt-wasix", "runtimeVersion": "0.2.0", - "addonAbiVersion": 1, + "addonAbiVersion": 2, "nodeApiVersion": 8, "profiles": [ "standard", diff --git a/src/runtimes/wasix-napi/packages/linux-x64-gnu/package.json b/src/runtimes/wasix-napi/packages/linux-x64-gnu/package.json index f81d8df46..eb0aba1c4 100644 --- a/src/runtimes/wasix-napi/packages/linux-x64-gnu/package.json +++ b/src/runtimes/wasix-napi/packages/linux-x64-gnu/package.json @@ -23,7 +23,7 @@ "target": "linux-x64-gnu", "runtimeProduct": "liboliphaunt-wasix", "runtimeVersion": "0.2.0", - "addonAbiVersion": 1, + "addonAbiVersion": 2, "nodeApiVersion": 8, "profiles": [ "standard", diff --git a/src/runtimes/wasix-napi/packages/win32-x64-msvc/package.json b/src/runtimes/wasix-napi/packages/win32-x64-msvc/package.json index 9437c884f..e1cf1997c 100644 --- a/src/runtimes/wasix-napi/packages/win32-x64-msvc/package.json +++ b/src/runtimes/wasix-napi/packages/win32-x64-msvc/package.json @@ -20,7 +20,7 @@ "target": "windows-x64-msvc", "runtimeProduct": "liboliphaunt-wasix", "runtimeVersion": "0.2.0", - "addonAbiVersion": 1, + "addonAbiVersion": 2, "nodeApiVersion": 8, "profiles": [ "standard", diff --git a/src/runtimes/wasix-napi/src/extension_package.rs b/src/runtimes/wasix-napi/src/extension_package.rs index a7ea2c029..c38df299c 100644 --- a/src/runtimes/wasix-napi/src/extension_package.rs +++ b/src/runtimes/wasix-napi/src/extension_package.rs @@ -8,6 +8,8 @@ use std::fs; use std::path::{Component, Path, PathBuf}; use std::sync::{Mutex, OnceLock}; +#[cfg(feature = "tools")] +use napi::bindgen_prelude::Task; use napi::{Error, Result}; #[cfg(feature = "tools")] use napi_derive::napi; @@ -25,8 +27,22 @@ pub struct NativeToolPackage { } #[cfg(feature = "tools")] -#[napi(js_name = "registerTools", catch_unwind)] -pub fn register_tools(selection: NativeToolPackage) -> Result<()> { +pub struct RegisterTools(pub(super) Option); + +#[cfg(feature = "tools")] +impl Task for RegisterTools { + type Output = (); + type JsValue = (); + fn compute(&mut self) -> Result<()> { + register_tools(self.0.take().expect("tool registration runs once")) + } + fn resolve(&mut self, _: napi::Env, _: ()) -> Result<()> { + Ok(()) + } +} + +#[cfg(feature = "tools")] +fn register_tools(selection: NativeToolPackage) -> Result<()> { let (root, manifest) = package(&selection.package_json)?; let metadata = &manifest["oliphaunt"]; let version = string(&manifest, "version")?; @@ -42,11 +58,14 @@ pub fn register_tools(selection: NativeToolPackage) -> Result<()> { for name in ["pg_dump", "psql"] { let module = &metadata["tools"][name]; let hash = string(module, "sha256")?; - let bytes = payload(&root, string(module, "path")?, hash)?; - if module["size"].as_u64() != Some(bytes.len() as u64) { - return Err(fail("installed tool module size mismatch")); - } - modules.push((name, bytes, hash.to_owned())); + let size = module["size"] + .as_u64() + .ok_or_else(|| fail("installed tool module size missing"))?; + modules.push(( + name, + (root.clone(), string(module, "path")?.to_owned(), size), + hash.to_owned(), + )); } let expected_name = format!("{name}-{}", target()); let (aot_root, aot_package) = package(&selection.aot_package_json)?; @@ -79,11 +98,11 @@ pub fn register_tools(selection: NativeToolPackage) -> Result<()> { } artifacts.push(( name.to_owned(), - payload( - &aot_root, - string(artifact, "path")?, - string(artifact, "sha256")?, - )?, + ( + aot_root.clone(), + string(artifact, "path")?.to_owned(), + string(artifact, "sha256")?.to_owned(), + ), )); } let key = format!( @@ -102,6 +121,22 @@ pub fn register_tools(selection: NativeToolPackage) -> Result<()> { let package = if let Some(package) = packages.get(&key) { **package } else { + let modules = modules + .into_iter() + .map(|(name, (root, path, size), hash)| { + let bytes = payload(&root, &path, &hash)?; + if bytes.len() as u64 != size { + return Err(fail("installed tool module size mismatch")); + } + Ok((name, bytes, hash)) + }) + .collect::>>()?; + let artifacts = artifacts + .into_iter() + .map(|(name, (root, path, hash))| { + payload(&root, &path, &hash).map(|bytes| (name, bytes)) + }) + .collect::>>()?; let modules = Box::leak( modules .into_iter() @@ -272,10 +307,6 @@ pub(super) fn load(selection: NativeExtensionPackage) -> Result { } let carrier = &metadata["carriers"][&selection.sql_name]; let archive_hash = string(carrier, "sha256")?; - let archive = payload(&root, string(carrier, "path")?, archive_hash)?; - if carrier["size"].as_u64() != Some(archive.len() as u64) { - return Err(fail("installed extension archive size mismatch")); - } let needs_aot = carrier["requiresAot"] .as_bool() .ok_or_else(|| fail("extension carrier is missing requiresAot"))?; @@ -322,11 +353,11 @@ pub(super) fn load(selection: NativeExtensionPackage) -> Result { } artifacts.push(( name.to_owned(), - payload( - &aot_root, - string(artifact, "path")?, - string(artifact, "sha256")?, - )?, + ( + aot_root.clone(), + string(artifact, "path")?.to_owned(), + string(artifact, "sha256")?.to_owned(), + ), )); } aot_manifest = @@ -347,6 +378,14 @@ pub(super) fn load(selection: NativeExtensionPackage) -> Result { if let Some(package) = packages.get(&key) { return Ok(extension.with_package(package)); } + let archive = payload(&root, string(carrier, "path")?, archive_hash)?; + if carrier["size"].as_u64() != Some(archive.len() as u64) { + return Err(fail("installed extension archive size mismatch")); + } + let artifacts = artifacts + .into_iter() + .map(|(name, (root, path, hash))| payload(&root, &path, &hash).map(|bytes| (name, bytes))) + .collect::>>()?; // Match the lifetime of imported native modules. Only validated packages // are retained, once per exact content identity, across worker environments. let archives = Box::leak( @@ -374,7 +413,7 @@ pub(super) fn load(selection: NativeExtensionPackage) -> Result { // It never accepts remote assets or a caller's executable bytes/digest pair. // Owner, exact version, host, runtime and all package-owned file identities // have been checked above; the runtime additionally validates AOT engine, - // source fingerprint, raw bytes and WebAssembly identity before deserializing. + // runtime version, raw bytes and WebAssembly identity before deserializing. let package = unsafe { ExtensionPackage::from_trusted_release( Box::leak(selection.product.into_boxed_str()), @@ -434,8 +473,9 @@ mod tests { .contains("does not match") ); fs::write(root.join("extensions/pgtap/extension.tar.zst"), b"corrupt").unwrap(); + assert_eq!(load(selection()).unwrap().sql_name(), "pgtap"); assert!( - load(selection()) + payload(&root, "extensions/pgtap/extension.tar.zst", &hash) .unwrap_err() .to_string() .contains("hash mismatch") diff --git a/src/runtimes/wasix-napi/src/icu_data.rs b/src/runtimes/wasix-napi/src/icu_data.rs index 0f31e84db..26267be51 100644 --- a/src/runtimes/wasix-napi/src/icu_data.rs +++ b/src/runtimes/wasix-napi/src/icu_data.rs @@ -1,4 +1,7 @@ -use napi::{Result, bindgen_prelude::Buffer}; +use napi::{ + Result, + bindgen_prelude::{Buffer, Either}, +}; use napi_derive::napi; use oliphaunt_wasix::IcuData; use sha2::{Digest, Sha256}; @@ -8,16 +11,39 @@ use std::sync::OnceLock; pub struct NativeIcuData { pub version: String, pub runtime_version: String, - pub archive: Buffer, + pub archive: Either, pub archive_sha256: String, pub data_tree_sha256: String, - pub seed_archive: Buffer, + pub seed_archive: Either, pub seed_archive_sha256: String, - pub seed_manifest: Buffer, + pub seed_manifest: Either, pub seed_manifest_sha256: String, } -static ICU: OnceLock = OnceLock::new(); +impl NativeIcuData { + // Detach caller-owned JavaScript memory before crossing to a Rust owner. + pub(super) fn snapshot(&mut self) { + for source in [ + &mut self.archive, + &mut self.seed_archive, + &mut self.seed_manifest, + ] { + if let Either::A(bytes) = source { + *bytes = Buffer::from(bytes.to_vec()); + } + } + } +} + +fn bytes(source: Either) -> Result> { + match source { + Either::A(bytes) => Ok(bytes.to_vec()), + Either::B(path) => std::fs::read(path) + .map_err(|error| super::invalid_argument(format!("read ICU package: {error}"))), + } +} + +static ICU: OnceLock<(String, IcuData)> = OnceLock::new(); pub(super) fn load(profile: &str, input: Option) -> Result> { let Some(input) = input else { @@ -33,16 +59,38 @@ pub(super) fn load(profile: &str, input: Option) -> Result