diff --git a/Cargo.lock b/Cargo.lock index b598ffd29..f57b25cfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1255,6 +1255,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -3328,6 +3334,37 @@ dependencies = [ "serde_json", ] +[[package]] +name = "oliphaunt-virtual-fs" +version = "0.3.1" +dependencies = [ + "async-trait", + "bytes", + "dashmap", + "derive_more", + "dunce", + "filetime", + "fs_extra", + "futures", + "getrandom 0.4.2", + "libc", + "pin-project-lite", + "pretty_assertions", + "replace_with", + "serde", + "shared-buffer", + "slab", + "tempfile", + "thiserror", + "tokio", + "tracing", + "tracing-test", + "virtual-mio", + "wasmer-package", + "web-time", + "webc", +] + [[package]] name = "oliphaunt-wasix" version = "0.3.1" @@ -3347,6 +3384,7 @@ dependencies = [ "liboliphaunt-wasix-portable", "oliphaunt-icu", "oliphaunt-query", + "oliphaunt-virtual-fs", "oliphaunt-wasix-tools", "oliphaunt-wasix-tools-aot-aarch64-apple-darwin", "oliphaunt-wasix-tools-aot-aarch64-unknown-linux-gnu", @@ -3364,7 +3402,6 @@ dependencies = [ "tokio", "tokio-postgres", "tracing", - "virtual-fs", "virtual-mio", "virtual-net", "wasmer-config", @@ -3495,6 +3532,7 @@ dependencies = [ "derive_more", "lz4_flex", "num_enum", + "oliphaunt-virtual-fs", "oliphaunt-wasmer", "oliphaunt-wasmer-wasix-types", "rkyv", @@ -3503,7 +3541,6 @@ dependencies = [ "shared-buffer", "thiserror", "tracing", - "virtual-fs", "virtual-net", "wasmer-config", ] @@ -3548,6 +3585,7 @@ dependencies = [ "linked_hash_set", "lz4_flex", "num_enum", + "oliphaunt-virtual-fs", "oliphaunt-wasmer", "oliphaunt-wasmer-journal", "oliphaunt-wasmer-wasix-types", @@ -3579,7 +3617,6 @@ dependencies = [ "tracing", "url", "urlencoding", - "virtual-fs", "virtual-mio", "virtual-net", "waker-fn", @@ -3944,6 +3981,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -5700,6 +5747,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", ] [[package]] @@ -5713,9 +5772,32 @@ dependencies = [ "once_cell", "regex-automata", "sharded-slab", + "smallvec", "thread_local", "tracing", "tracing-core", + "tracing-log", +] + +[[package]] +name = "tracing-test" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a4c448db514d4f24c5ddb9f73f2ee71bfb24c526cf0c570ba142d1119e0051" +dependencies = [ + "tracing-core", + "tracing-subscriber", + "tracing-test-macro", +] + +[[package]] +name = "tracing-test-macro" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d" +dependencies = [ + "quote", + "syn 2.0.117", ] [[package]] @@ -5972,6 +6054,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -6009,7 +6097,6 @@ dependencies = [ "tracing", "virtual-mio", "wasmer-package", - "web-time", "webc", ] @@ -7331,6 +7418,12 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index 7ff63baff..eb1e2c150 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ members = [ "src/wasix/runtime/engine/crates/wasmer-wasix-types", "src/wasix/runtime/engine/crates/wasmer-journal", "src/wasix/runtime/engine/crates/wasmer-wasix", + "src/wasix/runtime/engine/crates/virtual-fs", "src/wasix/runtime/engine/crates/v8-windows-x64-msvc", ] diff --git a/src/docs/architecture/database-storage.md b/src/docs/architecture/database-storage.md index e3677c503..153003649 100644 --- a/src/docs/architecture/database-storage.md +++ b/src/docs/architecture/database-storage.md @@ -43,6 +43,32 @@ PostgreSQL backend; its SDK-owned temporary directory must remain until that backend's physical lifetime ends. It is not durable storage and the operating system may reclaim it after process exit. +## Synchronization + +Persistent storage selects where files live, not whether every commit is forced +to disk. Embedded Native and WASIX default PostgreSQL's `fsync` to `off`. +Caller startup settings take precedence over saved PostgreSQL configuration, +which takes precedence over this default. No SDK durability flag or environment +variable duplicates that control. Full native servers and the separate WASIX +Postmaster retain PostgreSQL's `fsync=on` default. + +For disk synchronization, use the existing startup GUC (`fsync: 'on'` in +TypeScript or `.startup_guc("fsync", "on")` in Rust). With it off, a machine or +OS crash can lose committed data or corrupt the cluster. Turning it on later +does not retroactively make earlier unsynchronized writes safe; follow +PostgreSQL's procedure when changing a running database's durability policy. +WASIX host directories distinguish ordinary write flushing from explicit +`fdatasync` (data) and `fsync` (data and metadata); errors from explicit requests +are not suppressed by the default. Initial installation and physical restore +keep their checked publication barriers, independently of query settings. +Host directory barriers are checked on Unix; Windows helpers flush regular +files but do not implement a POSIX directory-sync barrier. + +Browser providers still complete their existing publication/flush boundary +before returning. IndexedDB's transaction guarantees and OPFS's per-file flushes +are different contracts; `fsync=on` alone does not make either a POSIX disk. +Memory storage has no durable medium to synchronize. + ## SDK spellings Each language keeps its native conventions instead of importing a cross-language @@ -98,6 +124,9 @@ staging generation; it cannot be mistaken for an existing database. Restore uses `destination`, not `root`. A restore destination is an external filesystem location receiving backup bytes; it is not an open database's storage selector. The current API accepts only a new or empty destination. +If native restore fails its final durability check after publication, it retains +the populated destination and reports that outcome. Do not blindly retry or +delete that destination: an error does not always mean nothing was published. ## WASIX persistence contract @@ -114,15 +143,22 @@ namespace never becomes ready. The initializing/ready phase and full-flush selector are provider details, not public operations or configuration. Applications that need a PostgreSQL checkpoint call `execute("CHECKPOINT")`; that statement completes the same ordinary provider boundary as another -successful operation. +successful operation. With the embedded default `fsync=off`, PostgreSQL skips +its file-sync barriers, so `CHECKPOINT` alone does not guarantee that every +database file has been flushed to host storage. Set PostgreSQL's `fsync=on` to +request those barriers; the browser provider's persistence guarantees still +apply rather than native filesystem guarantees. A checkpoint may also replace +metadata and trigger a full namespace publication; applications must not rely +on that incidental flush as a substitute for PostgreSQL's file-sync barriers. Each logical IndexedDB name owns an independent physical IndexedDB database. Compatibility metadata and path rows change in one atomic read-write transaction using the browser's default durability policy, so an aborted publication leaves the prior generation current. OPFS stores an opaque logical -namespace over flat backing files. Its direct path honors PostgreSQL file flushes -and drains WAL at operation boundaries; checkpoint, close, and namespace -publication flush WAL before ordinary files and `global/pg_control`. Its +namespace over flat backing files. Its direct path honors explicit PostgreSQL +file flushes and normally drains only WAL at operation boundaries. Initialization, +clean close, and namespace publication flush WAL before ordinary files and +`global/pg_control`, independently of the PostgreSQL `fsync` setting. Its portable path uses copy-on-write backing files and publishes the namespace state last. The direct path keeps a private preopened fast-path reserve. A larger creation burst is staged until the mandatory host boundary, where every staged diff --git a/src/examples/wasix/browser/opfs-crash-probe-worker.ts b/src/examples/wasix/browser/opfs-crash-probe-worker.ts index 7b2e47ea6..937fc7dc3 100644 --- a/src/examples/wasix/browser/opfs-crash-probe-worker.ts +++ b/src/examples/wasix/browser/opfs-crash-probe-worker.ts @@ -15,7 +15,12 @@ scope.addEventListener('message', (event: MessageEvent) => { }); async function prepareDurableState(name: string): Promise { - database = await Oliphaunt.open({ seed: standardSeed, storage: opfs(name) }); + // This probe requires checkpoint file barriers, not the embedded fsync=off default. + database = await Oliphaunt.open({ + seed: standardSeed, + storage: opfs(name), + startupGUCs: { fsync: 'on' }, + }); await database.queryRaw('CREATE TABLE opfs_crash_probe (answer integer NOT NULL)'); await database.queryRaw('INSERT INTO opfs_crash_probe VALUES (73)'); await database.queryRaw(` diff --git a/src/native/runtime/README.md b/src/native/runtime/README.md index 31a39f021..8ac4a2f82 100644 --- a/src/native/runtime/README.md +++ b/src/native/runtime/README.md @@ -113,8 +113,9 @@ need different PostgreSQL settings do not need a new C ABI; they pass validated `-c name=value` startup arguments through `OliphauntConfig.startup_args`. Later arguments win, so SDKs and benchmark harnesses can apply concrete PostgreSQL GUC overrides above the stable C boundary without inventing tuning profiles. -Direct startup defaults to `-F` (`fsync=off`). Persistent storage -alone does not provide crash safety: pass `-c fsync=on` when it is required. +Embedded startup defaults to `fsync=off`, without overriding saved PostgreSQL +configuration. Persistent storage alone does not provide crash safety: pass +`-c fsync=on` when disk synchronization is required. Full-server defaults are unchanged. SDKs must hydrate PGDATA from a packaged cluster seed before calling `oliphaunt_init`; the C boundary never runs `initdb` or initializes an empty diff --git a/src/native/runtime/include/oliphaunt.h b/src/native/runtime/include/oliphaunt.h index 407c0a9d9..9abb90ae1 100644 --- a/src/native/runtime/include/oliphaunt.h +++ b/src/native/runtime/include/oliphaunt.h @@ -147,7 +147,10 @@ typedef int32_t (*OliphauntStreamCallback)(void *context, const uint8_t *data, s /* Returns zero and sets read_len (zero at EOF), or nonzero on failure. * Read/write callbacks run synchronously and must not reenter this database. * A failed backup sink still runs backup-mode cleanup before returning. - * Restore writes only to private staging until the entire archive validates. */ + * Restore writes only to private staging until the entire archive validates. + * If final durability fails after publication, restore returns an error but + * retains the populated destination; retry must not overwrite it. This applies + * to both buffered and streamed restore. */ typedef int32_t (*OliphauntArchiveReadCallback)( void *context, uint8_t *data, size_t capacity, size_t *read_len); typedef struct OliphauntRestoreStreamOptions { diff --git a/src/native/runtime/moon.yml b/src/native/runtime/moon.yml index b8122944f..bfc78eb40 100644 --- a/src/native/runtime/moon.yml +++ b/src/native/runtime/moon.yml @@ -160,6 +160,7 @@ tasks: - /src/native/runtime/include/oliphaunt.h - /src/native/runtime/src/liboliphaunt_internal.h - /src/native/runtime/src/liboliphaunt_platform.h + - /src/native/runtime/src/liboliphaunt_archive.c - /src/native/runtime/src/liboliphaunt_archive_tar.c - /src/native/runtime/src/liboliphaunt_fs.c - /src/native/runtime/src/liboliphaunt_error.c diff --git a/src/native/runtime/postgres/series b/src/native/runtime/postgres/series index 80623f43a..e6ee5bb07 100644 --- a/src/native/runtime/postgres/series +++ b/src/native/runtime/postgres/series @@ -21,3 +21,4 @@ src/native/runtime/patches/postgresql-18.4/0019-liboliphaunt-link-windows-embedd src/native/runtime/patches/postgresql-18.4/0020-liboliphaunt-enforce-embedded-signal-boundary.patch src/native/runtime/patches/postgresql-18.4/0021-liboliphaunt-model-trusted-embedded-sessions.patch src/native/runtime/patches/postgresql-18.4/0022-liboliphaunt-preserve-host-process-boundaries.patch +src/third-party/postgres/patches/common/embedded-fsync-default.patch diff --git a/src/native/runtime/smoke/liboliphaunt_archive_stream.c b/src/native/runtime/smoke/liboliphaunt_archive_stream.c index f6cd519f5..36b8161b9 100644 --- a/src/native/runtime/smoke/liboliphaunt_archive_stream.c +++ b/src/native/runtime/smoke/liboliphaunt_archive_stream.c @@ -1,8 +1,60 @@ +#ifndef _DARWIN_C_SOURCE +#define _DARWIN_C_SOURCE +#endif + #include "liboliphaunt_internal.h" #include #include #include +static int restore_failure; +static const char *restore_destination; + +static int sync_restore_parent(OliphauntHandle *handle, const char *path) { + if (restore_failure == 2) { + set_error(handle, "injected parent sync I/O failure"); + return -1; + } + return oliphaunt_sync_directory(handle, path); +} + +static int sync_restore_tree(OliphauntHandle *handle, const char *path) { + if (restore_failure == 1) { + set_error(handle, "injected staging sync I/O failure"); + return -1; + } + return oliphaunt_sync_directory_tree(handle, path); +} + +static char *restore_parent_dup(const char *path) { + if (restore_failure == 3 && oliphaunt_path_exists(restore_destination)) return NULL; + return oliphaunt_path_parent_dup(path); +} + +/* Exercise both public restore entry points without a PostgreSQL producer. */ +#define oliphaunt_sync_directory sync_restore_parent +#define oliphaunt_sync_directory_tree sync_restore_tree +#define oliphaunt_path_parent_dup restore_parent_dup +#include "../src/liboliphaunt_archive.c" +#undef oliphaunt_sync_directory +#undef oliphaunt_sync_directory_tree +#undef oliphaunt_path_parent_dup + +/* This fixture restores one root at a time and does not test lock-key hashing. */ +void pg_sha256_init(void *ctx) { (void)ctx; } +void pg_sha256_update(void *ctx, const uint8_t *data, size_t len) { + (void)ctx; (void)data; (void)len; +} +void pg_sha256_final(void *ctx, uint8_t *dest) { (void)ctx; memset(dest, 0, 32); } + +/* Restore errors belong to the operation scope, never a resident database handle. */ +bool oliphaunt_try_begin_handle_call(OliphauntHandle *handle) { + (void)handle; + assert(!"restore must not use a database handle for error capture"); + return false; +} +void oliphaunt_end_handle_call(void) { assert(!"restore must not lease a database handle"); } + /* Windows portability headers remap read after the public ABI is declared. */ #ifndef read #define read _read @@ -33,6 +85,16 @@ static int32_t read_archive(void *context, uint8_t *data, size_t capacity, size_ return ferror(archive->file) ? -1 : 0; } +static void fixture_file(const char *root, const char *name, const char *contents) { + char *path = oliphaunt_join_path(root, name); + assert(path != NULL); + FILE *file = fopen(path, "wb"); + assert(file != NULL); + assert(fwrite(contents, 1, strlen(contents), file) == strlen(contents)); + assert(fclose(file) == 0); + free(path); +} + int main(int argc, char **argv) { assert(argc == 2); char *source = oliphaunt_join_path(argv[1], "source"); @@ -75,7 +137,9 @@ int main(int argc, char **argv) { .read_callback = read_archive, .context = &output, }; assert(oliphaunt_unpack_physical_archive_stream(NULL, options.read_callback, options.context, destination) == 0); + assert(oliphaunt_sync_directory_tree(NULL, destination) == 0); char *restored = oliphaunt_join_path(destination, "pgdata/large"); + assert(oliphaunt_sync_directory(NULL, restored) != 0); input = fopen(restored, "rb"); assert(input != NULL); for (size_t block = 0; block < 513; block++) { @@ -104,6 +168,68 @@ int main(int argc, char **argv) { assert(oliphaunt_archive_append_pgdata_tree(&streamed, NULL, source) != 0); assert(streamed.write_failed && streamed.data == NULL); fclose(output.file); + + /* A minimal valid archive exercises real staging, validation and publication. */ + for (size_t i = 0; i < 3; i++) { + const char *directories[] = {"base", "global", "pg_wal"}; + char *path = oliphaunt_join_path(source, directories[i]); + assert(oliphaunt_mkdir_p(path, 0700) == 0); + free(path); + } + fixture_file(source, "PG_VERSION", "18\n"); + fixture_file(source, "global/pg_control", "control"); + OliphauntByteBuffer physical = {0}; + assert(oliphaunt_archive_append_pgdata_tree(&physical, NULL, source) == 0); + assert(oliphaunt_archive_append_pg_control(&physical, NULL, source) == 0); + const uint8_t label[] = "backup\n"; + assert(oliphaunt_archive_append_bytes(&physical, NULL, "pgdata/backup_label", label, sizeof(label) - 1) == 0); + assert(append_default_backup_manifest(&physical, NULL) == 0); + assert(oliphaunt_archive_finish(&physical, NULL) == 0); + output = (ArchiveFile){.file = tmpfile()}; + assert(output.file != NULL); + assert(fwrite(physical.data, 1, physical.len, output.file) == physical.len); + restore_destination = destination; + for (int streaming = 0; streaming <= 1; streaming++) { + for (restore_failure = 0; restore_failure <= 3; restore_failure++) { + assert(oliphaunt_remove_tree(destination) == 0); + rewind(output.file); + output.transferred = 0; + OliphauntRestoreOptions memory = { + .abi_version = OLIPHAUNT_ABI_VERSION, .destination = destination, + .data = physical.data, .len = physical.len, + }; + OliphauntErrorCapture error = {0}; + int rc = streaming ? oliphaunt_restore_stream_with_error(&options, &error) + : oliphaunt_restore_with_error(&memory, &error); + if ((rc == 0) != (restore_failure == 0)) fprintf(stderr, "%s\n", error.message); + assert((rc == 0) == (restore_failure == 0)); + assert(oliphaunt_path_exists(destination) == (restore_failure != 1)); + if (restore_failure == 1) { + assert(strcmp(error.message, "injected staging sync I/O failure") == 0); + continue; + } + char *pgdata = oliphaunt_join_path(destination, "pgdata"); + assert(oliphaunt_validate_managed_root(NULL, pgdata) == 0); + free(pgdata); + if (restore_failure == 0) { + assert(error.message[0] == '\0'); + continue; + } + assert(strstr(error.message, "restore published; destination retained") != NULL); + assert(strstr(error.message, "final durability is unconfirmed") != NULL); + assert(strstr(error.message, restore_failure == 2 ? "injected parent sync I/O failure" + : "out of memory") != NULL); + rewind(output.file); + int previous_failure = restore_failure; + restore_failure = 0; + rc = streaming ? oliphaunt_restore_stream_with_error(&options, &error) + : oliphaunt_restore_with_error(&memory, &error); + restore_failure = previous_failure; + assert(rc != 0 && strstr(error.message, "already exists and is not empty") != NULL); + } + } + fclose(output.file); + free(physical.data); free(source); free(destination); free(file); free(restored); return 0; } diff --git a/src/native/runtime/smoke/liboliphaunt_generation_lifecycle.c b/src/native/runtime/smoke/liboliphaunt_generation_lifecycle.c index 2ef79af2e..2f49e330f 100644 --- a/src/native/runtime/smoke/liboliphaunt_generation_lifecycle.c +++ b/src/native/runtime/smoke/liboliphaunt_generation_lifecycle.c @@ -113,10 +113,12 @@ static int verify_backend_durability_arguments(void) { handle.startup_arg_count = count; OliphauntBackendArgv args = {0}; CHECK(oliphaunt_build_backend_argv(&handle, &args) == 0, "build durability argv"); - /* The embedded default disables fsync; later caller GUCs override it. */ - bool fsync_enabled = true; + /* Leave the boot default and saved configuration to PostgreSQL. */ + bool fsync_enabled = false; for (int i = 1; i < args.argc; i++) { - if (strcmp(args.argv[i], "-F") == 0) fsync_enabled = false; + CHECK(strcmp(args.argv[i], "-F") != 0, "do not override saved fsync configuration"); + CHECK(count != 0 || strncmp(args.argv[i], "fsync=", 6) != 0, + "do not force a default fsync startup GUC"); if (strcmp(args.argv[i], "fsync=off") == 0) fsync_enabled = false; if (strcmp(args.argv[i], "fsync=on") == 0) fsync_enabled = true; } diff --git a/src/native/runtime/src/liboliphaunt_archive.c b/src/native/runtime/src/liboliphaunt_archive.c index 56345e5cc..51c0cb5c8 100644 --- a/src/native/runtime/src/liboliphaunt_archive.c +++ b/src/native/runtime/src/liboliphaunt_archive.c @@ -758,7 +758,10 @@ static int validate_restored_pgdata(OliphauntHandle *handle, const char *staging set_error(handle, "out of memory resolving restored PGDATA"); return -1; } - int rc = oliphaunt_publish_native_root_descriptor(handle, pgdata); + int rc = oliphaunt_sync_directory_tree(handle, pgdata); + if (rc == 0) { + rc = oliphaunt_publish_native_root_descriptor(handle, pgdata); + } if (rc == 0) { rc = oliphaunt_validate_managed_root(handle, pgdata); } @@ -851,13 +854,36 @@ static int publish_restore_without_replacement(OliphauntHandle *handle, const ch set_error(handle, message); return -1; } +#ifdef _WIN32 + if (!MoveFileExA(staging_root, target_root, MOVEFILE_WRITE_THROUGH)) { + set_error(handle, "publish restored root failed"); + return -1; + } +#else if (rename(staging_root, target_root) != 0) { char message[1024]; snprintf(message, sizeof(message), "publish restored root %s: %s", target_root, strerror(errno)); set_error(handle, message); return -1; } - return 0; +#endif + char *parent = oliphaunt_path_parent_dup(target_root); + if (parent == NULL) { + set_error(handle, "restore published; destination retained, but final durability is unconfirmed: out of memory resolving restore parent"); + return -1; + } + int rc = oliphaunt_sync_directory(handle, parent); + free(parent); + if (rc != 0) { + const char prefix[] = "restore published; destination retained, but final durability is unconfirmed: "; + char cause[OLIPHAUNT_ERROR_CAPTURE_CAPACITY]; + char message[OLIPHAUNT_ERROR_CAPTURE_CAPACITY]; + oliphaunt_copy_last_error(handle, cause, sizeof(cause)); + snprintf(message, sizeof(message), "%s%.*s", prefix, + (int)(sizeof(message) - sizeof(prefix)), cause); + set_error(handle, message); + } + return rc; } static int32_t oliphaunt_restore_impl(const OliphauntRestoreStreamOptions *options) { diff --git a/src/native/runtime/src/liboliphaunt_fs.c b/src/native/runtime/src/liboliphaunt_fs.c index 3e6849ee2..a6b18fd3d 100644 --- a/src/native/runtime/src/liboliphaunt_fs.c +++ b/src/native/runtime/src/liboliphaunt_fs.c @@ -948,8 +948,8 @@ static int oliphaunt_read_small_file( return 0; } +int oliphaunt_sync_directory(OliphauntHandle *handle, const char *path) { #ifndef _WIN32 -static int oliphaunt_sync_directory(OliphauntHandle *handle, const char *path) { int flags = O_RDONLY | O_CLOEXEC; #ifdef O_DIRECTORY flags |= O_DIRECTORY; @@ -973,9 +973,67 @@ static int oliphaunt_sync_directory(OliphauntHandle *handle, const char *path) { set_error(handle, message); return -1; } +#else + /* Windows publication uses MOVEFILE_WRITE_THROUGH; directory FileHandles + * do not provide the POSIX fsync contract. */ + (void)handle; + (void)path; +#endif return 0; } + +/* Extracted regular files were already synced. Publish their directory entries + * bottom-up before the staged restore becomes visible. */ +int oliphaunt_sync_directory_tree(OliphauntHandle *handle, const char *path) { +#ifndef _WIN32 + DIR *directory = opendir(path); + if (directory == NULL) { + char message[1024]; + snprintf(message, sizeof(message), "open restored directory %s for sync: %s", path, strerror(errno)); + set_error(handle, message); + return -1; + } + int rc = 0; + struct dirent *entry; + errno = 0; + while ((entry = readdir(directory)) != NULL) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) continue; + char *child = oliphaunt_join_path(path, entry->d_name); + struct stat st; + if (child == NULL) { + set_error(handle, "out of memory resolving restored directory entry"); + rc = -1; + } else if (lstat(child, &st) != 0) { + char message[1024]; + snprintf(message, sizeof(message), "inspect restored directory entry %s: %s", child, strerror(errno)); + set_error(handle, message); + rc = -1; + } else if (S_ISDIR(st.st_mode)) { + rc = oliphaunt_sync_directory_tree(handle, child); + } else if (!S_ISREG(st.st_mode)) { + set_error(handle, "restored directory contains a non-regular file"); + rc = -1; + } + free(child); + if (rc != 0) break; + errno = 0; + } + if (rc == 0 && errno != 0) { + char message[1024]; + snprintf(message, sizeof(message), "read restored directory %s for sync: %s", path, strerror(errno)); + set_error(handle, message); + rc = -1; + } + if (closedir(directory) != 0 && rc == 0) { + char message[1024]; + snprintf(message, sizeof(message), "close restored directory %s after sync: %s", path, strerror(errno)); + set_error(handle, message); + rc = -1; + } + if (rc != 0) return rc; #endif + return oliphaunt_sync_directory(handle, path); +} static int oliphaunt_write_native_root_descriptor( OliphauntHandle *handle, diff --git a/src/native/runtime/src/liboliphaunt_internal.h b/src/native/runtime/src/liboliphaunt_internal.h index 7bcc6e00c..1343a9e65 100644 --- a/src/native/runtime/src/liboliphaunt_internal.h +++ b/src/native/runtime/src/liboliphaunt_internal.h @@ -363,6 +363,8 @@ char *oliphaunt_path_file_name_dup(const char *path); int oliphaunt_mkdir_p(const char *path, mode_t mode); int oliphaunt_remove_tree(const char *path); int oliphaunt_directory_is_empty(const char *path); +int oliphaunt_sync_directory(OliphauntHandle *handle, const char *path); +int oliphaunt_sync_directory_tree(OliphauntHandle *handle, const char *path); int oliphaunt_acquire_stable_root_lock(OliphauntHandle *handle, const char *root, int *out_fd, char **out_path); void oliphaunt_release_file_lock(int *fd, char **path); int oliphaunt_acquire_root_lock(OliphauntHandle *handle, const char *pgdata); diff --git a/src/native/runtime/src/liboliphaunt_runtime.c b/src/native/runtime/src/liboliphaunt_runtime.c index eef657aed..c2978c41f 100644 --- a/src/native/runtime/src/liboliphaunt_runtime.c +++ b/src/native/runtime/src/liboliphaunt_runtime.c @@ -13,7 +13,6 @@ #define DEFAULT_BACKEND_STACK_BYTES (8 * 1024 * 1024) static const char *const DEFAULT_BACKEND_ARGS[] = { - "-F", "-c", "search_path=public", "-c", diff --git a/src/native/sdks/rust/README.md b/src/native/sdks/rust/README.md index 36134fd1b..b1e4cd21b 100644 --- a/src/native/sdks/rust/README.md +++ b/src/native/sdks/rust/README.md @@ -29,7 +29,7 @@ fn main() -> Result<(), Box> { Default storage is a disposable temporary directory. Direct mode stays bound to its first root and configuration for the lifetime of the process, even after closing a handle. Use the quickstart's persistent-storage example for application data; run it as an alternative to this disposable example. Always close database handles explicitly. -Direct and broker modes retain the native `fsync=off` default. Persistent storage alone does not provide crash safety; set `.startup_guc("fsync", "on")` when it is required. Changing the default is tracked separately from patch consolidation. +Direct and broker modes default to `fsync=off` unless saved PostgreSQL configuration or caller startup settings select otherwise. Persistent storage alone does not provide crash safety; set `.startup_guc("fsync", "on")` when disk synchronization is required. Full-server mode retains PostgreSQL's `fsync=on` default. ## Build your integration diff --git a/src/native/sdks/ts/README.md b/src/native/sdks/ts/README.md index 213d021b7..e93bd8dc8 100644 --- a/src/native/sdks/ts/README.md +++ b/src/native/sdks/ts/README.md @@ -26,6 +26,8 @@ try { Default storage is a disposable temporary directory. Direct mode stays bound to its first root and configuration for the lifetime of the process, even after closing a handle. Use the quickstart's persistent-storage example for application data; run it as an alternative to this disposable example. Always close database handles explicitly. +Embedded execution defaults to `fsync=off`; saved PostgreSQL configuration and `startupGUCs` can override it. A persistent directory alone does not provide crash safety. Use `startupGUCs: { fsync: 'on' }` when disk synchronization is required. + ## Build your integration - [Guide](https://oliphaunt.dev/docs/sdk/typescript/guide): parameters, transactions, extensions, backups, and shutdown. diff --git a/src/native/sdks/ts/src/__tests__/filesystem-durability.test.ts b/src/native/sdks/ts/src/__tests__/filesystem-durability.test.ts index f7d6a5a32..e5c875dfd 100644 --- a/src/native/sdks/ts/src/__tests__/filesystem-durability.test.ts +++ b/src/native/sdks/ts/src/__tests__/filesystem-durability.test.ts @@ -11,6 +11,98 @@ describe('filesystem durability', () => { await Promise.all(roots.splice(0).map((root) => rm(root, { force: true, recursive: true }))); }); + it('skips Windows directory barriers without swallowing Unix permission or I/O failures', () => { + // Isolate module mocks from other SDK tests that use the real filesystem. + const result = Bun.spawnSync([ + process.execPath, + '--eval', + ` + import assert from 'node:assert/strict'; + import * as fs from 'node:fs/promises'; + import { mock } from 'bun:test'; + let code; + let failOpen; + let opens = 0; + let closes = 0; + const fail = () => { throw Object.assign(new Error(code), { code }); }; + mock.module('node:fs/promises', () => ({ ...fs, + open: async () => { + opens++; + if (failOpen) fail(); + return { sync: async () => fail(), close: async () => { closes++; } }; + } + })); + const { syncDirectory } = await import(${JSON.stringify(new URL('../native/filesystem-durability.ts', import.meta.url).href)}); + for (const platform of ['linux', 'win32']) { + Object.defineProperty(process, 'platform', { value: platform }); + for (failOpen of [true, false]) { + for (code of ['EPERM', 'EACCES', 'EISDIR', 'EIO', 'EINVAL', 'ENOTSUP']) { + const previousOpens = opens; + const previousCloses = closes; + if (platform === 'win32' || ['EINVAL', 'ENOTSUP'].includes(code)) await syncDirectory('/unused'); + else await assert.rejects(syncDirectory('/unused'), { code }); + assert.equal(opens - previousOpens, platform === 'win32' ? 0 : 1); + assert.equal(closes - previousCloses, platform === 'win32' || failOpen ? 0 : 1); + } + } + } + `, + ]); + expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0); + }); + + it('uses write-capable Windows file handles and propagates file-sync failures', () => { + const result = Bun.spawnSync([ + process.execPath, + '--eval', + ` + import assert from 'node:assert/strict'; + import * as fs from 'node:fs/promises'; + import { join } from 'node:path'; + import { tmpdir } from 'node:os'; + import { mock } from 'bun:test'; + const root = await fs.mkdtemp(join(tmpdir(), 'oliphaunt-file-sync-')); + const file = join(root, 'value'); + await fs.writeFile(file, 'retained'); + let code; + let closes = 0; + mock.module('node:fs/promises', () => ({ ...fs, + open: async (path, flags) => { + assert.equal(flags, path === file && process.platform === 'win32' ? 'r+' : 'r'); + return { + sync: async () => { if (code) throw Object.assign(new Error(code), { code }); }, + close: async () => { closes++; } + }; + } + })); + const { syncDirectoryTree } = await import(${JSON.stringify(new URL('../native/filesystem-durability.ts', import.meta.url).href)}); + try { + for (const platform of ['linux', 'win32']) { + Object.defineProperty(process, 'platform', { value: platform }); + code = undefined; + await syncDirectoryTree(root); + for (code of ['EPERM', 'EACCES', 'EIO', 'EINVAL', 'ENOTSUP']) { + const previousCloses = closes; + await assert.rejects(syncDirectoryTree(root), { code }); + assert.equal(closes - previousCloses, 1); + } + } + assert.equal(await fs.readFile(file, 'utf8'), 'retained'); + } finally { await fs.rm(root, { force: true, recursive: true }); } + `, + ]); + expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0); + }); + + it('flushes a nonempty staging tree on the current platform without changing file contents', async () => { + const root = await mkdtemp(join(tmpdir(), 'oliphaunt-staging-sync-')); + roots.push(root); + await mkdir(join(root, 'nested')); + await writeFile(join(root, 'nested', 'value'), 'retained'); + await expect(syncDirectoryTree(root)).resolves.toBeUndefined(); + expect(await Bun.file(join(root, 'nested', 'value')).text()).toBe('retained'); + }); + it.if(process.platform !== 'win32')( 'accepts packaged runtime symlinks without weakening PGDATA publication', async () => { diff --git a/src/native/sdks/ts/src/native/filesystem-durability.ts b/src/native/sdks/ts/src/native/filesystem-durability.ts index ea81cb8ad..344b88130 100644 --- a/src/native/sdks/ts/src/native/filesystem-durability.ts +++ b/src/native/sdks/ts/src/native/filesystem-durability.ts @@ -1,9 +1,11 @@ import { lstat, open, readdir, readlink, type FileHandle } from 'node:fs/promises'; import { join } from 'node:path'; -const UNSUPPORTED_DIRECTORY_SYNC_ERRORS = new Set(['EISDIR', 'EINVAL', 'EPERM', 'ENOTSUP']); +const UNSUPPORTED_DIRECTORY_SYNC_ERRORS = new Set(['EINVAL', 'ENOTSUP']); export async function syncDirectory(directory: string): Promise { + // Match the native Rust/C helpers: Windows has no POSIX directory-sync barrier. + if (process.platform === 'win32') return; let handle: FileHandle | undefined; try { handle = await open(directory, 'r'); @@ -43,7 +45,8 @@ async function syncTree(root: string, allowSymlinks: boolean): Promise { } else if (entryMetadata.isDirectory()) { await syncTree(file, allowSymlinks); } else if (entryMetadata.isFile()) { - const handle = await open(file, 'r'); + // FlushFileBuffers requires a write-capable handle on Windows; r+ does not truncate. + const handle = await open(file, process.platform === 'win32' ? 'r+' : 'r'); try { await handle.sync(); } finally { diff --git a/src/third-party/postgres/patches/common/embedded-fsync-default.patch b/src/third-party/postgres/patches/common/embedded-fsync-default.patch new file mode 100644 index 000000000..2a7c750aa --- /dev/null +++ b/src/third-party/postgres/patches/common/embedded-fsync-default.patch @@ -0,0 +1,47 @@ +From: Sid Jain +Date: Thu, 8 Oct 2026 00:00:00 +0000 +Subject: [PATCH] postgres: default embedded fsync off without overriding configuration + +Embedded applications opt into crash-safe disk synchronization through the +ordinary PostgreSQL fsync setting. Use a boot default, rather than -F on the +command line, so saved configuration, ALTER SYSTEM and caller startup settings +keep PostgreSQL's normal precedence and reload behavior. + +Keep the global initializer and GUC boot value aligned. Ordinary PostgreSQL +and the separate WASIX Postmaster retain the upstream fsync=on default. +--- + src/backend/utils/init/globals.c | 4 ++++ + src/backend/utils/misc/guc_tables.c | 4 ++++ + 2 files changed, 8 insertions(+) + +diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c +--- a/src/backend/utils/init/globals.c ++++ b/src/backend/utils/init/globals.c +@@ -126,7 +126,11 @@ + int DateOrder = DATEORDER_MDY; + int IntervalStyle = INTSTYLE_POSTGRES; + ++#if defined(OLIPHAUNT_EMBEDDED) || defined(OLIPHAUNT_WASM_SINGLE_USER) ++bool enableFsync = false; ++#else + bool enableFsync = true; ++#endif + bool allowSystemTableMods = false; + int work_mem = 4096; + double hash_mem_multiplier = 2.0; +diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c +--- a/src/backend/utils/misc/guc_tables.c ++++ b/src/backend/utils/misc/guc_tables.c +@@ -1166,6 +1166,10 @@ + }, + &enableFsync, ++#if defined(OLIPHAUNT_EMBEDDED) || defined(OLIPHAUNT_WASM_SINGLE_USER) ++ false, ++#else + true, ++#endif + NULL, NULL, NULL + }, + { +-- +2.51.0 diff --git a/src/wasix/runtime/engine/README.md b/src/wasix/runtime/engine/README.md index f791936fd..a87d64bf3 100644 --- a/src/wasix/runtime/engine/README.md +++ b/src/wasix/runtime/engine/README.md @@ -19,6 +19,8 @@ The ordered Wasmer patches cover engine delivery, C API ownership, native cache profiles, and shared memory/terminal lifetimes. The WASIX patches cover optional runner features, cached module loading, and terminal shutdown. The complete ordered series is the build and qualification unit. +The virtual filesystem patch separates write flushing from explicit data/full +sync requests, including through file wrappers and memory mounts. These are internal compatibility patches, not a promise to support every V8 backend API exposed upstream. diff --git a/src/wasix/runtime/engine/contract.mts b/src/wasix/runtime/engine/contract.mts index 514349bc5..f57d89a1e 100644 --- a/src/wasix/runtime/engine/contract.mts +++ b/src/wasix/runtime/engine/contract.mts @@ -3,6 +3,7 @@ export const ENGINE_SOURCE_CRATES = Object.freeze([ 'wasmer-wasix-types', 'wasmer-journal', 'wasmer-wasix', + 'virtual-fs', ]); export const ENGINE_PAYLOAD_PACKAGE = 'oliphaunt-wasmer-v8-windows-x64-msvc'; diff --git a/src/wasix/runtime/engine/crates/virtual-fs/Cargo.toml b/src/wasix/runtime/engine/crates/virtual-fs/Cargo.toml new file mode 100644 index 000000000..f9e6c1db5 --- /dev/null +++ b/src/wasix/runtime/engine/crates/virtual-fs/Cargo.toml @@ -0,0 +1,159 @@ +[package] +edition = "2024" +rust-version = "1.96" +name = "oliphaunt-virtual-fs" +version = "0.3.1" +authors = ["Wasmer Engineering Team "] +build = false +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Wasmer Virtual FileSystem" +homepage = "https://wasmer.io/" +readme = false +license = "MIT" +repository = "https://github.com/wasmerio/wasmer" + +[package.metadata.docs.rs] +rustc-args = [ + "--cfg", + "docsrs", +] + +[features] +default = [ + "host-fs", + "webc-fs", +] +enable-serde = [] +futures = [] +host-fs = [ + "libc", + "fs_extra", + "filetime", + "tokio/fs", + "tokio/io-std", + "tokio/rt", +] +js = [ + "dep:web-time", + "getrandom/wasm_js", +] +static-fs = [] +tracking = [] +webc-fs = ["webc"] + +[lib] +name = "virtual_fs" +path = "upstream/src/lib.rs" + +[dependencies.async-trait] +version = "0.1.68" + +[dependencies.bytes] +version = "1.11.1" + +[dependencies.dashmap] +version = "6.1.0" + +[dependencies.derive_more] +version = "2.0.1" +features = [ + "debug", + "display", +] + +[dependencies.dunce] +version = "1.0.4" + +[dependencies.filetime] +version = "0.2.18" +optional = true + +[dependencies.fs_extra] +version = "1.3.0" +optional = true + +[dependencies.futures] +version = "0.3.30" + +[dependencies.getrandom] +version = "0.4.1" + +[dependencies.libc] +version = "0.2.178" +optional = true +default-features = false + +[dependencies.pin-project-lite] +version = "0.2.10" + +[dependencies.replace_with] +version = "0.1.7" + +[dependencies.serde] +version = "1" +features = ["derive"] +optional = true +default-features = false + +[dependencies.shared-buffer] +version = "0.1.4" + +[dependencies.slab] +version = "0.4" + +[dependencies.thiserror] +version = "2.0.17" + +[dependencies.tokio] +version = "1.39.0" +features = [ + "io-util", + "sync", + "macros", +] +default-features = false + +[dependencies.tracing] +version = "0.1" +default-features = true + +[dependencies.virtual-mio] +version = "0.705.0" +default-features = false + +[dependencies.wasmer-package] +version = "0.705.0" + +[dependencies.web-time] +version = "1.1" +optional = true + +[dependencies.webc] +version = "=12.0.1" +features = [ + "v2", + "v3", +] +optional = true +default-features = false + +[dev-dependencies.pretty_assertions] +version = "1.4.0" + +[dev-dependencies.tempfile] +version = "3.6.0" + +[dev-dependencies.tokio] +version = "1.39.0" +features = [ + "io-util", + "rt", +] +default-features = false + +[dev-dependencies.tracing-test] +version = "0.2" diff --git a/src/wasix/runtime/engine/crates/wasmer-journal/Cargo.toml b/src/wasix/runtime/engine/crates/wasmer-journal/Cargo.toml index ec7edfbc1..b09d8f8cf 100644 --- a/src/wasix/runtime/engine/crates/wasmer-journal/Cargo.toml +++ b/src/wasix/runtime/engine/crates/wasmer-journal/Cargo.toml @@ -95,7 +95,9 @@ version = "0.1" default-features = false [dependencies.virtual-fs] -version = "0.705.0" +package = "oliphaunt-virtual-fs" +path = "../virtual-fs" +version = "*" default-features = false [dependencies.virtual-net] diff --git a/src/wasix/runtime/engine/crates/wasmer-wasix/Cargo.toml b/src/wasix/runtime/engine/crates/wasmer-wasix/Cargo.toml index 2720db3c8..7ea196cea 100644 --- a/src/wasix/runtime/engine/crates/wasmer-wasix/Cargo.toml +++ b/src/wasix/runtime/engine/crates/wasmer-wasix/Cargo.toml @@ -339,7 +339,9 @@ version = "2.5" version = "2" [dependencies.virtual-fs] -version = "0.705.0" +package = "oliphaunt-virtual-fs" +path = "../virtual-fs" +version = "*" features = ["webc-fs"] default-features = false diff --git a/src/wasix/runtime/engine/package.mts b/src/wasix/runtime/engine/package.mts index 863cf7281..449579ac3 100644 --- a/src/wasix/runtime/engine/package.mts +++ b/src/wasix/runtime/engine/package.mts @@ -178,7 +178,7 @@ fn main() { .replace(/^publish = false\n/gmu, '') .replace(/^version = "[^"]+"$/mu, `version = "${version}"`); text = text.replace( - /(package = "oliphaunt-wasmer[^"\n]*"\nversion = ")[^"]+/gu, + /(package = "oliphaunt-(?:wasmer[^"\n]*|virtual-fs)"\nversion = ")[^"]+/gu, `$1=${version}`, ); text = text.replace( diff --git a/src/wasix/runtime/engine/patches/virtual-fs/0001-explicit-file-synchronization.patch b/src/wasix/runtime/engine/patches/virtual-fs/0001-explicit-file-synchronization.patch new file mode 100644 index 000000000..245a25a4f --- /dev/null +++ b/src/wasix/runtime/engine/patches/virtual-fs/0001-explicit-file-synchronization.patch @@ -0,0 +1,291 @@ +From: Sid Jain +Date: Thu, 8 Oct 2026 00:00:00 +0000 +Subject: [PATCH] virtual-fs: separate write flushing from explicit synchronization + +AsyncWrite::flush drains pending writes; it is not a disk durability contract. +Add data-only and full sync operations without changing existing provider flush +behavior, and forward them through file wrappers and mounted files. Host files +finish pending writes before the checked operating-system sync. Memory files +retain their no-disk behavior. +--- +diff --git a/src/arc_box_file.rs b/src/arc_box_file.rs +index ba28b080..96bfc445 100644 +--- a/src/arc_box_file.rs ++++ b/src/arc_box_file.rs +@@ -87,6 +87,16 @@ impl AsyncRead for ArcBoxFile { + } + + impl VirtualFile for ArcBoxFile { ++ fn poll_sync_data(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ let mut inner = self.inner.lock().unwrap(); ++ Pin::new(inner.as_mut()).poll_sync_data(cx) ++ } ++ ++ fn poll_sync_all(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ let mut inner = self.inner.lock().unwrap(); ++ Pin::new(inner.as_mut()).poll_sync_all(cx) ++ } ++ + fn last_accessed(&self) -> u64 { + let inner = self.inner.lock().unwrap(); + inner.last_accessed() +diff --git a/src/arc_file.rs b/src/arc_file.rs +index c3b0fa97..723b4244 100644 +--- a/src/arc_file.rs ++++ b/src/arc_file.rs +@@ -103,6 +103,16 @@ impl VirtualFile for ArcFile + where + T: VirtualFile + Send + Sync + 'static, + { ++ fn poll_sync_data(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ let mut inner = self.inner.lock().unwrap(); ++ Pin::new(inner.as_mut()).poll_sync_data(cx) ++ } ++ ++ fn poll_sync_all(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ let mut inner = self.inner.lock().unwrap(); ++ Pin::new(inner.as_mut()).poll_sync_all(cx) ++ } ++ + fn last_accessed(&self) -> u64 { + let inner = self.inner.lock().unwrap(); + inner.last_accessed() +diff --git a/src/combine_file.rs b/src/combine_file.rs +index 8c12b83c..2a846199 100644 +--- a/src/combine_file.rs ++++ b/src/combine_file.rs +@@ -18,6 +18,14 @@ impl CombineFile { + } + + impl VirtualFile for CombineFile { ++ fn poll_sync_data(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ Pin::new(self.tx.as_mut()).poll_sync_data(cx) ++ } ++ ++ fn poll_sync_all(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ Pin::new(self.tx.as_mut()).poll_sync_all(cx) ++ } ++ + fn last_accessed(&self) -> u64 { + self.rx.last_accessed() + } +diff --git a/src/cow_file.rs b/src/cow_file.rs +index b3f31b86..d688c69e 100644 +--- a/src/cow_file.rs ++++ b/src/cow_file.rs +@@ -202,6 +202,20 @@ impl AsyncRead for CopyOnWriteFile { + } + + impl VirtualFile for CopyOnWriteFile { ++ fn poll_sync_data(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ match self.poll_copy_start_and_progress(cx) { ++ Poll::Ready(Ok(())) => Pin::new(self.state.inner_mut()).poll_sync_data(cx), ++ result => result, ++ } ++ } ++ ++ fn poll_sync_all(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ match self.poll_copy_start_and_progress(cx) { ++ Poll::Ready(Ok(())) => Pin::new(self.state.inner_mut()).poll_sync_all(cx), ++ result => result, ++ } ++ } ++ + fn last_accessed(&self) -> u64 { + self.last_accessed + } +diff --git a/src/dual_write_file.rs b/src/dual_write_file.rs +index b6f590a1..b02c4dee 100644 +--- a/src/dual_write_file.rs ++++ b/src/dual_write_file.rs +@@ -26,6 +26,14 @@ impl DualWriteFile { + } + + impl VirtualFile for DualWriteFile { ++ fn poll_sync_data(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ Pin::new(self.inner.as_mut()).poll_sync_data(cx) ++ } ++ ++ fn poll_sync_all(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ Pin::new(self.inner.as_mut()).poll_sync_all(cx) ++ } ++ + fn last_accessed(&self) -> u64 { + self.inner.last_accessed() + } +diff --git a/src/host_fs.rs b/src/host_fs.rs +index c765c4d3..f2ef546d 100644 +--- a/src/host_fs.rs ++++ b/src/host_fs.rs +@@ -417,6 +417,20 @@ impl File { + + #[async_trait::async_trait] + impl VirtualFile for File { ++ fn poll_sync_data(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ match self.as_mut().poll_flush(cx) { ++ Poll::Ready(Ok(())) => Poll::Ready(self.inner_std.sync_data()), ++ result => result, ++ } ++ } ++ ++ fn poll_sync_all(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ match self.as_mut().poll_flush(cx) { ++ Poll::Ready(Ok(())) => Poll::Ready(self.inner_std.sync_all()), ++ result => result, ++ } ++ } ++ + fn last_accessed(&self) -> u64 { + self.metadata() + .accessed() +diff --git a/src/lib.rs b/src/lib.rs +index e0987244..63fba5d1 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -345,6 +345,18 @@ impl<'a> OpenOptions<'a> { + pub trait VirtualFile: + fmt::Debug + AsyncRead + AsyncWrite + AsyncSeek + Unpin + Upcastable + Send + { ++ /// Synchronize data to the backing store, separately from draining writes. ++ /// Memory files and existing providers retain their flush contract; durable ++ /// host files must override this with a checked data synchronization. ++ fn poll_sync_data(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ self.poll_flush(cx) ++ } ++ ++ /// Synchronize data and metadata to the backing store. ++ fn poll_sync_all(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ self.poll_flush(cx) ++ } ++ + /// the last time the file was accessed in nanoseconds as a UNIX timestamp + fn last_accessed(&self) -> u64; + +diff --git a/src/mem_fs/file.rs b/src/mem_fs/file.rs +index 311d65a5..800d41d7 100644 +--- a/src/mem_fs/file.rs ++++ b/src/mem_fs/file.rs +@@ -54,6 +54,37 @@ impl Clone for FileHandle { + } + + impl FileHandle { ++ fn poll_sync(&mut self, cx: &mut Context<'_>, data_only: bool) -> Poll> { ++ let fs = self ++ .filesystem ++ .inner ++ .read() ++ .map_err(|_| io::Error::other("failed to acquire a read lock"))?; ++ let file = match fs.storage.get(self.inode) { ++ Some(Node::CustomFile(node)) => { ++ let mut file = node.file.lock().unwrap(); ++ return if data_only { ++ Pin::new(file.as_mut()).poll_sync_data(cx) ++ } else { ++ Pin::new(file.as_mut()).poll_sync_all(cx) ++ }; ++ } ++ Some(Node::ArcFile(_)) => { ++ drop(fs); ++ self.lazy_load_arc_file_mut().map_err(io::Error::from)? ++ } ++ Some(Node::File(_) | Node::OffloadedFile(_) | Node::ReadOnlyFile(_)) => { ++ return Poll::Ready(Ok(())); ++ } ++ _ => return Poll::Ready(Err(io::ErrorKind::NotFound.into())), ++ }; ++ if data_only { ++ Pin::new(file).poll_sync_data(cx) ++ } else { ++ Pin::new(file).poll_sync_all(cx) ++ } ++ } ++ + pub(super) fn new_opened( + inode: Inode, + filesystem: FileSystem, +@@ -116,6 +147,14 @@ impl FileHandle { + } + + impl VirtualFile for FileHandle { ++ fn poll_sync_data(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ self.get_mut().poll_sync(cx, true) ++ } ++ ++ fn poll_sync_all(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ self.get_mut().poll_sync(cx, false) ++ } ++ + fn last_accessed(&self) -> u64 { + let fs = match self.filesystem.inner.read() { + Ok(fs) => fs, +diff --git a/src/overlay_fs.rs b/src/overlay_fs.rs +index de9366bf..5292a9c0 100644 +--- a/src/overlay_fs.rs ++++ b/src/overlay_fs.rs +@@ -57,6 +57,14 @@ impl

VirtualFile for SecondaryFile

+ where + P: FileSystem + 'static, + { ++ fn poll_sync_data(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ Pin::new(self.inner.as_mut()).poll_sync_data(cx) ++ } ++ ++ fn poll_sync_all(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ Pin::new(self.inner.as_mut()).poll_sync_all(cx) ++ } ++ + fn last_accessed(&self) -> u64 { + self.inner.last_accessed() + } +@@ -1049,6 +1057,30 @@ where + where + P: FileSystem + 'static, + { ++ fn poll_sync_data(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ let copy = if self.new_size.is_some() { ++ self.poll_copy_start_and_progress(cx) ++ } else { ++ self.poll_copy_progress(cx) ++ }; ++ match copy { ++ Poll::Ready(Ok(())) => Pin::new(self.state.as_mut()).poll_sync_data(cx), ++ result => result, ++ } ++ } ++ ++ fn poll_sync_all(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ let copy = if self.new_size.is_some() { ++ self.poll_copy_start_and_progress(cx) ++ } else { ++ self.poll_copy_progress(cx) ++ }; ++ match copy { ++ Poll::Ready(Ok(())) => Pin::new(self.state.as_mut()).poll_sync_all(cx), ++ result => result, ++ } ++ } ++ + fn last_accessed(&self) -> u64 { + self.state.as_ref().last_accessed() + } +diff --git a/src/trace_fs.rs b/src/trace_fs.rs +index 667bc84c..2b6115f0 100644 +--- a/src/trace_fs.rs ++++ b/src/trace_fs.rs +@@ -114,6 +114,14 @@ struct TraceFile { + } + + impl VirtualFile for TraceFile { ++ fn poll_sync_data(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ Pin::new(self.file.as_mut()).poll_sync_data(cx) ++ } ++ ++ fn poll_sync_all(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { ++ Pin::new(self.file.as_mut()).poll_sync_all(cx) ++ } ++ + #[tracing::instrument(level = "trace", skip(self), fields(path=%self.path.display()))] + fn last_accessed(&self) -> u64 { + self.file.last_accessed() +-- +2.51.0 diff --git a/src/wasix/runtime/engine/patches/virtual-fs/series b/src/wasix/runtime/engine/patches/virtual-fs/series new file mode 100644 index 000000000..85787d1d4 --- /dev/null +++ b/src/wasix/runtime/engine/patches/virtual-fs/series @@ -0,0 +1 @@ +0001-explicit-file-synchronization.patch diff --git a/src/wasix/runtime/engine/patches/wasmer-wasix/0004-explicit-file-synchronization.patch b/src/wasix/runtime/engine/patches/wasmer-wasix/0004-explicit-file-synchronization.patch new file mode 100644 index 000000000..648fa84b1 --- /dev/null +++ b/src/wasix/runtime/engine/patches/wasmer-wasix/0004-explicit-file-synchronization.patch @@ -0,0 +1,64 @@ +From: Sid Jain +Date: Thu, 8 Oct 2026 00:00:00 +0000 +Subject: [PATCH] wasix: dispatch explicit file synchronization to the backing store + +Route fd_datasync and fd_sync to separate data-only and full backing-store +barriers. Ordinary flush/close remains a write-draining operation, so host +files need not force every close to disk. Preserve rights checks and report +synchronization failures with their actual errno. +--- +diff --git a/src/syscalls/wasi/fd_datasync.rs b/src/syscalls/wasi/fd_datasync.rs +index cc82f357..f2cd715f 100644 +--- a/src/syscalls/wasi/fd_datasync.rs ++++ b/src/syscalls/wasi/fd_datasync.rs +@@ -1,5 +1,4 @@ + use super::*; +-use crate::fs::FlushPoller; + use crate::syscalls::*; + + /// ### `fd_datasync()` +@@ -33,6 +32,13 @@ pub fn fd_datasync(mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd) -> Result, fd: WasiFd) -> Result { const root = path.resolve(import.meta.dir, '../../../..'); + mkdirSync(path.join(root, 'target'), { recursive: true }); const scratch = mkdtempSync(path.join(root, 'target/engine-patch-test-')); t.after(() => rmSync(scratch, { recursive: true, force: true })); const source = path.join(scratch, 'source'); diff --git a/src/wasix/runtime/engine/source.toml b/src/wasix/runtime/engine/source.toml index 6f8b03ec7..b8bbe7b9f 100644 --- a/src/wasix/runtime/engine/source.toml +++ b/src/wasix/runtime/engine/source.toml @@ -22,6 +22,12 @@ url = "https://static.crates.io/crates/wasmer-journal/wasmer-journal-0.705.0.cra sha256 = "d1ba6aab53ebc49f540787f2daf9a8865b0933de8a8d081b22a78b19259d279c" bytes = 73753 +[virtual-fs] +version = "0.705.0" +url = "https://static.crates.io/crates/virtual-fs/virtual-fs-0.705.0.crate" +sha256 = "412131c1369d812684470f44c4803d7447f36676bce6c265b5d0af2f9b1e7d9a" +bytes = 105056 + [wee8-windows] version = "11.9.7" v8-version = "13.6.233.17" diff --git a/src/wasix/runtime/engine/windows/build-caches.mts b/src/wasix/runtime/engine/windows/build-caches.mts index 88a3d2369..3286279fd 100644 --- a/src/wasix/runtime/engine/windows/build-caches.mts +++ b/src/wasix/runtime/engine/windows/build-caches.mts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { copyFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import { sha256File } from '../../../../third-party/tools/source-fetch-core.mts'; +import { runCacheProducer } from './cache-producer.mts'; const root = path.resolve(import.meta.dir, '../../../../..'); const engine = path.join(root, 'target/oliphaunt-wasix/engine/windows-x64-msvc'); @@ -16,10 +17,10 @@ for (const file of ['cache-producer.exe', 'oliphaunt_wee8.dll']) { } const env = { ...process.env, WINEPREFIX: path.join(work, 'wine-prefix'), WINEDEBUG: '-all' }; -function run(command: string[], emulated = false) { +function run(command: string[]) { const result = Bun.spawnSync(command, { cwd: work, - env: emulated ? { ...env, WINELOADERNOEXEC: '1' } : env, + env, timeout: 900_000, }); assert.equal(result.exitCode, 0, result.stdout.toString() + result.stderr.toString()); @@ -57,7 +58,10 @@ for (const product of products) { assert(input && destination, 'incomplete AOT producer input'); const guest = readFileSync(input); copyFileSync(input, path.join(work, 'guest.wasm')); - const log = run(['qemu-x86_64', '-cpu', 'Penryn', wine, 'cache-producer.exe', 'write'], true); + const log = await runCacheProducer( + ['qemu-x86_64', '-cpu', 'Penryn', wine, 'cache-producer.exe', 'write'], + { cwd: work, env: { ...env, WINELOADERNOEXEC: '1' } }, + ); assert.match(log, /cpu_mask=0xe /, 'the baseline must come from a real emulated SSE4.1 CPU'); assert.match(log, /native_deserialize=PASS/); const cache = readFileSync(path.join(work, 'cache.bin')); @@ -82,7 +86,10 @@ for (const product of products) { const digest = sha256File(input); const file = path.join(output, `${digest}.bin`); writeFileSync(file, cache); - const read = run(['qemu-x86_64', '-cpu', 'Penryn', wine, 'cache-producer.exe', 'read'], true); + const read = await runCacheProducer( + ['qemu-x86_64', '-cpu', 'Penryn', wine, 'cache-producer.exe', 'read'], + { cwd: work, env: { ...env, WINELOADERNOEXEC: '1' } }, + ); assert.match(read, /native_deserialize=PASS/, 'fresh-process native cache reader'); receipts.push({ product, diff --git a/src/wasix/runtime/engine/windows/cache-producer.c b/src/wasix/runtime/engine/windows/cache-producer.c index 4f37fcc75..2b632abae 100644 --- a/src/wasix/runtime/engine/windows/cache-producer.c +++ b/src/wasix/runtime/engine/windows/cache-producer.c @@ -55,10 +55,12 @@ static DWORD word(const unsigned char *bytes) { } static void finish(DWORD code) { - /* Wine/QEMU fails in process detach after all engine cleanup returns. - SDK applications retain ordinary Windows process shutdown. */ - TerminateProcess(GetCurrentProcess(), code); - ExitProcess(31); + /* The Linux producer supervisor owns shutdown: Wine/QEMU can crash even + during TerminateProcess. SDK applications do not use this helper. */ + print("producer_exit="); + number(code, 0); + print("\n"); + for (;;) Sleep(INFINITE); } static void fail(const char *message, DWORD code) { diff --git a/src/wasix/runtime/engine/windows/cache-producer.mts b/src/wasix/runtime/engine/windows/cache-producer.mts new file mode 100644 index 000000000..f5f1c817a --- /dev/null +++ b/src/wasix/runtime/engine/windows/cache-producer.mts @@ -0,0 +1,50 @@ +import assert from 'node:assert/strict'; + +/** Stop the emulated helper only after it explicitly reports its result. */ +export async function runCacheProducer( + command: string[], + options: { cwd: string; env?: NodeJS.ProcessEnv; timeoutMs?: number }, +): Promise { + const child = Bun.spawn(command, { + cwd: options.cwd, + env: options.env, + stdin: 'ignore', + stdout: 'pipe', + stderr: 'inherit', + }); + const reader = child.stdout.getReader(); + // Wine services can inherit stdout. A dead helper must not wait for their EOF. + const exited = child.exited.finally(() => reader.cancel()); + const decoder = new TextDecoder(); + let log = ''; + let result: number | undefined; + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + child.kill('SIGKILL'); + }, options.timeoutMs ?? 900_000); + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + log += decoder.decode(value, { stream: true }); + const completion = /(?:^|\n)producer_exit=(\d+)\n/.exec(log); + if (completion === null) continue; + result = Number(completion[1]); + child.kill('SIGKILL'); + break; + } + await exited; + assert(!timedOut, `cache producer timed out\n${log}`); + assert( + result !== undefined && child.signalCode === 'SIGKILL', + `cache producer exited before supervised completion (${child.exitCode}, ${child.signalCode})\n${log}`, + ); + assert.equal(result, 0, log); + return log; + } finally { + clearTimeout(timeout); + child.kill('SIGKILL'); + await exited; + } +} diff --git a/src/wasix/runtime/engine/windows/cache-producer.test.mts b/src/wasix/runtime/engine/windows/cache-producer.test.mts new file mode 100644 index 000000000..748099cdc --- /dev/null +++ b/src/wasix/runtime/engine/windows/cache-producer.test.mts @@ -0,0 +1,38 @@ +import { expect, test } from 'bun:test'; +import { runCacheProducer } from './cache-producer.mts'; + +function run(script: string, timeoutMs = 1_000) { + return runCacheProducer([process.execPath, '-e', script], { cwd: process.cwd(), timeoutMs }); +} + +test('supervises a fragmented completion after cleanup', async () => { + const log = await run(` + console.log('stage=emulated_producer_completed'); + process.stdout.write('producer_ex'); + setTimeout(() => process.stdout.write('it=0\\n'), 10); + setInterval(() => {}, 1000); + `); + expect(log).toContain('stage=emulated_producer_completed\nproducer_exit=0\n'); +}); + +test('does not turn native cache rejection into success', async () => { + await expect( + run("console.log('native_deserialize=REJECT\\nproducer_exit=24'); setInterval(() => {}, 1000)"), + ).rejects.toThrow('native_deserialize=REJECT'); +}); + +test('does not accept a success log followed by an unsupervised crash', async () => { + await expect( + run("console.log('native_deserialize=PASS'); process.kill(process.pid, 'SIGKILL')"), + ).rejects.toThrow('before supervised completion'); +}); + +test('requires an entire completion line', async () => { + await expect(run("process.stdout.write('producer_exit=0')")).rejects.toThrow( + 'before supervised completion', + ); +}); + +test('bounds a stalled helper', async () => { + await expect(run('setInterval(() => {}, 1000)', 100)).rejects.toThrow('timed out'); +}); diff --git a/src/wasix/runtime/moon.yml b/src/wasix/runtime/moon.yml index f97f56392..1ce320d9f 100644 --- a/src/wasix/runtime/moon.yml +++ b/src/wasix/runtime/moon.yml @@ -186,8 +186,15 @@ tasks: node --test src/wasix/runtime/tools/strong-random.test.mjs src/third-party/postgres/apply-series.test.mjs bash src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.test.sh bash src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.test.sh + bun test ./src/wasix/runtime/engine/prepare-sources.test.mts ./src/wasix/runtime/engine/windows/cache-producer.test.mts inputs: - /tools/dev/acquisition.sh + - engine/prepare-sources.mts + - engine/prepare-sources.test.mts + - engine/windows/cache-producer.mts + - engine/windows/cache-producer.test.mts + - engine/contract.mts + - engine/source.toml - assets/build/docker/**/* - assets/build/wasix_shim/**/* - tools/check-shim-abi.sh diff --git a/src/wasix/runtime/postgres/series b/src/wasix/runtime/postgres/series index 2810d4f23..bc8214aeb 100644 --- a/src/wasix/runtime/postgres/series +++ b/src/wasix/runtime/postgres/series @@ -30,3 +30,4 @@ src/wasix/runtime/assets/build/postgres/patches/0041-oliphaunt-wasix-honor-nonin src/wasix/runtime/assets/build/postgres/patches/0042-oliphaunt-wasix-use-explicit-wal-sync-operations.patch src/wasix/runtime/assets/build/postgres/patches/0043-oliphaunt-wasix-cache-jsonb-build-object-metadata.patch src/wasix/runtime/assets/build/postgres/patches/0044-oliphaunt-wasix-initialize-trusted-session-identity.patch +src/third-party/postgres/patches/common/embedded-fsync-default.patch diff --git a/src/wasix/runtime/release.toml b/src/wasix/runtime/release.toml index 577275239..b3cd5ef08 100644 --- a/src/wasix/runtime/release.toml +++ b/src/wasix/runtime/release.toml @@ -7,6 +7,7 @@ registry_packages = [ "crates:oliphaunt-wasmer-wasix-types", "crates:oliphaunt-wasmer-journal", "crates:oliphaunt-wasmer-wasix", + "crates:oliphaunt-virtual-fs", "crates:oliphaunt-wasmer-v8-windows-x64-msvc", "crates:liboliphaunt-wasix-portable", diff --git a/src/wasix/sdks/rust/Cargo.toml b/src/wasix/sdks/rust/Cargo.toml index e558e14f2..d1a371782 100644 --- a/src/wasix/sdks/rust/Cargo.toml +++ b/src/wasix/sdks/rust/Cargo.toml @@ -118,7 +118,7 @@ wasmer-wasix = { package = "oliphaunt-wasmer-wasix", version = "*", path = "../. "time", ] } wasmer-wasix-types = { package = "oliphaunt-wasmer-wasix-types", version = "*", path = "../../runtime/engine/crates/wasmer-wasix-types", default-features = false } -virtual-fs = { version = "=0.705.0", default-features = false } +virtual-fs = { package = "oliphaunt-virtual-fs", path = "../../runtime/engine/crates/virtual-fs", version = "*", default-features = false } virtual-mio = { version = "=0.705.0", default-features = false } virtual-net = { version = "=0.705.0", default-features = false } webc = "=12.0.1" diff --git a/src/wasix/sdks/rust/README.md b/src/wasix/sdks/rust/README.md index c03101969..2d7b2d2d6 100644 --- a/src/wasix/sdks/rust/README.md +++ b/src/wasix/sdks/rust/README.md @@ -29,6 +29,8 @@ fn main() -> Result<(), Box> { Default storage is a memory filesystem and is discarded on close. Use the quickstart's persistent-storage example for application data; run it as an alternative to this disposable example. Always close database handles explicitly. +Embedded execution defaults to `fsync=off`; saved PostgreSQL configuration and `.startup_guc("fsync", "on")` can override it. A persistent directory alone does not provide crash safety. Memory storage has no disk to synchronize. + SQL `statement_timeout` is not a reliable execution deadline for CPU-bound WASIX queries. Timing out a caller's future does not stop guest work. The host-selected username is the real PostgreSQL session principal: LOGIN/CONNECT restrictions, role defaults and login triggers apply. `RESET ROLE` and `DISCARD ALL` do not restore the bootstrap superuser. The host authenticates the caller; initialize new storage as `postgres` before selecting an existing application role. diff --git a/src/wasix/sdks/rust/src/oliphaunt/base.rs b/src/wasix/sdks/rust/src/oliphaunt/base.rs index 16aca7079..556fd329a 100644 --- a/src/wasix/sdks/rust/src/oliphaunt/base.rs +++ b/src/wasix/sdks/rust/src/oliphaunt/base.rs @@ -950,7 +950,7 @@ fn cluster_seed_publication_staging(pgdata: &Path) -> Result { Ok(parent.join(staging)) } -fn sync_publication_tree(path: &Path) -> Result<()> { +pub(super) fn sync_publication_tree(path: &Path) -> Result<()> { let mut files = Vec::new(); let mut directories = Vec::new(); collect_publication_entries(path, &mut files, &mut directories)?; diff --git a/src/wasix/sdks/rust/src/oliphaunt/data_dir.rs b/src/wasix/sdks/rust/src/oliphaunt/data_dir.rs index fa5f0a5dc..c7e4dcab1 100644 --- a/src/wasix/sdks/rust/src/oliphaunt/data_dir.rs +++ b/src/wasix/sdks/rust/src/oliphaunt/data_dir.rs @@ -12,7 +12,7 @@ use cap_std::fs::{Dir as CapDir, File as CapFile, OpenOptions as CapOpenOptions} use tar::{Archive, Builder, EntryType, Header}; use wasmer_wasix::virtual_fs::FileSystem as VirtualFileSystem; -use super::base::DirectoryLock; +use super::base::{DirectoryLock, sync_publication_tree}; use super::database_root_descriptor::{ PGDATA_DIRECTORY, PHYSICAL_FORMAT, POSTGRES_MAJOR, write_database_root_descriptor, }; @@ -559,6 +559,14 @@ pub(crate) fn restore_physical_archive(destination: &Path, bytes: &[u8]) -> Resu }, ) })?; + sync_publication_tree(&pgdata).map_err(|error| { + restore_error( + error, + StorageErrorCode::Unavailable, + StorageCommitState::Unchanged, + StorageErrorPhase::RestoreStaging, + ) + })?; write_database_root_descriptor(staging.path()).map_err(|error| { restore_error( error, diff --git a/src/wasix/sdks/rust/src/oliphaunt/postgres_mod.rs b/src/wasix/sdks/rust/src/oliphaunt/postgres_mod.rs index e634df795..de7d92c89 100644 --- a/src/wasix/sdks/rust/src/oliphaunt/postgres_mod.rs +++ b/src/wasix/sdks/rust/src/oliphaunt/postgres_mod.rs @@ -1117,12 +1117,7 @@ fn instantiate_wasix_module( add_pgdata_preopen(&mut builder)?; } add_oliphaunt_env(&mut builder, input.startup_config, input.runtime_layout); - add_oliphaunt_args( - &mut builder, - input.postgres_config, - input.startup_config, - input.pgdata_storage.is_durable_host_directory(), - )?; + add_oliphaunt_args(&mut builder, input.postgres_config, input.startup_config)?; constrain_single_backend_tasks(&mut builder); { @@ -2007,9 +2002,8 @@ fn add_oliphaunt_args( builder: &mut wasmer_wasix::WasiEnvBuilder, postgres_config: &PostgresConfig, startup_config: &StartupConfig, - durable_host_storage: bool, ) -> Result<()> { - for arg in oliphaunt_args(postgres_config, startup_config, durable_host_storage)? { + for arg in oliphaunt_args(postgres_config, startup_config)? { builder.add_arg(arg); } Ok(()) @@ -2018,14 +2012,10 @@ fn add_oliphaunt_args( fn oliphaunt_args( postgres_config: &PostgresConfig, startup_config: &StartupConfig, - durable_host_storage: bool, ) -> Result> { postgres_config.validate()?; startup_config.validate()?; let mut args = vec!["--single".to_owned()]; - if !durable_host_storage { - args.push("-F".to_owned()); - } args.extend(["-O", "-j"].map(str::to_owned)); for (name, value) in DEFAULT_STARTUP_GUCS { args.push("-c".to_owned()); @@ -2623,7 +2613,12 @@ mod tests { ..StartupConfig::default() }; - let args = oliphaunt_args(&PostgresConfig::default(), &startup, false)?; + let args = oliphaunt_args(&PostgresConfig::default(), &startup)?; + assert!( + !args + .iter() + .any(|arg| arg == "-F" || arg.starts_with("fsync=")) + ); assert!( args.windows(2) diff --git a/src/wasix/sdks/rust/src/oliphaunt/storage.rs b/src/wasix/sdks/rust/src/oliphaunt/storage.rs index 642c4ac52..aaddc1c64 100644 --- a/src/wasix/sdks/rust/src/oliphaunt/storage.rs +++ b/src/wasix/sdks/rust/src/oliphaunt/storage.rs @@ -84,10 +84,6 @@ impl StorageRoot { } } - pub(crate) fn is_durable_host_directory(&self) -> bool { - matches!(self, Self::HostDirectory(_)) - } - #[cfg(all(test, feature = "extensions"))] pub(crate) fn host_path(&self) -> Option<&Path> { match self { @@ -310,7 +306,6 @@ mod tests { let nested = Path::new("/nested"); let file = Path::new("/nested/file"); - assert!(host.is_durable_host_directory()); assert!(host.memory_filesystem().is_none()); host.create_dir_all(nested)?; std::fs::write(directory.path().join("nested/file"), b"host")?; @@ -320,7 +315,6 @@ mod tests { let memory = StorageRoot::memory(); let filesystem = memory.memory_filesystem().expect("memory filesystem"); - assert!(!memory.is_durable_host_directory()); memory.create_dir_all(nested)?; vfs_write(filesystem.as_ref(), file, b"memory")?; assert!(memory.is_dir(nested)); diff --git a/src/wasix/sdks/rust/src/oliphaunt/sync_host_fs.rs b/src/wasix/sdks/rust/src/oliphaunt/sync_host_fs.rs index 91bce4799..2d262c180 100644 --- a/src/wasix/sdks/rust/src/oliphaunt/sync_host_fs.rs +++ b/src/wasix/sdks/rust/src/oliphaunt/sync_host_fs.rs @@ -237,6 +237,14 @@ impl SyncHostFile { } impl VirtualFile for SyncHostFile { + fn poll_sync_data(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.file.sync_data()) + } + + fn poll_sync_all(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.file.sync_all()) + } + fn last_accessed(&self) -> u64 { self.metadata() .ok() @@ -323,7 +331,9 @@ impl AsyncWrite for SyncHostFile { } fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(self.get_mut().file.sync_all()) + // Positional writes finish synchronously. Only explicit guest sync + // requests need a durability barrier, not flush/close of every file. + Poll::Ready(self.get_mut().file.flush()) } fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { @@ -374,7 +384,7 @@ fn sync_directory(path: &Path) -> io::Result<()> { fn sync_directory(_path: &Path) -> io::Result<()> { // Windows does not provide a portable directory fsync operation: // FlushFileBuffers on a directory handle fails with ERROR_ACCESS_DENIED. - // File contents are still flushed by SyncHostFile::poll_flush. + // Explicit guest sync requests still synchronize file contents. Ok(()) } @@ -597,6 +607,61 @@ mod tests { byte[0] } + #[cfg(unix)] + #[test] + fn explicit_sync_errors_survive_shared_files_and_memory_mounts() { + use virtual_fs::FileSystem; + + // /dev/null accepts writes/flush but rejects durable synchronization. + let file = open_host(Path::new("/dev/null")); + let shared = virtual_fs::ArcBoxFile::new(Box::new(file)); + let mounted = virtual_fs::mem_fs::FileSystem::default(); + mounted + .insert_device_file("/probe".into(), Box::new(shared)) + .unwrap(); + let mut file = mounted + .new_open_options() + .write(true) + .open("/probe") + .unwrap(); + let mut cx = Context::from_waker(std::task::Waker::noop()); + assert!(matches!( + Pin::new(file.as_mut()).poll_flush(&mut cx), + Poll::Ready(Ok(())) + )); + assert!(matches!( + Pin::new(file.as_mut()).poll_sync_data(&mut cx), + Poll::Ready(Err(_)) + )); + assert!(matches!( + Pin::new(file.as_mut()).poll_sync_all(&mut cx), + Poll::Ready(Err(_)) + )); + } + + #[test] + fn explicit_sync_keeps_written_data_and_file_position() { + let root = TestDir::new(); + let path = root.path().join("data"); + fs::write(&path, b"abc").unwrap(); + let mut file = open_host(&path); + let mut cx = Context::from_waker(std::task::Waker::noop()); + assert!(matches!( + Pin::new(&mut file).poll_write(&mut cx, b"x"), + Poll::Ready(Ok(1)) + )); + assert!(matches!( + Pin::new(&mut file).poll_sync_data(&mut cx), + Poll::Ready(Ok(())) + )); + assert!(matches!( + Pin::new(&mut file).poll_sync_all(&mut cx), + Poll::Ready(Ok(())) + )); + assert_eq!(file.position, 1); + assert_eq!(fs::read(path).unwrap(), b"xbc"); + } + #[test] fn seek_end_observes_external_growth_and_truncation() { let root = TestDir::new(); diff --git a/src/wasix/sdks/rust/tests/resources.rs b/src/wasix/sdks/rust/tests/resources.rs index 200a4c461..53dd900c1 100644 --- a/src/wasix/sdks/rust/tests/resources.rs +++ b/src/wasix/sdks/rust/tests/resources.rs @@ -116,6 +116,67 @@ fn standard_seed_and_initializer_support_memory_directory_and_reopen() -> Result exercise(CatalogProfile::Standard) } +#[test] +#[ignore = "requires compiled WASIX runtime; run test-resources.sh"] +fn fsync_defaults_do_not_override_saved_configuration() -> Result<()> { + let workspace = tempfile::tempdir()?; + let root = workspace.path().join("database"); + let builder = Oliphaunt::builder().storage(DatabaseStorage::Directory(root.clone())); + let mut db = builder.clone().open()?; + assert_eq!(db.query("SHOW fsync")?.get_text(0, "fsync")?, Some("off")); + assert_eq!( + db.query("SELECT boot_val AS value FROM pg_settings WHERE name='fsync'")? + .get_text(0, "value")?, + Some("off") + ); + db.execute("ALTER SYSTEM SET fsync = on")?; + db.close()?; + drop(db); + + for method in ["fdatasync", "fsync"] { + let mut db = builder + .clone() + .startup_guc("wal_sync_method", method) + .open()?; + assert_eq!(db.query("SHOW fsync")?.get_text(0, "fsync")?, Some("on")); + db.execute("CREATE TABLE IF NOT EXISTS sync_probe (id int)")?; + db.execute("INSERT INTO sync_probe VALUES (1)")?; + db.execute("CHECKPOINT")?; + db.close()?; + } + + let mut db = builder.clone().startup_guc("fsync", "off").open()?; + assert_eq!(db.query("SHOW fsync")?.get_text(0, "fsync")?, Some("off")); + assert_eq!( + db.query("SELECT count(*)::text AS count FROM sync_probe")? + .get_text(0, "count")?, + Some("2") + ); + db.close()?; + drop(db); + + fs::write(root.join("pgdata/postgresql.auto.conf"), "")?; + for (value, expected) in [("0", "off"), ("yes", "on")] { + let mut db = builder.clone().startup_guc("fsync", value).open()?; + assert_eq!( + db.query("SHOW fsync")?.get_text(0, "fsync")?, + Some(expected) + ); + db.close()?; + } + assert!( + builder + .clone() + .startup_guc("fsync", "not-a-bool") + .open() + .is_err() + ); + let mut db = builder.open()?; + assert_eq!(db.query("SHOW fsync")?.get_text(0, "fsync")?, Some("off")); + db.close()?; + Ok(()) +} + #[test] #[ignore = "requires compiled WASIX runtime and separately produced resources; run test-resources.sh"] fn icu_seed_and_initializer_support_memory_directory_and_reopen() -> Result<()> { diff --git a/src/wasix/sdks/rust/tests/runtime_smoke.rs b/src/wasix/sdks/rust/tests/runtime_smoke.rs index 5ac5b2684..3e079ba71 100644 --- a/src/wasix/sdks/rust/tests/runtime_smoke.rs +++ b/src/wasix/sdks/rust/tests/runtime_smoke.rs @@ -217,7 +217,8 @@ fn direct_api_query_transaction_persistence_and_backup() -> Result<()> { current_setting('synchronous_commit') AS synchronous_commit", )?; assert_eq!(settings.get_text(0, "buffers")?, Some("128MB")); - for setting in ["fsync", "full_pages", "synchronous_commit"] { + assert_eq!(settings.get_text(0, "fsync")?, Some("off")); + for setting in ["full_pages", "synchronous_commit"] { assert_eq!(settings.get_text(0, setting)?, Some("on")); } let configuration = std::fs::read_to_string(source_root.join("pgdata/postgresql.conf"))?; diff --git a/src/wasix/sdks/ts/README.md b/src/wasix/sdks/ts/README.md index 0690c43ae..0e5f3ec4b 100644 --- a/src/wasix/sdks/ts/README.md +++ b/src/wasix/sdks/ts/README.md @@ -28,6 +28,8 @@ try { Default storage is a memory filesystem and is discarded on close. Use the quickstart's persistent-storage example for application data; run it as an alternative to this disposable example. Always close database handles explicitly. +Embedded execution defaults to `fsync=off`; saved PostgreSQL configuration and `startupGUCs` can override it. Use `startupGUCs: { fsync: 'on' }` to request disk synchronization. Browser storage still completes its provider publication boundary before an operation settles; its crash guarantees depend on IndexedDB or OPFS, not this PostgreSQL setting alone. + ## Build your integration - [Guide](https://oliphaunt.dev/docs/sdk/wasix-typescript/guide): parameters, transactions, extensions, backups, and shutdown. diff --git a/src/wasix/sdks/ts/src/__tests__/direct-client-common.test.ts b/src/wasix/sdks/ts/src/__tests__/direct-client-common.test.ts index 375c7ef31..b0a34f722 100644 --- a/src/wasix/sdks/ts/src/__tests__/direct-client-common.test.ts +++ b/src/wasix/sdks/ts/src/__tests__/direct-client-common.test.ts @@ -84,6 +84,14 @@ describe('direct WASIX session lifecycle', () => { ); }); + it('leaves the fsync default and saved configuration to PostgreSQL', () => { + const options = openOptions(); + expect(wasixPostgresArgs(options)).not.toContain('-F'); + expect(wasixPostgresArgs(options).some((arg) => arg.startsWith('fsync='))).toBe(false); + options.startupGUCs = { fsync: 'on' }; + expect(wasixPostgresArgs(options)).toContain('fsync=on'); + }); + it('uses PostgreSQL startup GUC name and value grammar', () => { const valid = openOptions(); valid.startupGUCs = { diff --git a/src/wasix/sdks/ts/src/__tests__/opfs-pool.test.ts b/src/wasix/sdks/ts/src/__tests__/opfs-pool.test.ts index afa6dadf6..b76b9bbef 100644 --- a/src/wasix/sdks/ts/src/__tests__/opfs-pool.test.ts +++ b/src/wasix/sdks/ts/src/__tests__/opfs-pool.test.ts @@ -135,9 +135,13 @@ describe('WASIX pooled OPFS storage', () => { write(pool, wal, Uint8Array.of(2)); write(pool, control, Uint8Array.of(2)); + io.failNextFlush = true; + expect(pool.request(OP.flush, '', new Uint8Array(), relation, 0, 0)[0]).not.toBe(0); requestOk(pool, OP.flush, '', new Uint8Array(), relation); expect(flushes.map(lastPathSegment)).toEqual([backing('base/value')]); flushes.length = 0; + io.failNextFlush = true; + await expect(pool.sync('operation')).rejects.toThrow('injected OPFS flush failure'); await pool.sync('operation'); expect(flushes.map(lastPathSegment)).toEqual([backing('pg_wal/0001')]); await pool.sync('full'); @@ -147,12 +151,33 @@ describe('WASIX pooled OPFS storage', () => { backing('global/pg_control'), ]); + // Without guest file-sync requests (fsync=off), an existing-file operation + // is not a full checkpoint barrier. Namespace creation below is different. flushes.length = 0; - const created = open(pool, 'base/new-relation', FLAG_WRITE | FLAG_CREATE); - write(pool, created, Uint8Array.of(3)); + write(pool, relation, Uint8Array.of(3)); write(pool, wal, Uint8Array.of(3)); write(pool, control, Uint8Array.of(3)); await pool.sync('operation'); + expect(flushes.map(lastPathSegment)).toEqual([backing('pg_wal/0001')]); + for (const descriptor of [relation, ordinary, wal, control]) { + requestOk(pool, OP.close, '', new Uint8Array(), descriptor); + } + expect(flushes.map(lastPathSegment)).toEqual([backing('pg_wal/0001')]); + await pool.sync('close'); + expect(flushes.map(lastPathSegment)).toEqual([ + backing('pg_wal/0001'), + backing('base/value'), + backing('global/pg_control'), + ]); + + const walAgain = open(pool, 'pg_wal/0001', FLAG_WRITE); + const controlAgain = open(pool, 'global/pg_control', FLAG_WRITE); + flushes.length = 0; + const created = open(pool, 'base/new-relation', FLAG_WRITE | FLAG_CREATE); + write(pool, created, Uint8Array.of(3)); + write(pool, walAgain, Uint8Array.of(3)); + write(pool, controlAgain, Uint8Array.of(3)); + await pool.sync('operation'); const updated = JSON.parse( await (await databaseDirectory(root, 'flush-order')).file('state.json').text(), ) as { entries: Array<{ path: string; backing?: string }> }; @@ -166,7 +191,7 @@ describe('WASIX pooled OPFS storage', () => { backing('global/pg_control'), ]); - for (const descriptor of [relation, ordinary, wal, control, created]) { + for (const descriptor of [walAgain, controlAgain, created]) { requestOk(pool, OP.close, '', new Uint8Array(), descriptor); } await pool.close(false); @@ -762,6 +787,7 @@ type FakeIo = { failNextAccessClose?: boolean; failNextStateCommit?: boolean; failNextDataCommit?: boolean; + failNextFlush?: boolean; flushes?: string[]; lockReleaseFailure?: Error; syncAccessErrorName?: string; @@ -920,6 +946,10 @@ class FakeFile { } }, flush: () => { + if (this.#io.failNextFlush === true) { + this.#io.failNextFlush = false; + throw new Error('injected OPFS flush failure'); + } this.#io.flushes?.push(this.#path); }, getSize: () => this.#bytes.byteLength, diff --git a/src/wasix/sdks/ts/src/hosts/browser/wasix-runtime.ts b/src/wasix/sdks/ts/src/hosts/browser/wasix-runtime.ts index 3cc59b1d9..e3138038c 100644 --- a/src/wasix/sdks/ts/src/hosts/browser/wasix-runtime.ts +++ b/src/wasix/sdks/ts/src/hosts/browser/wasix-runtime.ts @@ -116,7 +116,6 @@ function compareDirectoryDepth(left: string, right: string): number { /** @internal PostgreSQL argv shared by both execution surfaces. */ export function wasixPostgresArgs(options: SerializedOpenOptions): string[] { const args = ['--single']; - if (options.storage.kind === 'memory') args.push('-F'); args.push('-O', '-j'); const startupGUCs = normalizeWasixStartupGUCs(options.startupGUCs); for (const [configuredName, configuredValue] of Object.entries(startupGUCs)) {