diff --git a/.github/workflows/arm64-jobs.yml b/.github/workflows/arm64-jobs.yml index df5006f9..c337f2c8 100644 --- a/.github/workflows/arm64-jobs.yml +++ b/.github/workflows/arm64-jobs.yml @@ -201,8 +201,8 @@ jobs: SMALL_GUESTS: >- --lib --bins --test bash --test cli --test compiled --test cpio --test fs_backend --test native_kernel --test networking --test php --test project - --test python --test python_shell --test quickjs --test resolv - --test snapshot_mounts --test step --test wasmtime + --test python --test python_shell --test quickjs --test resize + --test resolv --test snapshot_mounts --test step --test wasmtime steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e299bc2..195c9ed0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ## [Prerelease] - Unreleased +### Added + +- `AppSandbox::snapshot_with_scratch_mb` takes a snapshot that restores with another scratch size: boot with what unpacking the rootfs needs, then run guest function calls with what they need, less or more. The snapshot records the size (`snapshot_scratch_mb`). A size too small for the kernel makes the restore fail (restoring a snapshot that fits recovers the sandbox) and a boot from the snapshot fail. A guest function call that runs out of scratch ends with its process killed (status 137 with a single-process driver, such as python's) or the guest aborted, depending on what runs out first. A restore recovers the sandbox. + +### Changed + +- The host places the transport arena at a fixed address below the top of scratch, so a snapshot keeps it when restored with a different scratch size. Snapshots saved by an earlier release are refused: save them again. A native Unikraft kernel (`--kernel`, `SandboxBuilder::from_kernel`) built against an earlier `plat/hyperlight` can allocate over the arena: rebuild it against this kernel tree. + +### Fixed + +- The kernel allocated scratch pages the host reserves at the top of scratch. It now stops below them. +- A guest that ran out of scratch during a copy-on-write fault crashed the VM (an unexpected exit) instead of aborting. +- On x86, a page fault the kernel could not handle early in boot or in its fault handler ended the guest as if it had finished: a guest function call that hit one looked complete. The guest aborts. + ## [v0.20.1] ### Changed diff --git a/Cargo.lock b/Cargo.lock index cb054426..eaf74775 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,22 +114,21 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" [[package]] name = "bindgen" -version = "0.72.1" +version = "0.73.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +checksum = "787ef8ef523575546b106a58213d6e6b06198a05c2f757258c68a74273670cfa" dependencies = [ "bitflags 2.13.1", "cexpr", "clang-sys", - "itertools", "log", "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", - "shlex 1.3.0", - "syn 2.0.119", + "shlex", + "syn 3.0.9", ] [[package]] @@ -210,7 +209,7 @@ checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] @@ -270,7 +269,7 @@ dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex 2.0.1", + "shlex", ] [[package]] @@ -358,7 +357,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] @@ -552,12 +551,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "either" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" - [[package]] name = "equivalent" version = "1.0.2" @@ -786,8 +779,7 @@ dependencies = [ [[package]] name = "hyperlight-common" version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360d5876788b26af66bf055bca6b4f2c9c930a0fdcaebda4bd7f808c6bd93068" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1#a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -806,8 +798,7 @@ dependencies = [ [[package]] name = "hyperlight-host" version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f8b85ddd688890477158b9921b228973d1b70c84b866698838df0f32d2aa44" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1#a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1" dependencies = [ "anyhow", "bindgen", @@ -953,15 +944,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.18" @@ -1337,12 +1319,12 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "prettyplease" -version = "0.2.37" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" dependencies = [ "proc-macro2", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] @@ -1642,7 +1624,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] @@ -1696,12 +1678,6 @@ dependencies = [ "dirs", ] -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "shlex" version = "2.0.1" @@ -1778,9 +1754,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.3" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -1836,7 +1812,7 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 18a9c0cf..360776df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,3 +103,10 @@ tracing-subscriber = { version = "0.3", default-features = false, features = ["r # The FlatBuffer decoder's external-value source, mirrored in the tests. anyhow = "1" bytes = "1" + +# hyperlight main, for scratch resize (#1947). crates.io takes no git +# dependency, so hyperlight-unikraft is not published until a hyperlight +# release replaces this. +[patch.crates-io] +hyperlight-host = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1" } +hyperlight-common = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1" } diff --git a/build.rs b/build.rs index 7a492c8a..651caafa 100644 --- a/build.rs +++ b/build.rs @@ -15,7 +15,7 @@ use sha2::{Digest, Sha256}; /// the guest MSR list, and the hyperlight-host release that reads a /// snapshot. Bump it when any of those changes in a way a snapshot would /// notice; a kernel change rolls the key by itself. -const SNAPSHOT_CONTRACT: u32 = 7; +const SNAPSHOT_CONTRACT: u32 = 8; /// The embedded kernel, whose code and host-call protocol a snapshot /// carries in its memory: one per architecture, named as Unikraft names diff --git a/demos/pptx-gen/Cargo.lock b/demos/pptx-gen/Cargo.lock index f8620f82..254293d6 100644 --- a/demos/pptx-gen/Cargo.lock +++ b/demos/pptx-gen/Cargo.lock @@ -68,7 +68,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -79,7 +79,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -180,22 +180,21 @@ checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" [[package]] name = "bindgen" -version = "0.72.1" +version = "0.73.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +checksum = "787ef8ef523575546b106a58213d6e6b06198a05c2f757258c68a74273670cfa" dependencies = [ "bitflags 2.13.1", "cexpr", "clang-sys", - "itertools", "log", "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", - "shlex 1.3.0", - "syn 2.0.119", + "shlex", + "syn 3.0.4", ] [[package]] @@ -293,7 +292,7 @@ dependencies = [ "cap-primitives", "cap-std", "io-lifetimes 3.0.1", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -310,7 +309,7 @@ dependencies = [ "maybe-owned", "rustix", "rustix-linux-procfs", - "windows-sys 0.61.2", + "windows-sys 0.60.2", "winx", ] @@ -335,7 +334,7 @@ dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex 2.0.1", + "shlex", ] [[package]] @@ -624,7 +623,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -644,12 +643,6 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" -[[package]] -name = "either" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" - [[package]] name = "equivalent" version = "1.0.2" @@ -663,7 +656,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1054,8 +1047,7 @@ dependencies = [ [[package]] name = "hyperlight-common" version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360d5876788b26af66bf055bca6b4f2c9c930a0fdcaebda4bd7f808c6bd93068" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1#a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -1074,8 +1066,7 @@ dependencies = [ [[package]] name = "hyperlight-host" version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f8b85ddd688890477158b9921b228973d1b70c84b866698838df0f32d2aa44" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1#a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1" dependencies = [ "anyhow", "bindgen", @@ -1345,15 +1336,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.18" @@ -1619,7 +1601,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1776,12 +1758,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.37" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" dependencies = [ "proc-macro2", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] @@ -1846,7 +1828,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2112,7 +2094,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2361,12 +2343,6 @@ dependencies = [ "dirs", ] -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "shlex" version = "2.0.1" @@ -2408,7 +2384,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2518,7 +2494,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3084,7 +3060,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] diff --git a/demos/pptx-gen/Cargo.toml b/demos/pptx-gen/Cargo.toml index f4d33fc3..1fce053e 100644 --- a/demos/pptx-gen/Cargo.toml +++ b/demos/pptx-gen/Cargo.toml @@ -15,3 +15,10 @@ dotenvy = "0.15" anyhow = "1" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +# hyperlight main, for scratch resize (#1947). crates.io takes no git +# dependency, so hyperlight-unikraft is not published until a hyperlight +# release replaces this. +[patch.crates-io] +hyperlight-host = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1" } +hyperlight-common = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1" } diff --git a/docs/execution.md b/docs/execution.md index 2ea65315..8293bcda 100644 --- a/docs/execution.md +++ b/docs/execution.md @@ -45,7 +45,8 @@ Everything a caller does with an [`AppSandbox`](../src/lib.rs) is one of these. | `call(function, input)` | `run(Exec::Call { function, input })`, returning the result of a function the guest defined ([calls.md](calls.md)). | | `join()` | `step` until the process exits, and hand back its status. | | `snapshot()` / `snapshot_to(dir)` | Captures the guest at the current boundary, a call in flight included; `snapshot_to` also writes it to a directory. | -| `restore(snapshot)` / `restore_from(dir)` | Replaces the guest with the snapshot in place and runs its `resume` entry. | +| `snapshot_with_scratch_mb(mb)` | Like `snapshot()`, but a sandbox restored from it gets `mb` MiB of scratch, less or more than it booted with. It refuses a size above hyperlight's maximum memory size or too small for the transport and page tables. A size too small for the kernel makes `restore`, or a boot from the snapshot, fail. A guest function call that runs out of scratch ends with its process killed (status 137 with a single-process driver, such as python's) or the guest aborted, depending on what runs out first. A restore recovers the sandbox. | +| `restore(snapshot)` / `restore_from(dir)` | Replaces the guest with the snapshot in place and runs its `resume` entry. The sandbox takes the snapshot's scratch size. | ## Three kinds of guest @@ -169,7 +170,7 @@ A server is the same: `join` parks on its listener between requests and returns A snapshot can be taken at any boundary, and a restored guest continues from exactly there. Two boundaries are worth naming. -**After boot, for a warm start.** With a driver image, `boot()` returns once the runtime is initialised and the driver is parked: the interpreter is up, and in the `agent` image the heavy imports are already done. A snapshot there is a warm image; every guest restored from it skips that work. +**After boot, for a warm start.** With a driver image, `boot()` returns once the runtime is initialised and the driver is parked: the interpreter is up, and in the `agent` image the heavy imports are already done. A snapshot there is a warm image; every guest restored from it skips that work. Booting needs room to unpack the rootfs, which guest function calls do not: `snapshot_with_scratch_mb` takes the warm image with less scratch, and every restore of it costs less. ```mermaid sequenceDiagram diff --git a/integrations/clickhouse/Cargo.lock b/integrations/clickhouse/Cargo.lock index 5926f30f..ff404a1f 100644 --- a/integrations/clickhouse/Cargo.lock +++ b/integrations/clickhouse/Cargo.lock @@ -46,22 +46,21 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "bindgen" -version = "0.72.1" +version = "0.73.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +checksum = "787ef8ef523575546b106a58213d6e6b06198a05c2f757258c68a74273670cfa" dependencies = [ "bitflags 2.13.2", "cexpr", "clang-sys", - "itertools", "log", "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", - "shlex 1.3.0", - "syn 2.0.119", + "shlex", + "syn 3.0.9", ] [[package]] @@ -159,7 +158,7 @@ dependencies = [ "cap-primitives", "cap-std", "io-lifetimes 3.0.1", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -176,7 +175,7 @@ dependencies = [ "maybe-owned", "rustix", "rustix-linux-procfs", - "windows-sys 0.61.2", + "windows-sys 0.60.2", "winx", ] @@ -201,7 +200,7 @@ dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex 2.0.1", + "shlex", ] [[package]] @@ -437,15 +436,9 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] -[[package]] -name = "either" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" - [[package]] name = "errno" version = "0.3.14" @@ -453,7 +446,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -614,8 +607,7 @@ dependencies = [ [[package]] name = "hyperlight-common" version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360d5876788b26af66bf055bca6b4f2c9c930a0fdcaebda4bd7f808c6bd93068" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1#a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -634,8 +626,7 @@ dependencies = [ [[package]] name = "hyperlight-host" version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f8b85ddd688890477158b9921b228973d1b70c84b866698838df0f32d2aa44" +source = "git+https://github.com/hyperlight-dev/hyperlight?rev=a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1#a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1" dependencies = [ "anyhow", "bindgen", @@ -755,15 +746,6 @@ version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.18" @@ -1099,12 +1081,12 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "prettyplease" -version = "0.2.37" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" dependencies = [ "proc-macro2", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] @@ -1267,7 +1249,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1390,12 +1372,6 @@ dependencies = [ "dirs", ] -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "shlex" version = "2.0.1" @@ -1479,7 +1455,7 @@ dependencies = [ "getrandom", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1678,7 +1654,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] diff --git a/integrations/clickhouse/Cargo.toml b/integrations/clickhouse/Cargo.toml index f17f82bc..5c4ef022 100644 --- a/integrations/clickhouse/Cargo.toml +++ b/integrations/clickhouse/Cargo.toml @@ -27,3 +27,10 @@ thiserror = "2" [dev-dependencies] tempfile = "3" + +# hyperlight main, for scratch resize (#1947). crates.io takes no git +# dependency, so hyperlight-unikraft is not published until a hyperlight +# release replaces this. +[patch.crates-io] +hyperlight-host = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1" } +hyperlight-common = { git = "https://github.com/hyperlight-dev/hyperlight", rev = "a4563dbd585d47bfc0664d7b9559fbf0cb51d2c1" } diff --git a/kernel/elfloader_hyperlight-arm64 b/kernel/elfloader_hyperlight-arm64 index 34ceebb2..bae5f69c 100755 Binary files a/kernel/elfloader_hyperlight-arm64 and b/kernel/elfloader_hyperlight-arm64 differ diff --git a/kernel/elfloader_hyperlight-x86_64 b/kernel/elfloader_hyperlight-x86_64 index dd713531..a6fc4313 100755 Binary files a/kernel/elfloader_hyperlight-x86_64 and b/kernel/elfloader_hyperlight-x86_64 differ diff --git a/kernel/unikraft b/kernel/unikraft index 0d9edd33..9f414729 160000 --- a/kernel/unikraft +++ b/kernel/unikraft @@ -1 +1 @@ -Subproject commit 0d9edd336844e3896c046cfce4fb77873144e8ee +Subproject commit 9f41472980edd6a93d2ec5f2a0e5f8f445d786d3 diff --git a/src/lib.rs b/src/lib.rs index 7018e60f..940e002b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,8 +26,10 @@ use std::time::{Duration, Instant}; pub use hyperlight_host; use hyperlight_host::{ - GuestBinary, HyperlightError, Sandbox, UninitializedSandbox, func::Registerable, - sandbox::SandboxConfiguration, sandbox::snapshot::OciTag, + GuestBinary, HyperlightError, Sandbox, UninitializedSandbox, + func::Registerable, + sandbox::SandboxConfiguration, + sandbox::snapshot::{OciTag, SnapshotOptions}, }; // Re-export snapshot types so dependents don't need hyperlight-host directly. @@ -679,11 +681,12 @@ fn record_scratch_mb(snapshot: Arc, scratch_mb: Option) -> Resu )?) } -/// Write a snapshot from [`AppSandbox::snapshot`] to `dir` as an OCI image -/// layout, named by this release and its [`SNAPSHOT_KEY`]. Another -/// process, or a later run of this one, reads it back with -/// [`load_snapshot`], [`SandboxBuilder::from_snapshot_dir`] or -/// [`AppSandbox::restore_from`]; any build with the same key can, since +/// Write a snapshot from [`AppSandbox::snapshot`] (or +/// [`AppSandbox::snapshot_with_scratch_mb`], whose scratch size it keeps) +/// to `dir` as an OCI image layout, named by this release and its +/// [`SNAPSHOT_KEY`]. Another process, or a later run of this one, reads +/// it back with [`load_snapshot`], [`SandboxBuilder::from_snapshot_dir`] +/// or [`AppSandbox::restore_from`]; any build with the same key can, since /// the key is the kernel and the host contract a snapshot depends on. /// [`AppSandbox::snapshot_to`] does both steps in one. pub fn save_snapshot(snapshot: &Snapshot, dir: impl AsRef) -> Result<()> { @@ -2117,6 +2120,13 @@ fn fstab_arg(mounts: &[Mount]) -> Result { Ok(format!(" vfs.fstab=[{entries}]")) } +/// MiB to bytes, saturating: a size that overflows is larger than any +/// hyperlight accepts, so it is rejected as too big, against the maximum +/// memory size. +fn mb_to_bytes(mb: usize) -> usize { + mb.saturating_mul(MIB) +} + /// Assemble the uninitialized sandbox and its [`GuestConfig`] from the /// pieces a [`SandboxBuilder`] gathered. `kernel` is `None` for the /// embedded [`KERNEL`], `Some` for an external one; `initrd` is `None` @@ -2135,7 +2145,7 @@ fn assemble_sandbox( guest_log_level: Option, console: Console, ) -> Result<(UninitializedSandbox, GuestConfig)> { - let scratch_size = scratch_mb * MIB; + let scratch_size = mb_to_bytes(scratch_mb); info!(scratch_mb, "guest memory"); let mut cfg = SandboxConfiguration::default(); cfg.set_scratch_size(scratch_size); @@ -2327,6 +2337,9 @@ impl SandboxBuilder { /// The [`mount`](Self::mount)s given here are the restored guest's: on /// its `resume` entry the kernel makes its mount table match them, so a /// snapshot saved without mounts serves any mount set. + /// + /// The sandbox gets the snapshot's scratch size: `mb` for a snapshot + /// from [`AppSandbox::snapshot_with_scratch_mb`]. pub fn from_snapshot(snapshot: Arc) -> Self { Self { snapshot: Some(snapshot), @@ -2335,7 +2348,7 @@ impl SandboxBuilder { } /// [`from_snapshot`](Self::from_snapshot) with the snapshot read from - /// `dir`, where [`AppSandbox::snapshot_to`] wrote it. + /// `dir`, where [`AppSandbox::snapshot_to`] or [`save_snapshot`] wrote it. pub fn from_snapshot_dir(dir: impl AsRef) -> Result { Ok(Self::from_snapshot(load_snapshot(dir)?)) } @@ -2964,9 +2977,7 @@ impl AppSandbox { /// driver ([`Error::NoDriver`]), or already has a call in flight /// ([`Error::CallInFlight`]: the kernel serves one at a time). pub fn submit(&mut self, exec: impl Into) -> Result<()> { - if let Some(Yield::Exited { status }) = self.exited { - return Err(Error::GuestExited { status }); - } + self.ensure_running()?; // The kernel would reject it too (nothing reads /dev/hlcall); // refusing here is earlier and says why. if !self.has_driver() { @@ -3111,13 +3122,63 @@ impl AppSandbox { /// Errors with [`Error::GuestExited`] once the guest process has /// exited: there is nothing left to resume. pub fn snapshot(&mut self) -> Result> { - if let Some(Yield::Exited { status }) = self.exited { - return Err(Error::GuestExited { status }); - } + self.ensure_running()?; let snapshot = self.sandbox.snapshot()?; record_scratch_mb(snapshot, self.config.scratch_size.map(|size| size / MIB)) } + /// [`snapshot`](Self::snapshot), but a sandbox restored from it gets + /// `mb` MiB of scratch. Booting needs room to unpack the rootfs; guest + /// function calls need far less, so restores of a smaller snapshot cost + /// less. A larger one gives guest function calls more room than the + /// sandbox booted with. The snapshot records `mb` + /// ([`snapshot_scratch_mb`]). + /// + /// - `mb` above hyperlight's maximum memory size errors with + /// `Error::Hyperlight(MemoryRequestTooBig)`. + /// - A size too small for the transport errors with + /// `Error::Hyperlight(MemoryRequestTooSmall)` before the guest is + /// entered. Room for the page tables is checked when the snapshot is + /// built, after any pending transport checkpoint has run in the guest; + /// that error, the same variant, leaves the sandbox usable. So does + /// the error for a larger size whose scratch would cover a guest + /// mapping, which the kernel does not make. + /// - Whether the kernel fits is not checked. If it does not, + /// [`restore`](Self::restore) fails with + /// `Error::Hyperlight(GuestAborted)`; until a restore of a snapshot + /// that fits recovers the sandbox, a guest function call fails with + /// [`Error::NoDriver`] and a snapshot with + /// `Error::Hyperlight(PoisonedSandbox)`. A new sandbox booted from + /// the snapshot fails to boot, with `Error::Hyperlight(GuestAborted)`. + /// - Scratch also holds the transport, page tables and the kernel's own + /// state, so a guest function call gets less than `mb`. One that + /// runs out either gets its process killed when the kernel's memory + /// budget runs out ([`Error::GuestExited`], status 137, with a + /// single-process driver such as python's), or aborts the guest + /// (`Error::Hyperlight(GuestAborted)`) when scratch itself runs out, + /// which copies of the snapshot's pages also draw on. A restore + /// recovers the sandbox from either. + /// - At the sandbox's own size this is [`snapshot`](Self::snapshot); + /// any other size captures a new snapshot each time, so keep the + /// result. + /// - Errors with [`Error::GuestExited`] once the guest process has + /// exited. + pub fn snapshot_with_scratch_mb(&mut self, mb: usize) -> Result> { + self.ensure_running()?; + let options = SnapshotOptions::default().scratch_size(mb_to_bytes(mb)); + let snapshot = self.sandbox.snapshot_with_options(&options)?; + record_scratch_mb(snapshot, Some(mb)) + } + + /// A guest that has exited cannot serve a guest function call or be + /// snapshotted. + fn ensure_running(&self) -> Result<()> { + match self.exited { + Some(Yield::Exited { status }) => Err(Error::GuestExited { status }), + _ => Ok(()), + } + } + /// [`snapshot`](Self::snapshot), then [`save_snapshot`] it to `dir`. /// The snapshot is also returned, for use in this process. pub fn snapshot_to(&mut self, dir: impl AsRef) -> Result> { @@ -3129,18 +3190,23 @@ impl AppSandbox { /// Restore a snapshot into this sandbox in place and put the restored /// guest right for this host with a `resume` entry, as /// [`SandboxBuilder::boot`] does, with this sandbox's mounts. + /// + /// The sandbox takes the snapshot's scratch size. If a resized + /// snapshot is too small for the kernel, the restore fails, and until a + /// restore of a snapshot that fits succeeds, guest function calls fail + /// with [`Error::NoDriver`], snapshots with + /// `Error::Hyperlight(PoisonedSandbox)`, and [`step`](Self::step) and + /// [`join`](Self::join) with [`Error::Deadlocked`]. pub fn restore(&mut self, snapshot: Arc) -> Result<()> { let recorded = snapshot_scratch_mb(&snapshot); let sandbox = &mut self.sandbox; self.config .profile .time("restore: hyperlight", || sandbox.restore(snapshot))?; - // A restore in place keeps the sandbox's memory layout, so its - // size stands; a snapshot that recorded one makes it known where - // it was not. - if let Some(mb) = recorded { - self.config.scratch_size = Some(mb * MIB); - } + // The sandbox takes the snapshot's scratch size, which can differ + // from its own: known where the snapshot recorded it, unknown + // where it did not. + self.config.scratch_size = recorded.map(mb_to_bytes); // The host sockets belong to the guest state just discarded; the // restored guest re-creates the ones it holds on its resume entry. if let Some(net) = &self.config.net { @@ -3153,7 +3219,7 @@ impl AppSandbox { } /// [`restore`](Self::restore) with the snapshot read from `dir`, where - /// [`snapshot_to`](Self::snapshot_to) wrote it. + /// [`snapshot_to`](Self::snapshot_to) or [`save_snapshot`] wrote it. pub fn restore_from(&mut self, dir: impl AsRef) -> Result<()> { self.restore(load_snapshot(dir)?) } @@ -3249,7 +3315,7 @@ fn restore_snapshot( // size the snapshot recorded, if it did. let config = GuestConfig::new( String::new(), - snapshot_scratch_mb(&snapshot).map(|mb| mb * MIB), + snapshot_scratch_mb(&snapshot).map(mb_to_bytes), 0, 0, mounts, diff --git a/tests/fixtures/native-kernel/helloworld-native_hyperlight-x86_64 b/tests/fixtures/native-kernel/helloworld-native_hyperlight-x86_64 index d924cfd3..7094853b 100755 Binary files a/tests/fixtures/native-kernel/helloworld-native_hyperlight-x86_64 and b/tests/fixtures/native-kernel/helloworld-native_hyperlight-x86_64 differ diff --git a/tests/resize.rs b/tests/resize.rs new file mode 100644 index 00000000..5516849b --- /dev/null +++ b/tests/resize.rs @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 The Hyperlight Authors. + +//! Snapshots that restore with a different scratch size than the guest +//! booted with. + +mod common; + +use common::{require_rootfs, temp_dir}; +use hyperlight_host::HyperlightError::{ + GuestAborted, MemoryRequestTooBig, MemoryRequestTooSmall, PoisonedSandbox, +}; +use hyperlight_unikraft::{Error, SandboxBuilder, save_snapshot, snapshot_scratch_mb}; + +const SCRIPT: &str = "def handler(request): + request['uri'] = '/redirected.html' + return request + +def big(request): + b = bytearray(int(request['mb']) << 20) + for i in range(0, len(b), 4096): + b[i] = 1 + return {'len': len(b)} +"; + +/// A guest function call that outgrew the scratch either has its process +/// killed, when the kernel's memory budget runs out first, or aborts the +/// guest, when scratch itself does. +fn outgrew(e: &Error) -> bool { + matches!( + e, + Error::GuestExited { status: 137 } | Error::Hyperlight(GuestAborted(..)) + ) +} + +const REQUEST: &str = r#"{"uri":"/index.html"}"#; +const BIG: &str = r#"{"mb":32}"#; +// More than the 256 MiB the sandbox boots with. +const HUGE: &str = r#"{"mb":320}"#; + +#[test] +fn a_resized_snapshot_restores_with_another_scratch_size() { + let mut sandbox = SandboxBuilder::from_initrd(require_rootfs("python")) + .scratch_mb(256) + .boot() + .unwrap(); + sandbox.run(SCRIPT).unwrap(); + let full = sandbox.snapshot().unwrap(); + assert_eq!(snapshot_scratch_mb(&full), Some(256)); + let same = sandbox.snapshot_with_scratch_mb(256).unwrap(); + assert_eq!(snapshot_scratch_mb(&same), Some(256)); + let small = sandbox.snapshot_with_scratch_mb(8).unwrap(); + assert_eq!(snapshot_scratch_mb(&small), Some(8)); + let grown = sandbox.snapshot_with_scratch_mb(512).unwrap(); + assert_eq!(snapshot_scratch_mb(&grown), Some(512)); + + // Sizes it cannot use are rejected before the guest is entered, so + // the sandbox runs a guest function call right after, with no restore. + let e = sandbox + .snapshot_with_scratch_mb(0) + .err() + .expect("0 MiB must be rejected"); + assert!( + matches!(e, Error::Hyperlight(MemoryRequestTooSmall(..))), + "{e}" + ); + // Saturates to usize::MAX bytes, above the maximum memory size. + let e = sandbox + .snapshot_with_scratch_mb(usize::MAX) + .err() + .expect("overflow must be rejected"); + assert!( + matches!(e, Error::Hyperlight(MemoryRequestTooBig(..))), + "{e}" + ); + let out = sandbox.call("handler", REQUEST).unwrap(); + assert!(out.contains("redirected"), "{out}"); + + for _ in 0..3 { + sandbox.restore(small.clone()).unwrap(); + let out = sandbox.call("handler", REQUEST).unwrap(); + assert!(out.contains("redirected"), "{out}"); + } + + // 32 MiB does not fit in 8 MiB of scratch, so the resize took effect, + // and a restore recovers the sandbox. + sandbox.restore(small.clone()).unwrap(); + let e = sandbox + .call("big", BIG) + .expect_err("32 MiB must not fit 8 MiB"); + assert!(outgrew(&e), "{e}"); + + // Restored with 8 MiB, the sandbox can still take a larger snapshot. + sandbox.restore(small.clone()).unwrap(); + let regrown = sandbox.snapshot_with_scratch_mb(64).unwrap(); + assert_eq!(snapshot_scratch_mb(®rown), Some(64)); + sandbox.restore(regrown).unwrap(); + let out = sandbox.call("big", BIG).unwrap(); + assert!(out.contains("33554432"), "{out}"); + + // 320 MiB does not fit the 256 MiB the sandbox booted with, but fits + // the grown snapshot. + sandbox.restore(full.clone()).unwrap(); + let e = sandbox + .call("big", HUGE) + .expect_err("320 MiB must not fit 256 MiB"); + assert!(outgrew(&e), "{e}"); + sandbox.restore(grown).unwrap(); + let out = sandbox.call("big", HUGE).unwrap(); + assert!(out.contains("335544320"), "{out}"); + + // The smallest whole MiB the host accepts is less than a MiB above its + // minimum (transport and page tables), while the kernel's frame + // allocator needs two 512 KiB chunks on top of that, so the restore + // fails, as does a boot from it, and restoring a larger snapshot + // recovers the sandbox. + sandbox.restore(full.clone()).unwrap(); + let tiny = (1..8) + .find_map(|mb| match sandbox.snapshot_with_scratch_mb(mb) { + Ok(snapshot) => Some(snapshot), + Err(Error::Hyperlight(MemoryRequestTooSmall(..))) => None, + Err(e) => panic!("{mb} MiB: {e}"), + }) + .expect("a size the host accepts"); + let e = sandbox + .restore(tiny.clone()) + .expect_err("the tiny restore must fail"); + assert!(matches!(e, Error::Hyperlight(GuestAborted(..))), "{e}"); + // Until a restore recovers it, the sandbox serves nothing: hluk has + // forgotten the driver, and hyperlight has poisoned the sandbox. + let e = sandbox + .call("handler", REQUEST) + .expect_err("a guest function call after a failed restore must fail"); + assert!(matches!(e, Error::NoDriver), "{e}"); + let e = sandbox + .snapshot() + .err() + .expect("a snapshot after a failed restore must fail"); + assert!(matches!(e, Error::Hyperlight(PoisonedSandbox)), "{e}"); + let e = SandboxBuilder::from_snapshot(tiny) + .boot() + .err() + .expect("the tiny boot must fail"); + assert!(matches!(e, Error::Hyperlight(GuestAborted(..))), "{e}"); + sandbox.restore(full).unwrap(); + let out = sandbox.call("handler", REQUEST).unwrap(); + assert!(out.contains("redirected"), "{out}"); + + // A saved resized snapshot keeps its size and its record, restored in + // place and in a new sandbox. + let dir = temp_dir("resize"); + save_snapshot(&small, dir.path().join("small")).unwrap(); + sandbox.restore_from(dir.path().join("small")).unwrap(); + let out = sandbox.call("handler", REQUEST).unwrap(); + assert!(out.contains("redirected"), "{out}"); + let e = sandbox + .call("big", BIG) + .expect_err("32 MiB must not fit 8 MiB"); + assert!(outgrew(&e), "{e}"); + + let mut fresh = SandboxBuilder::from_snapshot_dir(dir.path().join("small")) + .unwrap() + .boot() + .unwrap(); + let out = fresh.call("handler", REQUEST).unwrap(); + assert!(out.contains("redirected"), "{out}"); + // A plain snapshot of the restored sandbox records the size it took. + assert_eq!(snapshot_scratch_mb(&fresh.snapshot().unwrap()), Some(8)); + let e = fresh + .call("big", BIG) + .expect_err("32 MiB must not fit a fresh 8 MiB sandbox"); + assert!(outgrew(&e), "{e}"); +}