vmm: add mmap memory restore mode

docs/snapshot_restore.md

@@ -127,6 +127,35 @@ Current constraints for `memory_restore_mode=ondemand`:
 - `prefault=on` is not supported
 - the snapshot memory ranges must be page-aligned
 
+### Mmap restore
+
+With `memory_restore_mode=mmap`, guest RAM is created by mapping the snapshot
+memory file copy-on-write before any KVM memslot or device consumes the
+mapping: nothing is copied up front, pages fault in from the page cache — so
+many VMs restored from the same snapshot share it — and guest writes stay
+private to each VM.
+
+Current constraints for `memory_restore_mode=mmap`:
+
+- Plain private guest RAM only. Anything else falls back to the eager copy
+  (logged): `shared=on` or hugepages (global or per-zone), zones with
+  `host_numa_node`, `reserve`, `mergeable` or hotplug fields (a purely static
+  `id`+`size` zone is fine), resizable RAM (`hotplug_size`, virtio-mem), KSM
+  (`mergeable=on`), `--pvmemcontrol`, device passthrough
+  (`--device`/`--user-device`), and snapshot ranges that are not page-aligned
+  single-region extents. `reserve=on` and THP are re-applied to the mapped
+  region.
+- A snapshot memory file shorter than the saved ranges is rejected up front (it
+  would otherwise fault `SIGBUS` at run time).
+- `prefault` is rejected.
+- The snapshot memory file must remain on disk **and unchanged** for the
+  entire lifetime of the VM. This is stronger than `ondemand`: UFFD copies each
+  page into the original anonymous mapping and stops needing the file once every
+  page is populated, and a read error there is a controlled VM exit; the mmap
+  region stays file-backed forever, so truncating it delivers a synchronous
+  `SIGBUS` and any in-place edit corrupts the guest. The length check only
+  rejects a file that is already short at restore time.
+
 ## Restore a VM with new Net FDs
 For a VM created with FDs explicitly passed to NetConfig, a set of valid FDs
 need to be provided along with the VM restore command in the following syntax:

vmm/src/api/openapi/cloud-hypervisor.yaml

@@ -1484,7 +1484,7 @@ components:
 
     MemoryRestoreMode:
       type: string
-      enum: [Copy, OnDemand]
+      enum: [Copy, OnDemand, Mmap]
       default: Copy
 
     RestoreConfig:

vmm/src/config.rs

@@ -390,8 +390,8 @@ pub enum ValidationError {
     /// Number of FDs passed during Restore are incorrect to the NetConfig
     #[error("Number of Net FDs passed for '{0}' during Restore: {1}. Expected: {2}")]
     RestoreNetFdCountMismatch(String, usize, usize),
-    /// Prefault cannot be combined with on-demand restore
-    #[error("'prefault' cannot be combined with 'memory_restore_mode=ondemand'")]
+    /// Prefault requires the eager-copy restore mode
+    #[error("'prefault' requires 'memory_restore_mode=copy'")]
     InvalidRestorePrefaultWithOnDemand,
     /// Path provided in landlock-rules doesn't exist
     #[error("Path {0:?} provided in landlock-rules does not exist")]
@@ -2782,6 +2782,9 @@ pub enum MemoryRestoreMode {
     Copy,
     /// Restore lazily by faulting snapshot pages into guest RAM on demand.
     OnDemand,
+    /// Restore by mapping the snapshot memory file copy-on-write, sharing the
+    /// page cache across VMs restored from the same snapshot.
+    Mmap,
 }
 
 #[derive(Debug, Error)]
@@ -2797,6 +2800,7 @@ impl FromStr for MemoryRestoreMode {
         match s.to_lowercase().as_str() {
             "copy" => Ok(Self::Copy),
             "ondemand" => Ok(Self::OnDemand),
+            "mmap" => Ok(Self::Mmap),
             _ => Err(MemoryRestoreModeParseError::InvalidValue(s.to_owned())),
         }
     }
@@ -2817,11 +2821,11 @@ pub struct RestoreConfig {
 
 impl RestoreConfig {
     pub const SYNTAX: &'static str = "Restore from a VM snapshot. \
-        \nRestore parameters \"source_url=<source_url>,prefault=on|off,memory_restore_mode=copy|ondemand,\
+        \nRestore parameters \"source_url=<source_url>,prefault=on|off,memory_restore_mode=copy|ondemand|mmap,\
         net_fds=<list_of_net_ids_with_their_associated_fds>,resume=true|false\" \
         \n`source_url` should be a valid URL (e.g file:///foo/bar or tcp://192.168.1.10/foo) \
         \n`prefault` controls eager prefaulting for the copy-based restore path (disabled by default) \
-        \n`memory_restore_mode=copy` preserves the existing eager read-copy restore behavior, while `memory_restore_mode=ondemand` enables lazy demand paging and fails restore if userfaultfd support is unavailable \
+        \n`memory_restore_mode=copy` preserves the existing eager read-copy restore behavior, `memory_restore_mode=ondemand` enables lazy demand paging and fails restore if userfaultfd support is unavailable, and `memory_restore_mode=mmap` maps the snapshot file copy-on-write (plain private RAM only; falls back to copy otherwise) \
         \n`net_fds` is a list of net ids with new file descriptors. \
         Only net devices backed by FDs directly are needed as input.\
         \n `resume` controls whether the VM will be directly resumed after restore ";
@@ -2880,7 +2884,7 @@ impl RestoreConfig {
     // corresponding 'RestoreNetConfig' with a matched 'id' and expected
     // number of FDs.
     pub fn validate(&self, vm_config: &VmConfig) -> ValidationResult<()> {
-        if self.memory_restore_mode == MemoryRestoreMode::OnDemand && self.prefault {
+        if self.memory_restore_mode != MemoryRestoreMode::Copy && self.prefault {
             return Err(ValidationError::InvalidRestorePrefaultWithOnDemand);
         }
 
@@ -5333,6 +5337,18 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
             invalid_restore_mode.validate(&snapshot_vm_config),
             Err(ValidationError::InvalidRestorePrefaultWithOnDemand)
         );
+
+        let invalid_mmap_prefault = RestoreConfig {
+            source_url: PathBuf::from("/path/to/snapshot"),
+            prefault: true,
+            memory_restore_mode: MemoryRestoreMode::Mmap,
+            net_fds: None,
+            resume: false,
+        };
+        assert_eq!(
+            invalid_mmap_prefault.validate(&snapshot_vm_config),
+            Err(ValidationError::InvalidRestorePrefaultWithOnDemand)
+        );
     }
 
     fn platform_fixture() -> PlatformConfig {

vmm/src/memory_manager.rs

@@ -395,6 +395,10 @@ pub enum Error {
     #[error("Error copying snapshot into region")]
     SnapshotCopy(#[source] GuestMemoryError),
 
+    /// Error mapping snapshot file over guest RAM
+    #[error("Error mapping snapshot file over guest RAM")]
+    SnapshotMmap(#[source] io::Error),
+
     /// Failed to allocate MMIO address
     #[error("Failed to allocate MMIO address")]
     AllocateMmioAddress,
@@ -909,6 +913,47 @@ impl MemoryManager {
         Ok(())
     }
 
+    /// Restore guest memory by mapping the snapshot file copy-on-write over
+    /// the anonymous guest mappings — before any KVM memslot or device
+    /// consumes them, so overlay identity concerns do not apply. Falls back
+    /// to the eager copy whenever a range cannot be mapped safely; the
+    /// snapshot file must remain on disk for the VM lifetime.
+    fn mmap_saved_regions(
+        &mut self,
+        file_path: PathBuf,
+        saved_regions: &MemoryRangeTable,
+    ) -> Result<(), Error> {
+        if saved_regions.is_empty() {
+            return Ok(());
+        }
+        let guest_memory = self.guest_memory.memory();
+        if !mmap_restore_compatible(&guest_memory, saved_regions) {
+            info!("guest RAM unsuitable for mmap restore; falling back to copy");
+            drop(guest_memory);
+            return self.fill_saved_regions(file_path, saved_regions);
+        }
+        let memory_file = OpenOptions::new()
+            .read(true)
+            .open(file_path)
+            .map_err(Error::SnapshotOpen)?;
+        // A range mapped past EOF faults SIGBUS at run time, not restore time.
+        let mapped_len: u64 = saved_regions.regions().iter().map(|r| r.length).sum();
+        let file_len = memory_file.metadata().map_err(Error::SnapshotOpen)?.len();
+        if file_len < mapped_len {
+            return Err(Error::SnapshotMmap(io::Error::new(
+                io::ErrorKind::UnexpectedEof,
+                "snapshot memory file is shorter than the saved ranges",
+            )));
+        }
+        mmap_saved_ranges(
+            &guest_memory,
+            &memory_file,
+            saved_regions,
+            self.reserve,
+            self.thp,
+        )
+    }
+
     /// Restore guest memory using userfaultfd for lazy demand paging.
     ///
     /// Instead of reading the entire snapshot into guest RAM upfront (which
@@ -1905,16 +1950,20 @@ impl MemoryManager {
                 Default::default(),
             )?;
 
-            if memory_restore_mode == MemoryRestoreMode::OnDemand {
-                mm.lock().unwrap().restore_by_uffd(
+            match memory_restore_mode {
+                MemoryRestoreMode::OnDemand => mm.lock().unwrap().restore_by_uffd(
                     &memory_file_path,
                     &mem_snapshot.memory_ranges,
                     exit_evt,
-                )?;
-            } else {
-                mm.lock()
+                )?,
+                MemoryRestoreMode::Mmap => mm
+                    .lock()
+                    .unwrap()
+                    .mmap_saved_regions(memory_file_path, &mem_snapshot.memory_ranges)?,
+                MemoryRestoreMode::Copy => mm
+                    .lock()
                     .unwrap()
-                    .fill_saved_regions(memory_file_path, &mem_snapshot.memory_ranges)?;
+                    .fill_saved_regions(memory_file_path, &mem_snapshot.memory_ranges)?,
             }
 
             Ok(mm)
@@ -3391,3 +3440,197 @@ impl Migratable for MemoryManager {
         Ok(table)
     }
 }
+
+/// Reports whether every saved range is page-aligned and lies wholly inside a
+/// single plain private-anonymous guest region — the MAP_FIXED overlay
+/// preconditions. File-backed regions are rejected: their stale mapping
+/// metadata would misdirect a later snapshot or fd consumer.
+fn mmap_restore_compatible(
+    guest_memory: &GuestMemoryMmap,
+    saved_regions: &MemoryRangeTable,
+) -> bool {
+    // SAFETY: sysconf(_SC_PAGESIZE) has no failure mode relevant here.
+    let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as u64;
+    let mut file_offset: u64 = 0;
+    for range in saved_regions.regions() {
+        if !file_offset.is_multiple_of(page_size)
+            || !range.gpa.is_multiple_of(page_size)
+            || !range.length.is_multiple_of(page_size)
+        {
+            return false;
+        }
+        let Some(end) = range.gpa.checked_add(range.length) else {
+            return false;
+        };
+        let ok = guest_memory
+            .find_region(GuestAddress(range.gpa))
+            .is_some_and(|r| {
+                r.file_offset().is_none() && end <= r.start_addr().raw_value() + r.len()
+            });
+        if !ok {
+            return false;
+        }
+        let Some(next) = file_offset.checked_add(range.length) else {
+            return false;
+        };
+        file_offset = next;
+    }
+    true
+}
+
+/// Maps each saved range over its guest RAM window, re-applying reserve and
+/// THP policy; ranges must have passed [`mmap_restore_compatible`].
+fn mmap_saved_ranges(
+    guest_memory: &GuestMemoryMmap,
+    memory_file: &File,
+    saved_regions: &MemoryRangeTable,
+    reserve: bool,
+    thp: bool,
+) -> Result<(), Error> {
+    let reserve_flag = if reserve { 0 } else { libc::MAP_NORESERVE };
+    let mut file_offset: u64 = 0;
+    for range in saved_regions.regions() {
+        let host_addr = guest_memory
+            .get_host_address(GuestAddress(range.gpa))
+            .map_err(|e| Error::SnapshotMmap(io::Error::other(e)))?;
+        let length = range.length as usize;
+        // SAFETY: the window is page-aligned, wholly inside a live private
+        // anonymous region nothing consumes yet, so the MAP_FIXED replacement
+        // cannot clobber foreign mappings; the fd stays valid for the call.
+        let ret = unsafe {
+            libc::mmap(
+                host_addr.cast(),
+                length,
+                libc::PROT_READ | libc::PROT_WRITE,
+                libc::MAP_PRIVATE | libc::MAP_FIXED | reserve_flag,
+                memory_file.as_raw_fd(),
+                file_offset as libc::off_t,
+            )
+        };
+        if ret == libc::MAP_FAILED {
+            return Err(Error::SnapshotMmap(io::Error::last_os_error()));
+        }
+        if thp {
+            // SAFETY: ret/length name the private mapping just installed above.
+            let adv = unsafe { libc::madvise(ret, length, libc::MADV_HUGEPAGE) };
+            if adv != 0 {
+                warn!(
+                    "mmap restore: MADV_HUGEPAGE failed: {}",
+                    io::Error::last_os_error()
+                );
+            }
+        }
+        file_offset = file_offset.checked_add(range.length).ok_or_else(|| {
+            Error::SnapshotMmap(io::Error::other("snapshot range file offset overflow"))
+        })?;
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::io::{Read, Seek, SeekFrom, Write};
+
+    use vm_migration::protocol::{MemoryRange, MemoryRangeTable};
+
+    use super::*;
+
+    fn page_size() -> u64 {
+        // SAFETY: sysconf(_SC_PAGESIZE) has no failure mode relevant here.
+        unsafe { libc::sysconf(libc::_SC_PAGESIZE) as u64 }
+    }
+
+    #[test]
+    fn mmap_restore_maps_data_and_reads_holes_as_zero() {
+        let page = page_size();
+        let gm = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), (2 * page) as usize)]).unwrap();
+
+        let mut file = tempfile::tempfile().unwrap();
+        file.write_all(&vec![0xabu8; page as usize]).unwrap();
+        file.set_len(2 * page).unwrap();
+        file.seek(SeekFrom::Start(0)).unwrap();
+
+        let mut table = MemoryRangeTable::default();
+        table.push(MemoryRange {
+            gpa: 0,
+            length: 2 * page,
+        });
+
+        assert!(mmap_restore_compatible(&gm, &table));
+        mmap_saved_ranges(&gm, &file, &table, false, false).unwrap();
+
+        assert_eq!(gm.read_obj::<u8>(GuestAddress(0)).unwrap(), 0xab);
+        assert_eq!(gm.read_obj::<u8>(GuestAddress(page - 1)).unwrap(), 0xab);
+        assert_eq!(gm.read_obj::<u8>(GuestAddress(page)).unwrap(), 0);
+        assert_eq!(gm.read_obj::<u8>(GuestAddress(2 * page - 1)).unwrap(), 0);
+
+        // CoW: guest writes must not reach the file.
+        gm.write_obj::<u8>(0x5a, GuestAddress(0)).unwrap();
+        let mut back = [0u8; 1];
+        file.seek(SeekFrom::Start(0)).unwrap();
+        file.read_exact(&mut back).unwrap();
+        assert_eq!(back[0], 0xab);
+    }
+
+    #[test]
+    fn mmap_restore_rejects_unaligned_or_out_of_region_ranges() {
+        let page = page_size();
+        let gm = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), page as usize)]).unwrap();
+
+        let mut unaligned = MemoryRangeTable::default();
+        unaligned.push(MemoryRange {
+            gpa: 1,
+            length: page,
+        });
+        assert!(!mmap_restore_compatible(&gm, &unaligned));
+
+        let mut spanning = MemoryRangeTable::default();
+        spanning.push(MemoryRange {
+            gpa: 0,
+            length: 2 * page,
+        });
+        assert!(!mmap_restore_compatible(&gm, &spanning));
+    }
+
+    #[test]
+    fn mmap_restore_rejects_file_backed_region() {
+        let page = page_size();
+        let backing = tempfile::tempfile().unwrap();
+        backing.set_len(page).unwrap();
+        let region = MmapRegion::build(
+            Some(FileOffset::new(backing, 0)),
+            page as usize,
+            libc::PROT_READ | libc::PROT_WRITE,
+            libc::MAP_SHARED,
+        )
+        .unwrap();
+        let gm = GuestMemoryMmap::from_regions(vec![
+            GuestRegionMmap::new(region, GuestAddress(0)).unwrap(),
+        ])
+        .unwrap();
+
+        let mut table = MemoryRangeTable::default();
+        table.push(MemoryRange {
+            gpa: 0,
+            length: page,
+        });
+        assert!(!mmap_restore_compatible(&gm, &table));
+    }
+
+    #[test]
+    fn mmap_restore_honors_reserve_and_thp() {
+        let page = page_size();
+        let gm = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), page as usize)]).unwrap();
+        let mut file = tempfile::tempfile().unwrap();
+        file.write_all(&vec![0xcdu8; page as usize]).unwrap();
+        file.seek(SeekFrom::Start(0)).unwrap();
+
+        let mut table = MemoryRangeTable::default();
+        table.push(MemoryRange {
+            gpa: 0,
+            length: page,
+        });
+        mmap_saved_ranges(&gm, &file, &table, true, true).unwrap();
+        assert_eq!(gm.read_obj::<u8>(GuestAddress(0)).unwrap(), 0xcd);
+    }
+}