From b6b28ce3519387bea940934ec930654a8410c70d Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Fri, 30 Oct 2026 09:27:34 -0500 Subject: [PATCH 01/38] install: Add tested /var migration engine The /var copy needs careful filtering around exclusions and nested mounts, and must avoid publishing partial entries or overwriting existing data. Isolating the copy engine and its tests makes these filesystem guarantees reviewable before adding installer orchestration. Assisted-by: AI Signed-off-by: ckyrouac --- crates/lib/src/install.rs | 2 + crates/lib/src/install/migrate.rs | 1374 +++++++++++++++++++ crates/lib/src/install/migrate/mountinfo.rs | 322 +++++ 3 files changed, 1698 insertions(+) create mode 100644 crates/lib/src/install/migrate.rs create mode 100644 crates/lib/src/install/migrate/mountinfo.rs diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index 00d642f862..dabac7cef0 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -144,6 +144,8 @@ mod aleph; pub(crate) mod baseline; pub(crate) mod completion; pub(crate) mod config; +#[cfg(test)] +mod migrate; mod osbuild; pub(crate) mod osconfig; diff --git a/crates/lib/src/install/migrate.rs b/crates/lib/src/install/migrate.rs new file mode 100644 index 0000000000..e0dfa75324 --- /dev/null +++ b/crates/lib/src/install/migrate.rs @@ -0,0 +1,1374 @@ +//! # Package-mode to image-mode migration helpers +//! +//! This module implements the post-install state-preservation steps that make +//! `bootc install to-existing-root` useful for converting a live package-mode +//! (RPM/DEB) system to a bootc image-mode deployment without losing data. +//! +//! This migration is triggered by passing `--preserve-var` to +//! `bootc install to-existing-root`. It runs **after** the core install +//! (ostree deploy + bootupd) completes but **before** the first reboot, while +//! the package-mode environment is still the running OS. +//! +//! ## `/var` preservation (`--preserve-var`) +//! +//! After a plain `bootc install to-existing-root`, the new deployment's `/var` +//! is bound from `/ostree/deploy//var/` — an initially-empty +//! directory. The old package-mode `/var` is stranded at `/var` +//! (inside the container, the host root is mounted at `root_path`). +//! +//! This uses `cp -a` to copy each non-ephemeral subdirectory of `/var` into the +//! new deployment's stateroot `var/`. The copy is not a consistent snapshot of +//! live data; stop stateful services before migrating their data. +//! +//! Nested mountpoints are selected from one `/proc/self/mountinfo` snapshot +//! before the copy. This is a point-in-time, best-effort snapshot, not a +//! race-free guard: mounts added or removed beneath `/var` during copying can +//! race with the copy, just like concurrent file changes. +//! Each missing entry is copied to a unique sibling staging directory and +//! published atomically without replacing an existing destination. A failed +//! entry leaves no partial final entry and can be retried; this is not a global +//! transaction, so entries completely published before a later failure remain. +//! +//! Exclusions such as `tmp`, `log/journal`, `lib/containers`, or `lib/rpm` +//! can be supplied through configuration or repeated `--preserve-var-skip` +//! options. +//! +//! Rollback preservation is intentionally independent from `/var` migration. + +mod mountinfo; + +use std::path::{Component, Path}; +use std::{ + collections::BTreeSet, + ffi::OsStr, + os::fd::AsFd, + path::PathBuf, + process::{Command, Stdio}, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, +}; + +use anyhow::{Context, Result}; +use cap_std_ext::cap_std::ambient_authority; +use cap_std_ext::cap_std::fs::{Dir as CapStdDir, Metadata, MetadataExt}; +use cap_std_ext::cap_std::io_lifetimes::AsFilelike; +use cap_std_ext::cmdext::{CapStdExtCommandExt, CmdFds}; +#[cfg(test)] +use fn_error_context::context; + +/// Run GNU cp with directory handles inherited as fixed descriptors. Each +/// operand is `/proc/self/fd/N[/entry]`; cp's top-level paths therefore remain +/// rooted in the opened capabilities even if the original paths are renamed. +/// GNU cp still performs its own recursive traversal beneath each operand, so +/// this is not race-safe against a concurrent writer replacing entries while +/// the copy is in progress. +fn run_cp( + cp_program: &OsStr, + src_root: &CapStdDir, + src_names: &[&OsStr], + dst_root: &CapStdDir, + dst_name: Option<&OsStr>, + options: &[&str], + stdout: Stdio, + stderr: Stdio, +) -> Result { + let src_fd = Arc::new( + src_root + .as_filelike_view::() + .as_fd() + .try_clone_to_owned()?, + ); + let dst_fd = Arc::new( + dst_root + .as_filelike_view::() + .as_fd() + .try_clone_to_owned()?, + ); + let mut fds = CmdFds::new(); + fds.take_fd_n(src_fd, 3); + fds.take_fd_n(dst_fd, 4); + + let src_paths = src_names.iter().map(|name| { + let mut path = PathBuf::from("/proc/self/fd/3"); + path.push(name); + path + }); + let mut dst_path = PathBuf::from("/proc/self/fd/4"); + if let Some(name) = dst_name { + dst_path.push(name); + } + + let status = Command::new(cp_program) + .args(options) + .arg("--") + .args(src_paths) + .arg(dst_path) + .stdout(stdout) + .stderr(stderr) + .take_fds(fds) + .status() + .context("Running cp")?; + Ok(status) +} + +/// Copy a directory's ordinary children together with cp -a, recursing only +/// into directories whose descendants contain a mountpoint or exclusion. +/// Batching ordinary siblings in one cp invocation retains hardlinks between +/// those entries. Hardlinks crossing a mountpoint or exclusion split, or +/// crossing separate copies while merging into an existing destination +/// directory, may not remain linked. Untouched subtrees remain entirely under +/// cp -a's control. +fn copy_var_filtered( + src_dir: &CapStdDir, + dst_dir: &CapStdDir, + relative_dir: &Path, + exclusions: &[String], + mountpoints: &BTreeSet, + destination_created: bool, + cp_program: &OsStr, +) -> Result<()> { + // Existing directories are merged in place, so capture their own metadata + // before reading or copying children can alter their atime/mtime. The same + // wrapper restores source metadata on newly-created destination directories. + let source_metadata = if destination_created { + Some(snapshot_directory_metadata(src_dir).with_context(|| { + format!( + "Snapshotting source metadata for /var/{}", + relative_dir.display() + ) + })?) + } else { + None + }; + let destination_metadata = if destination_created { + None + } else { + Some(dst_dir.metadata(".").with_context(|| { + format!( + "Snapshotting existing destination timestamps for /var/{}", + relative_dir.display() + ) + })?) + }; + + let copy_result = copy_var_filtered_inner( + src_dir, + dst_dir, + relative_dir, + exclusions, + mountpoints, + cp_program, + ); + let restore_result = match (source_metadata.as_ref(), destination_metadata.as_ref()) { + (Some(metadata), _) => restore_directory_metadata(dst_dir, metadata), + (None, Some(metadata)) => restore_directory_timestamps(dst_dir, metadata), + (None, None) => Ok(()), + } + .with_context(|| { + format!( + "Restoring directory metadata for /var/{}", + relative_dir.display() + ) + }); + + match (copy_result, restore_result) { + (Ok(()), Ok(())) => Ok(()), + (Ok(()), Err(restore_error)) => Err(restore_error), + (Err(copy_error), Ok(())) => Err(copy_error), + (Err(copy_error), Err(restore_error)) => Err(copy_error.context(format!( + "Additionally failed to restore /var/{} metadata: {restore_error:#}", + relative_dir.display() + ))), + } +} + +fn copy_var_filtered_inner( + src_dir: &CapStdDir, + dst_dir: &CapStdDir, + relative_dir: &Path, + exclusions: &[String], + mountpoints: &BTreeSet, + cp_program: &OsStr, +) -> Result<()> { + let entries = src_dir + .read_dir(".") + .with_context(|| format!("Reading /var/{}", relative_dir.display()))?; + + let mut included_entries = Vec::new(); + for entry in entries { + let entry = entry.context("Reading entry in /var during migration")?; + let name = entry.file_name(); + let relative = relative_dir.join(&name); + + if exclusions + .iter() + .any(|excluded| Path::new(excluded) == relative) + { + println!(" Skipping {} (excluded)", relative.display()); + continue; + } + + if mountpoints.contains(&relative) { + println!( + " Creating empty mountpoint placeholder for {}", + relative.display() + ); + create_staged_placeholder(src_dir, dst_dir, &name)?; + continue; + } + + let has_nested_mount = mountpoints + .iter() + .any(|mountpoint| mountpoint != &relative && mountpoint.starts_with(&relative)); + let has_nested_exclusion = exclusions.iter().any(|excluded| { + let excluded = Path::new(excluded); + excluded != relative && excluded.starts_with(&relative) + }); + let source_metadata = src_dir.symlink_metadata(&name)?; + if source_metadata.is_dir() { + let child_src = open_child_dir_nofollow(src_dir, &name) + .with_context(|| format!("Opening /var/{}", relative.display()))?; + // Use the opened directory as the source of truth for metadata. + // This keeps the directory metadata and xattrs tied to the same + // inode even if its parent entry is replaced after the open. + let existing_dst = open_existing_child_dir(dst_dir, &name)?; + if let Some(child_dst) = existing_dst { + copy_var_filtered( + &child_src, + &child_dst, + &relative, + exclusions, + mountpoints, + false, + cp_program, + )?; + } else if destination_entry_exists(dst_dir, &name)? { + // A destination of a different type is an existing entry, not + // permission to replace it with the source directory. + continue; + } else if has_nested_mount || has_nested_exclusion { + copy_filtered_directory_staged( + &child_src, + dst_dir, + &name, + &relative, + exclusions, + mountpoints, + cp_program, + )?; + } else { + included_entries.push(name); + } + continue; + } + if destination_entry_exists(dst_dir, &name)? { + // Match cp --no-clobber without ever opening the existing object. + continue; + } + included_entries.push(name); + } + + if !included_entries.is_empty() { + let names = included_entries + .iter() + .map(|name| name.as_os_str()) + .collect::>(); + copy_batch_staged( + cp_program, + src_dir, + &names, + dst_dir, + &format!("/var/{}", relative_dir.display()), + )?; + } + + Ok(()) +} + +static NEXT_STAGING_ID: AtomicU64 = AtomicU64::new(0); + +/// Make an empty, uniquely-named staging directory beneath the destination +/// capability. Publishing from the same parent keeps the final rename atomic. +fn create_staging_dir(parent: &CapStdDir) -> Result<(std::ffi::OsString, CapStdDir)> { + use rustix::fs::{Mode, mkdirat}; + + let parent_fd = parent.as_filelike_view::(); + for _ in 0..100 { + let id = NEXT_STAGING_ID.fetch_add(1, Ordering::Relaxed); + let name = + std::ffi::OsString::from(format!(".bootc-preserve-var.{}.{}", std::process::id(), id)); + match mkdirat(parent_fd.as_fd(), &name, Mode::from_bits_truncate(0o700)) { + Ok(()) => match open_child_dir_nofollow(parent, &name) { + Ok(directory) => return Ok((name, directory)), + Err(error) => { + parent.remove_dir_all(&name).with_context(|| { + format!("Cleaning staging directory after open failure: {error}") + })?; + return Err(error).context("Opening /var migration staging directory"); + } + }, + Err(error) if error == rustix::io::Errno::EXIST => continue, + Err(error) => { + return Err(error).context("Creating /var migration staging directory"); + } + } + } + anyhow::bail!("Unable to allocate a unique /var migration staging directory") +} + +/// Ensure temporary staging entries are removed on both success and failure. +/// Any entries already published remain complete; transactions are per entry, +/// not a global rollback of earlier successful publications. +fn with_staging_dir( + parent: &CapStdDir, + operation: impl FnOnce(&OsStr, &CapStdDir) -> Result, +) -> Result { + let (name, directory) = create_staging_dir(parent)?; + let result = operation(&name, &directory); + drop(directory); + let cleanup = parent + .remove_dir_all(&name) + .context("Removing /var migration staging data"); + match (result, cleanup) { + (Ok(value), Ok(())) => Ok(value), + (Err(error), Ok(())) => Err(error), + (Ok(_), Err(error)) => Err(error), + (Err(error), Err(cleanup_error)) => Err(error.context(format!( + "Additionally failed to clean migration staging data: {cleanup_error:#}" + ))), + } +} + +fn destination_entry_exists(parent: &CapStdDir, name: &OsStr) -> Result { + match parent.symlink_metadata(name) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error.into()), + } +} + +fn publish_staged_entry( + staging: &CapStdDir, + destination: &CapStdDir, + name: &OsStr, +) -> Result { + use rustix::fs::{RenameFlags, renameat_with}; + + match renameat_with(staging, name, destination, name, RenameFlags::NOREPLACE) { + Ok(()) => Ok(true), + // A destination can appear after the initial no-clobber check. Merge + // staged directories into a same-type destination without replacing + // existing children; other collisions remain untouched. + Err(error) if error == rustix::io::Errno::EXIST => { + if staging.symlink_metadata(name)?.is_dir() { + if let Some(destination_dir) = open_existing_child_dir(destination, name)? { + let staging_dir = open_child_dir_nofollow(staging, name) + .context("Opening staged directory after publish collision")?; + merge_staged_directory(&staging_dir, &destination_dir)?; + } + } + Ok(false) + } + Err(error) => Err(error.into()), + } +} + +/// Move staged children into an existing destination directory without +/// replacing entries already present there. Keep the destination directory's +/// metadata unchanged; moving children updates its timestamps, so restore its +/// original metadata after the merge. +fn merge_staged_directory(staging: &CapStdDir, destination: &CapStdDir) -> Result<()> { + let metadata = snapshot_directory_metadata(destination)?; + let names = staging + .read_dir(".")? + .map(|entry| entry.map(|entry| entry.file_name())) + .collect::>>()?; + + let merge_result = (|| { + for name in names { + publish_staged_entry(staging, destination, &name)?; + } + Ok(()) + })(); + let restore_result = restore_directory_metadata(destination, &metadata); + + match (merge_result, restore_result) { + (Ok(()), Ok(())) => Ok(()), + (Ok(()), Err(error)) => Err(error), + (Err(error), Ok(())) => Err(error), + (Err(error), Err(restore_error)) => Err(error.context(format!( + "Additionally failed to restore merged destination directory metadata: {restore_error:#}" + ))), + } +} + +fn copy_batch_staged( + cp_program: &OsStr, + source: &CapStdDir, + names: &[&OsStr], + destination: &CapStdDir, + description: &str, +) -> Result<()> { + with_staging_dir(destination, |_, staging| { + let options = &["-a"][..]; + let status = run_cp( + cp_program, + source, + names, + staging, + None, + options, + Stdio::inherit(), + Stdio::inherit(), + )?; + anyhow::ensure!(status.success(), "cp failed while copying {description}"); + + for name in names { + publish_staged_entry(staging, destination, name)?; + } + Ok(()) + }) +} + +fn copy_filtered_directory_staged( + source: &CapStdDir, + destination_parent: &CapStdDir, + name: &OsStr, + relative: &Path, + exclusions: &[String], + mountpoints: &BTreeSet, + cp_program: &OsStr, +) -> Result<()> { + with_staging_dir(destination_parent, |_, staging| { + staging.create_dir(name)?; + let destination = open_child_dir_nofollow(staging, name)?; + copy_var_filtered( + source, + &destination, + relative, + exclusions, + mountpoints, + true, + cp_program, + )?; + publish_staged_entry(staging, destination_parent, name)?; + Ok(()) + }) +} + +fn create_staged_placeholder( + src_parent: &CapStdDir, + dst_parent: &CapStdDir, + name: &OsStr, +) -> Result<()> { + with_staging_dir(dst_parent, |_, staging| { + create_mountpoint_placeholder(src_parent, staging, name)?; + if !publish_staged_entry(staging, dst_parent, name)? { + validate_empty_placeholder(dst_parent, name, src_parent.symlink_metadata(name)?)?; + } + Ok(()) + }) +} + +fn validate_empty_placeholder( + destination_parent: &CapStdDir, + name: &OsStr, + source_metadata: Metadata, +) -> Result<()> { + let destination_metadata = destination_parent.symlink_metadata(name)?; + if source_metadata.is_dir() { + anyhow::ensure!( + destination_metadata.is_dir(), + "Mountpoint placeholder {} exists with the wrong type", + name.to_string_lossy() + ); + let destination = open_child_dir_nofollow(destination_parent, name)?; + let mut entries = destination.read_dir(".")?; + anyhow::ensure!( + entries.next().transpose()?.is_none(), + "Mountpoint placeholder {} is not empty", + name.to_string_lossy() + ); + } else { + anyhow::ensure!( + source_metadata.is_file() && destination_metadata.is_file(), + "Mountpoint placeholder {} exists with the wrong type", + name.to_string_lossy() + ); + anyhow::ensure!( + destination_metadata.len() == 0, + "Mountpoint placeholder {} is not empty", + name.to_string_lossy() + ); + } + Ok(()) +} + +fn create_mountpoint_placeholder( + src_parent: &CapStdDir, + dst_parent: &CapStdDir, + name: &OsStr, +) -> Result<()> { + let metadata = src_parent.symlink_metadata(name)?; + if metadata.is_dir() { + let src_dir = open_child_dir_nofollow(src_parent, name) + .context("Opening mountpoint source directory")?; + let directory_metadata = src_dir + .metadata(".") + .context("Reading mountpoint source directory metadata")?; + dst_parent.create_dir(name).with_context(|| { + format!( + "Creating mountpoint directory placeholder {}", + name.to_string_lossy() + ) + })?; + let dst_dir = open_child_dir_nofollow(dst_parent, name) + .context("Opening mountpoint directory placeholder")?; + preserve_directory_metadata(&src_dir, &dst_dir, &directory_metadata) + .context("Preserving mountpoint directory metadata")?; + return Ok(()); + } + anyhow::ensure!( + metadata.is_file(), + "Cannot create safe placeholder for non-file mountpoint {}", + name.to_string_lossy() + ); + use cap_std_ext::cap_std::fs::OpenOptions; + let dst_file = dst_parent + .open_with(name, OpenOptions::new().write(true).create_new(true)) + .context("Creating empty mountpoint file placeholder")?; + preserve_metadata( + src_parent, + name, + &metadata, + dst_file.as_filelike_view::().as_fd(), + ) + .context("Preserving mountpoint file metadata") +} + +/// Reproduce metadata for a directory which was split into filtered children +/// instead of being copied by `cp -a` as one subtree. +fn preserve_directory_metadata( + src_dir: &CapStdDir, + dst_dir: &CapStdDir, + metadata: &Metadata, +) -> Result<()> { + restore_directory_metadata( + dst_dir, + &snapshot_directory_metadata_from_source(src_dir, metadata)?, + ) +} + +struct DirectoryMetadataSnapshot { + metadata: Metadata, + xattrs: Vec<(std::ffi::OsString, Vec)>, +} + +fn snapshot_directory_metadata(directory: &CapStdDir) -> Result { + let metadata = directory.metadata(".")?; + snapshot_directory_metadata_from_source(directory, &metadata) +} + +fn snapshot_directory_metadata_from_source( + directory: &CapStdDir, + metadata: &Metadata, +) -> Result { + use cap_std_ext::dirext::CapStdExtDirExt; + + let mut xattrs = Vec::new(); + for attribute in directory.listxattrs(".")?.iter() { + if let Some(value) = directory.getxattr(".", attribute)? { + xattrs.push((attribute.to_os_string(), value)); + } + } + Ok(DirectoryMetadataSnapshot { + metadata: metadata.clone(), + xattrs, + }) +} + +fn restore_directory_metadata( + directory: &CapStdDir, + snapshot: &DirectoryMetadataSnapshot, +) -> Result<()> { + use rustix::fs::{Mode, fchmod}; + + let dst_fd = open_directory_for_metadata(directory)?; + let mut errors = Vec::new(); + if let Err(error) = preserve_metadata_owner(&snapshot.metadata, &dst_fd) { + errors.push(format!("owner/group: {error:#}")); + } + if let Err(error) = fchmod( + &dst_fd, + Mode::from_bits_truncate(snapshot.metadata.mode() & 0o7777), + ) { + errors.push(format!("mode: {error:#}")); + } + if let Err(error) = apply_xattrs_exact(&dst_fd, &snapshot.xattrs) { + errors.push(format!("xattrs: {error:#}")); + } + if let Err(error) = preserve_metadata_timestamps(&snapshot.metadata, &dst_fd) { + errors.push(format!("timestamps: {error:#}")); + } + if !errors.is_empty() { + anyhow::bail!("restoring directory metadata failed: {}", errors.join("; ")); + } + Ok(()) +} + +fn restore_directory_timestamps(directory: &CapStdDir, metadata: &Metadata) -> Result<()> { + let dst_fd = open_directory_for_metadata(directory)?; + preserve_metadata_timestamps(metadata, &dst_fd) +} + +/// Open a usable fd for fd-based metadata operations while remaining anchored +/// to the pinned directory capability. CapStdDir may hold an O_PATH fd, which +/// cannot be used with fchmod, xattr, or timestamp operations. +fn open_directory_for_metadata(directory: &CapStdDir) -> Result { + use rustix::fs::{Mode, OFlags, openat}; + + let directory_file = directory.as_filelike_view::(); + Ok(openat( + directory_file.as_fd(), + ".", + OFlags::CLOEXEC | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::RDONLY, + Mode::empty(), + )?) +} + +/// Preserve ownership, mode, timestamps, and available xattrs (including ACL +/// and SELinux labels) on a newly-created destination object. Callers only use +/// this for objects created by this migration, preserving the no-clobber rule. +/// For mountpoint file placeholders, source metadata and xattrs still use the +/// parent/name pair captured before the destination is opened; replacing that +/// source entry concurrently can therefore race those reads. Destination +/// updates remain anchored to the opened destination fd. Split directory +/// metadata and xattrs instead use the opened source directory handle. +fn preserve_metadata( + src_parent: &CapStdDir, + src_name: &OsStr, + metadata: &Metadata, + dst_fd: Fd, +) -> Result<()> { + use cap_std_ext::dirext::CapStdExtDirExt; + + let mut source_xattrs = Vec::new(); + for attribute in src_parent.listxattrs(src_name)?.iter() { + if let Some(value) = src_parent.getxattr(src_name, attribute)? { + source_xattrs.push((attribute.to_os_string(), value)); + } + } + preserve_metadata_owner_and_mode(metadata, &dst_fd)?; + apply_xattrs_exact(&dst_fd, &source_xattrs)?; + preserve_metadata_timestamps(metadata, &dst_fd) +} + +/// Make the destination xattr set exactly match the source snapshot. Removing +/// extras first matters for inherited ACLs and labels as well as user xattrs. +fn apply_xattrs_exact( + destination: Fd, + source_xattrs: &[(std::ffi::OsString, Vec)], +) -> Result<()> { + use rustix::fs::{fremovexattr, fsetxattr}; + use std::collections::BTreeSet; + + let destination_names = list_fd_xattrs(&destination)?; + let source_names = source_xattrs + .iter() + .map(|(name, _)| name.as_os_str()) + .collect::>(); + for name in destination_names { + if !source_names.contains(name.as_os_str()) { + fremovexattr(&destination, &name).with_context(|| { + format!("Removing destination xattr {}", name.to_string_lossy()) + })?; + } + } + for (name, value) in source_xattrs { + fsetxattr(&destination, name, value, rustix::fs::XattrFlags::empty()) + .with_context(|| format!("Setting source xattr {}", name.to_string_lossy()))?; + } + Ok(()) +} + +fn list_fd_xattrs(fd: Fd) -> Result> { + use std::os::unix::ffi::OsStringExt; + + // Linux caps the complete xattr-name list at XATTR_LIST_MAX (64 KiB). + let mut names = vec![0; 64 * 1024]; + let length = rustix::fs::flistxattr(fd, &mut names[..])?; + names.truncate(length); + if names.is_empty() { + return Ok(Vec::new()); + } + anyhow::ensure!(names.last() == Some(&0), "Malformed xattr name list"); + names.pop(); + names + .split(|byte| *byte == 0) + .filter(|name| !name.is_empty()) + .map(|name| Ok(std::ffi::OsString::from_vec(name.to_vec()))) + .collect() +} + +fn preserve_metadata_owner_and_mode(metadata: &Metadata, dst_fd: Fd) -> Result<()> { + use rustix::fs::{Mode, fchmod}; + + preserve_metadata_owner(metadata, &dst_fd)?; + fchmod(&dst_fd, Mode::from_bits_truncate(metadata.mode() & 0o7777))?; + Ok(()) +} + +fn preserve_metadata_owner(metadata: &Metadata, dst_fd: Fd) -> Result<()> { + use rustix::fs::{Gid, Uid, fchown}; + + let destination_metadata = rustix::fs::fstat(&dst_fd)?; + if destination_metadata.st_uid != metadata.uid() + || destination_metadata.st_gid != metadata.gid() + { + fchown( + &dst_fd, + Some(Uid::from_raw(metadata.uid())), + Some(Gid::from_raw(metadata.gid())), + )?; + } + Ok(()) +} + +fn preserve_metadata_timestamps(metadata: &Metadata, dst_fd: Fd) -> Result<()> { + use rustix::fs::{Timestamps, futimens}; + use rustix::time::Timespec; + + futimens( + &dst_fd, + &Timestamps { + last_access: Timespec { + tv_sec: metadata.atime(), + tv_nsec: metadata.atime_nsec() as _, + }, + last_modification: Timespec { + tv_sec: metadata.mtime(), + tv_nsec: metadata.mtime_nsec() as _, + }, + }, + )?; + Ok(()) +} + +/// Plain recursive copy of `/var` into the new deployment. +/// +/// # Known limitation +/// +/// This performs a full `cp -a` of every included subdirectory of the running +/// `/var` into the new deployment's ostree stateroot `var/`. For most workloads +/// this is fine, but it is **unsafe for databases and other applications that +/// keep open write handles into `/var`** (e.g. PostgreSQL in `/var/lib/pgsql`, +/// MySQL/MariaDB in `/var/lib/mysql`, SQLite databases under `/var/lib/*`). +/// Copying a live database with `cp -a` will almost certainly produce a +/// corrupted copy. +/// +/// The correct fix is to run this migration only after stopping all stateful +/// services that write to `/var`, or to use a consistent snapshot before +/// copying their data. +/// +/// Exclusions can be configured with install configuration or repeated +/// `--preserve-var-skip` options. Paths are relative to `/var` and may be one +/// component (such as `lib`) or exactly one child (such as `lib/pgsql`). `/var` +/// itself and deeper paths are not accepted. Excluded state is not migrated by +/// this command and must be handled separately. Exclusions do not make a live database copy +/// consistent: any database state that is copied while it is being written can +/// still be corrupted. Stop stateful services before copying their data, or use +/// a consistent snapshot. +/// +/// See: +#[cfg(test)] +#[context("Copying /var into new deployment (Strategy D)")] +fn preserve_var_copy( + src_var: &CapStdDir, + new_var: &CapStdDir, + exclusions: &[String], +) -> Result<()> { + preserve_var_copy_mounts(src_var, new_var, exclusions, &BTreeSet::new()) +} + +fn preserve_var_copy_mounts( + src_var: &CapStdDir, + new_var: &CapStdDir, + exclusions: &[String], + mountpoints: &BTreeSet, +) -> Result<()> { + copy_var_filtered( + src_var, + new_var, + Path::new(""), + exclusions, + mountpoints, + false, + OsStr::new("cp"), + ) + .context("Copying /var into new deployment (Strategy D)")?; + println!(" /var copy complete."); + Ok(()) +} + +/// Open a single source child directory without following a symlink swapped +/// into place after the caller's metadata check. The entry name comes directly +/// from `read_dir`, so this is one path component beneath the capability root. +fn open_child_dir_nofollow(parent: &CapStdDir, name: &OsStr) -> std::io::Result { + use rustix::fs::{Mode, OFlags}; + + let fd = rustix::fs::openat( + parent.as_filelike_view::().as_fd(), + name, + OFlags::CLOEXEC | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::RDONLY, + Mode::empty(), + )?; + Ok(CapStdDir::from_std_file(std::fs::File::from(fd))) +} + +#[cfg(test)] +fn open_or_create_child_dir(parent: &CapStdDir, name: &OsStr) -> Result<(CapStdDir, bool)> { + let created = match parent.create_dir(name) { + Ok(()) => true, + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + anyhow::ensure!( + parent.symlink_metadata(name)?.is_dir(), + "Destination child {} exists and is not a directory", + name.to_string_lossy() + ); + false + } + Err(e) => return Err(e).context("Creating destination child directory"), + }; + let dir = + open_child_dir_nofollow(parent, name).context("Opening destination child directory")?; + Ok((dir, created)) +} + +/// Open an existing destination directory without following symlinks. The +/// opened handle pins the directory used for the recursive merge. +fn open_existing_child_dir(parent: &CapStdDir, name: &OsStr) -> Result> { + let existing: Result> = match open_child_dir_nofollow(parent, name) { + Ok(directory) => Ok(Some(directory)), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) || error.raw_os_error() == Some(libc::ELOOP) => + { + Ok(None) + } + Err(error) => Err(error.into()), + }; + existing.with_context(|| { + format!( + "Opening existing destination directory {}", + name.to_string_lossy() + ) + }) +} + +/// Validate exclusions before touching either tree. Paths are relative to `/var`. +pub(crate) fn validate_exclusions(exclusions: &[String]) -> Result> { + exclusions + .iter() + .map(|path| { + let path = path.trim_end_matches('/'); + anyhow::ensure!(!path.is_empty(), "empty /var exclusion"); + anyhow::ensure!( + Path::new(path) + .components() + .all(|component| matches!(component, Component::Normal(_))), + "invalid /var exclusion {path:?}; expected a relative path" + ); + anyhow::ensure!( + Path::new(path).components().count() <= 2, + "unsupported nested /var exclusion {path:?}; use at most one child path" + ); + Ok(path.to_string()) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use cap_std_ext::dirext::CapStdExtDirExt; + use std::{ + fs, + os::unix::fs::{MetadataExt as StdMetadataExt, PermissionsExt, symlink}, + time::{Duration, SystemTime}, + }; + + fn cap_dir(path: &Path) -> CapStdDir { + CapStdDir::open_ambient_dir(path, ambient_authority()).unwrap() + } + + fn cap_dir_path(path: &Path) -> CapStdDir { + use rustix::fs::{Mode, OFlags, openat}; + + let directory = cap_dir(path); + let fd = openat( + directory.as_filelike_view::().as_fd(), + ".", + OFlags::CLOEXEC | OFlags::DIRECTORY | OFlags::PATH, + Mode::empty(), + ) + .unwrap(); + CapStdDir::from_std_file(std::fs::File::from(fd)) + } + + #[test] + fn exclusions_are_relative_and_normalized() { + assert_eq!( + validate_exclusions(&["lib/rpm/".into()]).unwrap(), + ["lib/rpm"] + ); + assert!(validate_exclusions(&["../etc".into()]).is_err()); + assert!(validate_exclusions(&["/etc".into()]).is_err()); + assert!(validate_exclusions(&["lib/containers/storage".into()]).is_err()); + } + + #[test] + fn directory_metadata_restoration_accepts_path_capability() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + fs::create_dir(&src_path)?; + fs::create_dir(&dst_path)?; + fs::set_permissions(&src_path, fs::Permissions::from_mode(0o751))?; + + let source = cap_dir(&src_path); + let has_user_xattr = source + .setxattr(".", "user.bootc_path_fd", b"metadata") + .is_ok(); + let destination = cap_dir_path(&dst_path); + let snapshot = snapshot_directory_metadata(&source)?; + restore_directory_metadata(&destination, &snapshot)?; + assert_eq!( + fs::metadata(&dst_path)?.permissions().mode() & 0o7777, + snapshot.metadata.mode() & 0o7777 + ); + if has_user_xattr { + assert_eq!( + destination.getxattr(".", "user.bootc_path_fd")?, + Some(b"metadata".to_vec()) + ); + } + + restore_directory_timestamps(&destination, &snapshot.metadata)?; + let restored = fs::metadata(&dst_path)?; + assert_eq!(restored.atime(), snapshot.metadata.atime()); + assert_eq!(restored.atime_nsec(), snapshot.metadata.atime_nsec()); + assert_eq!(restored.mtime(), snapshot.metadata.mtime()); + assert_eq!(restored.mtime_nsec(), snapshot.metadata.mtime_nsec()); + Ok(()) + } + + #[test] + fn split_directories_and_mountpoint_placeholders_keep_metadata() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + fs::create_dir_all(src_path.join("lib/bound"))?; + fs::create_dir(&dst_path)?; + fs::set_permissions(src_path.join("lib"), fs::Permissions::from_mode(0o751))?; + fs::set_permissions( + src_path.join("lib/bound"), + fs::Permissions::from_mode(0o711), + )?; + // Old atimes make traversal update them on relatime filesystems; the + // copy must use the metadata captured before reading child entries. + let expected_atime = SystemTime::UNIX_EPOCH + Duration::from_secs(1_600_000_001); + let expected_mtime = SystemTime::UNIX_EPOCH + Duration::from_secs(1_600_000_002); + for path in [src_path.join("lib"), src_path.join("lib/bound")] { + fs::File::open(path)?.set_times( + fs::FileTimes::new() + .set_accessed(expected_atime) + .set_modified(expected_mtime), + )?; + } + + let src = cap_dir(&src_path); + let dst = cap_dir(&dst_path); + let xattrs_supported = src + .setxattr("lib", "user.bootc_migrate_test", b"parent") + .is_ok() + && src + .setxattr("lib/bound", "user.bootc_migrate_test", b"placeholder") + .is_ok(); + preserve_var_copy_mounts( + &src, + &dst, + &[], + &BTreeSet::from([PathBuf::from("lib/bound")]), + )?; + + for (source, destination, expected_mode, expected_xattr) in [ + ("lib", "lib", 0o751, b"parent".as_slice()), + ("lib/bound", "lib/bound", 0o711, b"placeholder".as_slice()), + ] { + let source_metadata = fs::metadata(src_path.join(source))?; + let destination_metadata = fs::metadata(dst_path.join(destination))?; + assert_eq!(destination_metadata.mode() & 0o7777, expected_mode); + assert_eq!(destination_metadata.uid(), source_metadata.uid()); + assert_eq!(destination_metadata.gid(), source_metadata.gid()); + assert_eq!( + destination_metadata.atime(), + expected_atime + .duration_since(SystemTime::UNIX_EPOCH)? + .as_secs() as i64 + ); + assert_eq!( + destination_metadata.mtime(), + expected_mtime + .duration_since(SystemTime::UNIX_EPOCH)? + .as_secs() as i64 + ); + assert_eq!(destination_metadata.atime_nsec(), 0); + assert_eq!(destination_metadata.mtime_nsec(), 0); + if xattrs_supported { + assert_eq!( + dst.getxattr(destination, "user.bootc_migrate_test")?, + Some(expected_xattr.to_vec()) + ); + } + } + Ok(()) + } + + #[test] + fn split_directory_xattrs_match_source_exactly_when_supported() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + fs::create_dir(&src_path)?; + fs::create_dir(&dst_path)?; + let source = cap_dir(&src_path); + let destination = cap_dir(&dst_path); + if source + .setxattr(".", "user.bootc_expected", b"source") + .is_err() + || destination + .setxattr(".", "user.bootc_inherited", b"extra") + .is_err() + { + return Ok(()); + } + + let snapshot = snapshot_directory_metadata(&source)?; + restore_directory_metadata(&destination, &snapshot)?; + + assert_eq!( + destination.getxattr(".", "user.bootc_expected")?, + Some(b"source".to_vec()) + ); + assert_eq!(destination.getxattr(".", "user.bootc_inherited")?, None); + Ok(()) + } + + #[test] + fn filtered_copy_does_not_change_existing_destination_directory() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + fs::create_dir_all(src_path.join("tree/excluded"))?; + fs::write(src_path.join("tree/sentinel"), b"source")?; + fs::write(src_path.join("tree/new-child"), b"new child")?; + fs::write(src_path.join("root-new-child"), b"new root child")?; + fs::create_dir_all(dst_path.join("tree"))?; + fs::write(dst_path.join("tree/sentinel"), b"destination")?; + fs::set_permissions(src_path.join("tree"), fs::Permissions::from_mode(0o751))?; + fs::set_permissions(&dst_path, fs::Permissions::from_mode(0o710))?; + fs::set_permissions(dst_path.join("tree"), fs::Permissions::from_mode(0o700))?; + let expected_atime = SystemTime::UNIX_EPOCH + Duration::from_secs(1_600_000_101); + let expected_mtime = SystemTime::UNIX_EPOCH + Duration::from_secs(1_600_000_102); + let expected_root_atime = SystemTime::UNIX_EPOCH + Duration::from_secs(1_600_000_104); + let expected_root_mtime = SystemTime::UNIX_EPOCH + Duration::from_secs(1_600_000_103); + fs::File::open(dst_path.join("tree"))?.set_times( + fs::FileTimes::new() + .set_accessed(expected_atime) + .set_modified(expected_mtime), + )?; + fs::File::open(&dst_path)?.set_times( + fs::FileTimes::new() + .set_accessed(expected_root_atime) + .set_modified(expected_root_mtime), + )?; + let destination = cap_dir(&dst_path); + let xattrs_supported = destination + .setxattr("tree", "user.bootc_migrate_test", b"existing") + .is_ok(); + let expected_owner = fs::metadata(dst_path.join("tree"))?; + + preserve_var_copy_mounts( + &cap_dir(&src_path), + &destination, + &["tree/excluded".into()], + &BTreeSet::new(), + )?; + + let destination_metadata = fs::metadata(dst_path.join("tree"))?; + assert_eq!(destination_metadata.mode() & 0o7777, 0o700); + assert_eq!(destination_metadata.uid(), expected_owner.uid()); + assert_eq!(destination_metadata.gid(), expected_owner.gid()); + assert_eq!( + destination_metadata.atime(), + expected_atime + .duration_since(SystemTime::UNIX_EPOCH)? + .as_secs() as i64 + ); + assert_eq!( + destination_metadata.mtime(), + expected_mtime + .duration_since(SystemTime::UNIX_EPOCH)? + .as_secs() as i64 + ); + assert_eq!(fs::read(dst_path.join("tree/sentinel"))?, b"destination"); + assert_eq!(fs::read(dst_path.join("tree/new-child"))?, b"new child"); + assert_eq!( + fs::read(dst_path.join("root-new-child"))?, + b"new root child" + ); + assert_eq!(fs::read_dir(dst_path.join("tree"))?.count(), 2); + assert_eq!(destination_metadata.modified()?, expected_mtime); + + let root_metadata = fs::metadata(&dst_path)?; + assert_eq!(root_metadata.mode() & 0o7777, 0o710); + assert_eq!( + root_metadata.atime(), + expected_root_atime + .duration_since(SystemTime::UNIX_EPOCH)? + .as_secs() as i64 + ); + assert_eq!(root_metadata.modified()?, expected_root_mtime); + if xattrs_supported { + assert_eq!( + destination.getxattr("tree", "user.bootc_migrate_test")?, + Some(b"existing".to_vec()) + ); + } + Ok(()) + } + + #[test] + fn staged_directory_collision_merges_missing_children_without_clobbering() -> Result<()> { + let temp = tempfile::tempdir()?; + let destination_path = temp.path().join("destination"); + fs::create_dir(&destination_path)?; + let destination = cap_dir(&destination_path); + with_staging_dir(&destination, |_, staging| { + staging.create_dir("tree")?; + staging.write("tree/existing", b"staged collision")?; + staging.write("tree/missing", b"new child")?; + + // Simulate a concurrent creator winning after staging completed. + fs::create_dir(destination_path.join("tree"))?; + fs::write(destination_path.join("tree/existing"), b"destination")?; + fs::set_permissions( + destination_path.join("tree"), + fs::Permissions::from_mode(0o700), + )?; + let destination_mtime = SystemTime::UNIX_EPOCH + Duration::from_secs(1_600_000_301); + fs::File::open(destination_path.join("tree"))? + .set_times(fs::FileTimes::new().set_modified(destination_mtime))?; + let expected_owner = fs::metadata(destination_path.join("tree"))?; + + assert!(!publish_staged_entry( + staging, + &destination, + OsStr::new("tree") + )?); + assert_eq!( + fs::read(destination_path.join("tree/existing"))?, + b"destination" + ); + assert_eq!( + fs::read(destination_path.join("tree/missing"))?, + b"new child" + ); + + let metadata = fs::metadata(destination_path.join("tree"))?; + assert_eq!(metadata.mode() & 0o7777, 0o700); + assert_eq!(metadata.uid(), expected_owner.uid()); + assert_eq!(metadata.gid(), expected_owner.gid()); + assert_eq!(metadata.modified()?, destination_mtime); + Ok(()) + })?; + Ok(()) + } + + #[test] + fn mountpoint_placeholders_fail_without_reusing_existing_destinations() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + fs::create_dir_all(src_path.join("directory-mount"))?; + fs::write(src_path.join("file-mount"), b"source")?; + fs::create_dir_all(dst_path.join("directory-mount"))?; + fs::write(dst_path.join("directory-mount/sentinel"), b"keep")?; + fs::create_dir(dst_path.join("file-mount"))?; + fs::write(dst_path.join("file-mount/sentinel"), b"also keep")?; + + let src = cap_dir(&src_path); + let dst = cap_dir(&dst_path); + assert!(create_mountpoint_placeholder(&src, &dst, OsStr::new("directory-mount")).is_err()); + assert!(create_mountpoint_placeholder(&src, &dst, OsStr::new("file-mount")).is_err()); + assert_eq!( + fs::read(dst_path.join("directory-mount/sentinel"))?, + b"keep" + ); + assert_eq!( + fs::read(dst_path.join("file-mount/sentinel"))?, + b"also keep" + ); + Ok(()) + } + + #[test] + fn destination_directory_symlinks_are_rejected_without_following() -> Result<()> { + let temp = tempfile::tempdir()?; + let target = temp.path().join("target"); + fs::create_dir(&target)?; + fs::write(target.join("sentinel"), b"unchanged")?; + + let parent_path = temp.path().join("parent"); + fs::create_dir(&parent_path)?; + symlink(&target, parent_path.join("child-link"))?; + let parent = cap_dir(&parent_path); + assert!(open_or_create_child_dir(&parent, OsStr::new("child-link")).is_err()); + + let src_path = temp.path().join("src"); + fs::create_dir(&src_path)?; + fs::create_dir(src_path.join("mount"))?; + symlink(&target, parent_path.join("mount"))?; + assert!( + create_mountpoint_placeholder(&cap_dir(&src_path), &parent, OsStr::new("mount")) + .is_err() + ); + + assert_eq!(fs::read(target.join("sentinel"))?, b"unchanged"); + assert_eq!(fs::read_dir(target)?.count(), 1); + Ok(()) + } + + #[test] + fn plain_copy_skips_multiple_children_under_parent() -> Result<()> { + let exclusions = ["tmp".into(), "log/journal".into(), "log/private".into()]; + + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + fs::create_dir_all(src_path.join("tmp"))?; + fs::create_dir_all(src_path.join("log/journal"))?; + fs::create_dir_all(src_path.join("log/private"))?; + fs::create_dir_all(src_path.join("log/keep"))?; + fs::write(src_path.join("tmp/ignored"), b"tmp")?; + fs::write(src_path.join("log/journal/ignored"), b"journal")?; + fs::write(src_path.join("log/private/ignored"), b"private")?; + fs::write(src_path.join("log/keep/preserved"), b"keep")?; + fs::hard_link( + src_path.join("log/keep/preserved"), + src_path.join("log/keep/preserved-hardlink"), + )?; + fs::create_dir(&dst_path)?; + + preserve_var_copy(&cap_dir(&src_path), &cap_dir(&dst_path), &exclusions)?; + + assert!(!dst_path.join("tmp").exists()); + assert!(!dst_path.join("log/journal").exists()); + assert!(!dst_path.join("log/private").exists()); + assert_eq!(fs::read(dst_path.join("log/keep/preserved"))?, b"keep"); + assert_eq!( + fs::metadata(dst_path.join("log/keep/preserved"))?.ino(), + fs::metadata(dst_path.join("log/keep/preserved-hardlink"))?.ino() + ); + Ok(()) + } + + #[test] + fn copy_preserves_symlink_without_traversing_target() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + let outside = temp.path().join("outside"); + fs::create_dir(&src_path)?; + fs::create_dir(&dst_path)?; + fs::create_dir(&outside)?; + fs::write(outside.join("sentinel"), b"outside")?; + symlink(&outside, src_path.join("escape"))?; + + preserve_var_copy( + &cap_dir(&src_path), + &cap_dir(&dst_path), + &["escape/sentinel".into()], + )?; + + let copied = fs::symlink_metadata(dst_path.join("escape"))?; + assert!(copied.file_type().is_symlink()); + assert_eq!(fs::read_link(dst_path.join("escape"))?, outside); + // The target remains outside the destination tree; it was not copied. + assert!(!dst_path.join("sentinel").exists()); + Ok(()) + } + + #[test] + fn copy_preserves_archive_metadata_and_hardlinks() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + let src_state = src_path.join("state"); + fs::create_dir_all(&src_state)?; + fs::create_dir(&dst_path)?; + fs::write(src_state.join("one"), b"contents")?; + fs::hard_link(src_state.join("one"), src_state.join("two"))?; + fs::set_permissions(&src_state, fs::Permissions::from_mode(0o751))?; + fs::set_permissions(src_state.join("one"), fs::Permissions::from_mode(0o640))?; + let src_cap = cap_dir(&src_path); + let dst_cap = cap_dir(&dst_path); + let xattr_supported = src_cap + .setxattr("state/one", "user.bootc_migrate_test", b"xattr") + .is_ok(); + let expected_mtime = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000); + fs::File::open(&src_state)?.set_times(fs::FileTimes::new().set_modified(expected_mtime))?; + + preserve_var_copy(&src_cap, &dst_cap, &[])?; + + let copied_dir = fs::metadata(dst_path.join("state"))?; + let copied_one = fs::metadata(dst_path.join("state/one"))?; + let copied_two = fs::metadata(dst_path.join("state/two"))?; + assert_eq!(copied_dir.permissions().mode() & 0o7777, 0o751); + assert_eq!(copied_one.permissions().mode() & 0o7777, 0o640); + assert_eq!(copied_dir.modified()?, expected_mtime); + assert_eq!(copied_one.ino(), copied_two.ino()); + assert_eq!(copied_one.nlink(), 2); + assert_eq!(copied_one.uid(), fs::metadata(src_state.join("one"))?.uid()); + assert_eq!(copied_one.gid(), fs::metadata(src_state.join("one"))?.gid()); + if xattr_supported { + assert_eq!( + dst_cap.getxattr("state/one", "user.bootc_migrate_test")?, + Some(b"xattr".to_vec()) + ); + } + Ok(()) + } + + #[test] + fn copy_keeps_existing_collisions_untouched() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + fs::create_dir_all(src_path.join("state"))?; + fs::create_dir_all(dst_path.join("state"))?; + fs::write(src_path.join("state/existing"), b"source")?; + fs::write(src_path.join("state/new"), b"new")?; + fs::write(dst_path.join("state/existing"), b"destination")?; + + preserve_var_copy(&cap_dir(&src_path), &cap_dir(&dst_path), &[])?; + + assert_eq!(fs::read(dst_path.join("state/existing"))?, b"destination"); + assert_eq!(fs::read(dst_path.join("state/new"))?, b"new"); + Ok(()) + } +} diff --git a/crates/lib/src/install/migrate/mountinfo.rs b/crates/lib/src/install/migrate/mountinfo.rs new file mode 100644 index 0000000000..13eb5d3eb3 --- /dev/null +++ b/crates/lib/src/install/migrate/mountinfo.rs @@ -0,0 +1,322 @@ +//! Read `/proc/self/mountinfo` and select mounts nested beneath the source `/var`. +//! +//! The byte-oriented parser preserves non-UTF-8 Unix paths and decodes kernel +//! escapes. The `procfs` crate requires UTF-8 and leaves escaped mountpoint +//! paths undecoded; bootc-mount/findmnt exposes `String` targets and invokes +//! external `findmnt`, so neither preserves arbitrary path bytes. There is no +//! standard-library mountinfo API. + +use std::{ + collections::BTreeSet, + os::unix::ffi::OsStringExt, + path::{Component, Path, PathBuf}, +}; + +use anyhow::{Context, Result}; + +/// Read one snapshot and return the paths of mounts nested beneath `src_var`. +/// +/// `src_var` is the canonical path used to open the source directory; +/// mountinfo paths are already kernel-canonical. +pub(super) fn nested_mountpoints_for(src_var: &Path) -> Result> { + let snapshot = read_mountinfo_snapshot()?; + nested_mountpoints(src_var, parse_mountinfo(&snapshot)?) +} + +/// Read one complete installer mount namespace snapshot. +fn read_mountinfo_snapshot() -> Result> { + std::fs::read("/proc/self/mountinfo") + .context("Reading installer /proc/self/mountinfo for /var migration") +} + +fn nested_mountpoints( + src_var: &Path, + mountpoints: impl IntoIterator, +) -> Result> { + anyhow::ensure!(src_var.is_absolute(), "Source /var path must be absolute"); + let mut nested = BTreeSet::new(); + for mountpoint in mountpoints { + let Ok(relative) = mountpoint.strip_prefix(src_var) else { + continue; + }; + // The mount on /var itself is not a nested mount. Only child paths are + // filtered; Path::strip_prefix provides component-safe containment. + if relative.as_os_str().is_empty() { + continue; + } + anyhow::ensure!( + relative + .components() + .all(|component| matches!(component, Component::Normal(_))), + "Invalid nested mountpoint below {}: {}", + src_var.display(), + mountpoint.display() + ); + nested.insert(relative.to_path_buf()); + } + Ok(nested) +} + +/// Parse the byte-oriented Linux mountinfo format without requiring UTF-8. +/// Kernel pathname escapes are decoded before paths are constructed. +fn parse_mountinfo(data: &[u8]) -> Result> { + let mut mountpoints = Vec::new(); + let lines = data.split(|byte| *byte == b'\n'); + let line_count = lines.clone().count(); + for (index, line) in lines.enumerate() { + if line.is_empty() && index + 1 == line_count { + continue; // A final newline is conventional. + } + anyhow::ensure!(!line.is_empty(), "Empty mountinfo line {}", index + 1); + anyhow::ensure!( + !line.contains(&0), + "NUL byte in mountinfo line {}", + index + 1 + ); + let fields = line + .split(|byte| byte.is_ascii_whitespace()) + .filter(|field| !field.is_empty()) + .collect::>(); + let separator = fields + .iter() + .enumerate() + .skip(6) + .find_map(|(index, field)| (*field == b"-").then_some(index)) + .with_context(|| { + format!("Malformed mountinfo line {}: missing separator", index + 1) + })?; + anyhow::ensure!( + separator >= 6 && fields.len() == separator + 4, + "Malformed mountinfo line {}: invalid field count", + index + 1 + ); + anyhow::ensure!( + is_positive_decimal(fields[0]) + && is_decimal(fields[1]) + && is_device_number(fields[2]) + && !fields[5].is_empty() + && !fields[separator + 1].is_empty() + && !fields[separator + 2].is_empty() + && !fields[separator + 3].is_empty(), + "Malformed mountinfo line {}: invalid required field", + index + 1 + ); + // The root field is not used for mount selection. Decode it to ensure + // its escape syntax is valid, but don't impose pathname constraints: + // unlike the mountpoint, it may be relative or noncanonical. + decode_mountinfo_escapes(fields[3]) + .with_context(|| format!("Malformed root path on mountinfo line {}", index + 1))?; + let mountpoint = decode_mountinfo_path(fields[4]) + .with_context(|| format!("Malformed mountpoint on mountinfo line {}", index + 1))?; + decode_mountinfo_escapes(fields[separator + 2]) + .with_context(|| format!("Malformed source path on mountinfo line {}", index + 1))?; + mountpoints.push(mountpoint); + } + anyhow::ensure!(!mountpoints.is_empty(), "Mountinfo contained no records"); + Ok(mountpoints) +} + +fn is_decimal(value: &[u8]) -> bool { + !value.is_empty() + && value.iter().all(u8::is_ascii_digit) + && std::str::from_utf8(value) + .ok() + .and_then(|value| value.parse::().ok()) + .is_some() +} + +fn is_positive_decimal(value: &[u8]) -> bool { + is_decimal(value) + && std::str::from_utf8(value) + .ok() + .and_then(|value| value.parse::().ok()) + .is_some_and(|value| value > 0) +} + +fn is_device_number(value: &[u8]) -> bool { + let Some(separator) = value.iter().position(|byte| *byte == b':') else { + return false; + }; + is_decimal(&value[..separator]) && is_decimal(&value[separator + 1..]) +} + +fn decode_mountinfo_path(value: &[u8]) -> Result { + let decoded = decode_mountinfo_escapes(value)?; + anyhow::ensure!( + decoded.first() == Some(&b'/'), + "Mountinfo path is not absolute" + ); + let mut components = decoded[1..].split(|byte| *byte == b'/'); + anyhow::ensure!( + decoded.len() == 1 + || components.all(|component| { + !component.is_empty() && component != b"." && component != b".." + }), + "Mountinfo path is not canonical" + ); + Ok(PathBuf::from(std::ffi::OsString::from_vec(decoded))) +} + +fn decode_mountinfo_escapes(value: &[u8]) -> Result> { + let mut decoded = Vec::with_capacity(value.len()); + let mut index = 0; + while index < value.len() { + if value[index] != b'\\' { + decoded.push(value[index]); + index += 1; + continue; + } + anyhow::ensure!( + index + 3 < value.len() + && value[index + 1..index + 4] + .iter() + .all(|byte| (b'0'..=b'7').contains(byte)), + "Invalid octal escape in mountinfo path" + ); + // Mountinfo escapes are three octal digits. Decode wider than a byte + // so malformed values such as `\777` cannot overflow or wrap. + let escaped = u16::from(value[index + 1] - b'0') * 64 + + u16::from(value[index + 2] - b'0') * 8 + + u16::from(value[index + 3] - b'0'); + anyhow::ensure!( + escaped <= u16::from(u8::MAX), + "Octal escape exceeds one byte" + ); + anyhow::ensure!( + matches!(escaped, 0o11 | 0o12 | 0o40 | 0o134), + "Unsupported octal escape in mountinfo path" + ); + decoded.push(escaped as u8); + index += 4; + } + Ok(decoded) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{fs, os::unix::ffi::OsStrExt}; + + #[test] + fn mountinfo_decodes_escaped_and_non_utf8_paths() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_var = temp.path().join("var"); + fs::create_dir(&src_var)?; + let source = src_var.as_os_str().as_bytes(); + let mut mountinfo = format!( + "36 0 0:32 / {0} rw,relatime - tmpfs tmpfs rw\n37 25 0:33 / {0}/rpc\\040pipefs rw - nfs nfs rw\n", + src_var.display() + ) + .into_bytes(); + mountinfo.extend_from_slice(b"38 25 0:34 / "); + mountinfo.extend_from_slice(source); + mountinfo.extend_from_slice(b"/\xffstate rw - tmpfs tmpfs rw\n"); + let parsed = parse_mountinfo(&mountinfo)?; + let escaped_name = src_var.join("rpc pipefs"); + assert_eq!(parsed[1], escaped_name); + let mut non_utf8 = source.to_vec(); + non_utf8.extend_from_slice(b"/\xffstate"); + assert_eq!(parsed[2].as_os_str().as_bytes(), non_utf8); + + let nested = nested_mountpoints(&src_var, parsed)?; + assert_eq!( + nested, + BTreeSet::from([ + PathBuf::from("rpc pipefs"), + PathBuf::from(std::ffi::OsStr::from_bytes(b"\xffstate")) + ]) + ); + Ok(()) + } + + #[test] + fn unrelated_mount_records_do_not_change_nested_mount_selection() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_var = temp.path().join("target/var"); + fs::create_dir_all(&src_var)?; + let base = format!( + "36 0 0:32 / {0} rw - tmpfs tmpfs rw\n37 25 0:33 / {0}/lib/data rw - nfs nfs rw\n", + src_var.display() + ); + let with_unrelated = format!( + "21 0 0:21 / / rw - overlay overlay rw\n{base}38 25 0:34 / /unrelated rw - tmpfs tmpfs rw\n" + ); + + let selected = |mountinfo: &str| -> Result> { + nested_mountpoints(&src_var, parse_mountinfo(mountinfo.as_bytes())?) + }; + assert_eq!( + selected(&base)?, + BTreeSet::from([PathBuf::from("lib/data")]) + ); + assert_eq!(selected(&with_unrelated)?, selected(&base)?); + Ok(()) + } + + #[test] + fn mountinfo_allows_relative_and_noncanonical_roots_with_valid_nested_mountpoints() -> Result<()> + { + let temp = tempfile::tempdir()?; + let src_var = temp.path().join("target/var"); + fs::create_dir_all(&src_var)?; + + for root in ["/some/../root", "relative/root"] { + let mountinfo = format!( + "36 0 0:32 / {0} rw - tmpfs tmpfs rw\n37 25 0:33 {1} {0}/lib/data rw - nfs nfs rw\n", + src_var.display(), + root + ); + + assert_eq!( + nested_mountpoints(&src_var, parse_mountinfo(mountinfo.as_bytes())?)?, + BTreeSet::from([PathBuf::from("lib/data")]), + "root field {root:?} should not affect mountpoint selection" + ); + } + Ok(()) + } + + #[test] + fn mountinfo_rejects_malformed_records_and_escapes() { + for mountinfo in [ + b"".as_slice(), + b"not a mountinfo record\n", + b"36 25 0:32 /bad\\04x /target/var rw - tmpfs tmpfs rw\n", + b"36 25 0:32 / /target/var\\04x rw - tmpfs tmpfs rw\n", + b"36 25 0:32 / /target/var\\400 rw - tmpfs tmpfs rw\n", + b"36 25 0:32 / /target/var\\777 rw - tmpfs tmpfs rw\n", + b"36 25 0:32 / /target/var\\041 rw - tmpfs tmpfs rw\n", + b"36 25 0:32 / relative rw - tmpfs tmpfs rw\n", + b"36 25 0:32 / /target/var/../outside rw - tmpfs tmpfs rw\n", + b"36 25 0:32 / /target/var rw - tmpfs tmpfs rw extra\n", + ] { + assert!( + parse_mountinfo(mountinfo).is_err(), + "accepted {mountinfo:?}" + ); + } + } + + #[test] + fn nested_mountpoint_detection_is_component_safe() -> Result<()> { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir()?; + let src_var = temp.path().join("target/var"); + fs::create_dir_all(&src_var)?; + let alias = temp.path().join("var-alias"); + symlink(&src_var, &alias)?; + let paths = [ + src_var.clone(), + src_var.join("lib/data"), + temp.path().join("target/var-old/other"), + PathBuf::from("/elsewhere/var/mount"), + ]; + let canonical_src_var = fs::canonicalize(&alias)?; + assert_eq!( + nested_mountpoints(&canonical_src_var, paths)?, + BTreeSet::from([PathBuf::from("lib/data")]) + ); + Ok(()) + } +} From 109c6008e16ab3e4bbd244ac9b9a6353f66d38d0 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Mon, 2 Nov 2026 11:29:22 -0600 Subject: [PATCH 02/38] install: Separate bound-image resolution from state lookup Stored images should be validated before target mutations. Assisted-by: AI Signed-off-by: ckyrouac --- crates/lib/src/cli.rs | 9 +++- crates/lib/src/install.rs | 98 +++++++++++++++++++++++++++------------ 2 files changed, 76 insertions(+), 31 deletions(-) diff --git a/crates/lib/src/cli.rs b/crates/lib/src/cli.rs index 0059f45f82..106997548a 100644 --- a/crates/lib/src/cli.rs +++ b/crates/lib/src/cli.rs @@ -2545,8 +2545,13 @@ async fn run_from_opt(opt: Opt) -> Result { #[cfg(feature = "install-to-disk")] InstallOpts::ToDisk(opts) => crate::install::install_to_disk(opts).await, InstallOpts::ToFilesystem(opts) => { - crate::install::install_to_filesystem(opts, false, crate::install::Cleanup::Skip) - .await + crate::install::install_to_filesystem( + opts, + false, + crate::install::Cleanup::Skip, + false, + ) + .await } InstallOpts::ToExistingRoot(opts) => { crate::install::install_to_existing_root(opts).await diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index dabac7cef0..498c463a64 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -2188,35 +2188,41 @@ enum BoundImages { impl BoundImages { async fn from_state(state: &State) -> Result { - let bound_images = match state.config_opts.bound_images { - BoundImagesOpt::Skip => BoundImages::Skip, - others => { - let queried_images = crate::boundimage::query_bound_images(&state.container_root)?; - match others { - BoundImagesOpt::Stored => { - // Verify each bound image is present in the container storage - let mut r = Vec::with_capacity(queried_images.len()); - for image in queried_images { - let resolved = ResolvedBoundImage::from_image(&image).await?; - tracing::debug!("Resolved {}: {}", resolved.image, resolved.digest); - r.push(resolved) - } - BoundImages::Resolved(r) - } - BoundImagesOpt::Pull => { - // No need to resolve the images, we will pull them into the target later - BoundImages::Unresolved(queried_images) - } - BoundImagesOpt::Skip => anyhow::bail!("unreachable error"), + let bound_images_opt = state.config_opts.bound_images; + if bound_images_opt == BoundImagesOpt::Skip { + return Ok(BoundImages::Skip); + } + + let queried_images = crate::boundimage::query_bound_images(&state.container_root)?; + Self::from_queried(bound_images_opt, queried_images).await + } + + async fn from_queried( + bound_images_opt: BoundImagesOpt, + queried_images: Vec, + ) -> Result { + match bound_images_opt { + BoundImagesOpt::Stored => { + let mut resolved_images = Vec::with_capacity(queried_images.len()); + for image in queried_images { + let resolved = ResolvedBoundImage::from_image(&image).await?; + tracing::debug!("Resolved {}: {}", resolved.image, resolved.digest); + resolved_images.push(resolved); } + Ok(BoundImages::Resolved(resolved_images)) } - }; - - Ok(bound_images) + BoundImagesOpt::Pull => Ok(BoundImages::Unresolved(queried_images)), + BoundImagesOpt::Skip => Ok(BoundImages::Skip), + } } } -async fn ostree_install(state: &State, rootfs: &RootSetup, cleanup: Cleanup) -> Result<()> { +async fn ostree_install( + state: &State, + rootfs: &RootSetup, + cleanup: Cleanup, + pre_resolved_bound_images: Option, +) -> Result<()> { // We verify this upfront because it's currently required by bootupd let boot_uuid = rootfs .get_boot_uuid()? @@ -2224,7 +2230,10 @@ async fn ostree_install(state: &State, rootfs: &RootSetup, cleanup: Cleanup) -> .ok_or_else(|| anyhow!("No uuid for boot/root"))?; tracing::debug!("boot uuid={boot_uuid}"); - let bound_images = BoundImages::from_state(state).await?; + let bound_images = match pre_resolved_bound_images { + Some(bound_images) => bound_images, + None => BoundImages::from_state(state).await?, + }; // Initialize the ostree sysroot (repo, stateroot, etc.) @@ -2266,6 +2275,7 @@ async fn install_to_filesystem_impl( state: &State, rootfs: &mut RootSetup, cleanup: Cleanup, + pre_resolved_bound_images: Option, ) -> Result<()> { if matches!(state.selinux_state, SELinuxFinalState::ForceTargetDisabled) { rootfs.kargs.extend(&Cmdline::from("selinux=0")); @@ -2361,7 +2371,7 @@ async fn install_to_filesystem_impl( .context("SELinux labeling of composefs objects")?; } } else { - ostree_install(state, rootfs, cleanup).await?; + ostree_install(state, rootfs, cleanup, pre_resolved_bound_images).await?; // For s390x, we set zipl as the bootloader // this needs to be done after the ostree commit is deployed, @@ -2484,7 +2494,7 @@ pub(crate) async fn install_to_disk(mut opts: InstallToDiskOpts) -> Result<()> { (rootfs, loopback_dev) }; - install_to_filesystem_impl(&state, &mut rootfs, Cleanup::Skip).await?; + install_to_filesystem_impl(&state, &mut rootfs, Cleanup::Skip, None).await?; // Drop all data about the root except the bits we need to ensure any file descriptors etc. are closed. let (root_path, luksdev) = rootfs.into_storage(); @@ -2760,6 +2770,7 @@ pub(crate) async fn install_to_filesystem( opts: InstallToFilesystemOpts, targeting_host_root: bool, cleanup: Cleanup, + resolve_bound_images_early: bool, ) -> Result<()> { // Log the installation operation to systemd journal const INSTALL_FILESYSTEM_JOURNAL_ID: &str = "9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3"; @@ -2872,6 +2883,18 @@ pub(crate) async fn install_to_filesystem( ) .await?; + // Callers may opt in when stored bound images must be validated before + // target mutations. Other install flows retain their existing resolution + // timing by passing false. + let pre_resolved_bound_images = if targeting_host_root + && resolve_bound_images_early + && !state.composefs_options.composefs_backend + { + Some(BoundImages::from_state(&state).await?) + } else { + None + }; + // Check to see if this happens to be the real host root if !fsopts.acknowledge_destructive { warn_on_host_root(&target_rootfs_fd)?; @@ -3034,7 +3057,7 @@ pub(crate) async fn install_to_filesystem( skip_finalize, }; - install_to_filesystem_impl(&state, &mut rootfs, cleanup).await?; + install_to_filesystem_impl(&state, &mut rootfs, cleanup, pre_resolved_bound_images).await?; // Drop all data about the root except the path to ensure any file descriptors etc. are closed. drop(rootfs); @@ -3089,7 +3112,7 @@ pub(crate) async fn install_to_existing_root(opts: InstallToExistingRootOpts) -> composefs_opts: opts.composefs_opts, }; - install_to_filesystem(opts, true, cleanup).await + install_to_filesystem(opts, true, cleanup, false).await } /// Read the /boot entry from /etc/fstab, if it exists @@ -3672,4 +3695,21 @@ UUID=boot-uuid /boot ext4 defaults 0 0 Ok(()) } + + #[tokio::test] + async fn test_bound_images_pull_mode_stays_unresolved() { + let queried_images = vec![BoundImage { + image: "quay.io/example/app:latest".to_string(), + auth_file: None, + }]; + + let bound_images = BoundImages::from_queried(BoundImagesOpt::Pull, queried_images) + .await + .unwrap(); + let BoundImages::Unresolved(images) = bound_images else { + panic!("pull mode should defer bound image resolution until deployment"); + }; + assert_eq!(images.len(), 1); + assert_eq!(images[0].image, "quay.io/example/app:latest"); + } } From 1d905b52e98976d7499a1bbdc5df5130fc3fe961 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Fri, 30 Oct 2026 09:42:57 -0500 Subject: [PATCH 03/38] install: Wire /var migration into existing-root install Users converting an existing package-mode root need an opt-in path to move its state into the new OSTree deployment before reboot. Keep migration behind the existing-root install option and defer destructive cleanup until the copy succeeds, while preserving the established install behavior for other callers. Assisted-by: AI Signed-off-by: ckyrouac --- crates/lib/src/cli.rs | 20 ++-- crates/lib/src/install.rs | 179 +++++++++++++++++++++++++++--- crates/lib/src/install/config.rs | 31 ++++++ crates/lib/src/install/migrate.rs | 74 ++++++++++++ 4 files changed, 278 insertions(+), 26 deletions(-) diff --git a/crates/lib/src/cli.rs b/crates/lib/src/cli.rs index 106997548a..b282c325c0 100644 --- a/crates/lib/src/cli.rs +++ b/crates/lib/src/cli.rs @@ -380,6 +380,9 @@ pub(crate) enum InstallOpts { /// the running host root filesystem. Currently, the host root filesystem's `/boot` partition /// will be wiped, but the content of the existing root will otherwise be retained, and will /// need to be cleaned up if desired when rebooted into the new root. + /// + /// When migrating from a package-mode system, use `--preserve-var` to copy `/var` data + /// into the new deployment before rebooting. ToExistingRoot(crate::install::InstallToExistingRootOpts), /// Nondestructively create a fresh installation state inside an existing bootc system. /// @@ -2544,15 +2547,14 @@ async fn run_from_opt(opt: Opt) -> Result { InstallOpts::Mount(opts) => crate::mount::mount(opts).await, #[cfg(feature = "install-to-disk")] InstallOpts::ToDisk(opts) => crate::install::install_to_disk(opts).await, - InstallOpts::ToFilesystem(opts) => { - crate::install::install_to_filesystem( - opts, - false, - crate::install::Cleanup::Skip, - false, - ) - .await - } + InstallOpts::ToFilesystem(opts) => crate::install::install_to_filesystem( + opts, + false, + crate::install::Cleanup::Skip, + false, + ) + .await + .map(|_| ()), InstallOpts::ToExistingRoot(opts) => { crate::install::install_to_existing_root(opts).await } diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index 498c463a64..866128d86e 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -144,8 +144,7 @@ mod aleph; pub(crate) mod baseline; pub(crate) mod completion; pub(crate) mod config; -#[cfg(test)] -mod migrate; +pub(crate) mod migrate; mod osbuild; pub(crate) mod osconfig; @@ -667,6 +666,23 @@ pub(crate) struct InstallToExistingRootOpts { #[clap(flatten)] pub(crate) composefs_opts: InstallComposefsOpts, + + /// Preserve the running system's `/var` data into the new bootc deployment. + /// + /// Non-excluded content is copied with reflinks when supported, or with a + /// full copy otherwise. Reflinks are created per file and are not a + /// snapshot; stop stateful services or use a consistent snapshot before + /// copying live database data. + /// + /// `/var/tmp`, `/var/cache`, `/var/log/journal`, and `/var/lib/containers` + /// are always excluded. Repeat `--preserve-var-skip` to add paths relative + /// to `/var`. Nested mountpoints are represented by empty directories. + #[clap(long)] + pub(crate) preserve_var: bool, + + /// Relative paths below `/var` to leave out while preserving `/var`. + #[clap(long = "preserve-var-skip", value_name = "PATH")] + pub(crate) preserve_var_skip: Vec, } #[derive(Debug, clap::Parser, PartialEq, Eq)] @@ -2110,7 +2126,7 @@ async fn install_with_sysroot( boot_uuid: &str, bound_images: BoundImages, has_ostree: bool, -) -> Result<()> { +) -> Result { let ostree = storage.get_ostree()?; let c_storage = storage.get_ensure_imgstore()?; @@ -2177,7 +2193,7 @@ async fn install_with_sysroot( } } - Ok(()) + Ok(camino::Utf8PathBuf::from(deployment_path.to_string())) } enum BoundImages { @@ -2222,7 +2238,7 @@ async fn ostree_install( rootfs: &RootSetup, cleanup: Cleanup, pre_resolved_bound_images: Option, -) -> Result<()> { +) -> Result { // We verify this upfront because it's currently required by bootupd let boot_uuid = rootfs .get_boot_uuid()? @@ -2237,10 +2253,10 @@ async fn ostree_install( // Initialize the ostree sysroot (repo, stateroot, etc.) - { + let deployment_path = { let (sysroot, has_ostree) = initialize_ostree_root(state, rootfs).await?; - install_with_sysroot( + let deployment_path = install_with_sysroot( state, rootfs, &sysroot, @@ -2263,12 +2279,13 @@ async fn ostree_install( // We must drop the sysroot here in order to close any open file // descriptors. + deployment_path }; // Run this on every install as the penultimate step install_finalize(&rootfs.physical_root_path).await?; - Ok(()) + Ok(deployment_path) } async fn install_to_filesystem_impl( @@ -2276,7 +2293,7 @@ async fn install_to_filesystem_impl( rootfs: &mut RootSetup, cleanup: Cleanup, pre_resolved_bound_images: Option, -) -> Result<()> { +) -> Result> { if matches!(state.selinux_state, SELinuxFinalState::ForceTargetDisabled) { rootfs.kargs.extend(&Cmdline::from("selinux=0")); } @@ -2298,7 +2315,7 @@ async fn install_to_filesystem_impl( } } - if state.composefs_options.composefs_backend { + let deployment_path = if state.composefs_options.composefs_backend { let fetch_ref = state.source.composefs_fetch_reference(); let (manifest, _) = get_container_manifest_and_config(&fetch_ref).await?; // A capable filesystem gets a strict provisional repository. The @@ -2370,8 +2387,10 @@ async fn install_to_filesystem_impl( ) .context("SELinux labeling of composefs objects")?; } + None } else { - ostree_install(state, rootfs, cleanup, pre_resolved_bound_images).await?; + let deployment_path = + ostree_install(state, rootfs, cleanup, pre_resolved_bound_images).await?; // For s390x, we set zipl as the bootloader // this needs to be done after the ostree commit is deployed, @@ -2390,7 +2409,8 @@ async fn install_to_filesystem_impl( .run_capture_stderr() .context("Setting bootloader config to zipl")?; } - } + Some(deployment_path) + }; // As the very last step before filesystem finalization, do a full SELinux // relabel of the physical root filesystem. Any files that are already @@ -2412,7 +2432,7 @@ async fn install_to_filesystem_impl( } } - Ok(()) + Ok(deployment_path) } fn installation_complete() { @@ -2771,7 +2791,7 @@ pub(crate) async fn install_to_filesystem( targeting_host_root: bool, cleanup: Cleanup, resolve_bound_images_early: bool, -) -> Result<()> { +) -> Result> { // Log the installation operation to systemd journal const INSTALL_FILESYSTEM_JOURNAL_ID: &str = "9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3"; let source_image = opts @@ -2883,6 +2903,13 @@ pub(crate) async fn install_to_filesystem( ) .await?; + // The image can select the composefs backend during preparation, so check + // compatibility after that choice is resolved but before target mutation. + validate_migration_backend( + resolve_bound_images_early, + state.composefs_options.composefs_backend, + )?; + // Callers may opt in when stored bound images must be validated before // target mutations. Other install flows retain their existing resolution // timing by passing false. @@ -3057,13 +3084,41 @@ pub(crate) async fn install_to_filesystem( skip_finalize, }; - install_to_filesystem_impl(&state, &mut rootfs, cleanup, pre_resolved_bound_images).await?; + let deployment_path = + install_to_filesystem_impl(&state, &mut rootfs, cleanup, pre_resolved_bound_images).await?; // Drop all data about the root except the path to ensure any file descriptors etc. are closed. drop(rootfs); installation_complete(); + Ok(deployment_path) +} + +fn validate_migration_backend(require_ostree_backend: bool, composefs_backend: bool) -> Result<()> { + anyhow::ensure!( + !require_ostree_backend || !composefs_backend, + "--preserve-var requires the ostree backend" + ); + Ok(()) +} + +/// Resolve the physical sysroot when `root_path` points at an OSTree deployment. +fn physical_sysroot_path(root_path: &std::path::Path) -> std::path::PathBuf { + let possible_sysroot = root_path.join("sysroot"); + if possible_sysroot.join("ostree").exists() { + possible_sysroot + } else { + root_path.to_owned() + } +} + +fn write_destructive_cleanup_marker(root_path: &std::path::Path) -> Result<()> { + let physical_root = physical_sysroot_path(root_path); + let sysroot = Dir::open_ambient_dir(&physical_root, cap_std::ambient_authority()) + .with_context(|| format!("Opening physical sysroot {physical_root:?}"))?; + tracing::debug!("Writing {DESTRUCTIVE_CLEANUP}"); + sysroot.atomic_write(DESTRUCTIVE_CLEANUP, b"")?; Ok(()) } @@ -3097,7 +3152,23 @@ pub(crate) async fn install_to_existing_root(opts: InstallToExistingRootOpts) -> false => Cleanup::Skip, }; - let opts = InstallToFilesystemOpts { + let preserve_var = opts.preserve_var; + let defer_cleanup = opts.cleanup && preserve_var; + let root_path = std::path::PathBuf::from(opts.root_path.as_str()); + let mut preserve_var_skip = opts.preserve_var_skip; + if let Some(config) = config::load_config()? { + if let Some(config_skip) = config.preserve_var_skip { + preserve_var_skip.extend(config_skip); + } + } + let preserve_var_skip = migrate::validate_exclusions(&preserve_var_skip)?; + let require_ostree_backend = preserve_var; + validate_migration_backend( + require_ostree_backend, + opts.composefs_opts.composefs_backend, + )?; + + let fs_opts = InstallToFilesystemOpts { filesystem_opts: InstallTargetFilesystemOpts { root_path: opts.root_path, root_mount_spec: None, @@ -3112,7 +3183,36 @@ pub(crate) async fn install_to_existing_root(opts: InstallToExistingRootOpts) -> composefs_opts: opts.composefs_opts, }; - install_to_filesystem(opts, true, cleanup, false).await + let deployment_path = install_to_filesystem( + fs_opts, + true, + if defer_cleanup { + Cleanup::Skip + } else { + cleanup + }, + require_ostree_backend, + ) + .await?; + + if preserve_var { + println!(); + println!("Preserving /var..."); + let deployment_path = deployment_path + .as_ref() + .context("Install did not produce an ostree deployment")?; + let physical_root = physical_sysroot_path(&root_path); + let deploy_dir = physical_root.join(deployment_path); + let new_var = migrate::deployment_var_path(&deploy_dir)?; + migrate::preserve_var(&root_path.join("var"), &new_var, &preserve_var_skip) + .context("Post-install /var preservation")?; + } + + if defer_cleanup { + write_destructive_cleanup_marker(&root_path)?; + } + + Ok(()) } /// Read the /boot entry from /etc/fstab, if it exists @@ -3696,6 +3796,23 @@ UUID=boot-uuid /boot ext4 defaults 0 0 Ok(()) } + #[test] + fn test_migration_backend_compatibility() { + let cases = [ + (false, false, true), + (false, true, true), + (true, false, true), + (true, true, false), + ]; + for (require_ostree_backend, composefs_backend, valid) in cases { + assert_eq!( + validate_migration_backend(require_ostree_backend, composefs_backend).is_ok(), + valid, + "require_ostree_backend={require_ostree_backend}, composefs_backend={composefs_backend}" + ); + } + } + #[tokio::test] async fn test_bound_images_pull_mode_stays_unresolved() { let queried_images = vec![BoundImage { @@ -3712,4 +3829,32 @@ UUID=boot-uuid /boot ext4 defaults 0 0 assert_eq!(images.len(), 1); assert_eq!(images[0].image, "quay.io/example/app:latest"); } + + #[test] + fn test_destructive_cleanup_marker_uses_physical_sysroot() -> Result<()> { + let td = tempfile::tempdir()?; + let deployment_root = td.path(); + let physical_sysroot = deployment_root.join("sysroot"); + std::fs::create_dir_all(deployment_root.join("etc"))?; + std::fs::create_dir_all(physical_sysroot.join("ostree"))?; + std::fs::create_dir_all(physical_sysroot.join("etc"))?; + + write_destructive_cleanup_marker(deployment_root)?; + + assert!(physical_sysroot.join(DESTRUCTIVE_CLEANUP).is_file()); + assert!(!deployment_root.join(DESTRUCTIVE_CLEANUP).exists()); + Ok(()) + } + + #[test] + fn test_destructive_cleanup_marker_uses_root_without_nested_sysroot() -> Result<()> { + let td = tempfile::tempdir()?; + let root = td.path(); + std::fs::create_dir_all(root.join("etc"))?; + + write_destructive_cleanup_marker(root)?; + + assert!(root.join(DESTRUCTIVE_CLEANUP).is_file()); + Ok(()) + } } diff --git a/crates/lib/src/install/config.rs b/crates/lib/src/install/config.rs index 55fdd42f98..fc3e86c599 100644 --- a/crates/lib/src/install/config.rs +++ b/crates/lib/src/install/config.rs @@ -132,6 +132,8 @@ pub(crate) struct InstallConfiguration { /// Enforce that the containers-storage stack has a non-default /// (i.e. not `insecureAcceptAnything`) container image signature policy. pub(crate) enforce_container_sigpolicy: Option, + /// Paths below `/var` which should not be copied by `--preserve-var`. + pub(crate) preserve_var_skip: Option>, } fn merge_basic(s: &mut Option, o: Option, _env: &EnvProperties) { @@ -226,6 +228,11 @@ impl Mergeable for InstallConfiguration { other.enforce_container_sigpolicy, env, ); + if let Some(other_skip) = other.preserve_var_skip { + self.preserve_var_skip + .get_or_insert_with(Default::default) + .extend(other_skip) + } if let Some(other_kargs) = other.kargs { self.kargs .get_or_insert_with(Default::default) @@ -878,6 +885,30 @@ skip-boot-uuid = false // skip_boot_uuid should be overridden to true assert_eq!(install.bootupd.unwrap().skip_boot_uuid.unwrap(), true); } + + #[test] + fn test_merge_preserve_var_skip_appends_paths() { + let env = EnvProperties { + sys_arch: "x86_64".to_string(), + }; + let mut install: InstallConfiguration = toml::from_str( + r#"preserve-var-skip = ["lib/first"] +"#, + ) + .unwrap(); + let other: InstallConfiguration = toml::from_str( + r#"preserve-var-skip = ["lib/second"] +"#, + ) + .unwrap(); + + install.merge(other, &env); + + assert_eq!( + install.preserve_var_skip.unwrap(), + ["lib/first", "lib/second"] + ); + } } #[test] diff --git a/crates/lib/src/install/migrate.rs b/crates/lib/src/install/migrate.rs index e0dfa75324..da34c0d225 100644 --- a/crates/lib/src/install/migrate.rs +++ b/crates/lib/src/install/migrate.rs @@ -58,6 +58,67 @@ use cap_std_ext::cmdext::{CapStdExtCommandExt, CmdFds}; #[cfg(test)] use fn_error_context::context; +/// Preserve the running `/var` into the new deployment's `var/` directory. +/// +/// `src_var` is the running system's `/var` (at `/var`). +/// `new_var` is the new deployment's empty `var/` directory. +pub(crate) fn preserve_var(src_var: &Path, new_var: &Path, exclusions: &[String]) -> Result<()> { + // Resolve symlink aliases and path components before comparing against the + // absolute, kernel-resolved mountpoints in mountinfo. + let canonical_src_var = std::fs::canonicalize(src_var) + .with_context(|| format!("Canonicalizing source /var {}", src_var.display()))?; + let nested_mounts = mountinfo::nested_mountpoints_for(&canonical_src_var)?; + let src_var = CapStdDir::open_ambient_dir(&canonical_src_var, ambient_authority()) + .with_context(|| format!("Opening source {}", canonical_src_var.display()))?; + let new_var = open_or_create_var_dir(new_var)?; + + let mut effective_exclusions = ["tmp", "cache", "log/journal", "lib/containers"] + .into_iter() + .map(String::from) + .collect::>(); + effective_exclusions.extend(exclusions.iter().cloned()); + + println!(" Copying /var into the new deployment"); + preserve_var_copy_mounts(&src_var, &new_var, &effective_exclusions, &nested_mounts)?; + + Ok(()) +} + +/// Open the destination through its parent capability, creating just the +/// final `/var` directory if needed. The caller-supplied parent is the initial +/// ambient authority; all subsequent traversal is relative to opened handles. +fn open_or_create_var_dir(path: &Path) -> Result { + let parent = path.parent().context("Destination /var has no parent")?; + let name = path + .file_name() + .context("Destination /var has no final path component")?; + let parent = CapStdDir::open_ambient_dir(parent, ambient_authority()) + .with_context(|| format!("Opening destination parent {}", parent.display()))?; + match parent.create_dir(name) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + anyhow::ensure!( + parent.symlink_metadata(name)?.is_dir(), + "Destination {} exists and is not a directory", + path.display() + ); + } + Err(e) => return Err(e).with_context(|| format!("Creating {}", path.display())), + } + open_child_dir_nofollow(&parent, name) + .with_context(|| format!("Opening destination {}", path.display())) +} + +pub(crate) fn deployment_var_path(deploy_dir: &Path) -> Result { + let deploy_parent = deploy_dir + .parent() + .context("Deployment path has no parent")?; + let stateroot = deploy_parent + .parent() + .context("Deployment path has no stateroot")?; + Ok(stateroot.join("var")) +} + /// Run GNU cp with directory handles inherited as fixed descriptors. Each /// operand is `/proc/self/fd/N[/entry]`; cp's top-level paths therefore remain /// rooted in the opened capabilities even if the original paths are renamed. @@ -930,6 +991,15 @@ mod tests { assert!(validate_exclusions(&["lib/containers/storage".into()]).is_err()); } + #[test] + fn deployment_var_is_stateroot_var() { + let deploy = Path::new("/sysroot/ostree/deploy/default/deploy/checksum.0"); + assert_eq!( + deployment_var_path(deploy).unwrap(), + Path::new("/sysroot/ostree/deploy/default/var") + ); + } + #[test] fn directory_metadata_restoration_accepts_path_capability() -> Result<()> { let temp = tempfile::tempdir()?; @@ -1233,6 +1303,10 @@ mod tests { fs::create_dir(&target)?; fs::write(target.join("sentinel"), b"unchanged")?; + let var_link = temp.path().join("var-link"); + symlink(&target, &var_link)?; + assert!(open_or_create_var_dir(&var_link).is_err()); + let parent_path = temp.path().join("parent"); fs::create_dir(&parent_path)?; symlink(&target, parent_path.join("child-link"))?; From ef49df42e9bda2293439994eeb34dee795c936d9 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Fri, 30 Oct 2026 09:28:05 -0500 Subject: [PATCH 04/38] install: Use reflinks for /var migration when available Large /var trees are expensive to duplicate when the source and deployment filesystems support copy-on-write clones. Probe the actual source/destination pair, use reflinks when available, and retain a full-copy fallback for unsupported filesystems. Assisted-by: AI Signed-off-by: ckyrouac --- crates/lib/src/install/migrate.rs | 353 +++++++++++++++++++++++++++++- crates/lib/src/store/mod.rs | 2 + crates/lib/src/store/reflink.rs | 279 +++++++++++++++++++++++ 3 files changed, 625 insertions(+), 9 deletions(-) create mode 100644 crates/lib/src/store/reflink.rs diff --git a/crates/lib/src/install/migrate.rs b/crates/lib/src/install/migrate.rs index da34c0d225..2227cb85f9 100644 --- a/crates/lib/src/install/migrate.rs +++ b/crates/lib/src/install/migrate.rs @@ -16,9 +16,18 @@ //! directory. The old package-mode `/var` is stranded at `/var` //! (inside the container, the host root is mounted at `root_path`). //! -//! This uses `cp -a` to copy each non-ephemeral subdirectory of `/var` into the -//! new deployment's stateroot `var/`. The copy is not a consistent snapshot of -//! live data; stop stateful services before migrating their data. +//! Two strategies are tried in order: +//! +//! - **Strategy C — reflink copy** (btrfs / XFS with reflinks): `cp --reflink=always` +//! clones files copy-on-write, avoiding an immediate second copy of their data. +//! Files are cloned individually, so this is not an atomic snapshot of the +//! `/var` tree or of a live database. +//! +//! - **Strategy D — plain copy** (ext4 and other non-reflink filesystems): +//! `cp -a` copies each non-ephemeral subdirectory of `/var` into the new +//! deployment's stateroot `var/`. This is correct but slow for large `/var` +//! trees, and **unsafe for live databases** (see the known-limitation comment +//! on `preserve_var_copy`). //! //! Nested mountpoints are selected from one `/proc/self/mountinfo` snapshot //! before the copy. This is a point-in-time, best-effort snapshot, not a @@ -50,6 +59,7 @@ use std::{ }, }; +use crate::store::reflink::supports_reflink; use anyhow::{Context, Result}; use cap_std_ext::cap_std::ambient_authority; use cap_std_ext::cap_std::fs::{Dir as CapStdDir, Metadata, MetadataExt}; @@ -78,8 +88,14 @@ pub(crate) fn preserve_var(src_var: &Path, new_var: &Path, exclusions: &[String] .collect::>(); effective_exclusions.extend(exclusions.iter().cloned()); - println!(" Copying /var into the new deployment"); - preserve_var_copy_mounts(&src_var, &new_var, &effective_exclusions, &nested_mounts)?; + let copy_result = if supports_reflink(&src_var, &new_var)? { + println!(" Filesystem supports reflinks — using copy-on-write clone (Strategy C)"); + preserve_var_reflink_mounts(&src_var, &new_var, &effective_exclusions, &nested_mounts) + } else { + println!(" Filesystem does not support reflinks — falling back to full copy (Strategy D)"); + preserve_var_copy_mounts(&src_var, &new_var, &effective_exclusions, &nested_mounts) + }; + copy_result?; Ok(()) } @@ -174,6 +190,40 @@ fn run_cp( Ok(status) } +/// Strategy C: reflink-copy each top-level entry under `src_var` into `new_var`. +/// +/// Skips well-known ephemeral subdirectories. +#[cfg(test)] +#[context("Reflink-copying /var into new deployment (Strategy C)")] +fn preserve_var_reflink( + src_var: &CapStdDir, + new_var: &CapStdDir, + exclusions: &[String], +) -> Result<()> { + preserve_var_reflink_mounts(src_var, new_var, exclusions, &BTreeSet::new()) +} + +fn preserve_var_reflink_mounts( + src_var: &CapStdDir, + new_var: &CapStdDir, + exclusions: &[String], + mountpoints: &BTreeSet, +) -> Result<()> { + copy_var_filtered( + src_var, + new_var, + Path::new(""), + exclusions, + mountpoints, + true, + false, + OsStr::new("cp"), + ) + .context("Reflink-copying /var into new deployment (Strategy C)")?; + println!(" /var reflink copy complete."); + Ok(()) +} + /// Copy a directory's ordinary children together with cp -a, recursing only /// into directories whose descendants contain a mountpoint or exclusion. /// Batching ordinary siblings in one cp invocation retains hardlinks between @@ -187,6 +237,7 @@ fn copy_var_filtered( relative_dir: &Path, exclusions: &[String], mountpoints: &BTreeSet, + reflink: bool, destination_created: bool, cp_program: &OsStr, ) -> Result<()> { @@ -220,6 +271,7 @@ fn copy_var_filtered( relative_dir, exclusions, mountpoints, + reflink, cp_program, ); let restore_result = match (source_metadata.as_ref(), destination_metadata.as_ref()) { @@ -251,6 +303,7 @@ fn copy_var_filtered_inner( relative_dir: &Path, exclusions: &[String], mountpoints: &BTreeSet, + reflink: bool, cp_program: &OsStr, ) -> Result<()> { let entries = src_dir @@ -302,6 +355,7 @@ fn copy_var_filtered_inner( &relative, exclusions, mountpoints, + reflink, false, cp_program, )?; @@ -317,6 +371,7 @@ fn copy_var_filtered_inner( &relative, exclusions, mountpoints, + reflink, cp_program, )?; } else { @@ -341,6 +396,7 @@ fn copy_var_filtered_inner( src_dir, &names, dst_dir, + reflink, &format!("/var/{}", relative_dir.display()), )?; } @@ -470,10 +526,15 @@ fn copy_batch_staged( source: &CapStdDir, names: &[&OsStr], destination: &CapStdDir, + reflink: bool, description: &str, ) -> Result<()> { with_staging_dir(destination, |_, staging| { - let options = &["-a"][..]; + let options = if reflink { + &["--reflink=always", "-a"][..] + } else { + &["-a"][..] + }; let status = run_cp( cp_program, source, @@ -500,6 +561,7 @@ fn copy_filtered_directory_staged( relative: &Path, exclusions: &[String], mountpoints: &BTreeSet, + reflink: bool, cp_program: &OsStr, ) -> Result<()> { with_staging_dir(destination_parent, |_, staging| { @@ -511,6 +573,7 @@ fn copy_filtered_directory_staged( relative, exclusions, mountpoints, + reflink, true, cp_program, )?; @@ -817,7 +880,7 @@ fn preserve_metadata_timestamps(metadata: &Metadata, dst_fd: Fd) -> Re Ok(()) } -/// Plain recursive copy of `/var` into the new deployment. +/// Strategy D: plain recursive copy of `/var` for filesystems without reflink support. /// /// # Known limitation /// @@ -830,8 +893,10 @@ fn preserve_metadata_timestamps(metadata: &Metadata, dst_fd: Fd) -> Re /// corrupted copy. /// /// The correct fix is to run this migration only after stopping all stateful -/// services that write to `/var`, or to use a consistent snapshot before -/// copying their data. +/// services that write to `/var`, or — better — to migrate the filesystem to +/// btrfs or XFS (which support reflinks, Strategy C) to avoid copying every +/// file's data immediately. Reflinks clone files individually; they do not +/// provide an atomic snapshot of the `/var` tree or of a live database. /// /// Exclusions can be configured with install configuration or repeated /// `--preserve-var-skip` options. Paths are relative to `/var` and may be one @@ -866,6 +931,7 @@ fn preserve_var_copy_mounts( exclusions, mountpoints, false, + false, OsStr::new("cp"), ) .context("Copying /var into new deployment (Strategy D)")?; @@ -1036,6 +1102,63 @@ mod tests { Ok(()) } + #[test] + #[test] + fn both_copy_strategies_leave_nested_mountpoints_empty() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + fs::create_dir_all(src_path.join("lib/bound"))?; + fs::create_dir_all(src_path.join("lib/sibling"))?; + fs::write(src_path.join("lib/file-bound"), b"do not migrate")?; + fs::write( + src_path.join("lib/bound/mounted-sentinel"), + b"do not migrate", + )?; + fs::write(src_path.join("lib/sibling/sentinel"), b"migrate")?; + fs::create_dir(&dst_path)?; + let mounts = BTreeSet::from([PathBuf::from("lib/bound"), PathBuf::from("lib/file-bound")]); + let src = cap_dir(&src_path); + let dst = cap_dir(&dst_path); + + preserve_var_copy_mounts(&src, &dst, &[], &mounts)?; + assert!(dst_path.join("lib/bound").is_dir()); + assert_eq!(fs::read_dir(dst_path.join("lib/bound"))?.count(), 0); + assert_eq!(fs::read(dst_path.join("lib/file-bound"))?, b""); + assert_eq!(fs::read(dst_path.join("lib/sibling/sentinel"))?, b"migrate"); + + let excluded_dst_path = temp.path().join("excluded-dst"); + fs::create_dir(&excluded_dst_path)?; + preserve_var_copy_mounts( + &src, + &cap_dir(&excluded_dst_path), + &["lib/bound".into()], + &mounts, + )?; + assert!(!excluded_dst_path.join("lib/bound").exists()); + + let reflink_dst = temp.path().join("reflink-dst"); + fs::create_dir(&reflink_dst)?; + let reflink_dst = cap_dir(&reflink_dst); + if supports_reflink(&src, &reflink_dst)? { + preserve_var_reflink_mounts(&src, &reflink_dst, &[], &mounts)?; + assert!(reflink_dst.symlink_metadata("lib/bound")?.is_dir()); + assert_eq!( + fs::read_dir(temp.path().join("reflink-dst/lib/bound"))?.count(), + 0 + ); + assert_eq!( + fs::read(temp.path().join("reflink-dst/lib/file-bound"))?, + b"" + ); + assert_eq!( + fs::read(temp.path().join("reflink-dst/lib/sibling/sentinel"))?, + b"migrate" + ); + } + Ok(()) + } + #[test] fn split_directories_and_mountpoint_placeholders_keep_metadata() -> Result<()> { let temp = tempfile::tempdir()?; @@ -1269,6 +1392,185 @@ mod tests { Ok(()) } + #[test] + fn unfiltered_directory_collisions_preserve_destination_metadata() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + fs::create_dir_all(src_path.join("state"))?; + fs::write(src_path.join("state/existing"), b"source")?; + fs::write(src_path.join("state/new"), b"new")?; + fs::write(src_path.join("collision-file"), b"source collision")?; + fs::create_dir(src_path.join("directory-vs-file"))?; + fs::write(src_path.join("directory-vs-file/new"), b"must not replace")?; + fs::set_permissions(src_path.join("state"), fs::Permissions::from_mode(0o751))?; + let source_mtime = SystemTime::UNIX_EPOCH + Duration::from_secs(1_600_000_201); + fs::File::open(src_path.join("state"))? + .set_times(fs::FileTimes::new().set_modified(source_mtime))?; + + let src = cap_dir(&src_path); + for (strategy, reflink) in [("copy", false), ("reflink", true)] { + let dst_path = temp.path().join(format!("dst-{strategy}")); + fs::create_dir_all(dst_path.join("state"))?; + fs::write(dst_path.join("state/existing"), b"destination")?; + fs::write(dst_path.join("collision-file"), b"keep collision")?; + fs::write(dst_path.join("directory-vs-file"), b"keep file")?; + fs::set_permissions(dst_path.join("state"), fs::Permissions::from_mode(0o700))?; + let destination_mtime = SystemTime::UNIX_EPOCH + Duration::from_secs(1_600_000_202); + fs::File::open(dst_path.join("state"))? + .set_times(fs::FileTimes::new().set_modified(destination_mtime))?; + let dst = cap_dir(&dst_path); + if reflink && !supports_reflink(&src, &dst)? { + continue; + } + + if reflink { + preserve_var_reflink(&src, &dst, &[])?; + } else { + preserve_var_copy(&src, &dst, &[])?; + } + + let destination_metadata = fs::metadata(dst_path.join("state"))?; + assert_eq!(destination_metadata.mode() & 0o7777, 0o700); + assert_eq!(destination_metadata.modified()?, destination_mtime); + assert_eq!(fs::read(dst_path.join("state/existing"))?, b"destination"); + assert_eq!(fs::read(dst_path.join("state/new"))?, b"new"); + assert_eq!( + fs::read(dst_path.join("collision-file"))?, + b"keep collision" + ); + assert_eq!(fs::read(dst_path.join("directory-vs-file"))?, b"keep file"); + assert!(!dst_path.join("directory-vs-file/new").exists()); + } + Ok(()) + } + + #[test] + fn failed_copy_batch_is_cleaned_and_can_be_retried() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + fs::create_dir(&src_path)?; + fs::write(src_path.join("first"), b"linked data")?; + fs::hard_link(src_path.join("first"), src_path.join("second"))?; + let src = cap_dir(&src_path); + + let failing_cp = temp.path().join("cp-fails-after-copy"); + fs::write( + &failing_cp, + b"#!/bin/sh\nreflink=no\nwhile [ \"$1\" != -- ]; do\n [ \"$1\" = --reflink=always ] && reflink=yes\n shift\ndone\nshift\nsource=$1\nshift\nwhile [ \"$#\" -gt 1 ]; do shift; done\ndestination=$1\nif [ \"$reflink\" = yes ]; then\n /bin/cp --reflink=always -a -- \"$source\" \"$destination\"\nelse\n /bin/cp -a -- \"$source\" \"$destination\"\nfi\nexit 1\n", + )?; + fs::set_permissions(&failing_cp, fs::Permissions::from_mode(0o755))?; + + for (strategy, reflink) in [("copy", false), ("reflink", true)] { + let dst_path = temp.path().join(format!("dst-{strategy}")); + fs::create_dir(&dst_path)?; + let dst = cap_dir(&dst_path); + if reflink && !supports_reflink(&src, &dst)? { + continue; + } + + let error = copy_var_filtered( + &src, + &dst, + Path::new(""), + &[], + &BTreeSet::new(), + reflink, + false, + failing_cp.as_os_str(), + ); + assert!(error.is_err()); + assert_eq!(fs::read_dir(&dst_path)?.count(), 0); + + if reflink { + preserve_var_reflink(&src, &dst, &[])?; + } else { + preserve_var_copy(&src, &dst, &[])?; + } + let first = fs::metadata(dst_path.join("first"))?; + let second = fs::metadata(dst_path.join("second"))?; + assert_eq!(first.ino(), second.ino()); + assert_eq!(fs::read(dst_path.join("first"))?, b"linked data"); + } + Ok(()) + } + + #[test] + fn empty_mountpoint_placeholders_are_retry_safe_and_type_checked() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + fs::create_dir_all(src_path.join("directory-mount"))?; + fs::write(src_path.join("file-mount"), b"mounted data")?; + let src = cap_dir(&src_path); + + for (strategy, reflink) in [("copy", false), ("reflink", true)] { + let dst_path = temp.path().join(format!("dst-{strategy}")); + fs::create_dir(&dst_path)?; + let dst = cap_dir(&dst_path); + if reflink && !supports_reflink(&src, &dst)? { + continue; + } + + fs::create_dir(dst_path.join("directory-mount"))?; + fs::write(dst_path.join("file-mount"), b"")?; + let mounts = BTreeSet::from([ + PathBuf::from("directory-mount"), + PathBuf::from("file-mount"), + ]); + if reflink { + preserve_var_reflink_mounts(&src, &dst, &[], &mounts)?; + preserve_var_reflink_mounts(&src, &dst, &[], &mounts)?; + } else { + preserve_var_copy_mounts(&src, &dst, &[], &mounts)?; + preserve_var_copy_mounts(&src, &dst, &[], &mounts)?; + } + assert_eq!(fs::read_dir(dst_path.join("directory-mount"))?.count(), 0); + assert_eq!(fs::metadata(dst_path.join("file-mount"))?.len(), 0); + + fs::write(dst_path.join("directory-mount/sentinel"), b"keep")?; + let error = if reflink { + preserve_var_reflink_mounts(&src, &dst, &[], &mounts) + } else { + preserve_var_copy_mounts(&src, &dst, &[], &mounts) + }; + assert!(error.is_err()); + assert_eq!( + fs::read(dst_path.join("directory-mount/sentinel"))?, + b"keep" + ); + fs::remove_file(dst_path.join("directory-mount/sentinel"))?; + fs::write(dst_path.join("file-mount"), b"keep file")?; + let error = if reflink { + preserve_var_reflink_mounts(&src, &dst, &[], &mounts) + } else { + preserve_var_copy_mounts(&src, &dst, &[], &mounts) + }; + assert!(error.is_err()); + assert_eq!(fs::read(dst_path.join("file-mount"))?, b"keep file"); + + let wrong_type = temp.path().join(format!("wrong-type-{strategy}")); + fs::create_dir(&wrong_type)?; + fs::write(wrong_type.join("directory-mount"), b"keep wrong type")?; + fs::create_dir(wrong_type.join("file-mount"))?; + fs::write(wrong_type.join("file-mount/sentinel"), b"keep wrong type")?; + let wrong_dst = cap_dir(&wrong_type); + let error = if reflink { + preserve_var_reflink_mounts(&src, &wrong_dst, &[], &mounts) + } else { + preserve_var_copy_mounts(&src, &wrong_dst, &[], &mounts) + }; + assert!(error.is_err()); + assert_eq!( + fs::read(wrong_type.join("directory-mount"))?, + b"keep wrong type" + ); + assert_eq!( + fs::read(wrong_type.join("file-mount/sentinel"))?, + b"keep wrong type" + ); + } + Ok(()) + } + #[test] fn mountpoint_placeholders_fail_without_reusing_existing_destinations() -> Result<()> { let temp = tempfile::tempdir()?; @@ -1361,6 +1663,39 @@ mod tests { Ok(()) } + #[test] + fn reflink_strategy_skips_multiple_children_under_parent() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + fs::create_dir_all(src_path.join("log/journal"))?; + fs::create_dir_all(src_path.join("log/private"))?; + fs::create_dir_all(src_path.join("log/keep"))?; + fs::write(src_path.join("log/journal/ignored"), b"journal")?; + fs::write(src_path.join("log/private/ignored"), b"private")?; + fs::write(src_path.join("log/keep/preserved"), b"keep")?; + fs::create_dir(&dst_path)?; + let src = cap_dir(&src_path); + let dst = cap_dir(&dst_path); + let supports_reflink = supports_reflink(&src, &dst)?; + if !supports_reflink { + fs::remove_file(src_path.join("log/keep/preserved"))?; + fs::remove_dir(src_path.join("log/keep"))?; + } + + // Exercise Strategy C's exclusion routing on every filesystem and, if + // available, its reflink copy of a retained sibling. + preserve_var_reflink(&src, &dst, &["log/journal".into(), "log/private".into()])?; + + assert!(!dst_path.join("log/journal").exists()); + assert!(!dst_path.join("log/private").exists()); + assert_eq!( + dst_path.join("log/keep/preserved").exists(), + supports_reflink + ); + Ok(()) + } + #[test] fn copy_preserves_symlink_without_traversing_target() -> Result<()> { let temp = tempfile::tempdir()?; diff --git a/crates/lib/src/store/mod.rs b/crates/lib/src/store/mod.rs index b63ed6caa5..1e2a08af47 100644 --- a/crates/lib/src/store/mod.rs +++ b/crates/lib/src/store/mod.rs @@ -129,6 +129,8 @@ use crate::podstorage::CStorage; use crate::spec::{BootloaderKind, ImageStatus}; use crate::utils::{EfiError, deployment_fd, open_dir_remount_rw, read_uefi_var}; +pub(crate) mod reflink; + /// See pub type ComposefsRepository = composefs::repository::Repository; diff --git a/crates/lib/src/store/reflink.rs b/crates/lib/src/store/reflink.rs new file mode 100644 index 0000000000..148a40153b --- /dev/null +++ b/crates/lib/src/store/reflink.rs @@ -0,0 +1,279 @@ +//! Pairwise reflink capability probing for storage directories. + +use std::io::Write as _; + +use anyhow::{Context, Result}; +use cap_std_ext::cap_std::fs::{Dir as CapStdDir, File, OpenOptions}; +use cap_std_ext::cap_std::io_lifetimes::AsFilelike; +use rustix::fd::AsFd as _; + +/// Returns whether a file in `src` can be reflinked into `dst`. +/// +/// The result is scoped to this source/destination pair; it is deliberately +/// not cached. Temporary files are created exclusively beneath the supplied +/// directory capabilities and removed before returning. +pub(crate) fn supports_reflink(src: &CapStdDir, dst: &CapStdDir) -> Result { + probe_with(src, dst, |destination, source| { + let source_view = source.as_filelike_view::(); + let destination_view = destination.as_filelike_view::(); + let source_fd = source_view.as_fd(); + let destination_fd = destination_view.as_fd(); + rustix::fs::ioctl_ficlone(destination_fd, source_fd) + }) +} + +fn probe_with( + src: &CapStdDir, + dst: &CapStdDir, + clone: impl FnOnce(&File, &File) -> rustix::io::Result<()>, +) -> Result { + let mut source = match create_probe_file(src) { + Ok(source) => source, + Err(error) => return source_probe_failure(error), + }; + let mut destination = match create_probe_file(dst) { + Ok(destination) => destination, + Err(error) => { + source + .cleanup() + .context("Removing reflink probe source after destination creation failed")?; + return Err(error); + } + }; + + let probe_result = match source + .file + .as_mut() + .expect("probe source file is open") + .write_all(b"bootc reflink probe") + { + Err(error) => { + source_probe_failure(anyhow::Error::from(error).context("Writing reflink probe source")) + } + Ok(()) => match clone( + destination + .file + .as_ref() + .expect("probe destination file is open"), + source.file.as_ref().expect("probe source file is open"), + ) { + Ok(()) => Ok(true), + Err(error) if reflink_unsupported(error) => Ok(false), + Err(error) => Err(anyhow::Error::from(error).context("Probing FICLONE support")), + }, + }; + + // Attempt both removals even if one fails. Drop provides a best-effort + // fallback if the operation above returned early (for example, a write + // error). + let source_cleanup = source.cleanup(); + let destination_cleanup = destination.cleanup(); + source_cleanup.context("Removing reflink probe source")?; + destination_cleanup.context("Removing reflink probe destination")?; + probe_result +} + +fn reflink_unsupported(error: rustix::io::Errno) -> bool { + // Both files are newly created regular files, so EINVAL here indicates + // that FICLONE is unsupported rather than an invalid source type. + matches!( + error, + rustix::io::Errno::OPNOTSUPP | rustix::io::Errno::XDEV | rustix::io::Errno::INVAL + ) +} + +fn source_probe_access_error(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause.downcast_ref::().is_some_and(|error| { + matches!( + error.kind(), + std::io::ErrorKind::PermissionDenied | std::io::ErrorKind::ReadOnlyFilesystem + ) + }) + }) +} + +fn source_probe_failure(error: anyhow::Error) -> Result { + if source_probe_access_error(&error) { + tracing::debug!("Cannot prepare reflink probe source, using full copy: {error:#}"); + Ok(false) + } else { + Err(error) + } +} + +struct ProbeFile<'a> { + dir: &'a CapStdDir, + name: String, + file: Option, + removed: bool, +} + +impl ProbeFile<'_> { + fn cleanup(&mut self) -> std::io::Result<()> { + self.cleanup_with(|dir, name| dir.remove_file(name)) + } + + fn cleanup_with( + &mut self, + remove_file: impl FnOnce(&CapStdDir, &str) -> std::io::Result<()>, + ) -> std::io::Result<()> { + drop(self.file.take()); + match remove_file(self.dir, &self.name) { + Ok(()) => { + self.removed = true; + Ok(()) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + self.removed = true; + Ok(()) + } + Err(error) => Err(error), + } + } +} + +impl Drop for ProbeFile<'_> { + fn drop(&mut self) { + if !self.removed { + let _ = self.cleanup(); + } + } +} + +fn create_probe_file(dir: &CapStdDir) -> Result> { + loop { + let name = format!(".bootc-reflink-probe-{}", uuid::Uuid::new_v4()); + match create_probe_file_with_name(dir, &name) { + Ok(file) => return Ok(file), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error).context("Creating reflink probe file"), + } + } +} + +fn create_probe_file_with_name<'a>( + dir: &'a CapStdDir, + name: &str, +) -> std::io::Result> { + let file = dir.open_with( + name, + OpenOptions::new().read(true).write(true).create_new(true), + )?; + Ok(ProbeFile { + dir, + name: name.to_owned(), + file: Some(file), + removed: false, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use cap_std_ext::cap_std::ambient_authority; + use std::fs; + + fn cap_dir(path: &std::path::Path) -> CapStdDir { + CapStdDir::open_ambient_dir(path, ambient_authority()).unwrap() + } + + #[test] + fn probe_reports_success_and_cleans_up() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + fs::create_dir(&src_path)?; + fs::create_dir(&dst_path)?; + + let supported = probe_with(&cap_dir(&src_path), &cap_dir(&dst_path), |_, _| Ok(()))?; + assert!(supported); + assert!(fs::read_dir(&src_path)?.next().is_none()); + assert!(fs::read_dir(&dst_path)?.next().is_none()); + Ok(()) + } + + #[test] + fn probe_classifies_unsupported_and_propagates_unexpected_errors() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + fs::create_dir(&src_path)?; + fs::create_dir(&dst_path)?; + let src = cap_dir(&src_path); + let dst = cap_dir(&dst_path); + + assert!(!probe_with(&src, &dst, |_, _| { + Err(rustix::io::Errno::OPNOTSUPP) + })?); + assert!(!probe_with(&src, &dst, |_, _| { + Err(rustix::io::Errno::INVAL) + })?); + assert!(fs::read_dir(&src_path)?.next().is_none()); + assert!(fs::read_dir(&dst_path)?.next().is_none()); + + assert!(probe_with(&src, &dst, |_, _| Err(rustix::io::Errno::IO)).is_err()); + assert!(fs::read_dir(&src_path)?.next().is_none()); + assert!(fs::read_dir(&dst_path)?.next().is_none()); + Ok(()) + } + + #[test] + fn classifies_only_known_unsupported_errors() { + assert!(reflink_unsupported(rustix::io::Errno::OPNOTSUPP)); + assert!(reflink_unsupported(rustix::io::Errno::XDEV)); + assert!(reflink_unsupported(rustix::io::Errno::INVAL)); + assert!(!reflink_unsupported(rustix::io::Errno::IO)); + } + + #[test] + fn source_permission_and_read_only_errors_select_full_copy() { + for error in [ + std::io::Error::from(std::io::ErrorKind::PermissionDenied), + std::io::Error::from(std::io::ErrorKind::ReadOnlyFilesystem), + ] { + let error = anyhow::Error::from(error).context("Creating reflink probe file"); + assert!(source_probe_access_error(&error)); + assert!(!source_probe_failure(error).unwrap()); + } + assert!(source_probe_failure(anyhow::anyhow!("unrelated error")).is_err()); + } + + #[test] + fn cleanup_retries_unlink_on_drop_after_failure() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("src"); + fs::create_dir(&path)?; + let dir = cap_dir(&path); + let name = ".bootc-reflink-probe-retry"; + let mut probe = create_probe_file_with_name(&dir, name)?; + + let error = probe + .cleanup_with(|_, _| Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied))) + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + assert!(path.join(name).exists()); + + drop(probe); + assert!(!path.join(name).exists()); + Ok(()) + } + + #[test] + fn probe_file_creation_never_overwrites_existing_files() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("src"); + fs::create_dir(&path)?; + let existing = path.join(".bootc-reflink-probe-collision"); + fs::write(&existing, b"leave this alone")?; + + let error = + match create_probe_file_with_name(&cap_dir(&path), ".bootc-reflink-probe-collision") { + Ok(_) => panic!("create_new unexpectedly replaced an existing probe file"), + Err(error) => error, + }; + assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); + assert_eq!(fs::read(existing)?, b"leave this alone"); + Ok(()) + } +} From 168794c68f25f708dbc730d5b5120ea75248ff5b Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Fri, 30 Oct 2026 09:28:12 -0500 Subject: [PATCH 05/38] docs: Explain preserve-var migration behavior Users converting package-mode systems need to understand the copy strategy, required backend, exclusions, and lack of snapshot consistency before opting in. Assisted-by: AI Signed-off-by: ckyrouac --- docs/src/bootc-installation.7.md | 8 ++++++++ docs/src/man/bootc-install-config.5.md | 5 ++++- .../src/man/bootc-install-to-existing-root.8.md | 17 +++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/src/bootc-installation.7.md b/docs/src/bootc-installation.7.md index f143ab5102..1467ced9c6 100644 --- a/docs/src/bootc-installation.7.md +++ b/docs/src/bootc-installation.7.md @@ -294,6 +294,14 @@ and data from the old system (now accessible at `/sysroot`) to the new system. T useful for preserving network settings, user accounts, or application data from the previous installation. +With the OSTree backend, `--preserve-var` copies the old `/var` into the new +deployment before rebooting. It uses reflinks when supported and otherwise makes a +full copy; neither strategy is a consistent snapshot of live data. Stop stateful +services before copying their data. `/var/tmp`, `/var/cache`, `/var/log/journal`, +and `/var/lib/containers` are always excluded. Use repeatable +`--preserve-var-skip PATH` options or the `preserve-var-skip` install configuration +key for additional relative paths below `/var`. + ##### Before reboot: Injecting new configuration After running `bootc install to-existing-root`, you may want to inject diff --git a/docs/src/man/bootc-install-config.5.md b/docs/src/man/bootc-install-config.5.md index 7211c4aab7..35726137bd 100644 --- a/docs/src/man/bootc-install-config.5.md +++ b/docs/src/man/bootc-install-config.5.md @@ -51,7 +51,10 @@ The `install` section supports these subfields: path) includes a default policy which requires signatures. When `true`, image pulls will be rejected if the policy file specifies `insecureAcceptAnything` as the default. Defaults to `false`. - This is equivalent to the `--enforce-container-sigpolicy` CLI flag. + This is equivalent to the `--enforce-container-sigpolicy` CLI flag. +- `preserve-var-skip`: An array of relative paths below `/var` to exclude when + `bootc install to-existing-root --preserve-var` copies data. Each path may + contain at most one slash. The always-excluded paths cannot be re-included. # filesystem diff --git a/docs/src/man/bootc-install-to-existing-root.8.md b/docs/src/man/bootc-install-to-existing-root.8.md index d9694977c9..5baf2c280b 100644 --- a/docs/src/man/bootc-install-to-existing-root.8.md +++ b/docs/src/man/bootc-install-to-existing-root.8.md @@ -16,6 +16,15 @@ host root filesystem\'s `/boot` partition will be wiped, but the content of the existing root will otherwise be retained, and will need to be cleaned up if desired when rebooted into the new root. +The opt-in `--preserve-var` option copies the running system's `/var` into the +new deployment before rebooting. It requires the OSTree backend, uses reflinks +when supported, and otherwise performs a full copy. Reflinks are created per +file, not as an atomic snapshot; stop stateful services or use a consistent +snapshot before copying live database data. The paths `tmp`, `cache`, +`log/journal`, and `lib/containers` are always excluded. Repeat +`--preserve-var-skip` to exclude additional paths relative to `/var`; each path +may contain at most one slash, for example `lib/rpm`. + ## Managing configuration: before and after reboot When using `to-existing-root`, there are two distinct scenarios for managing @@ -216,6 +225,14 @@ of migrating the fstab entries. See the "Injecting kernel arguments" section abo Add the bootc-destructive-cleanup systemd service to delete files from the previous install on first boot +**--preserve-var** + + Preserve the running system's `/var` data into the new bootc deployment + +**--preserve-var-skip**=*PATH* + + Relative paths below `/var` to leave out while preserving `/var` + **--composefs-backend** Use the composefs backend instead of ostree. This is the default for images with a UKI, and for images with /usr/lib/composefs/setup-root-conf.toml and no ostree prepare-root.conf From 9e1a3006b0fb6fb94addd242b26afa48dee73977 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 22 Oct 2026 10:11:06 -0500 Subject: [PATCH 06/38] tmt: Test package-mode /var preservation A /var migration is useful only if copied data remains available after the first reboot. Add a single-guest plan that invokes --preserve-var and verifies a host marker after boot, while keeping TMT workdirs accessible when /var/tmp is excluded. Assisted-by: AI Signed-off-by: ckyrouac --- .packit.yaml | 44 +++ Justfile | 27 ++ hack/provision-packit.sh | 8 +- tmt/plans/package-mode.fmf | 141 +++++++++ .../test-to-existing-root-preserve-var.sh | 295 ++++++++++++++++++ .../test-50-to-existing-root-preserve-var.fmf | 10 + 6 files changed, 523 insertions(+), 2 deletions(-) create mode 100644 tmt/plans/package-mode.fmf create mode 100755 tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh create mode 100644 tmt/tests/test-50-to-existing-root-preserve-var.fmf diff --git a/.packit.yaml b/.packit.yaml index 4ad646476f..e7ce623edc 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -102,6 +102,28 @@ jobs: context: running_env: "packit" + # Run package-mode migration coverage on a single guest alongside the main + # tier-1 integration plan. + - job: tests + trigger: pull_request + identifier: package-mode-tier-1 + require: + label: + present: + - ci/tier-1 + targets: + - centos-stream-10-x86_64 + - centos-stream-10-aarch64 + - fedora-latest-stable-x86_64 + - fedora-latest-stable-aarch64 + tmt_plan: /tmt/plans/package-mode + tf_extra_params: + environments: + - tmt: + context: + running_env: "packit" + package_source: "artifacts" + # ci/merge: full target set - job: tests trigger: pull_request @@ -124,6 +146,28 @@ jobs: context: running_env: "packit" + - job: tests + trigger: pull_request + identifier: package-mode-full + require: + label: + present: + - ci/merge + targets: + - centos-stream-9-x86_64 + - centos-stream-9-aarch64 + - centos-stream-10-x86_64 + - centos-stream-10-aarch64 + - fedora-all-x86_64 + - fedora-all-aarch64 + tmt_plan: /tmt/plans/package-mode + tf_extra_params: + environments: + - tmt: + context: + running_env: "packit" + package_source: "artifacts" + - job: propose_downstream trigger: release dist_git_branches: diff --git a/Justfile b/Justfile index c23f160d33..890cc45255 100644 --- a/Justfile +++ b/Justfile @@ -229,6 +229,33 @@ build-ukiboot-update-test-image: build-ukiboot-test-image test-ukiboot: build-ukiboot-update-test-image cargo xtask run-tmt --env=BOOTC_variant=composefs --filesystem=ext4 --boot-type=aboot --seal-state=unsealed --upgrade-image={{ukiboot_img}}:v2 {{ukiboot_img}}:v1 aboot +# Run the package-mode migration plan against the current checkout image. +# The VM cannot share controller Podman storage, so transfer it as an OCI archive. +[group('testing')] +test-tmt-package-mode: + #!/bin/bash + set -euo pipefail + just build + archive_dir=$(mktemp -d -- tmt/plans/.package-mode-local.XXXXXX) + cleanup() { + rm -rf -- "$archive_dir" + } + trap cleanup EXIT + trap 'cleanup; exit 130' INT + trap 'cleanup; exit 143' TERM + archive_path="$archive_dir/bootc.oci" + if [[ -e "$archive_path" ]]; then + echo "Refusing to overwrite existing archive: $archive_path" >&2 + exit 1 + fi + podman save --format oci-archive --output "$archive_path" localhost/bootc + tmt \ + --context=running_env=packit \ + --context=package_source=local \ + --context="test_disk_image=${test_disk_image:-centos-stream-10}" \ + run --environment "PACKAGE_MODE_IMAGE_ARCHIVE=$archive_path" \ + plans --name '^/tmt/plans/package-mode$' + # Split out from `test-container` because, unlike the container integration tests, # unit tests don't depend on variant/filesystem/bootloader/boot_type/seal_state, so # CI runs this once per OS (in the `package` job) instead of once per test-integration diff --git a/hack/provision-packit.sh b/hack/provision-packit.sh index a12a1a674e..905d51981c 100755 --- a/hack/provision-packit.sh +++ b/hack/provision-packit.sh @@ -15,6 +15,7 @@ trap 'rm -rf -- "$BOOTC_TEMPDIR"' EXIT cp -a . "$BOOTC_TEMPDIR" # Keep testing farm run folder +mkdir -p /var/ARTIFACTS cp -r /var/ARTIFACTS "$BOOTC_TEMPDIR" # Copy bootc repo @@ -95,6 +96,9 @@ fi # Keep these in sync with what's used in hack/lbi; ubi9/podman is pinned, see hack/lbi/podman.image podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:9.8-1793032728 -# Run system-reinstall-bootc # TODO make it more scriptable instead of expect + send -./system-reinstall-bootc.exp +if [[ ${BOOTC_PROVISION_PACKIT_SKIP_REINSTALL:-0} == 1 ]]; then + echo "BOOTC_PROVISION_PACKIT_SKIP_REINSTALL=1: skipping system-reinstall-bootc conversion and reboot" +else + ./system-reinstall-bootc.exp +fi diff --git a/tmt/plans/package-mode.fmf b/tmt/plans/package-mode.fmf new file mode 100644 index 0000000000..7930c556f4 --- /dev/null +++ b/tmt/plans/package-mode.fmf @@ -0,0 +1,141 @@ +summary: Test package-mode to-existing-root migration flags +environment: + PACKAGE_SOURCE: $@{package_source} +provision: + how: virtual + image: $@{test_disk_image} + user: root +prepare: + # The VM cannot share controller Podman storage; TMT syncs the OCI archive + # through its workdir before prepare runs. + - how: install + order: 93 + package: + - podman + - jq + - bootc + when: package_source == local + # Packit build-only preparation needs the full integration toolchain. + - how: install + order: 93 + package: + - podman + - skopeo + - jq + - bootc + - system-reinstall-bootc + - zstd + - cpio + - rpm-build + - expect + - ansible-core + - parted + - lvm2 + - dosfstools + - e2fsprogs + when: package_source == artifacts + # Reject missing/unknown sources before any mode-specific setup runs. + - how: shell + order: 95 + script: | + case "${PACKAGE_SOURCE:-}" in + artifacts|local) ;; + *) + printf 'package_source must be artifacts or local (got %q)\n' "${PACKAGE_SOURCE:-}" >&2 + exit 1 + ;; + esac + # Load the TMT-synced archive into rootful guest Podman, then clean its copy. + - how: shell + order: 96 + script: | + set -eu + archive_relative=${PACKAGE_MODE_IMAGE_ARCHIVE:-} + case "$archive_relative" in + tmt/plans/.package-mode-local.*/bootc.oci) ;; + *) + printf 'Unexpected PACKAGE_MODE_IMAGE_ARCHIVE path: %s\n' "$archive_relative" >&2 + exit 1 + ;; + esac + archive_directory_relative=${archive_relative%/*} + temporary_name=${archive_directory_relative##*/} + case "$temporary_name" in + .package-mode-local.*) ;; + *) + printf 'Unexpected package-mode archive directory: %s\n' "$archive_directory_relative" >&2 + exit 1 + ;; + esac + if [ "$archive_directory_relative" != "tmt/plans/$temporary_name" ]; then + printf 'Package-mode archive must be directly under tmt/plans: %s\n' "$archive_relative" >&2 + exit 1 + fi + + tree_root=$(realpath -e -- "${TMT_TREE:?TMT_TREE is required}") + if [ ! -d "$tree_root" ]; then + printf 'TMT_TREE is not a directory: %s\n' "$tree_root" >&2 + exit 1 + fi + plans_root=$(realpath -e -- "$tree_root/tmt/plans") + plans_root_prefix=${tree_root%/}/ + if [ "$plans_root" = "$tree_root" ]; then + printf 'TMT plans directory resolves to the worktree root: %s\n' "$plans_root" >&2 + exit 1 + fi + case "$plans_root" in + "$plans_root_prefix"*) ;; + *) + printf 'TMT plans directory resolves outside the worktree: %s\n' "$plans_root" >&2 + exit 1 + ;; + esac + archive_directory=$(realpath -m -- "$tree_root/$archive_directory_relative") + if [ "$archive_directory" != "$plans_root/$temporary_name" ]; then + printf 'Package-mode archive directory resolves outside tmt/plans: %s\n' "$archive_relative" >&2 + exit 1 + fi + cleanup() { + rm -rf -- "$archive_directory" + } + trap cleanup EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + archive_path=$(realpath -e -- "$archive_directory/bootc.oci") + if [ "$archive_path" != "$archive_directory/bootc.oci" ]; then + printf 'Package-mode archive must not resolve outside its temporary directory: %s\n' "$archive_relative" >&2 + exit 1 + fi + if [ ! -f "$archive_path" ]; then + printf 'Package-mode archive is not a regular file: %s\n' "$archive_relative" >&2 + exit 1 + fi + podman --remote=false load --input "$archive_path" + podman --remote=false image exists localhost/bootc + when: package_source == local + - how: shell + order: 98 + script: + - mkdir -p bootc && cp /var/share/test-artifacts/*.src.rpm bootc + # Packit artifacts are already staged in the guest before extraction. + - cd bootc && rpm2cpio *.src.rpm > pkg.cpio && cpio -idmv < pkg.cpio && rm pkg.cpio && rm -f *-vendor.tar.zstd && zstd -d *.tar.zstd && tar -xvf *.tar -C . --strip-components=1 && ls -al + - cd bootc/hack && BOOTC_PROVISION_PACKIT_SKIP_REINSTALL=1 ./provision-packit.sh + when: package_source == artifacts + # Both sources now provide localhost/bootc in the guest's rootful storage. + # Pull the LBI references needed to install the image. + - how: shell + order: 99 + script: | + podman --remote=false image exists localhost/bootc + podman --remote=false pull quay.io/curl/curl:latest + podman --remote=false pull quay.io/curl/curl-base:latest + podman --remote=false pull registry.access.redhat.com/ubi9/podman:latest + when: package_source == artifacts or package_source == local +execute: + how: tmt +discover: + how: fmf + prune: true + test: + - /tmt/tests/test-50-to-existing-root-preserve-var diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh new file mode 100755 index 0000000000..779df53189 --- /dev/null +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -0,0 +1,295 @@ +#!/bin/bash +# Both local and Packit runs prepare one package-mode guest, then convert and +# reboot that same guest; local loads the checkout image, Packit builds it. +set -xeuo pipefail + +target_image=localhost/bootc +var_marker=/var/lib/bootc-tmt-preserve-var/sentinel +tmt_plan_data_suffix=/tmt/plans/package-mode/data +tmt_workdir_parent=/var/tmp/tmt +tmt_workdir_root=/var/lib/bootc-tmt-workdirs +tmt_artifacts_parent=/var/ARTIFACTS +tmt_test_data_sentinel_name=workdir-persistence-sentinel +tmt_test_data_post_reboot_marker_name=post-reboot-marker + +get_tmt_run_dir() { + local plan_data=${TMT_PLAN_DATA:-} + local run_dir + local resolved_plan_data + local persistent_run_dir + local link_target + + if [[ "$plan_data" != *"$tmt_plan_data_suffix" ]]; then + printf 'TMT_PLAN_DATA must end in %s (got %q)\n' "$tmt_plan_data_suffix" "$plan_data" >&2 + return 1 + fi + run_dir=${plan_data%"$tmt_plan_data_suffix"} + if [[ ( ! $run_dir =~ ^/var/tmp/tmt/run-[[:alnum:]_.-]+$ && + ! $run_dir =~ ^/var/ARTIFACTS/work-[[:alnum:]_.-]+$ ) || $run_dir == *..* ]]; then + printf 'Expected a safe TMT run directory under %s or %s (got %q)\n' \ + "$tmt_workdir_parent" "$tmt_artifacts_parent" "$run_dir" >&2 + return 1 + fi + case ${TMT_REBOOT_COUNT:-0} in + 0) + if ! resolved_plan_data=$(realpath -e -- "$plan_data"); then + printf 'TMT_PLAN_DATA must be an existing path without symlinked components (got %q)\n' \ + "$plan_data" >&2 + return 1 + fi + if [[ $resolved_plan_data != "$plan_data" ]]; then + printf 'TMT_PLAN_DATA must be a canonical path without symlinked components (got %q, resolves to %q)\n' \ + "$plan_data" "$resolved_plan_data" >&2 + return 1 + fi + ;; + 1) + if [[ $run_dir == /var/tmp/tmt/* ]]; then + persistent_run_dir=$tmt_workdir_root/${run_dir##*/} + if [[ ! -L $run_dir ]] || ! link_target=$(readlink -- "$run_dir") || + [[ $link_target != "$persistent_run_dir" ]]; then + printf 'TMT run directory must link directly to %s (got %q)\n' \ + "$persistent_run_dir" "$(readlink -- "$run_dir" 2>/dev/null || true)" >&2 + return 1 + fi + if ! resolved_plan_data=$(realpath -e -- "$plan_data") || + [[ $resolved_plan_data != "$persistent_run_dir$tmt_plan_data_suffix" ]]; then + printf 'TMT_PLAN_DATA must resolve to %s (got %q, resolves to %q)\n' \ + "$persistent_run_dir$tmt_plan_data_suffix" "$plan_data" "${resolved_plan_data:-}" >&2 + return 1 + fi + else + # Packit/Testing Farm keeps its workdir under /var/ARTIFACTS. + # It must remain an ordinary directory, with no symlinked path + # components, and stay on the filesystem preserve-var copies. + if [[ -L $run_dir || ! -d $run_dir ]] || + ! resolved_plan_data=$(realpath -e -- "$plan_data") || + [[ $resolved_plan_data != "$plan_data" ]]; then + printf 'TMT workdir and plan data must remain canonical, real paths after reboot (got %q)\n' \ + "$plan_data" >&2 + return 1 + fi + verify_tmt_workdir_in_preserved_var "$run_dir" || return 1 + fi + ;; + *) + printf 'Unexpected TMT_REBOOT_COUNT=%s\n' "${TMT_REBOOT_COUNT}" >&2 + return 1 + ;; + esac + + printf '%s\n' "$run_dir" +} + +verify_tmt_workdir_in_preserved_var() { + local run_dir=$1 + local mount_target + + if ! mount_target=$(findmnt --noheadings --output TARGET --target "$run_dir"); then + printf 'Could not determine the filesystem containing TMT workdir %s\n' \ + "$run_dir" >&2 + return 1 + fi + # /var itself may be a separate filesystem; otherwise the containing root + # filesystem is copied as part of /var. Any mount below either is excluded + # from preserve-var and would make this workdir disappear across reboot. + if [[ $mount_target != / && $mount_target != /var ]]; then + printf 'TMT workdir %s is on nested mount %s, which --preserve-var does not copy\n' \ + "$run_dir" "$mount_target" >&2 + return 1 + fi + return 0 +} + +case ${TMT_REBOOT_COUNT:-0} in + 0) + podman --remote=false image exists "$target_image" + + # The guest must still be the original package-mode system. + test ! -e /run/ostree-booted + + mkdir -p "${var_marker%/*}" + + # TMT connects as root. Package-mode hosts keep root's home at /root, + # while bootc images link /root to /var/roothome, so preserve its key + # under /var for TMT to reconnect after the migration reboot. + install -d -m 0700 /var/roothome/.ssh + install -m 0600 /root/.ssh/authorized_keys /var/roothome/.ssh/authorized_keys + restorecon -RF /var/roothome/.ssh + + printf '%s\n' preserved-var | tee "$var_marker" + + tmt_run_dir=$(get_tmt_run_dir) + tmt_run_id=${tmt_run_dir##*/} + if [[ $tmt_run_dir == /var/ARTIFACTS/* ]]; then + tmt_run_layout=artifacts + else + tmt_run_layout=var-tmp + fi + if [[ ! -d "$tmt_run_dir" || -L "$tmt_run_dir" ]]; then + printf 'TMT run directory is missing or not a directory: %s\n' "$tmt_run_dir" >&2 + exit 1 + fi + tmt_run_dir_identity=$(stat -c '%d:%i' -- "$tmt_run_dir") + persistent_tmt_run_dir=$tmt_workdir_root/$tmt_run_id + if [[ -e "$persistent_tmt_run_dir" || -L "$persistent_tmt_run_dir" ]]; then + printf 'Persistent TMT workdir already exists: %s\n' "$persistent_tmt_run_dir" >&2 + exit 1 + fi + test -d "$TMT_TEST_DATA" + tmt_test_data_sentinel=$TMT_TEST_DATA/$tmt_test_data_sentinel_name + test ! -e "$tmt_test_data_sentinel" + printf '%s\n' workdir-persistence-sentinel > "$tmt_test_data_sentinel" + if [[ $tmt_run_layout == var-tmp ]]; then + # --preserve-var excludes /var/tmp. Stage a full copy on /var, then + # redirect the original path so TMT's post-reboot writes survive. + install -d -m 0755 "$tmt_workdir_root" + staging_parent=$(mktemp -d "$tmt_workdir_root/.${tmt_run_id}.XXXXXX") + trap 'if [[ -n ${staging_parent:-} ]]; then rm -rf -- "$staging_parent"; fi' EXIT + cp -a -- "$tmt_run_dir" "$staging_parent/" + staged_run_dir=$staging_parent/$tmt_run_id + mv -T --no-clobber -- "$staged_run_dir" "$persistent_tmt_run_dir" + if [[ -e "$staged_run_dir" || -L "$staged_run_dir" ]]; then + printf 'Persistent TMT workdir appeared during staging: %s\n' \ + "$persistent_tmt_run_dir" >&2 + exit 1 + fi + rmdir -- "$staging_parent" + staging_parent= + trap - EXIT + + install -d -m 0755 /etc/tmpfiles.d + printf 'd %s 0755 root root -\nL+ %s - - - - %s\n' \ + "$tmt_workdir_parent" "$tmt_run_dir" "$persistent_tmt_run_dir" \ + > /etc/tmpfiles.d/bootc-tmt-workdir.conf + + disposable_run_dir=$tmt_workdir_parent/.${tmt_run_id}.original.$$ + if [[ -e "$disposable_run_dir" || -L "$disposable_run_dir" ]]; then + printf 'Disposable TMT workdir already exists: %s\n' "$disposable_run_dir" >&2 + exit 1 + fi + + tmt_run_dir_moved=0 + tmt_run_dir_swap_complete=0 + cleanup_tmt_run_dir_swap() { + local status=$? + local disposable_identity + local original_identity + + trap - EXIT HUP INT TERM + if (( tmt_run_dir_moved )); then + # A signal can arrive after ln succeeds but before the flag is + # updated, so recognize the completed swap from the link too. + if [[ -L $tmt_run_dir ]] && + [[ $(readlink -- "$tmt_run_dir") == "$persistent_tmt_run_dir" ]]; then + tmt_run_dir_swap_complete=1 + fi + + if (( ! tmt_run_dir_swap_complete )); then + disposable_identity=$(stat -c '%d:%i' -- "$disposable_run_dir" 2>/dev/null || true) + if [[ $disposable_identity == "$tmt_run_dir_identity" ]]; then + if [[ ! -e $tmt_run_dir && ! -L $tmt_run_dir ]]; then + # --no-clobber ensures a replacement path created after + # the existence check is never overwritten. + mv -T --no-clobber -- "$disposable_run_dir" "$tmt_run_dir" || true + original_identity=$(stat -c '%d:%i' -- "$tmt_run_dir" 2>/dev/null || true) + if [[ $original_identity != "$tmt_run_dir_identity" ]]; then + disposable_identity=$(stat -c '%d:%i' -- "$disposable_run_dir" 2>/dev/null || true) + if [[ $disposable_identity == "$tmt_run_dir_identity" ]]; then + printf 'Could not restore TMT workdir because no-replace skipped the move; the original remains at %s\n' \ + "$disposable_run_dir" >&2 + else + printf 'Failed to restore TMT workdir from %s to %s\n' \ + "$disposable_run_dir" "$tmt_run_dir" >&2 + fi + fi + else + printf 'Cannot restore TMT workdir because the original path now exists; the original remains at %s\n' \ + "$disposable_run_dir" >&2 + fi + elif [[ -e $disposable_run_dir || -L $disposable_run_dir ]]; then + printf 'Leaving replacement at disposable TMT workdir path untouched: %s\n' \ + "$disposable_run_dir" >&2 + fi + fi + fi + exit "$status" + } + trap cleanup_tmt_run_dir_swap EXIT + trap 'exit 1' HUP INT TERM + tmt_run_dir_moved=1 + mv -T --no-clobber -- "$tmt_run_dir" "$disposable_run_dir" + disposable_run_dir_identity=$(stat -c '%d:%i' -- "$disposable_run_dir" 2>/dev/null || true) + if [[ $disposable_run_dir_identity != "$tmt_run_dir_identity" ]]; then + tmt_run_dir_moved=0 + printf 'Failed to move original TMT workdir to %s; the destination may have appeared concurrently\n' \ + "$disposable_run_dir" >&2 + exit 1 + fi + ln -s -- "$persistent_tmt_run_dir" "$tmt_run_dir" + tmt_run_dir_swap_complete=1 + # Keep the old tree: it is disposable under /var/tmp and vanishes on + # reboot, while retaining it keeps this shell's cwd valid until we leave. + tmt_run_dir_moved=0 + trap - EXIT HUP INT TERM + else + # /var/ARTIFACTS is already inside preserved /var. Do not move or + # symlink the Testing Farm workdir; preserve-var must carry it. + verify_tmt_workdir_in_preserved_var "$tmt_run_dir" + fi + + # The shell may still be inside the renamed workdir; leave it before + # installing or rebooting so the old cwd is no longer in use. + cd / + + # Run the build-under-test as a privileged installer, with the + # package-mode root mounted at /target. + podman --remote=false run --rm --privileged --pid=host --user=root:root \ + -v /dev:/dev \ + -v /:/target \ + -v /boot:/target/boot \ + -v /var/lib/containers:/var/lib/containers \ + --security-opt label=type:unconfined_t \ + "$target_image" \ + bootc install to-existing-root --preserve-var --acknowledge-destructive + + bootc status + # Ask the guest to reboot itself; this is more reliable than + # testcloud's ACPI soft-poweroff for a full bootc deployment reboot. + tmt-reboot -c 'systemctl reboot' + ;; + 1) + test -e /run/ostree-booted + + tmt_run_dir=$(get_tmt_run_dir) + tmt_run_id=${tmt_run_dir##*/} + if [[ $tmt_run_dir == /var/ARTIFACTS/* ]]; then + # The Packit workdir itself must have survived preserve-var intact. + test -d "$tmt_run_dir" + test ! -L "$tmt_run_dir" + verify_tmt_workdir_in_preserved_var "$tmt_run_dir" + else + persistent_tmt_run_dir=$tmt_workdir_root/$tmt_run_id + test -L "$tmt_run_dir" + test "$(readlink -f -- "$tmt_run_dir")" = "$persistent_tmt_run_dir" + fi + test -d "$TMT_PLAN_DATA" + test -d "$TMT_TEST_DATA" + test -f "$TMT_TEST_METADATA" + tmt_test_data_sentinel=$TMT_TEST_DATA/$tmt_test_data_sentinel_name + tmt_test_data_post_reboot_marker=$TMT_TEST_DATA/$tmt_test_data_post_reboot_marker_name + test -f "$tmt_test_data_sentinel" + test "$(<"$tmt_test_data_sentinel")" = workdir-persistence-sentinel + printf '%s\n' post-reboot-workdir-writable > "$tmt_test_data_post_reboot_marker" + test "$(<"$tmt_test_data_post_reboot_marker")" = post-reboot-workdir-writable + + booted_image=$(bootc status --json | jq -r '.status.booted.image.image.image') + test "$booted_image" = "$target_image" + + test "$(<"$var_marker")" = preserved-var + ;; + *) + printf 'Unexpected TMT_REBOOT_COUNT=%s\n' "$TMT_REBOOT_COUNT" >&2 + exit 1 + ;; +esac diff --git a/tmt/tests/test-50-to-existing-root-preserve-var.fmf b/tmt/tests/test-50-to-existing-root-preserve-var.fmf new file mode 100644 index 0000000000..305222ad67 --- /dev/null +++ b/tmt/tests/test-50-to-existing-root-preserve-var.fmf @@ -0,0 +1,10 @@ +# The VM cannot share controller Podman storage, so TMT syncs the local OCI +# archive to the single guest before migration. It reboots once, then verifies +# preserved /var data. +summary: Package-mode /var preservation +duration: 45m +adjust: + - when: running_env == image_mode + enabled: false + because: this migration test requires a package-mode target +test: bash booted/package-mode/test-to-existing-root-preserve-var.sh From c778103e4226f40758446b1af3c2147110c2145e Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 22 Oct 2026 10:21:16 -0500 Subject: [PATCH 07/38] tmt: Install preserve-var workdir rule in deployment The package-mode host's /etc is not migrated by to-existing-root, so a tmpfiles rule written there disappears on reboot. Write and verify the rule in the mounted deployment's persistent /etc before reboot instead. Assisted-by: AI Signed-off-by: ckyrouac --- .../test-to-existing-root-preserve-var.sh | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index 779df53189..4bd9365609 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -158,11 +158,6 @@ case ${TMT_REBOOT_COUNT:-0} in staging_parent= trap - EXIT - install -d -m 0755 /etc/tmpfiles.d - printf 'd %s 0755 root root -\nL+ %s - - - - %s\n' \ - "$tmt_workdir_parent" "$tmt_run_dir" "$persistent_tmt_run_dir" \ - > /etc/tmpfiles.d/bootc-tmt-workdir.conf - disposable_run_dir=$tmt_workdir_parent/.${tmt_run_id}.original.$$ if [[ -e "$disposable_run_dir" || -L "$disposable_run_dir" ]]; then printf 'Disposable TMT workdir already exists: %s\n' "$disposable_run_dir" >&2 @@ -253,6 +248,46 @@ case ${TMT_REBOOT_COUNT:-0} in "$target_image" \ bootc install to-existing-root --preserve-var --acknowledge-destructive + if [[ $tmt_run_layout == var-tmp ]]; then + # to-existing-root does not migrate the old host's /etc. Add the + # tmpfiles rule to the new deployment's persistent /etc instead. + podman --remote=false run --rm --privileged --pid=host --user=root:root \ + -v /dev:/dev \ + -v /:/target \ + -v /boot:/target/boot \ + -v /var/lib/containers:/var/lib/containers \ + --security-opt label=type:unconfined_t \ + "$target_image" \ + bash -euo pipefail -c ' + deployment=/mnt/bootc-tmt-deployment + cleanup() { + local status=$? + trap - EXIT + if ! umount -R "$deployment"; then + printf "Failed to unmount deployment target %s\n" "$deployment" >&2 + status=1 + fi + exit "$status" + } + trap cleanup EXIT + + workdir_parent=$1 + run_dir=$2 + persistent_run_dir=$3 + mkdir -p "$deployment" + bootc install mount --sysroot /target --latest "$deployment" + rule=$deployment/etc/tmpfiles.d/bootc-tmt-workdir.conf + install -d -m 0755 "${rule%/*}" + tmpfiles_rule=( + "d $workdir_parent 0755 root root -" + "L+ $run_dir - - - - $persistent_run_dir" + ) + printf '%s\n' "${tmpfiles_rule[@]}" > "$rule" + printf '%s\n' "${tmpfiles_rule[@]}" | cmp - "$rule" + ' install-tmt-tmpfiles-rule \ + "$tmt_workdir_parent" "$tmt_run_dir" "$persistent_tmt_run_dir" + fi + bootc status # Ask the guest to reboot itself; this is more reliable than # testcloud's ACPI soft-poweroff for a full bootc deployment reboot. @@ -270,6 +305,7 @@ case ${TMT_REBOOT_COUNT:-0} in verify_tmt_workdir_in_preserved_var "$tmt_run_dir" else persistent_tmt_run_dir=$tmt_workdir_root/$tmt_run_id + test -f /etc/tmpfiles.d/bootc-tmt-workdir.conf test -L "$tmt_run_dir" test "$(readlink -f -- "$tmt_run_dir")" = "$persistent_tmt_run_dir" fi From f7d04efdfd854890f5c3b7ab611182ed67123816 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 22 Oct 2026 10:51:14 -0500 Subject: [PATCH 08/38] tests: Restore nested bind-mount preserve-var coverage The split package-mode test no longer exercised how preserve-var handles a nested bind mount. Add a derived image with an enabled mount and verify copied data and the rebooted mount behavior. Assisted-by: AI Signed-off-by: ckyrouac --- tmt/plans/package-mode.fmf | 23 +++++++++++++ .../test-to-existing-root-preserve-var.sh | 33 ++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/tmt/plans/package-mode.fmf b/tmt/plans/package-mode.fmf index 7930c556f4..9966633eff 100644 --- a/tmt/plans/package-mode.fmf +++ b/tmt/plans/package-mode.fmf @@ -131,6 +131,29 @@ prepare: podman --remote=false pull quay.io/curl/curl:latest podman --remote=false pull quay.io/curl/curl-base:latest podman --remote=false pull registry.access.redhat.com/ubi9/podman:latest + workdir=$(mktemp -d) + trap 'rm -rf -- "$workdir"' EXIT + cat > "$workdir/Containerfile" <<'EOF' + FROM localhost/bootc + RUN unit='var-lib-bootc\x2dtmt\x2dpreserve\x2dvar\x2dbind-target.mount' && \ + mkdir -p /usr/etc/systemd/system/local-fs.target.wants && \ + printf '%s\n' \ + '[Unit]' \ + 'Description=bootc TMT preserve-var bind mount' \ + '' \ + '[Mount]' \ + 'What=/var/lib/bootc-tmt-preserve-var-bind/source' \ + 'Where=/var/lib/bootc-tmt-preserve-var-bind/target' \ + 'Type=none' \ + 'Options=bind' \ + '' \ + '[Install]' \ + 'WantedBy=local-fs.target' \ + > "/usr/etc/systemd/system/$unit" && \ + ln -s "../$unit" "/usr/etc/systemd/system/local-fs.target.wants/$unit" + EOF + podman --remote=false build -t localhost/bootc-preserve-var-test "$workdir" + podman --remote=false image exists localhost/bootc-preserve-var-test when: package_source == artifacts or package_source == local execute: how: tmt diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index 4bd9365609..4bcae52089 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -3,8 +3,11 @@ # reboot that same guest; local loads the checkout image, Packit builds it. set -xeuo pipefail -target_image=localhost/bootc +target_image=localhost/bootc-preserve-var-test var_marker=/var/lib/bootc-tmt-preserve-var/sentinel +bind_dir=/var/lib/bootc-tmt-preserve-var-bind +bind_source=$bind_dir/source +bind_target=$bind_dir/target tmt_plan_data_suffix=/tmt/plans/package-mode/data tmt_workdir_parent=/var/tmp/tmt tmt_workdir_root=/var/lib/bootc-tmt-workdirs @@ -118,6 +121,13 @@ case ${TMT_REBOOT_COUNT:-0} in restorecon -RF /var/roothome/.ssh printf '%s\n' preserved-var | tee "$var_marker" + mkdir -p "$bind_source" "$bind_target" + printf '%s\n' bind-source-data > "$bind_source/sentinel" + printf '%s\n' mounted-only-data > "$bind_source/mounted-sentinel" + printf '%s\n' underlying-target-data > "$bind_target/underlying-sentinel" + mount --bind "$bind_source" "$bind_target" + test "$(<"$bind_target/mounted-sentinel")" = mounted-only-data + test ! -e "$bind_target/underlying-sentinel" tmt_run_dir=$(get_tmt_run_dir) tmt_run_id=${tmt_run_dir##*/} @@ -248,6 +258,23 @@ case ${TMT_REBOOT_COUNT:-0} in "$target_image" \ bootc install to-existing-root --preserve-var --acknowledge-destructive + # Find the new stateroot by the copied source sentinel; generated + # stateroot names are intentionally not assumed. + new_var= + for candidate in /ostree/deploy/*/var; do + if [[ -f "$candidate${bind_source#/var}/sentinel" ]]; then + new_var=$candidate + break + fi + done + test -n "$new_var" + test "$(<"$new_var${bind_source#/var}/sentinel")" = bind-source-data + bind_target_in_new_var=$new_var${bind_target#/var} + test -d "$bind_target_in_new_var" + test -z "$(find "$bind_target_in_new_var" -mindepth 1 -print -quit)" + test ! -e "$bind_target_in_new_var/mounted-sentinel" + test ! -e "$bind_target_in_new_var/underlying-sentinel" + if [[ $tmt_run_layout == var-tmp ]]; then # to-existing-root does not migrate the old host's /etc. Add the # tmpfiles rule to the new deployment's persistent /etc instead. @@ -296,6 +323,10 @@ case ${TMT_REBOOT_COUNT:-0} in 1) test -e /run/ostree-booted + mountpoint -q "$bind_target" + test "$(<"$bind_target/mounted-sentinel")" = mounted-only-data + test ! -e "$bind_target/underlying-sentinel" + tmt_run_dir=$(get_tmt_run_dir) tmt_run_id=${tmt_run_dir##*/} if [[ $tmt_run_dir == /var/ARTIFACTS/* ]]; then From 4fa58a9025dc2a830a6d349280db01e37d6b7d87 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Tue, 27 Oct 2026 10:17:50 -0500 Subject: [PATCH 09/38] lib: Harden preserve-var migration safety Reject wipe mode before installation can mutate the running source tree. Make no-clobber collisions visible and clarify the copy, staging, and crash-durability guarantees. Assisted-by: AI Signed-off-by: ckyrouac --- crates/lib/src/install.rs | 63 ++++++++++--- crates/lib/src/install/migrate.rs | 144 ++++++++++++++++++------------ 2 files changed, 138 insertions(+), 69 deletions(-) diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index 866128d86e..fa27a29f76 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -677,6 +677,14 @@ pub(crate) struct InstallToExistingRootOpts { /// `/var/tmp`, `/var/cache`, `/var/log/journal`, and `/var/lib/containers` /// are always excluded. Repeat `--preserve-var-skip` to add paths relative /// to `/var`. Nested mountpoints are represented by empty directories. + /// + /// Do not combine this with `--replace=wipe`: wiping the target would erase + /// the source `/var` before it can be copied. An abrupt termination may + /// leave hidden `.bootc-preserve-var.*` staging directories in the new + /// `/var`; they are not automatically removed later. Verify an individual + /// entry is an abandoned migration artifact before manually removing it. + /// Staged publication is atomic for visibility, but is not guaranteed + /// durable across sudden power loss. #[clap(long)] pub(crate) preserve_var: bool, @@ -2790,8 +2798,10 @@ pub(crate) async fn install_to_filesystem( opts: InstallToFilesystemOpts, targeting_host_root: bool, cleanup: Cleanup, - resolve_bound_images_early: bool, + require_ostree_backend: bool, ) -> Result> { + validate_preserve_var_replace(require_ostree_backend, opts.filesystem_opts.replace)?; + // Log the installation operation to systemd journal const INSTALL_FILESYSTEM_JOURNAL_ID: &str = "9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3"; let source_image = opts @@ -2906,17 +2916,13 @@ pub(crate) async fn install_to_filesystem( // The image can select the composefs backend during preparation, so check // compatibility after that choice is resolved but before target mutation. validate_migration_backend( - resolve_bound_images_early, + require_ostree_backend, state.composefs_options.composefs_backend, )?; - // Callers may opt in when stored bound images must be validated before - // target mutations. Other install flows retain their existing resolution - // timing by passing false. - let pre_resolved_bound_images = if targeting_host_root - && resolve_bound_images_early - && !state.composefs_options.composefs_backend - { + // Resolve stored bound images before replacement or boot-directory cleanup + // can mutate the existing root. Pull mode remains unresolved until deploy. + let pre_resolved_bound_images = if targeting_host_root && require_ostree_backend { Some(BoundImages::from_state(&state).await?) } else { None @@ -2931,8 +2937,10 @@ pub(crate) async fn install_to_filesystem( Some(ReplaceMode::Wipe) => { let rootfs_fd = rootfs_fd.try_clone()?; println!("Wiping contents of root"); - tokio::task::spawn_blocking(move || remove_all_in_dir_no_xdev(&rootfs_fd, true)) - .await??; + tokio::task::spawn_blocking(move || { + wipe_install_target(&rootfs_fd, require_ostree_backend) + }) + .await??; } Some(ReplaceMode::Alongside) => { clean_boot_directories(&target_rootfs_fd, &target_root_path, is_already_ostree)? @@ -3103,6 +3111,20 @@ fn validate_migration_backend(require_ostree_backend: bool, composefs_backend: b Ok(()) } +fn validate_preserve_var_replace(preserve_var: bool, replace: Option) -> Result<()> { + anyhow::ensure!( + !(preserve_var && matches!(replace, Some(ReplaceMode::Wipe))), + "--preserve-var cannot be used with --replace=wipe: wiping the target would erase the source /var before it can be copied" + ); + Ok(()) +} + +fn wipe_install_target(rootfs: &Dir, preserve_var: bool) -> Result<()> { + validate_preserve_var_replace(preserve_var, Some(ReplaceMode::Wipe))?; + remove_all_in_dir_no_xdev(rootfs, true)?; + Ok(()) +} + /// Resolve the physical sysroot when `root_path` points at an OSTree deployment. fn physical_sysroot_path(root_path: &std::path::Path) -> std::path::PathBuf { let possible_sysroot = root_path.join("sysroot"); @@ -3123,6 +3145,10 @@ fn write_destructive_cleanup_marker(root_path: &std::path::Path) -> Result<()> { } pub(crate) async fn install_to_existing_root(opts: InstallToExistingRootOpts) -> Result<()> { + // Reject this before configuration loading, mounting, preparation, or any + // target changes: a wipe would destroy the running source /var. + validate_preserve_var_replace(opts.preserve_var, opts.replace)?; + // Log the existing root installation operation to systemd journal const INSTALL_EXISTING_ROOT_JOURNAL_ID: &str = "7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f1"; let source_image = opts @@ -3813,6 +3839,21 @@ UUID=boot-uuid /boot ext4 defaults 0 0 } } + #[test] + fn preserve_var_rejects_wipe_before_target_contents_can_be_removed() -> Result<()> { + let target = tempfile::tempdir()?; + let sentinel = target.path().join("var/state.db"); + std::fs::create_dir_all(sentinel.parent().unwrap())?; + std::fs::write(&sentinel, b"must remain untouched")?; + + let rootfs = Dir::open_ambient_dir(target.path(), cap_std::ambient_authority())?; + let error = wipe_install_target(&rootfs, true).unwrap_err(); + + assert!(error.to_string().contains("--preserve-var")); + assert_eq!(std::fs::read(&sentinel)?, b"must remain untouched"); + Ok(()) + } + #[tokio::test] async fn test_bound_images_pull_mode_stays_unresolved() { let queried_images = vec![BoundImage { diff --git a/crates/lib/src/install/migrate.rs b/crates/lib/src/install/migrate.rs index 2227cb85f9..00b54be0e7 100644 --- a/crates/lib/src/install/migrate.rs +++ b/crates/lib/src/install/migrate.rs @@ -27,7 +27,7 @@ //! `cp -a` copies each non-ephemeral subdirectory of `/var` into the new //! deployment's stateroot `var/`. This is correct but slow for large `/var` //! trees, and **unsafe for live databases** (see the known-limitation comment -//! on `preserve_var_copy`). +//! on `preserve_var_copy_mounts`). //! //! Nested mountpoints are selected from one `/proc/self/mountinfo` snapshot //! before the copy. This is a point-in-time, best-effort snapshot, not a @@ -35,12 +35,27 @@ //! race with the copy, just like concurrent file changes. //! Each missing entry is copied to a unique sibling staging directory and //! published atomically without replacing an existing destination. A failed -//! entry leaves no partial final entry and can be retried; this is not a global -//! transaction, so entries completely published before a later failure remain. +//! entry leaves no partial final entry and can be retried; there is no global +//! rollback, so entries completely published before a later failure remain. //! -//! Exclusions such as `tmp`, `log/journal`, `lib/containers`, or `lib/rpm` -//! can be supplied through configuration or repeated `--preserve-var-skip` -//! options. +//! The built-in exclusions are `tmp`, `cache`, `log/journal`, and +//! `lib/containers`. Additional exclusions can be supplied through +//! configuration or repeated `--preserve-var-skip` options; `lib/rpm` is not +//! excluded unless explicitly configured. +//! +//! Staging directories use names beginning `.bootc-preserve-var.` beneath the +//! destination `/var`. They are cleaned up during normal success and error +//! handling, but an abrupt termination can leave them behind. They are not +//! automatically removed on a later run because a matching entry could be +//! unrelated. Before manually removing one, ensure no migration is running and +//! verify the individual entry is an artifact of an interrupted migration; do +//! not blindly remove every similarly named entry. +//! +//! Staged rename publication is atomic for namespace visibility, but this code +//! does not fsync copied files or directories. It therefore makes no guarantee +//! that a published entry survives sudden power loss or kernel failure. This is +//! per-entry publication, not a transaction: entries published before a later +//! error remain. //! //! Rollback preservation is intentionally independent from `/var` migration. @@ -217,7 +232,7 @@ fn preserve_var_reflink_mounts( mountpoints, true, false, - OsStr::new("cp"), + OsStr::new(GNU_CP), ) .context("Reflink-copying /var into new deployment (Strategy C)")?; println!(" /var reflink copy complete."); @@ -329,7 +344,7 @@ fn copy_var_filtered_inner( " Creating empty mountpoint placeholder for {}", relative.display() ); - create_staged_placeholder(src_dir, dst_dir, &name)?; + create_staged_placeholder(src_dir, dst_dir, &name, &relative)?; continue; } @@ -362,6 +377,10 @@ fn copy_var_filtered_inner( } else if destination_entry_exists(dst_dir, &name)? { // A destination of a different type is an existing entry, not // permission to replace it with the source directory. + println!( + " Skipping destination collision at /var path {:?}", + relative + ); continue; } else if has_nested_mount || has_nested_exclusion { copy_filtered_directory_staged( @@ -381,6 +400,10 @@ fn copy_var_filtered_inner( } if destination_entry_exists(dst_dir, &name)? { // Match cp --no-clobber without ever opening the existing object. + println!( + " Skipping destination collision at /var path {:?}", + relative + ); continue; } included_entries.push(name); @@ -391,20 +414,22 @@ fn copy_var_filtered_inner( .iter() .map(|name| name.as_os_str()) .collect::>(); - copy_batch_staged( - cp_program, - src_dir, - &names, - dst_dir, - reflink, - &format!("/var/{}", relative_dir.display()), - )?; + copy_batch_staged(cp_program, src_dir, &names, dst_dir, reflink, relative_dir)?; } Ok(()) } static NEXT_STAGING_ID: AtomicU64 = AtomicU64::new(0); +// Bound collision retries so a pathological destination cannot make staging +// allocation loop forever. The process id plus monotonic id makes collisions +// exceptional; one hundred attempts leave room for stale entries while still +// failing promptly if the namespace is obstructed. +const STAGING_NAME_ATTEMPTS: u32 = 100; + +// Do not resolve privileged copy commands through a caller-controlled PATH. +// Lower-level helpers still accept an executable parameter for test injection. +const GNU_CP: &str = "/usr/bin/cp"; /// Make an empty, uniquely-named staging directory beneath the destination /// capability. Publishing from the same parent keeps the final rename atomic. @@ -412,7 +437,7 @@ fn create_staging_dir(parent: &CapStdDir) -> Result<(std::ffi::OsString, CapStdD use rustix::fs::{Mode, mkdirat}; let parent_fd = parent.as_filelike_view::(); - for _ in 0..100 { + for _ in 0..STAGING_NAME_ATTEMPTS { let id = NEXT_STAGING_ID.fetch_add(1, Ordering::Relaxed); let name = std::ffi::OsString::from(format!(".bootc-preserve-var.{}.{}", std::process::id(), id)); @@ -436,8 +461,8 @@ fn create_staging_dir(parent: &CapStdDir) -> Result<(std::ffi::OsString, CapStdD } /// Ensure temporary staging entries are removed on both success and failure. -/// Any entries already published remain complete; transactions are per entry, -/// not a global rollback of earlier successful publications. +/// Any entries already published remain complete; there is no global rollback +/// of earlier successful publications. fn with_staging_dir( parent: &CapStdDir, operation: impl FnOnce(&OsStr, &CapStdDir) -> Result, @@ -470,6 +495,7 @@ fn publish_staged_entry( staging: &CapStdDir, destination: &CapStdDir, name: &OsStr, + relative: &Path, ) -> Result { use rustix::fs::{RenameFlags, renameat_with}; @@ -479,11 +505,15 @@ fn publish_staged_entry( // staged directories into a same-type destination without replacing // existing children; other collisions remain untouched. Err(error) if error == rustix::io::Errno::EXIST => { + println!( + " Destination collision at /var path {:?}; keeping existing entries", + relative + ); if staging.symlink_metadata(name)?.is_dir() { if let Some(destination_dir) = open_existing_child_dir(destination, name)? { let staging_dir = open_child_dir_nofollow(staging, name) .context("Opening staged directory after publish collision")?; - merge_staged_directory(&staging_dir, &destination_dir)?; + merge_staged_directory(&staging_dir, &destination_dir, relative)?; } } Ok(false) @@ -496,7 +526,11 @@ fn publish_staged_entry( /// replacing entries already present there. Keep the destination directory's /// metadata unchanged; moving children updates its timestamps, so restore its /// original metadata after the merge. -fn merge_staged_directory(staging: &CapStdDir, destination: &CapStdDir) -> Result<()> { +fn merge_staged_directory( + staging: &CapStdDir, + destination: &CapStdDir, + relative_dir: &Path, +) -> Result<()> { let metadata = snapshot_directory_metadata(destination)?; let names = staging .read_dir(".")? @@ -505,7 +539,7 @@ fn merge_staged_directory(staging: &CapStdDir, destination: &CapStdDir) -> Resul let merge_result = (|| { for name in names { - publish_staged_entry(staging, destination, &name)?; + publish_staged_entry(staging, destination, &name, &relative_dir.join(&name))?; } Ok(()) })(); @@ -527,7 +561,7 @@ fn copy_batch_staged( names: &[&OsStr], destination: &CapStdDir, reflink: bool, - description: &str, + relative_dir: &Path, ) -> Result<()> { with_staging_dir(destination, |_, staging| { let options = if reflink { @@ -545,10 +579,14 @@ fn copy_batch_staged( Stdio::inherit(), Stdio::inherit(), )?; - anyhow::ensure!(status.success(), "cp failed while copying {description}"); + anyhow::ensure!( + status.success(), + "cp failed while copying /var/{}", + relative_dir.display() + ); for name in names { - publish_staged_entry(staging, destination, name)?; + publish_staged_entry(staging, destination, name, &relative_dir.join(name))?; } Ok(()) }) @@ -577,7 +615,7 @@ fn copy_filtered_directory_staged( true, cp_program, )?; - publish_staged_entry(staging, destination_parent, name)?; + publish_staged_entry(staging, destination_parent, name, relative)?; Ok(()) }) } @@ -586,10 +624,11 @@ fn create_staged_placeholder( src_parent: &CapStdDir, dst_parent: &CapStdDir, name: &OsStr, + relative: &Path, ) -> Result<()> { with_staging_dir(dst_parent, |_, staging| { create_mountpoint_placeholder(src_parent, staging, name)?; - if !publish_staged_entry(staging, dst_parent, name)? { + if !publish_staged_entry(staging, dst_parent, name, relative)? { validate_empty_placeholder(dst_parent, name, src_parent.symlink_metadata(name)?)?; } Ok(()) @@ -880,34 +919,6 @@ fn preserve_metadata_timestamps(metadata: &Metadata, dst_fd: Fd) -> Re Ok(()) } -/// Strategy D: plain recursive copy of `/var` for filesystems without reflink support. -/// -/// # Known limitation -/// -/// This performs a full `cp -a` of every included subdirectory of the running -/// `/var` into the new deployment's ostree stateroot `var/`. For most workloads -/// this is fine, but it is **unsafe for databases and other applications that -/// keep open write handles into `/var`** (e.g. PostgreSQL in `/var/lib/pgsql`, -/// MySQL/MariaDB in `/var/lib/mysql`, SQLite databases under `/var/lib/*`). -/// Copying a live database with `cp -a` will almost certainly produce a -/// corrupted copy. -/// -/// The correct fix is to run this migration only after stopping all stateful -/// services that write to `/var`, or — better — to migrate the filesystem to -/// btrfs or XFS (which support reflinks, Strategy C) to avoid copying every -/// file's data immediately. Reflinks clone files individually; they do not -/// provide an atomic snapshot of the `/var` tree or of a live database. -/// -/// Exclusions can be configured with install configuration or repeated -/// `--preserve-var-skip` options. Paths are relative to `/var` and may be one -/// component (such as `lib`) or exactly one child (such as `lib/pgsql`). `/var` -/// itself and deeper paths are not accepted. Excluded state is not migrated by -/// this command and must be handled separately. Exclusions do not make a live database copy -/// consistent: any database state that is copied while it is being written can -/// still be corrupted. Stop stateful services before copying their data, or use -/// a consistent snapshot. -/// -/// See: #[cfg(test)] #[context("Copying /var into new deployment (Strategy D)")] fn preserve_var_copy( @@ -918,6 +929,23 @@ fn preserve_var_copy( preserve_var_copy_mounts(src_var, new_var, exclusions, &BTreeSet::new()) } +/// Strategy D: plain recursive copy of `/var` for filesystems without reflink support. +/// +/// This performs `cp -a` for included state and is **unsafe for databases and +/// other applications with open write handles into `/var`** (for example, +/// PostgreSQL in `/var/lib/pgsql`, MySQL/MariaDB in `/var/lib/mysql`, or SQLite +/// databases under `/var/lib/*`). A live database copy may be corrupted. +/// Stop stateful services before copying, or use a consistent snapshot. +/// Reflinks (Strategy C) also clone files individually and are not an atomic +/// snapshot of `/var` or a live database. +/// +/// Additional exclusions can be configured or supplied with repeated +/// `--preserve-var-skip` options. Paths are relative to `/var`, with one +/// component or exactly one child component allowed. Excluded state is not +/// migrated and must be handled separately. Exclusions do not make a copied +/// live database consistent. +/// +/// See: fn preserve_var_copy_mounts( src_var: &CapStdDir, new_var: &CapStdDir, @@ -932,7 +960,7 @@ fn preserve_var_copy_mounts( mountpoints, false, false, - OsStr::new("cp"), + OsStr::new(GNU_CP), ) .context("Copying /var into new deployment (Strategy D)")?; println!(" /var copy complete."); @@ -1102,7 +1130,6 @@ mod tests { Ok(()) } - #[test] #[test] fn both_copy_strategies_leave_nested_mountpoints_empty() -> Result<()> { let temp = tempfile::tempdir()?; @@ -1371,7 +1398,8 @@ mod tests { assert!(!publish_staged_entry( staging, &destination, - OsStr::new("tree") + OsStr::new("tree"), + Path::new("tree") )?); assert_eq!( fs::read(destination_path.join("tree/existing"))?, From 03064354214f73641bd3286b81b35685fcacd423 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Tue, 27 Oct 2026 10:34:45 -0500 Subject: [PATCH 10/38] docs: Clarify preserve-var crash durability Clarify that a completed preserve-var copy is not guaranteed to survive sudden power loss or kernel failure, while retaining the consistent-live-snapshot caveat. Assisted-by: AI Signed-off-by: ckyrouac --- docs/src/bootc-installation.7.md | 3 ++- docs/src/man/bootc-install-to-existing-root.8.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/src/bootc-installation.7.md b/docs/src/bootc-installation.7.md index 1467ced9c6..35de8d39ac 100644 --- a/docs/src/bootc-installation.7.md +++ b/docs/src/bootc-installation.7.md @@ -297,7 +297,8 @@ previous installation. With the OSTree backend, `--preserve-var` copies the old `/var` into the new deployment before rebooting. It uses reflinks when supported and otherwise makes a full copy; neither strategy is a consistent snapshot of live data. Stop stateful -services before copying their data. `/var/tmp`, `/var/cache`, `/var/log/journal`, +services before copying their data. A completed copy is not guaranteed to survive +sudden power loss or a kernel failure. `/var/tmp`, `/var/cache`, `/var/log/journal`, and `/var/lib/containers` are always excluded. Use repeatable `--preserve-var-skip PATH` options or the `preserve-var-skip` install configuration key for additional relative paths below `/var`. diff --git a/docs/src/man/bootc-install-to-existing-root.8.md b/docs/src/man/bootc-install-to-existing-root.8.md index 5baf2c280b..9c86b86e57 100644 --- a/docs/src/man/bootc-install-to-existing-root.8.md +++ b/docs/src/man/bootc-install-to-existing-root.8.md @@ -20,7 +20,8 @@ The opt-in `--preserve-var` option copies the running system's `/var` into the new deployment before rebooting. It requires the OSTree backend, uses reflinks when supported, and otherwise performs a full copy. Reflinks are created per file, not as an atomic snapshot; stop stateful services or use a consistent -snapshot before copying live database data. The paths `tmp`, `cache`, +snapshot before copying live database data. A completed copy is not guaranteed +to survive sudden power loss or a kernel failure. The paths `tmp`, `cache`, `log/journal`, and `lib/containers` are always excluded. Repeat `--preserve-var-skip` to exclude additional paths relative to `/var`; each path may contain at most one slash, for example `lib/rpm`. From 86a65e0ea0546704b484a04c650a6732fa006d71 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Tue, 27 Oct 2026 10:44:51 -0500 Subject: [PATCH 11/38] docs: Clarify preserve-var migration guarantees Explain that atomic per-entry namespace visibility does not make the migration a tree-wide snapshot or transaction, and distinguish both from crash durability. This helps operators reason about partial migrations and live database data. Assisted-by: AI Signed-off-by: ckyrouac --- docs/src/bootc-installation.7.md | 12 ++++++++---- docs/src/man/bootc-install-to-existing-root.8.md | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/src/bootc-installation.7.md b/docs/src/bootc-installation.7.md index 35de8d39ac..51ec5e0669 100644 --- a/docs/src/bootc-installation.7.md +++ b/docs/src/bootc-installation.7.md @@ -296,10 +296,14 @@ previous installation. With the OSTree backend, `--preserve-var` copies the old `/var` into the new deployment before rebooting. It uses reflinks when supported and otherwise makes a -full copy; neither strategy is a consistent snapshot of live data. Stop stateful -services before copying their data. A completed copy is not guaranteed to survive -sudden power loss or a kernel failure. `/var/tmp`, `/var/cache`, `/var/log/journal`, -and `/var/lib/containers` are always excluded. Use repeatable +full copy. Each new entry is staged and renamed into place atomically for +namespace visibility. This is not a tree-wide point-in-time snapshot or a +global all-or-nothing transaction: the copy proceeds over time, and a later +failure can leave a partial migration. Neither strategy provides a consistent +snapshot of live data; stop stateful services before copying their data. There +is no crash-durability guarantee: copied content and renames are not guaranteed +to survive sudden power loss or a kernel failure. `/var/tmp`, `/var/cache`, +`/var/log/journal`, and `/var/lib/containers` are always excluded. Use repeatable `--preserve-var-skip PATH` options or the `preserve-var-skip` install configuration key for additional relative paths below `/var`. diff --git a/docs/src/man/bootc-install-to-existing-root.8.md b/docs/src/man/bootc-install-to-existing-root.8.md index 9c86b86e57..1ba30f7cb6 100644 --- a/docs/src/man/bootc-install-to-existing-root.8.md +++ b/docs/src/man/bootc-install-to-existing-root.8.md @@ -19,10 +19,14 @@ to be cleaned up if desired when rebooted into the new root. The opt-in `--preserve-var` option copies the running system's `/var` into the new deployment before rebooting. It requires the OSTree backend, uses reflinks when supported, and otherwise performs a full copy. Reflinks are created per -file, not as an atomic snapshot; stop stateful services or use a consistent -snapshot before copying live database data. A completed copy is not guaranteed -to survive sudden power loss or a kernel failure. The paths `tmp`, `cache`, -`log/journal`, and `lib/containers` are always excluded. Repeat +file. Each new entry is staged and renamed into place atomically for namespace +visibility, but the migration is not a tree-wide point-in-time snapshot or a +global all-or-nothing transaction: it proceeds over time, and a later failure +can leave a partial migration. The copy is not a consistent snapshot of live +data; stop stateful services or use a consistent snapshot before copying live +database data. There is no crash-durability guarantee: copied content and +renames are not guaranteed to survive sudden power loss or a kernel failure. +The paths `tmp`, `cache`, `log/journal`, and `lib/containers` are always excluded. Repeat `--preserve-var-skip` to exclude additional paths relative to `/var`; each path may contain at most one slash, for example `lib/rpm`. From 592ad9a8bd1b0bcc1ac0a4f405f7d9152cc2a9a7 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 10:23:53 -0600 Subject: [PATCH 12/38] install: Preflight preserve-var sources before install Check source /var and nested mountpoint types before installation can replace the old bootable state. Reuse the resulting plan for the post-install copy, while documenting that copy and capacity failures can still occur late. Assisted-by: AI --- crates/lib/src/install.rs | 51 +++-- crates/lib/src/install/migrate.rs | 154 ++++++++++++-- crates/lib/src/install/migrate/mountinfo.rs | 218 +++++++------------- 3 files changed, 251 insertions(+), 172 deletions(-) diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index fa27a29f76..6d374add31 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -2825,21 +2825,8 @@ pub(crate) async fn install_to_filesystem( // And the last bit of state here is the fsopts, which we also destructure now. let mut fsopts = opts.filesystem_opts; - // If we're doing an alongside install, automatically set up the host rootfs - // mount if it wasn't done already. - if targeting_host_root - && fsopts.root_path.as_str() == ALONGSIDE_ROOT_MOUNT - && !fsopts.root_path.try_exists()? - { - tracing::debug!("Mounting host / to {ALONGSIDE_ROOT_MOUNT}"); - std::fs::create_dir(ALONGSIDE_ROOT_MOUNT)?; - bootc_mount::bind_mount_from_pidns( - bootc_mount::PID1, - "/".into(), - ALONGSIDE_ROOT_MOUNT.into(), - true, - ) - .context("Mounting host / to {ALONGSIDE_ROOT_MOUNT}")?; + if targeting_host_root { + mount_alongside_root_if_needed(&fsopts.root_path)?; } let target_root_path = fsopts.root_path.clone(); @@ -3103,6 +3090,23 @@ pub(crate) async fn install_to_filesystem( Ok(deployment_path) } +/// Set up the host root mount for the default alongside target if it has not +/// already been provided by the caller (e.g. with `-v /:/target`). +fn mount_alongside_root_if_needed(root_path: &Utf8Path) -> Result<()> { + if root_path.as_str() == ALONGSIDE_ROOT_MOUNT && !root_path.try_exists()? { + tracing::debug!("Mounting host / to {ALONGSIDE_ROOT_MOUNT}"); + std::fs::create_dir(ALONGSIDE_ROOT_MOUNT)?; + bootc_mount::bind_mount_from_pidns( + bootc_mount::PID1, + "/".into(), + ALONGSIDE_ROOT_MOUNT.into(), + true, + ) + .context("Mounting host / to {ALONGSIDE_ROOT_MOUNT}")?; + } + Ok(()) +} + fn validate_migration_backend(require_ostree_backend: bool, composefs_backend: bool) -> Result<()> { anyhow::ensure!( !require_ostree_backend || !composefs_backend, @@ -3194,6 +3198,19 @@ pub(crate) async fn install_to_existing_root(opts: InstallToExistingRootOpts) -> opts.composefs_opts.composefs_backend, )?; + // Inspect source /var before installation can remove or replace the old + // system's bootable state. The actual copy runs afterward and can still + // fail, for example from concurrent changes or insufficient space. + let var_migration = if preserve_var { + mount_alongside_root_if_needed(&opts.root_path)?; + Some(migrate::VarMigrationPlan::prepare( + &root_path.join("var"), + &preserve_var_skip, + )?) + } else { + None + }; + let fs_opts = InstallToFilesystemOpts { filesystem_opts: InstallTargetFilesystemOpts { root_path: opts.root_path, @@ -3221,7 +3238,7 @@ pub(crate) async fn install_to_existing_root(opts: InstallToExistingRootOpts) -> ) .await?; - if preserve_var { + if let Some(var_migration) = var_migration { println!(); println!("Preserving /var..."); let deployment_path = deployment_path @@ -3230,7 +3247,7 @@ pub(crate) async fn install_to_existing_root(opts: InstallToExistingRootOpts) -> let physical_root = physical_sysroot_path(&root_path); let deploy_dir = physical_root.join(deployment_path); let new_var = migrate::deployment_var_path(&deploy_dir)?; - migrate::preserve_var(&root_path.join("var"), &new_var, &preserve_var_skip) + migrate::preserve_var(&var_migration, &new_var) .context("Post-install /var preservation")?; } diff --git a/crates/lib/src/install/migrate.rs b/crates/lib/src/install/migrate.rs index 00b54be0e7..0c814b54de 100644 --- a/crates/lib/src/install/migrate.rs +++ b/crates/lib/src/install/migrate.rs @@ -83,32 +83,99 @@ use cap_std_ext::cmdext::{CapStdExtCommandExt, CmdFds}; #[cfg(test)] use fn_error_context::context; +/// Paths below `/var` which `--preserve-var` never copies. +const ALWAYS_EXCLUDED: &[&str] = &["tmp", "cache", "log/journal", "lib/containers"]; + +/// Source-side work identified before the install starts. Copying remains a +/// post-install operation and can still fail (for example, due to space or +/// concurrent source changes). +#[derive(Debug)] +pub(crate) struct VarMigrationPlan { + /// Canonical path of the running system's `/var`. + src_var: PathBuf, + /// Mounts below `src_var`, relative to it, which get empty placeholders. + nested_mounts: BTreeSet, + /// Built-in and requested exclusions, relative to `src_var`. + exclusions: Vec, +} + +impl VarMigrationPlan { + /// Resolve and inspect source `/var` before installation touches the target. + pub(crate) fn prepare(src_var: &Path, exclusions: &[String]) -> Result { + let canonical_src_var = std::fs::canonicalize(src_var) + .with_context(|| format!("Canonicalizing source /var {}", src_var.display()))?; + let source = CapStdDir::open_ambient_dir(&canonical_src_var, ambient_authority()) + .with_context(|| format!("Opening source {}", canonical_src_var.display()))?; + for entry in source + .entries() + .with_context(|| format!("Listing source {}", canonical_src_var.display()))? + { + entry.with_context(|| format!("Reading source {}", canonical_src_var.display()))?; + } + + let nested_mounts = mountinfo::nested_mountpoints_for(&canonical_src_var)?; + let exclusions = ALWAYS_EXCLUDED + .iter() + .map(|path| (*path).to_string()) + .chain(exclusions.iter().cloned()) + .collect::>(); + + validate_mountpoint_types(&canonical_src_var, &exclusions, &nested_mounts)?; + + Ok(Self { + src_var: canonical_src_var, + nested_mounts, + exclusions, + }) + } +} + +/// Only directories and regular files can receive safe placeholders. Do not +/// inspect mountpoints the filtered copy will never reach. +fn validate_mountpoint_types( + src_var: &Path, + exclusions: &[String], + nested_mounts: &BTreeSet, +) -> Result<()> { + for mount in nested_mounts { + let is_excluded = exclusions + .iter() + .any(|excluded| mount.starts_with(excluded)); + let is_below_another_mount = nested_mounts + .iter() + .any(|other| other != mount && mount.starts_with(other)); + if is_excluded || is_below_another_mount { + continue; + } + let metadata = std::fs::symlink_metadata(src_var.join(mount)) + .with_context(|| format!("Inspecting mountpoint /var/{}", mount.display()))?; + anyhow::ensure!( + metadata.is_dir() || metadata.is_file(), + "/var/{} is a mountpoint that is neither a directory nor a file; exclude it with --preserve-var-skip", + mount.display() + ); + } + Ok(()) +} + /// Preserve the running `/var` into the new deployment's `var/` directory. /// -/// `src_var` is the running system's `/var` (at `/var`). -/// `new_var` is the new deployment's empty `var/` directory. -pub(crate) fn preserve_var(src_var: &Path, new_var: &Path, exclusions: &[String]) -> Result<()> { - // Resolve symlink aliases and path components before comparing against the - // absolute, kernel-resolved mountpoints in mountinfo. - let canonical_src_var = std::fs::canonicalize(src_var) - .with_context(|| format!("Canonicalizing source /var {}", src_var.display()))?; - let nested_mounts = mountinfo::nested_mountpoints_for(&canonical_src_var)?; - let src_var = CapStdDir::open_ambient_dir(&canonical_src_var, ambient_authority()) - .with_context(|| format!("Opening source {}", canonical_src_var.display()))?; +/// `new_var` is the new deployment's `var/` directory, which may have +/// image-seeded content. +pub(crate) fn preserve_var(plan: &VarMigrationPlan, new_var: &Path) -> Result<()> { + let src_var = CapStdDir::open_ambient_dir(&plan.src_var, ambient_authority()) + .with_context(|| format!("Opening source {}", plan.src_var.display()))?; let new_var = open_or_create_var_dir(new_var)?; - let mut effective_exclusions = ["tmp", "cache", "log/journal", "lib/containers"] - .into_iter() - .map(String::from) - .collect::>(); - effective_exclusions.extend(exclusions.iter().cloned()); + let effective_exclusions = &plan.exclusions; + let nested_mounts = &plan.nested_mounts; let copy_result = if supports_reflink(&src_var, &new_var)? { println!(" Filesystem supports reflinks — using copy-on-write clone (Strategy C)"); - preserve_var_reflink_mounts(&src_var, &new_var, &effective_exclusions, &nested_mounts) + preserve_var_reflink_mounts(&src_var, &new_var, effective_exclusions, nested_mounts) } else { println!(" Filesystem does not support reflinks — falling back to full copy (Strategy D)"); - preserve_var_copy_mounts(&src_var, &new_var, &effective_exclusions, &nested_mounts) + preserve_var_copy_mounts(&src_var, &new_var, effective_exclusions, nested_mounts) }; copy_result?; @@ -1085,6 +1152,59 @@ mod tests { assert!(validate_exclusions(&["lib/containers/storage".into()]).is_err()); } + #[test] + fn migration_plan_preflights_source_and_combines_exclusions() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_var = temp.path().join("var"); + fs::create_dir(&src_var)?; + fs::write(src_var.join("sentinel"), b"source remains untouched")?; + + let plan = VarMigrationPlan::prepare(&src_var, &["lib/rpm".to_string()])?; + assert_eq!(plan.src_var, fs::canonicalize(&src_var)?); + assert_eq!( + plan.exclusions, + ["tmp", "cache", "log/journal", "lib/containers", "lib/rpm"] + ); + assert_eq!( + fs::read(src_var.join("sentinel"))?, + b"source remains untouched" + ); + + let error = VarMigrationPlan::prepare(&temp.path().join("missing"), &[]).unwrap_err(); + assert!( + format!("{error:#}").contains("Canonicalizing source /var"), + "{error:#}" + ); + Ok(()) + } + + #[test] + fn preflight_checks_only_reachable_mountpoint_types() -> Result<()> { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir()?; + let src_var = temp.path().join("var"); + fs::create_dir_all(src_var.join("directory"))?; + fs::write(src_var.join("file"), b"")?; + symlink("file", src_var.join("symlink"))?; + let mounts = BTreeSet::from([ + PathBuf::from("directory"), + PathBuf::from("file"), + PathBuf::from("symlink"), + PathBuf::from("tmp/excluded"), + PathBuf::from("directory/nested"), + ]); + + let error = validate_mountpoint_types(&src_var, &["tmp".to_string()], &mounts).unwrap_err(); + assert!(format!("{error:#}").contains("/var/symlink")); + validate_mountpoint_types( + &src_var, + &["tmp".to_string(), "symlink".to_string()], + &mounts, + )?; + Ok(()) + } + #[test] fn deployment_var_is_stateroot_var() { let deploy = Path::new("/sysroot/ostree/deploy/default/deploy/checksum.0"); diff --git a/crates/lib/src/install/migrate/mountinfo.rs b/crates/lib/src/install/migrate/mountinfo.rs index 13eb5d3eb3..6b82bfd381 100644 --- a/crates/lib/src/install/migrate/mountinfo.rs +++ b/crates/lib/src/install/migrate/mountinfo.rs @@ -14,6 +14,9 @@ use std::{ use anyhow::{Context, Result}; +/// The mountpoint is the fifth space-separated mountinfo field. +const MOUNTINFO_MOUNTPOINT_FIELD: usize = 4; + /// Read one snapshot and return the paths of mounts nested beneath `src_var`. /// /// `src_var` is the canonical path used to open the source directory; @@ -57,139 +60,59 @@ fn nested_mountpoints( Ok(nested) } -/// Parse the byte-oriented Linux mountinfo format without requiring UTF-8. -/// Kernel pathname escapes are decoded before paths are constructed. +/// Extract mountpoints from the byte-oriented Linux mountinfo format without +/// requiring UTF-8. Only field 5 is relevant; in particular, empty fields and +/// malformed trailing fields in records for unrelated mounts are ignored. +/// Literal spaces are the field delimiter, so CR, form-feed, and other path +/// bytes remain intact. Kernel pathname escapes are decoded before paths are +/// constructed. fn parse_mountinfo(data: &[u8]) -> Result> { - let mut mountpoints = Vec::new(); - let lines = data.split(|byte| *byte == b'\n'); - let line_count = lines.clone().count(); - for (index, line) in lines.enumerate() { - if line.is_empty() && index + 1 == line_count { - continue; // A final newline is conventional. - } - anyhow::ensure!(!line.is_empty(), "Empty mountinfo line {}", index + 1); - anyhow::ensure!( - !line.contains(&0), - "NUL byte in mountinfo line {}", - index + 1 - ); - let fields = line - .split(|byte| byte.is_ascii_whitespace()) - .filter(|field| !field.is_empty()) - .collect::>(); - let separator = fields - .iter() - .enumerate() - .skip(6) - .find_map(|(index, field)| (*field == b"-").then_some(index)) - .with_context(|| { - format!("Malformed mountinfo line {}: missing separator", index + 1) - })?; - anyhow::ensure!( - separator >= 6 && fields.len() == separator + 4, - "Malformed mountinfo line {}: invalid field count", - index + 1 - ); - anyhow::ensure!( - is_positive_decimal(fields[0]) - && is_decimal(fields[1]) - && is_device_number(fields[2]) - && !fields[5].is_empty() - && !fields[separator + 1].is_empty() - && !fields[separator + 2].is_empty() - && !fields[separator + 3].is_empty(), - "Malformed mountinfo line {}: invalid required field", - index + 1 - ); - // The root field is not used for mount selection. Decode it to ensure - // its escape syntax is valid, but don't impose pathname constraints: - // unlike the mountpoint, it may be relative or noncanonical. - decode_mountinfo_escapes(fields[3]) - .with_context(|| format!("Malformed root path on mountinfo line {}", index + 1))?; - let mountpoint = decode_mountinfo_path(fields[4]) - .with_context(|| format!("Malformed mountpoint on mountinfo line {}", index + 1))?; - decode_mountinfo_escapes(fields[separator + 2]) - .with_context(|| format!("Malformed source path on mountinfo line {}", index + 1))?; - mountpoints.push(mountpoint); - } + let mountpoints = data + .split(|byte| *byte == b'\n') + .enumerate() + .filter(|(_, line)| !line.is_empty()) + .map(|(index, line)| { + let field = line + .split(|byte| *byte == b' ') + .nth(MOUNTINFO_MOUNTPOINT_FIELD) + .with_context(|| { + format!("Malformed mountinfo line {}: no mountpoint", index + 1) + })?; + Ok(PathBuf::from(std::ffi::OsString::from_vec( + decode_mountinfo_escapes(field), + ))) + }) + .collect::>>()?; anyhow::ensure!(!mountpoints.is_empty(), "Mountinfo contained no records"); Ok(mountpoints) } -fn is_decimal(value: &[u8]) -> bool { - !value.is_empty() - && value.iter().all(u8::is_ascii_digit) - && std::str::from_utf8(value) - .ok() - .and_then(|value| value.parse::().ok()) - .is_some() -} - -fn is_positive_decimal(value: &[u8]) -> bool { - is_decimal(value) - && std::str::from_utf8(value) - .ok() - .and_then(|value| value.parse::().ok()) - .is_some_and(|value| value > 0) -} - -fn is_device_number(value: &[u8]) -> bool { - let Some(separator) = value.iter().position(|byte| *byte == b':') else { - return false; - }; - is_decimal(&value[..separator]) && is_decimal(&value[separator + 1..]) -} - -fn decode_mountinfo_path(value: &[u8]) -> Result { - let decoded = decode_mountinfo_escapes(value)?; - anyhow::ensure!( - decoded.first() == Some(&b'/'), - "Mountinfo path is not absolute" - ); - let mut components = decoded[1..].split(|byte| *byte == b'/'); - anyhow::ensure!( - decoded.len() == 1 - || components.all(|component| { - !component.is_empty() && component != b"." && component != b".." - }), - "Mountinfo path is not canonical" - ); - Ok(PathBuf::from(std::ffi::OsString::from_vec(decoded))) -} - -fn decode_mountinfo_escapes(value: &[u8]) -> Result> { +/// Decode the kernel's `\ooo` escapes for space, tab, newline, and backslash. +/// Unknown or malformed escape sequences are retained as literal bytes. +fn decode_mountinfo_escapes(value: &[u8]) -> Vec { let mut decoded = Vec::with_capacity(value.len()); - let mut index = 0; - while index < value.len() { - if value[index] != b'\\' { - decoded.push(value[index]); - index += 1; - continue; - } - anyhow::ensure!( - index + 3 < value.len() - && value[index + 1..index + 4] + let mut rest = value; + while let Some((&byte, tail)) = rest.split_first() { + let escaped = (byte == b'\\') + .then(|| tail.get(..3)) + .flatten() + .filter(|digits| digits.iter().all(|digit| (b'0'..=b'7').contains(digit))) + .and_then(|digits| { + let value = digits .iter() - .all(|byte| (b'0'..=b'7').contains(byte)), - "Invalid octal escape in mountinfo path" - ); - // Mountinfo escapes are three octal digits. Decode wider than a byte - // so malformed values such as `\777` cannot overflow or wrap. - let escaped = u16::from(value[index + 1] - b'0') * 64 - + u16::from(value[index + 2] - b'0') * 8 - + u16::from(value[index + 3] - b'0'); - anyhow::ensure!( - escaped <= u16::from(u8::MAX), - "Octal escape exceeds one byte" - ); - anyhow::ensure!( - matches!(escaped, 0o11 | 0o12 | 0o40 | 0o134), - "Unsupported octal escape in mountinfo path" - ); - decoded.push(escaped as u8); - index += 4; + .fold(0u16, |acc, digit| acc * 8 + u16::from(digit - b'0')); + u8::try_from(value).ok() + }) + .filter(|value| matches!(value, b'\t' | b'\n' | b' ' | b'\\')); + if let Some(escaped) = escaped { + decoded.push(escaped); + rest = &tail[3..]; + } else { + decoded.push(byte); + rest = tail; + } } - Ok(decoded) + decoded } #[cfg(test)] @@ -229,6 +152,19 @@ mod tests { Ok(()) } + #[test] + fn mountinfo_preserves_cr_form_feed_and_decodes_kernel_escapes() -> Result<()> { + use std::os::unix::ffi::OsStrExt; + + let parsed = parse_mountinfo( + b"1 0 0:1 / /target/var rw - tmpfs tmpfs rw\n\ + 2 1 0:2 / /target/var/a\\040b\\011c\\012d\\134e\r\x0cf rw - x x rw\n", + )?; + let mountpoint = parsed[1].as_os_str().as_bytes(); + assert_eq!(mountpoint, b"/target/var/a b\tc\nd\\e\r\x0cf"); + Ok(()) + } + #[test] fn unrelated_mount_records_do_not_change_nested_mount_selection() -> Result<()> { let temp = tempfile::tempdir()?; @@ -277,24 +213,30 @@ mod tests { } #[test] - fn mountinfo_rejects_malformed_records_and_escapes() { - for mountinfo in [ - b"".as_slice(), - b"not a mountinfo record\n", - b"36 25 0:32 /bad\\04x /target/var rw - tmpfs tmpfs rw\n", - b"36 25 0:32 / /target/var\\04x rw - tmpfs tmpfs rw\n", - b"36 25 0:32 / /target/var\\400 rw - tmpfs tmpfs rw\n", - b"36 25 0:32 / /target/var\\777 rw - tmpfs tmpfs rw\n", - b"36 25 0:32 / /target/var\\041 rw - tmpfs tmpfs rw\n", - b"36 25 0:32 / relative rw - tmpfs tmpfs rw\n", - b"36 25 0:32 / /target/var/../outside rw - tmpfs tmpfs rw\n", - b"36 25 0:32 / /target/var rw - tmpfs tmpfs rw extra\n", - ] { + fn mountinfo_ignores_empty_source_and_unrelated_malformed_trailing_fields() -> Result<()> { + let parsed = parse_mountinfo( + b"36 25 0:32 / /target/var rw - tmpfs tmpfs rw\n\ + 37 36 0:33 / /target/var/lib/mounted rw,relatime shared:1 - tmpfs rw\n\ + 38 36 0:34 / /unrelated rw - missing-fields\n", + )?; + assert_eq!( + parsed, + [ + PathBuf::from("/target/var"), + PathBuf::from("/target/var/lib/mounted"), + PathBuf::from("/unrelated") + ] + ); + let nested = nested_mountpoints(Path::new("/target/var"), parsed)?; + assert_eq!(nested, BTreeSet::from([PathBuf::from("lib/mounted")])); + + for mountinfo in [b"".as_slice(), b"\n", b"36 25 0:32 /\n"] { assert!( parse_mountinfo(mountinfo).is_err(), "accepted {mountinfo:?}" ); } + Ok(()) } #[test] From 6128c9db25e2658fcd4cb5506a0344e3d0beb98d Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 10:26:09 -0600 Subject: [PATCH 13/38] install: Merge image-seeded preserve-var mountpoints The new deployment's /var can already contain image data at nested mount destinations. Retain those entries when their types match, preserve policy-provided SELinux labels, and let cp fall back for individual files that cannot be reflinked. Assisted-by: AI --- crates/lib/src/install.rs | 8 +- crates/lib/src/install/migrate.rs | 125 +++++++++++++++++++----------- 2 files changed, 82 insertions(+), 51 deletions(-) diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index 6d374add31..0cc3204f19 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -669,10 +669,10 @@ pub(crate) struct InstallToExistingRootOpts { /// Preserve the running system's `/var` data into the new bootc deployment. /// - /// Non-excluded content is copied with reflinks when supported, or with a - /// full copy otherwise. Reflinks are created per file and are not a - /// snapshot; stop stateful services or use a consistent snapshot before - /// copying live database data. + /// Non-excluded content is copied with reflinks where individual files + /// support them, falling back to ordinary copies as needed. Reflinks are + /// created per file and are not a snapshot; stop stateful services or use + /// a consistent snapshot before copying live database data. /// /// `/var/tmp`, `/var/cache`, `/var/log/journal`, and `/var/lib/containers` /// are always excluded. Repeat `--preserve-var-skip` to add paths relative diff --git a/crates/lib/src/install/migrate.rs b/crates/lib/src/install/migrate.rs index 0c814b54de..acd7f4c64b 100644 --- a/crates/lib/src/install/migrate.rs +++ b/crates/lib/src/install/migrate.rs @@ -18,10 +18,10 @@ //! //! Two strategies are tried in order: //! -//! - **Strategy C — reflink copy** (btrfs / XFS with reflinks): `cp --reflink=always` -//! clones files copy-on-write, avoiding an immediate second copy of their data. -//! Files are cloned individually, so this is not an atomic snapshot of the -//! `/var` tree or of a live database. +//! - **Strategy C — reflink copy** (btrfs / XFS with reflinks): `cp --reflink=auto` +//! clones files copy-on-write where possible and falls back to a regular copy +//! for individual files that cannot be cloned. Files are handled individually, +//! so this is not an atomic snapshot of the `/var` tree or of a live database. //! //! - **Strategy D — plain copy** (ext4 and other non-reflink filesystems): //! `cp -a` copies each non-ephemeral subdirectory of `/var` into the new @@ -497,6 +497,7 @@ const STAGING_NAME_ATTEMPTS: u32 = 100; // Do not resolve privileged copy commands through a caller-controlled PATH. // Lower-level helpers still accept an executable parameter for test injection. const GNU_CP: &str = "/usr/bin/cp"; +const CP_REFLINK_OPTIONS: &[&str] = &["--reflink=auto", "-a"]; /// Make an empty, uniquely-named staging directory beneath the destination /// capability. Publishing from the same parent keeps the final rename atomic. @@ -632,7 +633,7 @@ fn copy_batch_staged( ) -> Result<()> { with_staging_dir(destination, |_, staging| { let options = if reflink { - &["--reflink=always", "-a"][..] + CP_REFLINK_OPTIONS } else { &["-a"][..] }; @@ -696,41 +697,43 @@ fn create_staged_placeholder( with_staging_dir(dst_parent, |_, staging| { create_mountpoint_placeholder(src_parent, staging, name)?; if !publish_staged_entry(staging, dst_parent, name, relative)? { - validate_empty_placeholder(dst_parent, name, src_parent.symlink_metadata(name)?)?; + validate_existing_placeholder( + dst_parent, + name, + relative, + src_parent.symlink_metadata(name)?, + )?; } Ok(()) }) } -fn validate_empty_placeholder( +fn validate_existing_placeholder( destination_parent: &CapStdDir, name: &OsStr, + relative: &Path, source_metadata: Metadata, ) -> Result<()> { let destination_metadata = destination_parent.symlink_metadata(name)?; - if source_metadata.is_dir() { - anyhow::ensure!( - destination_metadata.is_dir(), - "Mountpoint placeholder {} exists with the wrong type", - name.to_string_lossy() - ); - let destination = open_child_dir_nofollow(destination_parent, name)?; - let mut entries = destination.read_dir(".")?; - anyhow::ensure!( - entries.next().transpose()?.is_none(), - "Mountpoint placeholder {} is not empty", - name.to_string_lossy() - ); + let same_type = (source_metadata.is_dir() && destination_metadata.is_dir()) + || (source_metadata.is_file() && destination_metadata.is_file()); + anyhow::ensure!( + same_type, + "Mountpoint placeholder /var/{} already exists in the new deployment with a different type", + relative.display() + ); + let has_content = if destination_metadata.is_dir() { + open_child_dir_nofollow(destination_parent, name)? + .entries()? + .next() + .is_some() } else { - anyhow::ensure!( - source_metadata.is_file() && destination_metadata.is_file(), - "Mountpoint placeholder {} exists with the wrong type", - name.to_string_lossy() - ); - anyhow::ensure!( - destination_metadata.len() == 0, - "Mountpoint placeholder {} is not empty", - name.to_string_lossy() + destination_metadata.len() > 0 + }; + if has_content { + println!( + " Keeping the image's content at mountpoint /var/{}; the mount hides it at boot", + relative.display() ); } Ok(()) @@ -895,8 +898,12 @@ fn preserve_metadata( preserve_metadata_timestamps(metadata, &dst_fd) } -/// Make the destination xattr set exactly match the source snapshot. Removing -/// extras first matters for inherited ACLs and labels as well as user xattrs. +const SELINUX_XATTR: &str = "security.selinux"; + +/// Make the destination xattr set match the source snapshot, except that a +/// destination SELinux label is retained when the source has no label. +/// Removing extras first matters for inherited ACLs and labels as well as user +/// xattrs. fn apply_xattrs_exact( destination: Fd, source_xattrs: &[(std::ffi::OsString, Vec)], @@ -910,7 +917,8 @@ fn apply_xattrs_exact( .map(|(name, _)| name.as_os_str()) .collect::>(); for name in destination_names { - if !source_names.contains(name.as_os_str()) { + if !source_names.contains(name.as_os_str()) && name.as_os_str() != OsStr::new(SELINUX_XATTR) + { fremovexattr(&destination, &name).with_context(|| { format!("Removing destination xattr {}", name.to_string_lossy()) })?; @@ -1408,6 +1416,30 @@ mod tests { Ok(()) } + #[test] + fn apply_xattrs_exact_preserves_destination_selinux_when_source_lacks_it() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("destination"); + fs::write(&path, b"")?; + let destination = cap_dir(temp.path()); + if destination + .setxattr("destination", SELINUX_XATTR, b"test-label") + .is_err() + { + // Some test filesystems or unprivileged environments cannot set + // security.selinux; exercise the behavior where supported. + return Ok(()); + } + + let file = fs::File::open(&path)?; + apply_xattrs_exact(&file, &[])?; + assert_eq!( + destination.getxattr("destination", SELINUX_XATTR)?, + Some(b"test-label".to_vec()) + ); + Ok(()) + } + #[test] fn filtered_copy_does_not_change_existing_destination_directory() -> Result<()> { let temp = tempfile::tempdir()?; @@ -1604,7 +1636,7 @@ mod tests { let failing_cp = temp.path().join("cp-fails-after-copy"); fs::write( &failing_cp, - b"#!/bin/sh\nreflink=no\nwhile [ \"$1\" != -- ]; do\n [ \"$1\" = --reflink=always ] && reflink=yes\n shift\ndone\nshift\nsource=$1\nshift\nwhile [ \"$#\" -gt 1 ]; do shift; done\ndestination=$1\nif [ \"$reflink\" = yes ]; then\n /bin/cp --reflink=always -a -- \"$source\" \"$destination\"\nelse\n /bin/cp -a -- \"$source\" \"$destination\"\nfi\nexit 1\n", + b"#!/bin/sh\nreflink=no\nwhile [ \"$1\" != -- ]; do\n [ \"$1\" = --reflink=auto ] && reflink=yes\n shift\ndone\nshift\nsource=$1\nshift\nwhile [ \"$#\" -gt 1 ]; do shift; done\ndestination=$1\nif [ \"$reflink\" = yes ]; then\n /bin/cp --reflink=auto -a -- \"$source\" \"$destination\"\nelse\n /bin/cp -a -- \"$source\" \"$destination\"\nfi\nexit 1\n", )?; fs::set_permissions(&failing_cp, fs::Permissions::from_mode(0o755))?; @@ -1643,7 +1675,12 @@ mod tests { } #[test] - fn empty_mountpoint_placeholders_are_retry_safe_and_type_checked() -> Result<()> { + fn reflink_copy_uses_auto_fallback_for_individual_files() { + assert_eq!(CP_REFLINK_OPTIONS, ["--reflink=auto", "-a"]); + } + + #[test] + fn mountpoint_placeholders_reuse_seeded_destinations_and_check_type() -> Result<()> { let temp = tempfile::tempdir()?; let src_path = temp.path().join("src"); fs::create_dir_all(src_path.join("directory-mount"))?; @@ -1674,25 +1711,19 @@ mod tests { assert_eq!(fs::read_dir(dst_path.join("directory-mount"))?.count(), 0); assert_eq!(fs::metadata(dst_path.join("file-mount"))?.len(), 0); + // Image-seeded content at a destination that will be hidden by + // the mount is safe to retain when its type matches. fs::write(dst_path.join("directory-mount/sentinel"), b"keep")?; - let error = if reflink { - preserve_var_reflink_mounts(&src, &dst, &[], &mounts) + fs::write(dst_path.join("file-mount"), b"keep file")?; + if reflink { + preserve_var_reflink_mounts(&src, &dst, &[], &mounts)?; } else { - preserve_var_copy_mounts(&src, &dst, &[], &mounts) - }; - assert!(error.is_err()); + preserve_var_copy_mounts(&src, &dst, &[], &mounts)?; + } assert_eq!( fs::read(dst_path.join("directory-mount/sentinel"))?, b"keep" ); - fs::remove_file(dst_path.join("directory-mount/sentinel"))?; - fs::write(dst_path.join("file-mount"), b"keep file")?; - let error = if reflink { - preserve_var_reflink_mounts(&src, &dst, &[], &mounts) - } else { - preserve_var_copy_mounts(&src, &dst, &[], &mounts) - }; - assert!(error.is_err()); assert_eq!(fs::read(dst_path.join("file-mount"))?, b"keep file"); let wrong_type = temp.path().join(format!("wrong-type-{strategy}")); From de01e7639e955961afbccbef1b7e024581a98d21 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 10:27:06 -0600 Subject: [PATCH 14/38] install: Gate preserve-var exclusions Skip paths only affect preserve-var installs. Require the CLI option to be paired with --preserve-var and defer reading and validating config exclusions until that mode is active. Assisted-by: AI --- crates/lib/src/install.rs | 80 ++++++++++++++++++++++++++++---- crates/lib/src/install/config.rs | 2 +- 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index 0cc3204f19..4a9730d731 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -689,7 +689,12 @@ pub(crate) struct InstallToExistingRootOpts { pub(crate) preserve_var: bool, /// Relative paths below `/var` to leave out while preserving `/var`. - #[clap(long = "preserve-var-skip", value_name = "PATH")] + /// Requires `--preserve-var`. + #[clap( + long = "preserve-var-skip", + value_name = "PATH", + requires = "preserve_var" + )] pub(crate) preserve_var_skip: Vec, } @@ -3115,6 +3120,22 @@ fn validate_migration_backend(require_ostree_backend: bool, composefs_backend: b Ok(()) } +/// Resolve skip paths only for preserve-var installs. The loader is deferred +/// so an unrelated to-existing-root install never validates these options. +fn resolve_preserve_var_skip( + preserve_var: bool, + mut cli_skip: Vec, + load_config_skip: impl FnOnce() -> Result>>, +) -> Result> { + if !preserve_var { + return Ok(Vec::new()); + } + if let Some(config_skip) = load_config_skip()? { + cli_skip.extend(config_skip); + } + migrate::validate_exclusions(&cli_skip) +} + fn validate_preserve_var_replace(preserve_var: bool, replace: Option) -> Result<()> { anyhow::ensure!( !(preserve_var && matches!(replace, Some(ReplaceMode::Wipe))), @@ -3185,13 +3206,10 @@ pub(crate) async fn install_to_existing_root(opts: InstallToExistingRootOpts) -> let preserve_var = opts.preserve_var; let defer_cleanup = opts.cleanup && preserve_var; let root_path = std::path::PathBuf::from(opts.root_path.as_str()); - let mut preserve_var_skip = opts.preserve_var_skip; - if let Some(config) = config::load_config()? { - if let Some(config_skip) = config.preserve_var_skip { - preserve_var_skip.extend(config_skip); - } - } - let preserve_var_skip = migrate::validate_exclusions(&preserve_var_skip)?; + let preserve_var_skip = + resolve_preserve_var_skip(preserve_var, opts.preserve_var_skip, || { + Ok(config::load_config()?.and_then(|config| config.preserve_var_skip)) + })?; let require_ostree_backend = preserve_var; validate_migration_backend( require_ostree_backend, @@ -3437,6 +3455,7 @@ pub(crate) async fn install_finalize(target: &Utf8Path) -> Result<()> { #[cfg(test)] mod tests { use super::*; + use clap::Parser; #[test] fn test_composefs_opts_validate() { @@ -3856,6 +3875,51 @@ UUID=boot-uuid /boot ext4 defaults 0 0 } } + #[test] + fn preserve_var_skip_requires_preserve_var() { + assert!( + InstallToExistingRootOpts::try_parse_from(["bootc", "--preserve-var-skip", "lib/rpm"]) + .is_err() + ); + assert!( + InstallToExistingRootOpts::try_parse_from([ + "bootc", + "--preserve-var", + "--preserve-var-skip", + "lib/rpm" + ]) + .is_ok() + ); + } + + #[test] + fn preserve_var_skip_config_is_ignored_unless_preserving_var() -> Result<()> { + let mut config_loaded = false; + let ordinary = resolve_preserve_var_skip(false, Vec::new(), || { + config_loaded = true; + Ok(Some(vec!["../invalid".to_string()])) + })?; + assert!(ordinary.is_empty()); + assert!( + !config_loaded, + "ordinary installs must not load skip config" + ); + + let error = resolve_preserve_var_skip(true, Vec::new(), || { + Ok(Some(vec!["../invalid".to_string()])) + }) + .unwrap_err(); + assert!(format!("{error:#}").contains("invalid /var exclusion")); + + assert_eq!( + resolve_preserve_var_skip(true, vec!["lib/cli".to_string()], || { + Ok(Some(vec!["lib/config".to_string()])) + })?, + ["lib/cli", "lib/config"] + ); + Ok(()) + } + #[test] fn preserve_var_rejects_wipe_before_target_contents_can_be_removed() -> Result<()> { let target = tempfile::tempdir()?; diff --git a/crates/lib/src/install/config.rs b/crates/lib/src/install/config.rs index fc3e86c599..8595c74e16 100644 --- a/crates/lib/src/install/config.rs +++ b/crates/lib/src/install/config.rs @@ -132,7 +132,7 @@ pub(crate) struct InstallConfiguration { /// Enforce that the containers-storage stack has a non-default /// (i.e. not `insecureAcceptAnything`) container image signature policy. pub(crate) enforce_container_sigpolicy: Option, - /// Paths below `/var` which should not be copied by `--preserve-var`. + /// Additional paths below `/var` to skip when `--preserve-var` is enabled. pub(crate) preserve_var_skip: Option>, } From 4f2e9423a90cae3ad982b5975417dd5b14fb9f0a Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 10:46:42 -0600 Subject: [PATCH 15/38] install: Relabel preserved /var from target policy HostDisabled installs can skip physical-root relabeling of old /var, leaving preserved data with source labels. Relabel staged migration entries against the installed deployment's logical /var paths before publication, without changing seeded collision winners or unrelated metadata. Assisted-by: AI --- crates/lib/src/install.rs | 8 +- crates/lib/src/install/migrate.rs | 457 +++++++++++++++++++++++++++++- crates/lib/src/lsm.rs | 41 +++ 3 files changed, 498 insertions(+), 8 deletions(-) diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index 4a9730d731..b7a74e140a 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -3265,7 +3265,13 @@ pub(crate) async fn install_to_existing_root(opts: InstallToExistingRootOpts) -> let physical_root = physical_sysroot_path(&root_path); let deploy_dir = physical_root.join(deployment_path); let new_var = migrate::deployment_var_path(&deploy_dir)?; - migrate::preserve_var(&var_migration, &new_var) + let sysroot = Dir::open_ambient_dir(&physical_root, cap_std::ambient_authority()) + .with_context(|| format!("Opening physical sysroot {physical_root:?}"))?; + let deployment_dir = sysroot + .open_dir(deployment_path) + .with_context(|| format!("Opening installed deployment {deployment_path:?}"))?; + let target_policy = crate::lsm::new_sepolicy_at(&deployment_dir)?; + migrate::preserve_var(&var_migration, &new_var, target_policy.as_ref()) .context("Post-install /var preservation")?; } diff --git a/crates/lib/src/install/migrate.rs b/crates/lib/src/install/migrate.rs index acd7f4c64b..10e56a6a17 100644 --- a/crates/lib/src/install/migrate.rs +++ b/crates/lib/src/install/migrate.rs @@ -65,7 +65,7 @@ use std::path::{Component, Path}; use std::{ collections::BTreeSet, ffi::OsStr, - os::fd::AsFd, + os::fd::{AsFd, AsRawFd}, path::PathBuf, process::{Command, Stdio}, sync::{ @@ -162,20 +162,40 @@ fn validate_mountpoint_types( /// /// `new_var` is the new deployment's `var/` directory, which may have /// image-seeded content. -pub(crate) fn preserve_var(plan: &VarMigrationPlan, new_var: &Path) -> Result<()> { +pub(crate) fn preserve_var( + plan: &VarMigrationPlan, + new_var: &Path, + policy: Option<&ostree_ext::ostree::SePolicy>, +) -> Result<()> { let src_var = CapStdDir::open_ambient_dir(&plan.src_var, ambient_authority()) .with_context(|| format!("Opening source {}", plan.src_var.display()))?; let new_var = open_or_create_var_dir(new_var)?; + let labeler = policy.map(|policy| TargetPolicyLabeler { policy }); + let labeler = labeler + .as_ref() + .map(|labeler| labeler as &dyn StagedLabeler); let effective_exclusions = &plan.exclusions; let nested_mounts = &plan.nested_mounts; let copy_result = if supports_reflink(&src_var, &new_var)? { println!(" Filesystem supports reflinks — using copy-on-write clone (Strategy C)"); - preserve_var_reflink_mounts(&src_var, &new_var, effective_exclusions, nested_mounts) + preserve_var_reflink_mounts_labeled( + &src_var, + &new_var, + effective_exclusions, + nested_mounts, + labeler, + ) } else { println!(" Filesystem does not support reflinks — falling back to full copy (Strategy D)"); - preserve_var_copy_mounts(&src_var, &new_var, effective_exclusions, nested_mounts) + preserve_var_copy_mounts_labeled( + &src_var, + &new_var, + effective_exclusions, + nested_mounts, + labeler, + ) }; copy_result?; @@ -285,11 +305,22 @@ fn preserve_var_reflink( preserve_var_reflink_mounts(src_var, new_var, exclusions, &BTreeSet::new()) } +#[cfg(test)] fn preserve_var_reflink_mounts( src_var: &CapStdDir, new_var: &CapStdDir, exclusions: &[String], mountpoints: &BTreeSet, +) -> Result<()> { + preserve_var_reflink_mounts_labeled(src_var, new_var, exclusions, mountpoints, None) +} + +fn preserve_var_reflink_mounts_labeled( + src_var: &CapStdDir, + new_var: &CapStdDir, + exclusions: &[String], + mountpoints: &BTreeSet, + labeler: Option<&dyn StagedLabeler>, ) -> Result<()> { copy_var_filtered( src_var, @@ -299,6 +330,7 @@ fn preserve_var_reflink_mounts( mountpoints, true, false, + labeler, OsStr::new(GNU_CP), ) .context("Reflink-copying /var into new deployment (Strategy C)")?; @@ -321,6 +353,7 @@ fn copy_var_filtered( mountpoints: &BTreeSet, reflink: bool, destination_created: bool, + labeler: Option<&dyn StagedLabeler>, cp_program: &OsStr, ) -> Result<()> { // Existing directories are merged in place, so capture their own metadata @@ -354,6 +387,7 @@ fn copy_var_filtered( exclusions, mountpoints, reflink, + labeler, cp_program, ); let restore_result = match (source_metadata.as_ref(), destination_metadata.as_ref()) { @@ -386,6 +420,7 @@ fn copy_var_filtered_inner( exclusions: &[String], mountpoints: &BTreeSet, reflink: bool, + labeler: Option<&dyn StagedLabeler>, cp_program: &OsStr, ) -> Result<()> { let entries = src_dir @@ -411,7 +446,7 @@ fn copy_var_filtered_inner( " Creating empty mountpoint placeholder for {}", relative.display() ); - create_staged_placeholder(src_dir, dst_dir, &name, &relative)?; + create_staged_placeholder(src_dir, dst_dir, &name, &relative, labeler)?; continue; } @@ -439,6 +474,7 @@ fn copy_var_filtered_inner( mountpoints, reflink, false, + labeler, cp_program, )?; } else if destination_entry_exists(dst_dir, &name)? { @@ -458,6 +494,7 @@ fn copy_var_filtered_inner( exclusions, mountpoints, reflink, + labeler, cp_program, )?; } else { @@ -481,7 +518,16 @@ fn copy_var_filtered_inner( .iter() .map(|name| name.as_os_str()) .collect::>(); - copy_batch_staged(cp_program, src_dir, &names, dst_dir, reflink, relative_dir)?; + copy_batch_staged( + cp_program, + src_dir, + &names, + dst_dir, + reflink, + relative_dir, + mountpoints, + labeler, + )?; } Ok(()) @@ -590,6 +636,183 @@ fn publish_staged_entry( } } +/// The target used to apply a policy label to one staged inode. Regular files +/// and directories use an already-open fd; symlinks and other special files +/// use a no-follow path anchored at the opened parent directory. +enum StagedLabelTarget<'a> { + Fd(std::os::fd::BorrowedFd<'a>), + NoFollowPath(&'a CapStdDir, &'a OsStr), +} + +trait StagedLabeler { + fn lookup(&self, logical_path: &Path, mode: u32) -> Result>>; + fn apply(&self, logical_path: &Path, target: StagedLabelTarget<'_>, label: &[u8]) + -> Result<()>; +} + +struct TargetPolicyLabeler<'a> { + policy: &'a ostree_ext::ostree::SePolicy, +} + +impl StagedLabeler for TargetPolicyLabeler<'_> { + fn lookup(&self, logical_path: &Path, mode: u32) -> Result>> { + crate::lsm::optional_label_os(self.policy, logical_path, mode) + } + + fn apply( + &self, + _logical_path: &Path, + target: StagedLabelTarget<'_>, + label: &[u8], + ) -> Result<()> { + match target { + StagedLabelTarget::Fd(fd) => crate::lsm::set_security_selinux(fd, label), + StagedLabelTarget::NoFollowPath(parent, name) => { + let parent_fd = parent.as_filelike_view::(); + let mut path = PathBuf::from(format!("/proc/self/fd/{}", parent_fd.as_raw_fd())); + path.push(name); + rustix::fs::lsetxattr(path, SELINUX_XATTR, label, rustix::fs::XattrFlags::empty()) + .context("lsetxattr(security.selinux)")?; + Ok(()) + } + } + } +} + +const VAR_LOGICAL_ROOT: &str = "/var"; + +fn logical_var_path(relative: &Path) -> PathBuf { + Path::new(VAR_LOGICAL_ROOT).join(relative) +} + +fn lookup_and_apply_label( + labeler: &dyn StagedLabeler, + logical_path: &Path, + mode: u32, + target: StagedLabelTarget<'_>, +) -> Result<()> { + let label = labeler + .lookup(logical_path, mode)? + .with_context(|| format!("No SELinux policy label for {logical_path:?}"))?; + labeler.apply(logical_path, target, &label) +} + +/// Relabel only staged entries. Logical lookup paths are composed from the +/// source-relative path, never from the private staging directory name. +fn relabel_staged_names( + labeler: &dyn StagedLabeler, + staging: &CapStdDir, + names: &[&OsStr], + relative_dir: &Path, + mountpoints: &BTreeSet, +) -> Result<()> { + let staging_device = staging.metadata(".")?.dev(); + for name in names { + let relative_path = relative_dir.join(name); + relabel_staged_entry( + labeler, + staging, + name, + &logical_var_path(&relative_path), + mountpoints, + Some(staging_device), + false, + )?; + } + Ok(()) +} + +/// Relabel one staged inode and, for same-device directories, recursively +/// relabel its staged children without following symlinks. Device changes and +/// known nested-mount placeholders are labeled at the mount root but never +/// traversed. +fn relabel_staged_entry( + labeler: &dyn StagedLabeler, + parent: &CapStdDir, + name: &OsStr, + logical_path: &Path, + mountpoints: &BTreeSet, + root_device: Option, + stop_at_entry: bool, +) -> Result<()> { + use rustix::fs::{Mode, OFlags, openat}; + + let metadata = parent + .symlink_metadata(name) + .with_context(|| format!("Inspecting staged /var entry {logical_path:?} before relabel"))?; + if metadata.is_dir() { + let directory = open_child_dir_nofollow(parent, name) + .with_context(|| format!("Opening staged directory {logical_path:?}"))?; + let metadata = directory.metadata(".")?; + let device = root_device.unwrap_or_else(|| metadata.dev()); + let fd = directory.as_filelike_view::(); + lookup_and_apply_label( + labeler, + logical_path, + metadata.mode(), + StagedLabelTarget::Fd(fd.as_fd()), + )?; + + let relative = logical_path + .strip_prefix(VAR_LOGICAL_ROOT) + .with_context(|| format!("Invalid logical migration path {logical_path:?}"))?; + if stop_at_entry || metadata.dev() != device || mountpoints.contains(relative) { + return Ok(()); + } + for entry in directory.read_dir(".")? { + let entry = entry?; + let child_name = entry.file_name(); + relabel_staged_entry( + labeler, + &directory, + &child_name, + &logical_path.join(&child_name), + mountpoints, + Some(device), + false, + )?; + } + return Ok(()); + } + + if metadata.is_symlink() { + return lookup_and_apply_label( + labeler, + logical_path, + metadata.mode(), + StagedLabelTarget::NoFollowPath(parent, name), + ); + } + + if metadata.is_file() { + let fd = openat( + parent.as_filelike_view::().as_fd(), + name, + OFlags::CLOEXEC | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::RDONLY, + Mode::empty(), + ) + .with_context(|| format!("Opening staged file {logical_path:?} without following links"))?; + let stat = rustix::fs::fstat(&fd)?; + anyhow::ensure!( + stat.st_mode & libc::S_IFMT == libc::S_IFREG, + "Staged entry {logical_path:?} changed type before relabel" + ); + return lookup_and_apply_label( + labeler, + logical_path, + stat.st_mode, + StagedLabelTarget::Fd(fd.as_fd()), + ); + } + + lookup_and_apply_label( + labeler, + logical_path, + metadata.mode(), + StagedLabelTarget::NoFollowPath(parent, name), + ) +} + /// Move staged children into an existing destination directory without /// replacing entries already present there. Keep the destination directory's /// metadata unchanged; moving children updates its timestamps, so restore its @@ -630,6 +853,8 @@ fn copy_batch_staged( destination: &CapStdDir, reflink: bool, relative_dir: &Path, + mountpoints: &BTreeSet, + labeler: Option<&dyn StagedLabeler>, ) -> Result<()> { with_staging_dir(destination, |_, staging| { let options = if reflink { @@ -653,6 +878,10 @@ fn copy_batch_staged( relative_dir.display() ); + if let Some(labeler) = labeler { + relabel_staged_names(labeler, staging, names, relative_dir, mountpoints)?; + } + for name in names { publish_staged_entry(staging, destination, name, &relative_dir.join(name))?; } @@ -668,6 +897,7 @@ fn copy_filtered_directory_staged( exclusions: &[String], mountpoints: &BTreeSet, reflink: bool, + labeler: Option<&dyn StagedLabeler>, cp_program: &OsStr, ) -> Result<()> { with_staging_dir(destination_parent, |_, staging| { @@ -681,8 +911,20 @@ fn copy_filtered_directory_staged( mountpoints, reflink, true, + labeler, cp_program, )?; + if let Some(labeler) = labeler { + relabel_staged_entry( + labeler, + staging, + name, + &logical_var_path(relative), + mountpoints, + Some(staging.metadata(".")?.dev()), + false, + )?; + } publish_staged_entry(staging, destination_parent, name, relative)?; Ok(()) }) @@ -693,9 +935,21 @@ fn create_staged_placeholder( dst_parent: &CapStdDir, name: &OsStr, relative: &Path, + labeler: Option<&dyn StagedLabeler>, ) -> Result<()> { with_staging_dir(dst_parent, |_, staging| { create_mountpoint_placeholder(src_parent, staging, name)?; + if let Some(labeler) = labeler { + relabel_staged_entry( + labeler, + staging, + name, + &logical_var_path(relative), + &BTreeSet::from([relative.to_owned()]), + Some(staging.metadata(".")?.dev()), + true, + )?; + } if !publish_staged_entry(staging, dst_parent, name, relative)? { validate_existing_placeholder( dst_parent, @@ -1021,11 +1275,22 @@ fn preserve_var_copy( /// live database consistent. /// /// See: +#[cfg(test)] fn preserve_var_copy_mounts( src_var: &CapStdDir, new_var: &CapStdDir, exclusions: &[String], mountpoints: &BTreeSet, +) -> Result<()> { + preserve_var_copy_mounts_labeled(src_var, new_var, exclusions, mountpoints, None) +} + +fn preserve_var_copy_mounts_labeled( + src_var: &CapStdDir, + new_var: &CapStdDir, + exclusions: &[String], + mountpoints: &BTreeSet, + labeler: Option<&dyn StagedLabeler>, ) -> Result<()> { copy_var_filtered( src_var, @@ -1035,6 +1300,7 @@ fn preserve_var_copy_mounts( mountpoints, false, false, + labeler, OsStr::new(GNU_CP), ) .context("Copying /var into new deployment (Strategy D)")?; @@ -1127,10 +1393,48 @@ mod tests { use cap_std_ext::dirext::CapStdExtDirExt; use std::{ fs, - os::unix::fs::{MetadataExt as StdMetadataExt, PermissionsExt, symlink}, + os::unix::{ + ffi::{OsStrExt, OsStringExt}, + fs::{MetadataExt as StdMetadataExt, PermissionsExt, symlink}, + }, + sync::Mutex, time::{Duration, SystemTime}, }; + #[derive(Default)] + struct FakeLabeler { + lookups: Mutex>, + applied: Mutex>, + fail_lookup: bool, + fail_apply: bool, + } + + impl StagedLabeler for FakeLabeler { + fn lookup(&self, logical_path: &Path, _mode: u32) -> Result>> { + self.lookups.lock().unwrap().push(logical_path.to_owned()); + anyhow::ensure!( + logical_path.starts_with(VAR_LOGICAL_ROOT), + "policy lookup used non-logical path {logical_path:?}" + ); + anyhow::ensure!(!self.fail_lookup, "injected SELinux policy lookup failure"); + Ok(Some(b"test-label".to_vec())) + } + + fn apply( + &self, + logical_path: &Path, + _target: StagedLabelTarget<'_>, + _label: &[u8], + ) -> Result<()> { + self.applied.lock().unwrap().push(logical_path.to_owned()); + anyhow::ensure!( + !self.fail_apply, + "injected SELinux label application failure" + ); + Ok(()) + } + } + fn cap_dir(path: &Path) -> CapStdDir { CapStdDir::open_ambient_dir(path, ambient_authority()).unwrap() } @@ -1149,6 +1453,144 @@ mod tests { CapStdDir::from_std_file(std::fs::File::from(fd)) } + #[test] + fn relabels_only_staged_entries_with_logical_paths_for_both_copy_strategies() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + fs::create_dir_all(src_path.join("state"))?; + fs::write(src_path.join("state/existing"), b"source collision")?; + fs::write(src_path.join("state/new"), b"new source state")?; + let non_utf8_name = std::ffi::OsString::from_vec(vec![b'n', 0xff]); + fs::write(src_path.join(&non_utf8_name), b"non-UTF8 name")?; + symlink("state", src_path.join("link-to-state"))?; + let source = cap_dir(&src_path); + + for (strategy, reflink) in [("copy", false), ("reflink", true)] { + let dst_path = temp.path().join(format!("dst-{strategy}")); + fs::create_dir_all(dst_path.join("state"))?; + fs::write(dst_path.join("state/existing"), b"seeded winner")?; + let destination = cap_dir(&dst_path); + let labeler = FakeLabeler::default(); + if reflink { + preserve_var_reflink_mounts_labeled( + &source, + &destination, + &[], + &BTreeSet::new(), + Some(&labeler), + )?; + } else { + preserve_var_copy_mounts_labeled( + &source, + &destination, + &[], + &BTreeSet::new(), + Some(&labeler), + )?; + } + + assert_eq!(fs::read(dst_path.join("state/existing"))?, b"seeded winner"); + let lookups = labeler.lookups.lock().unwrap(); + assert!( + lookups + .iter() + .all(|path| path.starts_with(VAR_LOGICAL_ROOT)) + ); + assert!(lookups.iter().all(|path| { + !path + .as_os_str() + .as_bytes() + .windows(b".bootc-preserve-var.".len()) + .any(|window| window == b".bootc-preserve-var.") + })); + assert!(lookups.contains(&PathBuf::from("/var/state/new"))); + assert!(!lookups.contains(&PathBuf::from("/var/state/existing"))); + assert!(!lookups.contains(&PathBuf::from("/var/state"))); + assert!(lookups.contains(&PathBuf::from("/var/link-to-state"))); + assert!(!lookups.contains(&PathBuf::from("/var/link-to-state/new"))); + assert!( + lookups + .iter() + .any(|path| path.as_os_str().as_bytes().ends_with(&[b'n', 0xff])) + ); + } + Ok(()) + } + + #[test] + fn relabels_empty_mount_placeholders_without_walking_into_them() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + let dst_path = temp.path().join("dst"); + fs::create_dir_all(src_path.join("tree/bound"))?; + fs::write(src_path.join("tree/bound/mounted-only"), b"not copied")?; + fs::write(src_path.join("tree/sibling"), b"copied")?; + fs::create_dir(&dst_path)?; + let source = cap_dir(&src_path); + let destination = cap_dir(&dst_path); + let labeler = FakeLabeler::default(); + let mountpoints = BTreeSet::from([PathBuf::from("tree/bound")]); + + preserve_var_copy_mounts_labeled(&source, &destination, &[], &mountpoints, Some(&labeler))?; + + assert_eq!(fs::read_dir(dst_path.join("tree/bound"))?.count(), 0); + let lookups = labeler.lookups.lock().unwrap(); + assert!(lookups.contains(&PathBuf::from("/var/tree"))); + assert!(lookups.contains(&PathBuf::from("/var/tree/bound"))); + assert!(lookups.contains(&PathBuf::from("/var/tree/sibling"))); + assert!(!lookups.contains(&PathBuf::from("/var/tree/bound/mounted-only"))); + Ok(()) + } + + #[test] + fn label_failure_cleans_stage_and_does_not_publish_for_both_strategies() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + fs::create_dir(&src_path)?; + fs::write(src_path.join("new-entry"), b"state")?; + let source = cap_dir(&src_path); + + for (failure, fail_lookup, fail_apply) in [("lookup", true, false), ("apply", false, true)] + { + for (strategy, reflink) in [("copy", false), ("reflink", true)] { + let dst_path = temp.path().join(format!("dst-{failure}-{strategy}")); + fs::create_dir(&dst_path)?; + let destination = cap_dir(&dst_path); + let labeler = FakeLabeler { + fail_lookup, + fail_apply, + ..FakeLabeler::default() + }; + let error = if reflink { + preserve_var_reflink_mounts_labeled( + &source, + &destination, + &[], + &BTreeSet::new(), + Some(&labeler), + ) + } else { + preserve_var_copy_mounts_labeled( + &source, + &destination, + &[], + &BTreeSet::new(), + Some(&labeler), + ) + } + .unwrap_err(); + let expected_error = if fail_lookup { + "SELinux policy lookup failure" + } else { + "SELinux label application failure" + }; + assert!(format!("{error:#}").contains(expected_error)); + assert_eq!(fs::read_dir(&dst_path)?.count(), 0); + } + } + Ok(()) + } + #[test] fn exclusions_are_relative_and_normalized() { assert_eq!( @@ -1656,6 +2098,7 @@ mod tests { &BTreeSet::new(), reflink, false, + None, failing_cp.as_os_str(), ); assert!(error.is_err()); diff --git a/crates/lib/src/lsm.rs b/crates/lib/src/lsm.rs index 772358f02a..ac0a1a1297 100644 --- a/crates/lib/src/lsm.rs +++ b/crates/lib/src/lsm.rs @@ -3,6 +3,7 @@ use std::borrow::Cow; use std::ffi::CString; use std::io::Write; use std::os::fd::AsRawFd; +use std::os::unix::ffi::OsStrExt; use std::os::unix::process::CommandExt; use std::path::Path; use std::process::Command; @@ -280,6 +281,46 @@ pub(crate) fn optional_label( .map_err(Into::into) } +/// Look up a policy label for a Unix path without requiring the path to be +/// UTF-8. OSTree's safe wrapper accepts `&str`, while the underlying API takes +/// a NUL-terminated byte string. This narrowly-scoped FFI call owns both +/// returned GLib allocations before returning, including the label on the +/// unusual case where OSTree sets both outputs. +#[allow(unsafe_code)] +pub(crate) fn optional_label_os( + policy: &ostree::SePolicy, + path: &Path, + mode: u32, +) -> Result>> { + use ostree::glib::translate::{ToGlibPtr, from_glib_full}; + + let path = std::ffi::CString::new(path.as_os_str().as_bytes()) + .context("SELinux policy path contains an interior NUL")?; + let mut out_label = std::ptr::null_mut(); + let mut error = std::ptr::null_mut(); + // SAFETY: `path` is NUL-terminated and lives through the call; `policy` + // keeps the instance alive; all output pointers are initialized and their + // GLib-owned values are immediately converted with `from_glib_full` below. + let succeeded = unsafe { + ostree::ffi::ostree_sepolicy_get_label( + policy.to_glib_none().0, + path.as_ptr(), + mode, + &mut out_label, + std::ptr::null_mut(), + &mut error, + ) + }; + // Convert both outputs so even an unexpected label on error is freed. + let label: Option = unsafe { from_glib_full(out_label) }; + let error: Option = unsafe { from_glib_full(error) }; + debug_assert_eq!(succeeded == ostree::glib::ffi::GFALSE, error.is_some()); + if let Some(error) = error { + return Err(error.into()); + } + Ok(label.map(|label| label.as_bytes().to_vec())) +} + /// A thin wrapper for invoking fsetxattr(security.selinux) pub(crate) fn set_security_selinux(fd: std::os::fd::BorrowedFd, label: &[u8]) -> Result<()> { rustix::fs::fsetxattr( From c978fe89740ca26084bc4cc9364bbf70d5a7dbf5 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 11:08:14 -0600 Subject: [PATCH 16/38] tests: Expand package-mode preserve-var coverage The migration test needs to exercise source preflight, separate and temporary mounts, persistent application data, and a complete reboot without losing TMT state. Keep the TMT run directory at a real path under preserved /var/lib and pass the configured test image through guest preparation. Assisted-by: AI --- Justfile | 20 +- tmt/plans/package-mode.fmf | 69 ++- .../test-to-existing-root-preserve-var.sh | 410 ++++++++---------- .../test-50-to-existing-root-preserve-var.fmf | 6 +- 4 files changed, 247 insertions(+), 258 deletions(-) diff --git a/Justfile b/Justfile index 890cc45255..f511c34360 100644 --- a/Justfile +++ b/Justfile @@ -232,9 +232,10 @@ test-ukiboot: build-ukiboot-update-test-image # Run the package-mode migration plan against the current checkout image. # The VM cannot share controller Podman storage, so transfer it as an OCI archive. [group('testing')] -test-tmt-package-mode: +test-tmt-package-mode image="centos-stream-10": #!/bin/bash set -euo pipefail + lbi_images="{{lbi_images}}" just build archive_dir=$(mktemp -d -- tmt/plans/.package-mode-local.XXXXXX) cleanup() { @@ -248,12 +249,23 @@ test-tmt-package-mode: echo "Refusing to overwrite existing archive: $archive_path" >&2 exit 1 fi - podman save --format oci-archive --output "$archive_path" localhost/bootc + podman save --format oci-archive --output "$archive_path" {{base_img}} + # TMT syncs the plan workdir to the same absolute guest path. Use a + # world-writable root so non-root local runs can place it under preserved + # /var/lib on the guest without a symlink or a separate VM harness. + if [[ "$EUID" -eq 0 ]]; then + install -d -m 1777 /var/lib/bootc-tmt-workdirs + else + sudo install -d -m 1777 /var/lib/bootc-tmt-workdirs + fi tmt \ --context=running_env=packit \ --context=package_source=local \ - --context="test_disk_image=${test_disk_image:-centos-stream-10}" \ - run --environment "PACKAGE_MODE_IMAGE_ARCHIVE=$archive_path" \ + --context="test_disk_image={{image}}" \ + run --workdir-root /var/lib/bootc-tmt-workdirs \ + --environment "PACKAGE_MODE_IMAGE_NAME={{base_img}}" \ + --environment "PACKAGE_MODE_LBI_IMAGES=${lbi_images// /,}" \ + --environment "PACKAGE_MODE_IMAGE_ARCHIVE=$archive_path" \ plans --name '^/tmt/plans/package-mode$' # Split out from `test-container` because, unlike the container integration tests, diff --git a/tmt/plans/package-mode.fmf b/tmt/plans/package-mode.fmf index 9966633eff..dac2eb149e 100644 --- a/tmt/plans/package-mode.fmf +++ b/tmt/plans/package-mode.fmf @@ -1,6 +1,10 @@ -summary: Test package-mode to-existing-root migration flags +summary: Package-mode preserve-var integration test environment: PACKAGE_SOURCE: $@{package_source} + PACKAGE_MODE_IMAGE_NAME: localhost/bootc + # Keep these comma-separated refs aligned with Justfile's lbi_images. The + # podman LBI uses the signed version rather than the moving :latest tag. + PACKAGE_MODE_LBI_IMAGES: quay.io/curl/curl:latest,quay.io/curl/curl-base:latest,registry.access.redhat.com/ubi9/podman:9.8-1793032728 provision: how: virtual image: $@{test_disk_image} @@ -13,6 +17,8 @@ prepare: package: - podman - jq + - e2fsprogs + - util-linux - bootc when: package_source == local # Packit build-only preparation needs the full integration toolchain. @@ -33,6 +39,7 @@ prepare: - lvm2 - dosfstools - e2fsprogs + - util-linux when: package_source == artifacts # Reject missing/unknown sources before any mode-specific setup runs. - how: shell @@ -112,7 +119,7 @@ prepare: exit 1 fi podman --remote=false load --input "$archive_path" - podman --remote=false image exists localhost/bootc + podman --remote=false image exists "${PACKAGE_MODE_IMAGE_NAME:-localhost/bootc}" when: package_source == local - how: shell order: 98 @@ -122,22 +129,28 @@ prepare: - cd bootc && rpm2cpio *.src.rpm > pkg.cpio && cpio -idmv < pkg.cpio && rm pkg.cpio && rm -f *-vendor.tar.zstd && zstd -d *.tar.zstd && tar -xvf *.tar -C . --strip-components=1 && ls -al - cd bootc/hack && BOOTC_PROVISION_PACKIT_SKIP_REINSTALL=1 ./provision-packit.sh when: package_source == artifacts - # Both sources now provide localhost/bootc in the guest's rootful storage. + # Both sources now provide the configured test image in guest rootful storage. # Pull the LBI references needed to install the image. - how: shell order: 99 script: | - podman --remote=false image exists localhost/bootc - podman --remote=false pull quay.io/curl/curl:latest - podman --remote=false pull quay.io/curl/curl-base:latest - podman --remote=false pull registry.access.redhat.com/ubi9/podman:latest + set -eu + image_name=${PACKAGE_MODE_IMAGE_NAME:-localhost/bootc} + podman --remote=false image exists "$image_name" + IFS=, read -r -a lbi_images <<< "${PACKAGE_MODE_LBI_IMAGES:?PACKAGE_MODE_LBI_IMAGES is required}" + for image in "${lbi_images[@]}"; do + podman --remote=false pull "$image" + done workdir=$(mktemp -d) trap 'rm -rf -- "$workdir"' EXIT - cat > "$workdir/Containerfile" <<'EOF' - FROM localhost/bootc - RUN unit='var-lib-bootc\x2dtmt\x2dpreserve\x2dvar\x2dbind-target.mount' && \ - mkdir -p /usr/etc/systemd/system/local-fs.target.wants && \ - printf '%s\n' \ + cat > "$workdir/Containerfile" < "/usr/etc/systemd/system/$unit" && \ - ln -s "../$unit" "/usr/etc/systemd/system/local-fs.target.wants/$unit" + > "/etc/systemd/system/\$bind_unit" && \ + ln -s "../\$bind_unit" "/etc/systemd/system/local-fs.target.wants/\$bind_unit" && \ + printf "%s\n" \ + '[Unit]' \ + 'Description=bootc TMT preserve-var /var/log filesystem' \ + '' \ + '[Mount]' \ + 'What=/var/lib/bootc-tmt-preserve-var/logfs.img' \ + 'Where=/var/log' \ + 'Type=ext4' \ + 'Options=loop' \ + '' \ + '[Install]' \ + 'WantedBy=local-fs.target' \ + > /etc/systemd/system/var-log.mount && \ + ln -s ../var-log.mount /etc/systemd/system/local-fs.target.wants/var-log.mount && \ + printf "%s\n" \ + '[Unit]' \ + 'Description=bootc TMT preserve-var empty tmpfs' \ + '' \ + '[Mount]' \ + 'What=tmpfs' \ + 'Where=/var/lib/bootc-tmt-preserve-var/empty_tmpfs' \ + 'Type=tmpfs' \ + 'Options=mode=0755,size=16m' \ + '' \ + '[Install]' \ + 'WantedBy=local-fs.target' \ + > "/etc/systemd/system/\$tmpfs_unit" && \ + ln -s "../\$tmpfs_unit" "/etc/systemd/system/local-fs.target.wants/\$tmpfs_unit" EOF podman --remote=false build -t localhost/bootc-preserve-var-test "$workdir" podman --remote=false image exists localhost/bootc-preserve-var-test diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index 4bcae52089..c3a4f9474b 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -1,16 +1,19 @@ #!/bin/bash # Both local and Packit runs prepare one package-mode guest, then convert and -# reboot that same guest; local loads the checkout image, Packit builds it. +# reboot that same guest; local TMT workdirs live under preserved /var/lib. set -xeuo pipefail target_image=localhost/bootc-preserve-var-test -var_marker=/var/lib/bootc-tmt-preserve-var/sentinel +var_state=/var/lib/bootc-tmt-preserve-var +app_state=$var_state/app-state/state +log_fs_image=$var_state/logfs.img +empty_tmpfs=$var_state/empty_tmpfs bind_dir=/var/lib/bootc-tmt-preserve-var-bind bind_source=$bind_dir/source bind_target=$bind_dir/target +preflight_mountpoint=$var_state/preflight-special tmt_plan_data_suffix=/tmt/plans/package-mode/data -tmt_workdir_parent=/var/tmp/tmt -tmt_workdir_root=/var/lib/bootc-tmt-workdirs +tmt_local_workdir_root=/var/lib/bootc-tmt-workdirs tmt_artifacts_parent=/var/ARTIFACTS tmt_test_data_sentinel_name=workdir-persistence-sentinel tmt_test_data_post_reboot_marker_name=post-reboot-marker @@ -19,68 +22,34 @@ get_tmt_run_dir() { local plan_data=${TMT_PLAN_DATA:-} local run_dir local resolved_plan_data - local persistent_run_dir - local link_target - if [[ "$plan_data" != *"$tmt_plan_data_suffix" ]]; then + if [[ $plan_data != *"$tmt_plan_data_suffix" ]]; then printf 'TMT_PLAN_DATA must end in %s (got %q)\n' "$tmt_plan_data_suffix" "$plan_data" >&2 return 1 fi run_dir=${plan_data%"$tmt_plan_data_suffix"} - if [[ ( ! $run_dir =~ ^/var/tmp/tmt/run-[[:alnum:]_.-]+$ && - ! $run_dir =~ ^/var/ARTIFACTS/work-[[:alnum:]_.-]+$ ) || $run_dir == *..* ]]; then - printf 'Expected a safe TMT run directory under %s or %s (got %q)\n' \ - "$tmt_workdir_parent" "$tmt_artifacts_parent" "$run_dir" >&2 + if [[ $run_dir =~ ^/var/lib/bootc-tmt-workdirs/run-[[:alnum:]_.-]+$ || + $run_dir =~ ^/var/ARTIFACTS/work-[[:alnum:]_.-]+$ ]]; then + : + else + printf 'Expected TMT workdir under %s or %s (got %q)\n' \ + "$tmt_local_workdir_root" "$tmt_artifacts_parent" "$run_dir" >&2 return 1 fi case ${TMT_REBOOT_COUNT:-0} in - 0) - if ! resolved_plan_data=$(realpath -e -- "$plan_data"); then - printf 'TMT_PLAN_DATA must be an existing path without symlinked components (got %q)\n' \ - "$plan_data" >&2 + 0|1) + if ! resolved_plan_data=$(realpath -e -- "$plan_data") || + [[ $resolved_plan_data != "$plan_data" ]]; then + printf 'TMT_PLAN_DATA must be a canonical real path (got %q, resolves to %q)\n' \ + "$plan_data" "${resolved_plan_data:-}" >&2 return 1 fi - if [[ $resolved_plan_data != "$plan_data" ]]; then - printf 'TMT_PLAN_DATA must be a canonical path without symlinked components (got %q, resolves to %q)\n' \ - "$plan_data" "$resolved_plan_data" >&2 - return 1 - fi - ;; - 1) - if [[ $run_dir == /var/tmp/tmt/* ]]; then - persistent_run_dir=$tmt_workdir_root/${run_dir##*/} - if [[ ! -L $run_dir ]] || ! link_target=$(readlink -- "$run_dir") || - [[ $link_target != "$persistent_run_dir" ]]; then - printf 'TMT run directory must link directly to %s (got %q)\n' \ - "$persistent_run_dir" "$(readlink -- "$run_dir" 2>/dev/null || true)" >&2 - return 1 - fi - if ! resolved_plan_data=$(realpath -e -- "$plan_data") || - [[ $resolved_plan_data != "$persistent_run_dir$tmt_plan_data_suffix" ]]; then - printf 'TMT_PLAN_DATA must resolve to %s (got %q, resolves to %q)\n' \ - "$persistent_run_dir$tmt_plan_data_suffix" "$plan_data" "${resolved_plan_data:-}" >&2 - return 1 - fi - else - # Packit/Testing Farm keeps its workdir under /var/ARTIFACTS. - # It must remain an ordinary directory, with no symlinked path - # components, and stay on the filesystem preserve-var copies. - if [[ -L $run_dir || ! -d $run_dir ]] || - ! resolved_plan_data=$(realpath -e -- "$plan_data") || - [[ $resolved_plan_data != "$plan_data" ]]; then - printf 'TMT workdir and plan data must remain canonical, real paths after reboot (got %q)\n' \ - "$plan_data" >&2 - return 1 - fi - verify_tmt_workdir_in_preserved_var "$run_dir" || return 1 - fi ;; *) printf 'Unexpected TMT_REBOOT_COUNT=%s\n' "${TMT_REBOOT_COUNT}" >&2 return 1 ;; esac - printf '%s\n' "$run_dir" } @@ -93,267 +62,234 @@ verify_tmt_workdir_in_preserved_var() { "$run_dir" >&2 return 1 fi - # /var itself may be a separate filesystem; otherwise the containing root - # filesystem is copied as part of /var. Any mount below either is excluded - # from preserve-var and would make this workdir disappear across reboot. + # A nested mount is excluded from preserve-var even if it is below /var. if [[ $mount_target != / && $mount_target != /var ]]; then printf 'TMT workdir %s is on nested mount %s, which --preserve-var does not copy\n' \ "$run_dir" "$mount_target" >&2 return 1 fi - return 0 } case ${TMT_REBOOT_COUNT:-0} in 0) podman --remote=false image exists "$target_image" - # The guest must still be the original package-mode system. + # The guest is still the original package-mode system. test ! -e /run/ostree-booted + tmt_run_dir=$(get_tmt_run_dir) + if [[ ! -d $tmt_run_dir || -L $tmt_run_dir ]]; then + printf 'TMT workdir is missing or not a real directory: %s\n' "$tmt_run_dir" >&2 + exit 1 + fi + verify_tmt_workdir_in_preserved_var "$tmt_run_dir" - mkdir -p "${var_marker%/*}" - - # TMT connects as root. Package-mode hosts keep root's home at /root, - # while bootc images link /root to /var/roothome, so preserve its key - # under /var for TMT to reconnect after the migration reboot. - install -d -m 0700 /var/roothome/.ssh - install -m 0600 /root/.ssh/authorized_keys /var/roothome/.ssh/authorized_keys - restorecon -RF /var/roothome/.ssh - - printf '%s\n' preserved-var | tee "$var_marker" - mkdir -p "$bind_source" "$bind_target" + install -d -m 0755 "$var_state" "${app_state%/*}" "$bind_source" "$bind_target" + printf '%s\n' preserved-application-state > "$app_state" printf '%s\n' bind-source-data > "$bind_source/sentinel" - printf '%s\n' mounted-only-data > "$bind_source/mounted-sentinel" + printf '%s\n' bind-mounted-data > "$bind_source/mounted-sentinel" printf '%s\n' underlying-target-data > "$bind_target/underlying-sentinel" mount --bind "$bind_source" "$bind_target" - test "$(<"$bind_target/mounted-sentinel")" = mounted-only-data + test "$(<"$bind_target/mounted-sentinel")" = bind-mounted-data test ! -e "$bind_target/underlying-sentinel" - tmt_run_dir=$(get_tmt_run_dir) - tmt_run_id=${tmt_run_dir##*/} - if [[ $tmt_run_dir == /var/ARTIFACTS/* ]]; then - tmt_run_layout=artifacts - else - tmt_run_layout=var-tmp - fi - if [[ ! -d "$tmt_run_dir" || -L "$tmt_run_dir" ]]; then - printf 'TMT run directory is missing or not a directory: %s\n' "$tmt_run_dir" >&2 + # Seed a database using the new deployment's own PostgreSQL version so + # the test also works when the guest and bootc base use different OSes. + podman --remote=false run --rm --security-opt label=disable \ + -v /var/lib/pgsql:/var/lib/pgsql "$target_image" \ + bash -euo pipefail -c ' + postgresql-setup --initdb + runuser -u postgres -- pg_ctl -D /var/lib/pgsql/data \ + -o "-c listen_addresses=localhost" -w start + runuser -u postgres -- psql -v ON_ERROR_STOP=1 \ + -c "CREATE DATABASE bootc_preserve_var" + runuser -u postgres -- psql -v ON_ERROR_STOP=1 -d bootc_preserve_var \ + -c "CREATE TABLE app_state (id integer PRIMARY KEY, value text NOT NULL); \ + INSERT INTO app_state VALUES (1, \$\$postgres-state-preserved\$\$)" + runuser -u postgres -- pg_ctl -D /var/lib/pgsql/data -m fast -w stop + ' + + # Exercise source preflight with a nested mount whose source is a + # device node. No target boot configuration may be changed on failure. + truncate -s 128M "$log_fs_image" + mkfs.ext4 -F -m 0 "$log_fs_image" + if mountpoint -q /var/log; then + printf '/var/log is already a mountpoint before test setup\n' >&2 exit 1 fi - tmt_run_dir_identity=$(stat -c '%d:%i' -- "$tmt_run_dir") - persistent_tmt_run_dir=$tmt_workdir_root/$tmt_run_id - if [[ -e "$persistent_tmt_run_dir" || -L "$persistent_tmt_run_dir" ]]; then - printf 'Persistent TMT workdir already exists: %s\n' "$persistent_tmt_run_dir" >&2 + mount -o loop "$log_fs_image" /var/log + printf '%s\n' separate-var-log-filesystem > /var/log/bootc-tmt-sentinel + install -d -m 0755 "$empty_tmpfs" + mount -t tmpfs -o size=16m tmpfs "$empty_tmpfs" + test -z "$(find "$empty_tmpfs" -mindepth 1 -print -quit)" + : > "$preflight_mountpoint" + mount --bind /dev/null "$preflight_mountpoint" + + boot_snapshot=$(mktemp /run/boot-config-before.XXXXXX) + boot_snapshot_after=$(mktemp /run/boot-config-after.XXXXXX) + tar --sort=name --numeric-owner --xattrs --acls -cf "$boot_snapshot" -C /boot . + if podman --remote=false run --rm --privileged --pid=host --user=root:root \ + -v /dev:/dev \ + -v /:/target \ + -v /boot:/target/boot \ + -v /var/lib/containers:/var/lib/containers \ + --security-opt label=type:unconfined_t \ + "$target_image" \ + bootc install to-existing-root --preserve-var --acknowledge-destructive \ + > /run/preserve-var-preflight.log 2>&1; then + printf 'preserve-var unexpectedly accepted the special-file mountpoint\n' >&2 exit 1 fi - test -d "$TMT_TEST_DATA" + grep -F "$preflight_mountpoint is a mountpoint that is neither a directory nor a file" \ + /run/preserve-var-preflight.log + tar --sort=name --numeric-owner --xattrs --acls -cf "$boot_snapshot_after" -C /boot . + cmp "$boot_snapshot" "$boot_snapshot_after" + umount "$preflight_mountpoint" + rm -f "$preflight_mountpoint" "$boot_snapshot" "$boot_snapshot_after" + + # TMT 1.69 rsyncs to the same absolute path on the guest. Keeping its + # real workdir under /var/lib avoids both excluded /var/tmp and symlinks. + test -f "$TMT_TEST_METADATA" tmt_test_data_sentinel=$TMT_TEST_DATA/$tmt_test_data_sentinel_name test ! -e "$tmt_test_data_sentinel" printf '%s\n' workdir-persistence-sentinel > "$tmt_test_data_sentinel" - if [[ $tmt_run_layout == var-tmp ]]; then - # --preserve-var excludes /var/tmp. Stage a full copy on /var, then - # redirect the original path so TMT's post-reboot writes survive. - install -d -m 0755 "$tmt_workdir_root" - staging_parent=$(mktemp -d "$tmt_workdir_root/.${tmt_run_id}.XXXXXX") - trap 'if [[ -n ${staging_parent:-} ]]; then rm -rf -- "$staging_parent"; fi' EXIT - cp -a -- "$tmt_run_dir" "$staging_parent/" - staged_run_dir=$staging_parent/$tmt_run_id - mv -T --no-clobber -- "$staged_run_dir" "$persistent_tmt_run_dir" - if [[ -e "$staged_run_dir" || -L "$staged_run_dir" ]]; then - printf 'Persistent TMT workdir appeared during staging: %s\n' \ - "$persistent_tmt_run_dir" >&2 - exit 1 - fi - rmdir -- "$staging_parent" - staging_parent= - trap - EXIT - - disposable_run_dir=$tmt_workdir_parent/.${tmt_run_id}.original.$$ - if [[ -e "$disposable_run_dir" || -L "$disposable_run_dir" ]]; then - printf 'Disposable TMT workdir already exists: %s\n' "$disposable_run_dir" >&2 - exit 1 - fi - - tmt_run_dir_moved=0 - tmt_run_dir_swap_complete=0 - cleanup_tmt_run_dir_swap() { - local status=$? - local disposable_identity - local original_identity - - trap - EXIT HUP INT TERM - if (( tmt_run_dir_moved )); then - # A signal can arrive after ln succeeds but before the flag is - # updated, so recognize the completed swap from the link too. - if [[ -L $tmt_run_dir ]] && - [[ $(readlink -- "$tmt_run_dir") == "$persistent_tmt_run_dir" ]]; then - tmt_run_dir_swap_complete=1 - fi - - if (( ! tmt_run_dir_swap_complete )); then - disposable_identity=$(stat -c '%d:%i' -- "$disposable_run_dir" 2>/dev/null || true) - if [[ $disposable_identity == "$tmt_run_dir_identity" ]]; then - if [[ ! -e $tmt_run_dir && ! -L $tmt_run_dir ]]; then - # --no-clobber ensures a replacement path created after - # the existence check is never overwritten. - mv -T --no-clobber -- "$disposable_run_dir" "$tmt_run_dir" || true - original_identity=$(stat -c '%d:%i' -- "$tmt_run_dir" 2>/dev/null || true) - if [[ $original_identity != "$tmt_run_dir_identity" ]]; then - disposable_identity=$(stat -c '%d:%i' -- "$disposable_run_dir" 2>/dev/null || true) - if [[ $disposable_identity == "$tmt_run_dir_identity" ]]; then - printf 'Could not restore TMT workdir because no-replace skipped the move; the original remains at %s\n' \ - "$disposable_run_dir" >&2 - else - printf 'Failed to restore TMT workdir from %s to %s\n' \ - "$disposable_run_dir" "$tmt_run_dir" >&2 - fi - fi - else - printf 'Cannot restore TMT workdir because the original path now exists; the original remains at %s\n' \ - "$disposable_run_dir" >&2 - fi - elif [[ -e $disposable_run_dir || -L $disposable_run_dir ]]; then - printf 'Leaving replacement at disposable TMT workdir path untouched: %s\n' \ - "$disposable_run_dir" >&2 - fi - fi - fi - exit "$status" - } - trap cleanup_tmt_run_dir_swap EXIT - trap 'exit 1' HUP INT TERM - tmt_run_dir_moved=1 - mv -T --no-clobber -- "$tmt_run_dir" "$disposable_run_dir" - disposable_run_dir_identity=$(stat -c '%d:%i' -- "$disposable_run_dir" 2>/dev/null || true) - if [[ $disposable_run_dir_identity != "$tmt_run_dir_identity" ]]; then - tmt_run_dir_moved=0 - printf 'Failed to move original TMT workdir to %s; the destination may have appeared concurrently\n' \ - "$disposable_run_dir" >&2 - exit 1 - fi - ln -s -- "$persistent_tmt_run_dir" "$tmt_run_dir" - tmt_run_dir_swap_complete=1 - # Keep the old tree: it is disposable under /var/tmp and vanishes on - # reboot, while retaining it keeps this shell's cwd valid until we leave. - tmt_run_dir_moved=0 - trap - EXIT HUP INT TERM - else - # /var/ARTIFACTS is already inside preserved /var. Do not move or - # symlink the Testing Farm workdir; preserve-var must carry it. - verify_tmt_workdir_in_preserved_var "$tmt_run_dir" - fi - # The shell may still be inside the renamed workdir; leave it before - # installing or rebooting so the old cwd is no longer in use. - cd / + # TMT connects as root. Preserve its authorized key in /var so it can + # reconnect after the migration reboot. + install -d -m 0700 /var/roothome/.ssh + install -m 0600 /root/.ssh/authorized_keys /var/roothome/.ssh/authorized_keys + restorecon -RF /var/roothome/.ssh - # Run the build-under-test as a privileged installer, with the - # package-mode root mounted at /target. - podman --remote=false run --rm --privileged --pid=host --user=root:root \ + # Keep the backing image for the separate /var/log filesystem stable + # while preserve-var copies its image file from /var/lib. + fsfreeze -f /var/log + if ! podman --remote=false run --rm --privileged --pid=host --user=root:root \ -v /dev:/dev \ -v /:/target \ -v /boot:/target/boot \ -v /var/lib/containers:/var/lib/containers \ --security-opt label=type:unconfined_t \ "$target_image" \ - bootc install to-existing-root --preserve-var --acknowledge-destructive + bootc install to-existing-root --preserve-var --acknowledge-destructive; then + fsfreeze -u /var/log + exit 1 + fi + fsfreeze -u /var/log - # Find the new stateroot by the copied source sentinel; generated - # stateroot names are intentionally not assumed. + # Find the generated stateroot by the copied bind-source sentinel. new_var= for candidate in /ostree/deploy/*/var; do - if [[ -f "$candidate${bind_source#/var}/sentinel" ]]; then + if [[ -f $candidate${bind_source#/var}/sentinel ]]; then new_var=$candidate break fi done test -n "$new_var" test "$(<"$new_var${bind_source#/var}/sentinel")" = bind-source-data + test "$(<"$new_var${app_state#/var}")" = preserved-application-state + test -f "$new_var${log_fs_image#/var}" bind_target_in_new_var=$new_var${bind_target#/var} test -d "$bind_target_in_new_var" test -z "$(find "$bind_target_in_new_var" -mindepth 1 -print -quit)" - test ! -e "$bind_target_in_new_var/mounted-sentinel" - test ! -e "$bind_target_in_new_var/underlying-sentinel" - if [[ $tmt_run_layout == var-tmp ]]; then - # to-existing-root does not migrate the old host's /etc. Add the - # tmpfiles rule to the new deployment's persistent /etc instead. - podman --remote=false run --rm --privileged --pid=host --user=root:root \ - -v /dev:/dev \ - -v /:/target \ - -v /boot:/target/boot \ - -v /var/lib/containers:/var/lib/containers \ - --security-opt label=type:unconfined_t \ - "$target_image" \ - bash -euo pipefail -c ' - deployment=/mnt/bootc-tmt-deployment - cleanup() { - local status=$? - trap - EXIT - if ! umount -R "$deployment"; then - printf "Failed to unmount deployment target %s\n" "$deployment" >&2 - status=1 - fi - exit "$status" - } - trap cleanup EXIT + # to-existing-root does not migrate the old host's /etc. Inject + # test-only SSH host keys and TMT tmpfiles config into the new /etc. + # The host-key generation is only for this VM test, not product behavior. + podman --remote=false run --rm --privileged --pid=host --user=root:root \ + -v /dev:/dev \ + -v /:/target \ + -v /boot:/target/boot \ + -v /var/lib/containers:/var/lib/containers \ + --security-opt label=type:unconfined_t \ + "$target_image" \ + bash -euo pipefail -c ' + deployment=/run/bootc-tmt-deployment + cleanup() { + local status=$? + trap - EXIT + if ! umount -R "$deployment"; then + printf "Failed to unmount deployment target %s\n" "$deployment" >&2 + status=1 + fi + exit "$status" + } + trap cleanup EXIT + + local_workdir_root=$1 + run_dir=$2 + mkdir -p "$deployment" + bootc install mount --sysroot /target --latest "$deployment" + + # New /etc must provide keys because old /etc/ssh is not migrated. + install -d -m 0755 "$deployment/etc/ssh" + ssh-keygen -q -t ed25519 -N "" \ + -f "$deployment/etc/ssh/ssh_host_ed25519_key" - workdir_parent=$1 - run_dir=$2 - persistent_run_dir=$3 - mkdir -p "$deployment" - bootc install mount --sysroot /target --latest "$deployment" + if [[ $local_workdir_root == 1 ]]; then rule=$deployment/etc/tmpfiles.d/bootc-tmt-workdir.conf install -d -m 0755 "${rule%/*}" tmpfiles_rule=( - "d $workdir_parent 0755 root root -" - "L+ $run_dir - - - - $persistent_run_dir" + "d /var/lib/bootc-tmt-workdirs - - - - -" + "d $run_dir - - - - -" ) - printf '%s\n' "${tmpfiles_rule[@]}" > "$rule" - printf '%s\n' "${tmpfiles_rule[@]}" | cmp - "$rule" - ' install-tmt-tmpfiles-rule \ - "$tmt_workdir_parent" "$tmt_run_dir" "$persistent_tmt_run_dir" - fi + printf "%s\n" "${tmpfiles_rule[@]}" > "$rule" + printf "%s\n" "${tmpfiles_rule[@]}" | cmp - "$rule" + fi + ' install-test-configuration \ + "$([[ $tmt_run_dir == "$tmt_local_workdir_root"/* ]] && printf 1 || printf 0)" \ + "$tmt_run_dir" bootc status - # Ask the guest to reboot itself; this is more reliable than - # testcloud's ACPI soft-poweroff for a full bootc deployment reboot. + # Reboot the guest itself; this avoids testcloud's ACPI soft-poweroff. tmt-reboot -c 'systemctl reboot' ;; 1) test -e /run/ostree-booted - - mountpoint -q "$bind_target" - test "$(<"$bind_target/mounted-sentinel")" = mounted-only-data - test ! -e "$bind_target/underlying-sentinel" - tmt_run_dir=$(get_tmt_run_dir) - tmt_run_id=${tmt_run_dir##*/} - if [[ $tmt_run_dir == /var/ARTIFACTS/* ]]; then - # The Packit workdir itself must have survived preserve-var intact. + if [[ $tmt_run_dir == "$tmt_local_workdir_root"/* ]]; then test -d "$tmt_run_dir" test ! -L "$tmt_run_dir" verify_tmt_workdir_in_preserved_var "$tmt_run_dir" + expected_tmpfiles_rule=( + "d /var/lib/bootc-tmt-workdirs - - - - -" + "d $tmt_run_dir - - - - -" + ) + printf "%s\n" "${expected_tmpfiles_rule[@]}" | \ + cmp - /etc/tmpfiles.d/bootc-tmt-workdir.conf else - persistent_tmt_run_dir=$tmt_workdir_root/$tmt_run_id - test -f /etc/tmpfiles.d/bootc-tmt-workdir.conf - test -L "$tmt_run_dir" - test "$(readlink -f -- "$tmt_run_dir")" = "$persistent_tmt_run_dir" + test -d "$tmt_run_dir" + test ! -L "$tmt_run_dir" + verify_tmt_workdir_in_preserved_var "$tmt_run_dir" fi + test -d "$TMT_PLAN_DATA" test -d "$TMT_TEST_DATA" test -f "$TMT_TEST_METADATA" tmt_test_data_sentinel=$TMT_TEST_DATA/$tmt_test_data_sentinel_name tmt_test_data_post_reboot_marker=$TMT_TEST_DATA/$tmt_test_data_post_reboot_marker_name - test -f "$tmt_test_data_sentinel" test "$(<"$tmt_test_data_sentinel")" = workdir-persistence-sentinel printf '%s\n' post-reboot-workdir-writable > "$tmt_test_data_post_reboot_marker" test "$(<"$tmt_test_data_post_reboot_marker")" = post-reboot-workdir-writable + mountpoint -q "$bind_target" + test "$(<"$bind_target/mounted-sentinel")" = bind-mounted-data + test ! -e "$bind_target/underlying-sentinel" + mountpoint -q /var/log + test "$(findmnt --noheadings --output FSTYPE --target /var/log)" = ext4 + test "$(&2 diff --git a/tmt/tests/test-50-to-existing-root-preserve-var.fmf b/tmt/tests/test-50-to-existing-root-preserve-var.fmf index 305222ad67..83c8b5aafc 100644 --- a/tmt/tests/test-50-to-existing-root-preserve-var.fmf +++ b/tmt/tests/test-50-to-existing-root-preserve-var.fmf @@ -1,7 +1,7 @@ # The VM cannot share controller Podman storage, so TMT syncs the local OCI -# archive to the single guest before migration. It reboots once, then verifies -# preserved /var data. -summary: Package-mode /var preservation +# archive to one guest before migration. The test checks preflight failure, +# separate and temporary mounts, stateful data, then reboots and verifies them. +summary: Package-mode preserve-var state and mounts duration: 45m adjust: - when: running_env == image_mode From 42a594ceec03741e7fab405cb9379c6f95b46b42 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 11:08:18 -0600 Subject: [PATCH 17/38] docs: Explain preserve-var SSH host key handling The install guide distinguishes preserved root authorized keys from host keys in the old /etc, and warns that retained cloud-init state can suppress host-key generation. The TMT guide documents the Fedora 44 and CentOS Stream 10 VM commands and the durable local workdir prerequisite. Assisted-by: AI --- docs/src/bootc-installation.7.md | 8 ++++++++ tmt/README.md | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/docs/src/bootc-installation.7.md b/docs/src/bootc-installation.7.md index 51ec5e0669..3d542e7184 100644 --- a/docs/src/bootc-installation.7.md +++ b/docs/src/bootc-installation.7.md @@ -376,6 +376,14 @@ to automatically inherit root's SSH keys (which may have been injected from e.g. cloud instance userdata via a tool like `cloud-init`). To do this, add `--root-ssh-authorized-keys /target/root/.ssh/authorized_keys` to the install command. +This only preserves root's **authorized keys** for inbound access; SSH host keys +from the old `/etc/ssh` are not migrated. Provision host keys for the new system +before rebooting, or arrange for the new system to generate them on first boot. +If `/var/lib/cloud` is preserved, cloud-init may consider its initialization +already complete and skip first-boot key generation. An image that relies on +cloud-init for SSH host keys can therefore boot without usable SSH service +credentials unless they are provisioned separately. + See also the [bootc-install-to-existing-root(8)](man/bootc-install-to-existing-root.8.md) man page for more details. diff --git a/tmt/README.md b/tmt/README.md index 6744b17269..43a0818178 100644 --- a/tmt/README.md +++ b/tmt/README.md @@ -4,3 +4,28 @@ In the bootc CI, integration tests are executed via Packit on the Testing Farm. See [CONTRIBUTING.md](../CONTRIBUTING.md#running-tmt-integration-tests) for instructions on running them locally. + +## Package-mode preserve-var test + +The package-mode migration test can be run against both supported local VM +images: + +```bash +just test-tmt-package-mode fedora-44 +just test-tmt-package-mode centos-stream-10 +``` + +Each invocation builds the configured `BOOTC_image` (default +`localhost/bootc`), transfers it as an OCI archive, and runs the same TMT plan. +The recipe asks for `sudo` once to create `/var/lib/bootc-tmt-workdirs`; TMT 1.69 +syncs to the same absolute path on the guest, where `/var/lib` is covered by +`--preserve-var`. This avoids using the excluded `/var/tmp` tree and avoids +replacing TMT's real run directory with a symlink. TMT run logs and the +testcloud image cache are kept under that workdir root. + +If sudo cannot prompt in the current shell, create the root once interactively +and retry: + +```bash +sudo install -d -m 1777 /var/lib/bootc-tmt-workdirs +``` From 27417bcf8e97309ca907b760e16092bf7b1570f2 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 11:17:00 -0600 Subject: [PATCH 18/38] install: Check hardlink labels before relabeling GNU cp -a can preserve aliases whose logical paths map to different SELinux labels. Applying those labels in traversal order would leave the shared inode mislabeled, so validate every staged alias before changing labels or publishing the staged entries. Assisted-by: AI --- crates/lib/src/install/migrate.rs | 278 +++++++++++++++++++++++++++--- 1 file changed, 257 insertions(+), 21 deletions(-) diff --git a/crates/lib/src/install/migrate.rs b/crates/lib/src/install/migrate.rs index 10e56a6a17..e58cd63dc7 100644 --- a/crates/lib/src/install/migrate.rs +++ b/crates/lib/src/install/migrate.rs @@ -63,7 +63,7 @@ mod mountinfo; use std::path::{Component, Path}; use std::{ - collections::BTreeSet, + collections::{BTreeSet, HashMap}, ffi::OsStr, os::fd::{AsFd, AsRawFd}, path::PathBuf, @@ -685,16 +685,45 @@ fn logical_var_path(relative: &Path) -> PathBuf { Path::new(VAR_LOGICAL_ROOT).join(relative) } -fn lookup_and_apply_label( +type StagedLabels = HashMap>; +type RegularFileLabels = HashMap<(libc::dev_t, libc::ino_t), (PathBuf, Vec)>; + +fn collect_label( labeler: &dyn StagedLabeler, logical_path: &Path, mode: u32, - target: StagedLabelTarget<'_>, + regular_file_id: Option<(libc::dev_t, libc::ino_t)>, + labels: &mut StagedLabels, + regular_file_labels: &mut RegularFileLabels, ) -> Result<()> { let label = labeler .lookup(logical_path, mode)? .with_context(|| format!("No SELinux policy label for {logical_path:?}"))?; - labeler.apply(logical_path, target, &label) + + if let Some(file_id) = regular_file_id { + if let Some((alias_path, alias_label)) = regular_file_labels.get(&file_id) { + anyhow::ensure!( + alias_label == &label, + "Hardlinked staged regular files {alias_path:?} and {logical_path:?} resolve to different SELinux labels" + ); + } else { + regular_file_labels.insert(file_id, (logical_path.to_owned(), label.clone())); + } + } + labels.insert(logical_path.to_owned(), label); + Ok(()) +} + +fn apply_staged_label( + labeler: &dyn StagedLabeler, + labels: &StagedLabels, + logical_path: &Path, + target: StagedLabelTarget<'_>, +) -> Result<()> { + let label = labels + .get(logical_path) + .with_context(|| format!("No collected SELinux label for {logical_path:?}"))?; + labeler.apply(logical_path, target, label) } /// Relabel only staged entries. Logical lookup paths are composed from the @@ -707,9 +736,25 @@ fn relabel_staged_names( mountpoints: &BTreeSet, ) -> Result<()> { let staging_device = staging.metadata(".")?.dev(); + let mut labels = StagedLabels::new(); + let mut regular_file_labels = RegularFileLabels::new(); + for name in names { + let relative_path = relative_dir.join(name); + collect_staged_entry_labels( + labeler, + staging, + name, + &logical_var_path(&relative_path), + mountpoints, + Some(staging_device), + false, + &mut labels, + &mut regular_file_labels, + )?; + } for name in names { let relative_path = relative_dir.join(name); - relabel_staged_entry( + apply_staged_entry_labels( labeler, staging, name, @@ -717,16 +762,16 @@ fn relabel_staged_names( mountpoints, Some(staging_device), false, + &labels, )?; } Ok(()) } -/// Relabel one staged inode and, for same-device directories, recursively -/// relabel its staged children without following symlinks. Device changes and -/// known nested-mount placeholders are labeled at the mount root but never -/// traversed. -fn relabel_staged_entry( +/// Look up labels for staged entries without applying them. Regular-file inode +/// identities are grouped across this staging tree so aliases must agree +/// before any labels are changed. +fn collect_staged_entry_labels( labeler: &dyn StagedLabeler, parent: &CapStdDir, name: &OsStr, @@ -734,6 +779,8 @@ fn relabel_staged_entry( mountpoints: &BTreeSet, root_device: Option, stop_at_entry: bool, + labels: &mut StagedLabels, + regular_file_labels: &mut RegularFileLabels, ) -> Result<()> { use rustix::fs::{Mode, OFlags, openat}; @@ -745,12 +792,13 @@ fn relabel_staged_entry( .with_context(|| format!("Opening staged directory {logical_path:?}"))?; let metadata = directory.metadata(".")?; let device = root_device.unwrap_or_else(|| metadata.dev()); - let fd = directory.as_filelike_view::(); - lookup_and_apply_label( + collect_label( labeler, logical_path, metadata.mode(), - StagedLabelTarget::Fd(fd.as_fd()), + None, + labels, + regular_file_labels, )?; let relative = logical_path @@ -762,7 +810,7 @@ fn relabel_staged_entry( for entry in directory.read_dir(".")? { let entry = entry?; let child_name = entry.file_name(); - relabel_staged_entry( + collect_staged_entry_labels( labeler, &directory, &child_name, @@ -770,17 +818,21 @@ fn relabel_staged_entry( mountpoints, Some(device), false, + labels, + regular_file_labels, )?; } return Ok(()); } if metadata.is_symlink() { - return lookup_and_apply_label( + return collect_label( labeler, logical_path, metadata.mode(), - StagedLabelTarget::NoFollowPath(parent, name), + None, + labels, + regular_file_labels, ); } @@ -797,22 +849,152 @@ fn relabel_staged_entry( stat.st_mode & libc::S_IFMT == libc::S_IFREG, "Staged entry {logical_path:?} changed type before relabel" ); - return lookup_and_apply_label( + return collect_label( labeler, logical_path, stat.st_mode, - StagedLabelTarget::Fd(fd.as_fd()), + Some((stat.st_dev, stat.st_ino)), + labels, + regular_file_labels, ); } - lookup_and_apply_label( + collect_label( labeler, logical_path, metadata.mode(), + None, + labels, + regular_file_labels, + ) +} + +/// Apply previously collected labels to staged entries without following +/// symlinks. This is intentionally a separate walk from label lookup so all +/// hardlink aliases can be checked before the first inode is relabeled. +fn apply_staged_entry_labels( + labeler: &dyn StagedLabeler, + parent: &CapStdDir, + name: &OsStr, + logical_path: &Path, + mountpoints: &BTreeSet, + root_device: Option, + stop_at_entry: bool, + labels: &StagedLabels, +) -> Result<()> { + use rustix::fs::{Mode, OFlags, openat}; + + let metadata = parent + .symlink_metadata(name) + .with_context(|| format!("Inspecting staged /var entry {logical_path:?} before relabel"))?; + if metadata.is_dir() { + let directory = open_child_dir_nofollow(parent, name) + .with_context(|| format!("Opening staged directory {logical_path:?}"))?; + let metadata = directory.metadata(".")?; + let device = root_device.unwrap_or_else(|| metadata.dev()); + let fd = directory.as_filelike_view::(); + apply_staged_label( + labeler, + labels, + logical_path, + StagedLabelTarget::Fd(fd.as_fd()), + )?; + + let relative = logical_path + .strip_prefix(VAR_LOGICAL_ROOT) + .with_context(|| format!("Invalid logical migration path {logical_path:?}"))?; + if stop_at_entry || metadata.dev() != device || mountpoints.contains(relative) { + return Ok(()); + } + for entry in directory.read_dir(".")? { + let entry = entry?; + let child_name = entry.file_name(); + apply_staged_entry_labels( + labeler, + &directory, + &child_name, + &logical_path.join(&child_name), + mountpoints, + Some(device), + false, + labels, + )?; + } + return Ok(()); + } + + if metadata.is_symlink() { + return apply_staged_label( + labeler, + labels, + logical_path, + StagedLabelTarget::NoFollowPath(parent, name), + ); + } + + if metadata.is_file() { + let fd = openat( + parent.as_filelike_view::().as_fd(), + name, + OFlags::CLOEXEC | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::RDONLY, + Mode::empty(), + ) + .with_context(|| format!("Opening staged file {logical_path:?} without following links"))?; + let stat = rustix::fs::fstat(&fd)?; + anyhow::ensure!( + stat.st_mode & libc::S_IFMT == libc::S_IFREG, + "Staged entry {logical_path:?} changed type before relabel" + ); + return apply_staged_label( + labeler, + labels, + logical_path, + StagedLabelTarget::Fd(fd.as_fd()), + ); + } + + apply_staged_label( + labeler, + labels, + logical_path, StagedLabelTarget::NoFollowPath(parent, name), ) } +fn relabel_staged_entry( + labeler: &dyn StagedLabeler, + parent: &CapStdDir, + name: &OsStr, + logical_path: &Path, + mountpoints: &BTreeSet, + root_device: Option, + stop_at_entry: bool, +) -> Result<()> { + let mut labels = StagedLabels::new(); + let mut regular_file_labels = RegularFileLabels::new(); + collect_staged_entry_labels( + labeler, + parent, + name, + logical_path, + mountpoints, + root_device, + stop_at_entry, + &mut labels, + &mut regular_file_labels, + )?; + apply_staged_entry_labels( + labeler, + parent, + name, + logical_path, + mountpoints, + root_device, + stop_at_entry, + &labels, + ) +} + /// Move staged children into an existing destination directory without /// replacing entries already present there. Keep the destination directory's /// metadata unchanged; moving children updates its timestamps, so restore its @@ -911,7 +1093,7 @@ fn copy_filtered_directory_staged( mountpoints, reflink, true, - labeler, + None, cp_program, )?; if let Some(labeler) = labeler { @@ -1407,6 +1589,7 @@ mod tests { applied: Mutex>, fail_lookup: bool, fail_apply: bool, + different_labels: bool, } impl StagedLabeler for FakeLabeler { @@ -1417,7 +1600,12 @@ mod tests { "policy lookup used non-logical path {logical_path:?}" ); anyhow::ensure!(!self.fail_lookup, "injected SELinux policy lookup failure"); - Ok(Some(b"test-label".to_vec())) + let label = if self.different_labels && logical_path == Path::new("/var/alias-b") { + b"different-label".to_vec() + } else { + b"test-label".to_vec() + }; + Ok(Some(label)) } fn apply( @@ -1591,6 +1779,54 @@ mod tests { Ok(()) } + #[test] + fn hardlinked_paths_require_matching_labels_before_publication() -> Result<()> { + let temp = tempfile::tempdir()?; + let src_path = temp.path().join("src"); + fs::create_dir(&src_path)?; + fs::write(src_path.join("alias-a"), b"shared state")?; + fs::hard_link(src_path.join("alias-a"), src_path.join("alias-b"))?; + let source = cap_dir(&src_path); + + let mismatch_path = temp.path().join("dst-mismatch"); + fs::create_dir(&mismatch_path)?; + let mismatch_destination = cap_dir(&mismatch_path); + let mismatch_labeler = FakeLabeler { + different_labels: true, + ..FakeLabeler::default() + }; + let error = preserve_var_copy_mounts_labeled( + &source, + &mismatch_destination, + &[], + &BTreeSet::new(), + Some(&mismatch_labeler), + ) + .unwrap_err(); + assert!(format!("{error:#}").contains("resolve to different SELinux labels")); + assert!(mismatch_labeler.applied.lock().unwrap().is_empty()); + assert_eq!(fs::read_dir(&mismatch_path)?.count(), 0); + + let matching_path = temp.path().join("dst-matching"); + fs::create_dir(&matching_path)?; + let matching_destination = cap_dir(&matching_path); + let matching_labeler = FakeLabeler::default(); + preserve_var_copy_mounts_labeled( + &source, + &matching_destination, + &[], + &BTreeSet::new(), + Some(&matching_labeler), + )?; + let alias_a = fs::metadata(matching_path.join("alias-a"))?; + let alias_b = fs::metadata(matching_path.join("alias-b"))?; + assert_eq!(alias_a.ino(), alias_b.ino()); + let applied = matching_labeler.applied.lock().unwrap(); + assert!(applied.contains(&PathBuf::from("/var/alias-a"))); + assert!(applied.contains(&PathBuf::from("/var/alias-b"))); + Ok(()) + } + #[test] fn exclusions_are_relative_and_normalized() { assert_eq!( From 53af74ab884c8520f79f8a2e74c23f2b8189304b Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 11:27:10 -0600 Subject: [PATCH 19/38] tests: Fix package-mode TMT workdir setup Local package-mode tests should run without root access. Keep TMT workdirs in the developer cache, verify the guest mirror is preserved across migration, and encode the LBI list in a TMT-compatible environment file. Assisted-by: AI --- Justfile | 24 +-- tmt/README.md | 27 ++-- .../test-to-existing-root-preserve-var.sh | 142 +++++++++++++++--- 3 files changed, 142 insertions(+), 51 deletions(-) diff --git a/Justfile b/Justfile index f511c34360..2081f0fddc 100644 --- a/Justfile +++ b/Justfile @@ -238,34 +238,36 @@ test-tmt-package-mode image="centos-stream-10": lbi_images="{{lbi_images}}" just build archive_dir=$(mktemp -d -- tmt/plans/.package-mode-local.XXXXXX) + archive_path="$archive_dir/bootc.oci" + environment_file="$archive_dir/package-mode.env" cleanup() { rm -rf -- "$archive_dir" } trap cleanup EXIT trap 'cleanup; exit 130' INT trap 'cleanup; exit 143' TERM - archive_path="$archive_dir/bootc.oci" if [[ -e "$archive_path" ]]; then echo "Refusing to overwrite existing archive: $archive_path" >&2 exit 1 fi podman save --format oci-archive --output "$archive_path" {{base_img}} - # TMT syncs the plan workdir to the same absolute guest path. Use a - # world-writable root so non-root local runs can place it under preserved - # /var/lib on the guest without a symlink or a separate VM harness. - if [[ "$EUID" -eq 0 ]]; then - install -d -m 1777 /var/lib/bootc-tmt-workdirs - else - sudo install -d -m 1777 /var/lib/bootc-tmt-workdirs - fi + # TMT syncs its workdir to the same absolute guest path. On bootc guests, + # /home resolves to /var/home, which preserve-var carries across migration. + workdir_root="$HOME/.cache/bootc-tmt-workdirs" + mkdir -p -- "$workdir_root" + workdir_root=$(realpath -e -- "$workdir_root") + # The test consumes comma-separated refs, and tmt 1.69 supports these in + # a dotenv environment file without splitting the value on whitespace. + printf 'PACKAGE_MODE_LBI_IMAGES=%s\n' "${lbi_images// /,}" > "$environment_file" tmt \ --context=running_env=packit \ --context=package_source=local \ --context="test_disk_image={{image}}" \ - run --workdir-root /var/lib/bootc-tmt-workdirs \ + run --workdir-root "$workdir_root" \ + --environment-file "$environment_file" \ --environment "PACKAGE_MODE_IMAGE_NAME={{base_img}}" \ - --environment "PACKAGE_MODE_LBI_IMAGES=${lbi_images// /,}" \ --environment "PACKAGE_MODE_IMAGE_ARCHIVE=$archive_path" \ + --environment "PACKAGE_MODE_TMT_WORKDIR_ROOT=$workdir_root" \ plans --name '^/tmt/plans/package-mode$' # Split out from `test-container` because, unlike the container integration tests, diff --git a/tmt/README.md b/tmt/README.md index 43a0818178..847bc0d0a8 100644 --- a/tmt/README.md +++ b/tmt/README.md @@ -7,25 +7,20 @@ instructions on running them locally. ## Package-mode preserve-var test -The package-mode migration test can be run against both supported local VM -images: +Run the package-mode migration test against either supported local VM image. +The command builds the configured `BOOTC_image` (default `localhost/bootc`), +transfers it as an OCI archive, and runs the TMT plan. It does not require +`sudo`: ```bash just test-tmt-package-mode fedora-44 just test-tmt-package-mode centos-stream-10 ``` -Each invocation builds the configured `BOOTC_image` (default -`localhost/bootc`), transfers it as an OCI archive, and runs the same TMT plan. -The recipe asks for `sudo` once to create `/var/lib/bootc-tmt-workdirs`; TMT 1.69 -syncs to the same absolute path on the guest, where `/var/lib` is covered by -`--preserve-var`. This avoids using the excluded `/var/tmp` tree and avoids -replacing TMT's real run directory with a symlink. TMT run logs and the -testcloud image cache are kept under that workdir root. - -If sudo cannot prompt in the current shell, create the root once interactively -and retry: - -```bash -sudo install -d -m 1777 /var/lib/bootc-tmt-workdirs -``` +The controller workdir is created under `$HOME/.cache/bootc-tmt-workdirs`. TMT +1.69 mirrors it to +the same absolute guest path; on bootc guests `/home/...` resolves to +`/var/home/...`, which is preserved by the migration. The test checks the +resolved path is under preserved `/var` and rejects nested mounts, so it does +not rely on `/var/tmp` or symlinks. TMT run logs and the testcloud image cache +remain under that workdir root. diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index c3a4f9474b..835a9c02ad 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -1,6 +1,6 @@ #!/bin/bash # Both local and Packit runs prepare one package-mode guest, then convert and -# reboot that same guest; local TMT workdirs live under preserved /var/lib. +# reboot that same guest; local TMT workdirs live under preserved /var. set -xeuo pipefail target_image=localhost/bootc-preserve-var-test @@ -8,12 +8,12 @@ var_state=/var/lib/bootc-tmt-preserve-var app_state=$var_state/app-state/state log_fs_image=$var_state/logfs.img empty_tmpfs=$var_state/empty_tmpfs +empty_tmpfs_marker=$empty_tmpfs/marker bind_dir=/var/lib/bootc-tmt-preserve-var-bind bind_source=$bind_dir/source bind_target=$bind_dir/target preflight_mountpoint=$var_state/preflight-special tmt_plan_data_suffix=/tmt/plans/package-mode/data -tmt_local_workdir_root=/var/lib/bootc-tmt-workdirs tmt_artifacts_parent=/var/ARTIFACTS tmt_test_data_sentinel_name=workdir-persistence-sentinel tmt_test_data_post_reboot_marker_name=post-reboot-marker @@ -22,29 +22,106 @@ get_tmt_run_dir() { local plan_data=${TMT_PLAN_DATA:-} local run_dir local resolved_plan_data + local resolved_run_dir + local workdir_root=${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} + local resolved_workdir_root + local run_id if [[ $plan_data != *"$tmt_plan_data_suffix" ]]; then printf 'TMT_PLAN_DATA must end in %s (got %q)\n' "$tmt_plan_data_suffix" "$plan_data" >&2 return 1 fi run_dir=${plan_data%"$tmt_plan_data_suffix"} - if [[ $run_dir =~ ^/var/lib/bootc-tmt-workdirs/run-[[:alnum:]_.-]+$ || - $run_dir =~ ^/var/ARTIFACTS/work-[[:alnum:]_.-]+$ ]]; then - : + if [[ -n $workdir_root ]]; then + if [[ $workdir_root != /* || $workdir_root == */ ]]; then + printf 'PACKAGE_MODE_TMT_WORKDIR_ROOT must be an absolute path without a trailing slash (got %q)\n' \ + "$workdir_root" >&2 + return 1 + fi + resolved_workdir_root=$(realpath -e -- "$workdir_root") || { + printf 'TMT workdir root does not exist: %s\n' "$workdir_root" >&2 + return 1 + } + case $resolved_workdir_root in + /var/*) ;; + *) + printf 'TMT workdir root must resolve under preserved /var (got %q -> %q)\n' \ + "$workdir_root" "$resolved_workdir_root" >&2 + return 1 + ;; + esac + case $resolved_workdir_root in + /var/tmp|/var/tmp/*|/var/cache|/var/cache/*|/var/log/journal|/var/log/journal/*|/var/lib/containers|/var/lib/containers/*) + printf 'TMT workdir root is excluded from preserve-var: %s\n' \ + "$resolved_workdir_root" >&2 + return 1 + ;; + esac + case $run_dir in + "$workdir_root"/run-*) + run_id=${run_dir#"$workdir_root"/} + ;; + "$resolved_workdir_root"/run-*) + run_id=${run_dir#"$resolved_workdir_root"/} + ;; + *) + printf 'Expected TMT run directory under configured root %q (got %q)\n' \ + "$workdir_root" "$run_dir" >&2 + return 1 + ;; + esac + if [[ ! $run_id =~ ^run-[[:alnum:]_.-]+$ ]]; then + printf 'Unexpected TMT run directory under configured root: %q\n' \ + "$run_dir" >&2 + return 1 + fi + resolved_run_dir=$(realpath -e -- "$run_dir") || { + printf 'TMT run directory does not exist: %s\n' "$run_dir" >&2 + return 1 + } + if [[ $resolved_run_dir != "$resolved_workdir_root/$run_id" ]]; then + printf 'TMT run directory must be a real directory below its configured root (got %q -> %q)\n' \ + "$run_dir" "$resolved_run_dir" >&2 + return 1 + fi + resolved_plan_data=$(realpath -e -- "$plan_data") || { + printf 'TMT_PLAN_DATA does not exist: %s\n' "$plan_data" >&2 + return 1 + } + if [[ $resolved_plan_data != "$resolved_run_dir$tmt_plan_data_suffix" ]]; then + printf 'TMT_PLAN_DATA resolves outside its run directory (got %q -> %q)\n' \ + "$plan_data" "$resolved_plan_data" >&2 + return 1 + fi + elif [[ $run_dir =~ ^/var/ARTIFACTS/work-[[:alnum:]_.-]+$ ]]; then + resolved_run_dir=$(realpath -e -- "$run_dir") || { + printf 'TMT artifact workdir does not exist: %s\n' "$run_dir" >&2 + return 1 + } + case $resolved_run_dir in + /var/*) ;; + *) + printf 'TMT artifact workdir must resolve under /var (got %q)\n' \ + "$resolved_run_dir" >&2 + return 1 + ;; + esac + resolved_plan_data=$(realpath -e -- "$plan_data") || { + printf 'TMT_PLAN_DATA does not exist: %s\n' "$plan_data" >&2 + return 1 + } + if [[ $resolved_plan_data != "$resolved_run_dir$tmt_plan_data_suffix" ]]; then + printf 'TMT_PLAN_DATA resolves outside its run directory (got %q -> %q)\n' \ + "$plan_data" "$resolved_plan_data" >&2 + return 1 + fi else - printf 'Expected TMT workdir under %s or %s (got %q)\n' \ - "$tmt_local_workdir_root" "$tmt_artifacts_parent" "$run_dir" >&2 + printf 'Expected TMT workdir under PACKAGE_MODE_TMT_WORKDIR_ROOT or %s (got %q)\n' \ + "$tmt_artifacts_parent" "$run_dir" >&2 return 1 fi case ${TMT_REBOOT_COUNT:-0} in - 0|1) - if ! resolved_plan_data=$(realpath -e -- "$plan_data") || - [[ $resolved_plan_data != "$plan_data" ]]; then - printf 'TMT_PLAN_DATA must be a canonical real path (got %q, resolves to %q)\n' \ - "$plan_data" "${resolved_plan_data:-}" >&2 - return 1 - fi - ;; + 0|1) ;; *) printf 'Unexpected TMT_REBOOT_COUNT=%s\n' "${TMT_REBOOT_COUNT}" >&2 return 1 @@ -70,6 +147,18 @@ verify_tmt_workdir_in_preserved_var() { fi } +is_local_tmt_workdir() { + local workdir_root=${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} + local resolved_workdir_root + + [[ -n $workdir_root ]] || return 1 + if [[ $1 == "$workdir_root"/run-* ]]; then + return 0 + fi + resolved_workdir_root=$(realpath -e -- "$workdir_root") || return 1 + [[ $1 == "$resolved_workdir_root"/run-* ]] +} + case ${TMT_REBOOT_COUNT:-0} in 0) podman --remote=false image exists "$target_image" @@ -119,8 +208,10 @@ case ${TMT_REBOOT_COUNT:-0} in mount -o loop "$log_fs_image" /var/log printf '%s\n' separate-var-log-filesystem > /var/log/bootc-tmt-sentinel install -d -m 0755 "$empty_tmpfs" - mount -t tmpfs -o size=16m tmpfs "$empty_tmpfs" + mount -t tmpfs -o size=16m '' "$empty_tmpfs" test -z "$(find "$empty_tmpfs" -mindepth 1 -print -quit)" + printf '%s\n' transient-tmpfs-data > "$empty_tmpfs_marker" + test "$(<"$empty_tmpfs_marker")" = transient-tmpfs-data : > "$preflight_mountpoint" mount --bind /dev/null "$preflight_mountpoint" @@ -146,8 +237,8 @@ case ${TMT_REBOOT_COUNT:-0} in umount "$preflight_mountpoint" rm -f "$preflight_mountpoint" "$boot_snapshot" "$boot_snapshot_after" - # TMT 1.69 rsyncs to the same absolute path on the guest. Keeping its - # real workdir under /var/lib avoids both excluded /var/tmp and symlinks. + # TMT 1.69 rsyncs to the same absolute path on the guest. The configured + # controller path resolves under /var here (including /home -> /var/home). test -f "$TMT_TEST_METADATA" tmt_test_data_sentinel=$TMT_TEST_DATA/$tmt_test_data_sentinel_name test ! -e "$tmt_test_data_sentinel" @@ -216,6 +307,7 @@ case ${TMT_REBOOT_COUNT:-0} in local_workdir_root=$1 run_dir=$2 + workdir_root=$3 mkdir -p "$deployment" bootc install mount --sysroot /target --latest "$deployment" @@ -228,15 +320,16 @@ case ${TMT_REBOOT_COUNT:-0} in rule=$deployment/etc/tmpfiles.d/bootc-tmt-workdir.conf install -d -m 0755 "${rule%/*}" tmpfiles_rule=( - "d /var/lib/bootc-tmt-workdirs - - - - -" + "d $workdir_root - - - - -" "d $run_dir - - - - -" ) printf "%s\n" "${tmpfiles_rule[@]}" > "$rule" printf "%s\n" "${tmpfiles_rule[@]}" | cmp - "$rule" fi ' install-test-configuration \ - "$([[ $tmt_run_dir == "$tmt_local_workdir_root"/* ]] && printf 1 || printf 0)" \ - "$tmt_run_dir" + "$(is_local_tmt_workdir "$tmt_run_dir" && printf 1 || printf 0)" \ + "$(realpath -e -- "$tmt_run_dir")" \ + "$(if [[ -n ${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} ]]; then realpath -e -- "$PACKAGE_MODE_TMT_WORKDIR_ROOT"; fi)" bootc status # Reboot the guest itself; this avoids testcloud's ACPI soft-poweroff. @@ -245,13 +338,13 @@ case ${TMT_REBOOT_COUNT:-0} in 1) test -e /run/ostree-booted tmt_run_dir=$(get_tmt_run_dir) - if [[ $tmt_run_dir == "$tmt_local_workdir_root"/* ]]; then + if is_local_tmt_workdir "$tmt_run_dir"; then test -d "$tmt_run_dir" test ! -L "$tmt_run_dir" verify_tmt_workdir_in_preserved_var "$tmt_run_dir" expected_tmpfiles_rule=( - "d /var/lib/bootc-tmt-workdirs - - - - -" - "d $tmt_run_dir - - - - -" + "d $(realpath -e -- "$PACKAGE_MODE_TMT_WORKDIR_ROOT") - - - - -" + "d $(realpath -e -- "$tmt_run_dir") - - - - -" ) printf "%s\n" "${expected_tmpfiles_rule[@]}" | \ cmp - /etc/tmpfiles.d/bootc-tmt-workdir.conf @@ -278,6 +371,7 @@ case ${TMT_REBOOT_COUNT:-0} in test "$( Date: Thu, 12 Nov 2026 11:36:23 -0600 Subject: [PATCH 20/38] tests: Harden preserve-var TMT workdir cleanup The Packit artifact workdir path needs the same preservation safeguards as local TMT workdirs, and a failed migration must not leave /var/log frozen. Validate the artifact directory and make freeze cleanup run on normal exits and signals while retaining existing traps and exit status. Assisted-by: AI --- .../test-to-existing-root-preserve-var.sh | 122 ++++++++++++++++-- 1 file changed, 113 insertions(+), 9 deletions(-) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index 835a9c02ad..6bf4645d3f 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -18,6 +18,97 @@ tmt_artifacts_parent=/var/ARTIFACTS tmt_test_data_sentinel_name=workdir-persistence-sentinel tmt_test_data_post_reboot_marker_name=post-reboot-marker +is_always_excluded_var_path() { + case $1 in + /var/tmp|/var/tmp/*|/var/cache|/var/cache/*|/var/log/journal|/var/log/journal/*|/var/lib/containers|/var/lib/containers/*) + return 0 + ;; + esac + return 1 +} + +verify_tmt_workdir_directory() { + local run_dir=$1 + local canonical_run_dir=$2 + + if [[ ! -d $run_dir || -L $run_dir ]]; then + printf 'TMT workdir is missing or not a real directory: %s\n' "$run_dir" >&2 + return 1 + fi + if is_always_excluded_var_path "$canonical_run_dir"; then + printf 'TMT workdir is excluded from preserve-var: %s\n' \ + "$canonical_run_dir" >&2 + return 1 + fi +} + +get_trap_action() { + local signal=$1 + local trap_definition + + trap_definition=$(trap -p "$signal") + if [[ -n $trap_definition ]]; then + eval "set -- $trap_definition" + printf '%s' "$3" + fi +} + +restore_signal_trap() { + local signal=$1 + local previous_signal_trap=$2 + + if [[ -n $previous_signal_trap ]]; then + trap -- "$previous_signal_trap" "$signal" + else + trap - "$signal" + fi +} + +restore_log_freeze_signal_traps() { + restore_signal_trap HUP "${previous_hup_trap:-}" + restore_signal_trap INT "${previous_int_trap:-}" + restore_signal_trap TERM "${previous_term_trap:-}" +} + +thaw_log_filesystem() { + if (( log_fs_frozen )); then + fsfreeze -u /var/log || return 1 + log_fs_frozen=0 + restore_log_freeze_signal_traps + fi +} + +cleanup_log_freeze() { + local status=$? + + trap - EXIT + if ! thaw_log_filesystem; then + printf 'Failed to thaw /var/log during cleanup\n' >&2 + if (( status == 0 )); then + status=1 + fi + fi + if [[ -n $previous_exit_trap ]]; then + eval "$previous_exit_trap" || : + fi + exit "$status" +} + +handle_log_freeze_signal() { + local status=$1 + local signal=$2 + local previous_signal_trap=$3 + + trap - "$signal" + if ! thaw_log_filesystem; then + printf 'Failed to thaw /var/log after %s\n' "$signal" >&2 + fi + if [[ -n $previous_signal_trap ]]; then + eval "$previous_signal_trap" || : + fi + exit "$status" +} + get_tmt_run_dir() { local plan_data=${TMT_PLAN_DATA:-} local run_dir @@ -50,13 +141,11 @@ get_tmt_run_dir() { return 1 ;; esac - case $resolved_workdir_root in - /var/tmp|/var/tmp/*|/var/cache|/var/cache/*|/var/log/journal|/var/log/journal/*|/var/lib/containers|/var/lib/containers/*) - printf 'TMT workdir root is excluded from preserve-var: %s\n' \ - "$resolved_workdir_root" >&2 - return 1 - ;; - esac + if is_always_excluded_var_path "$resolved_workdir_root"; then + printf 'TMT workdir root is excluded from preserve-var: %s\n' \ + "$resolved_workdir_root" >&2 + return 1 + fi case $run_dir in "$workdir_root"/run-*) run_id=${run_dir#"$workdir_root"/} @@ -84,6 +173,7 @@ get_tmt_run_dir() { "$run_dir" "$resolved_run_dir" >&2 return 1 fi + verify_tmt_workdir_directory "$run_dir" "$resolved_run_dir" || return 1 resolved_plan_data=$(realpath -e -- "$plan_data") || { printf 'TMT_PLAN_DATA does not exist: %s\n' "$plan_data" >&2 return 1 @@ -98,6 +188,12 @@ get_tmt_run_dir() { printf 'TMT artifact workdir does not exist: %s\n' "$run_dir" >&2 return 1 } + verify_tmt_workdir_directory "$run_dir" "$resolved_run_dir" || return 1 + if [[ $resolved_run_dir != "$run_dir" ]]; then + printf 'TMT artifact workdir must be canonical (got %q -> %q)\n' \ + "$run_dir" "$resolved_run_dir" >&2 + return 1 + fi case $resolved_run_dir in /var/*) ;; *) @@ -253,6 +349,15 @@ case ${TMT_REBOOT_COUNT:-0} in # Keep the backing image for the separate /var/log filesystem stable # while preserve-var copies its image file from /var/lib. fsfreeze -f /var/log + log_fs_frozen=1 + previous_exit_trap=$(get_trap_action EXIT) + previous_hup_trap=$(get_trap_action HUP) + previous_int_trap=$(get_trap_action INT) + previous_term_trap=$(get_trap_action TERM) + trap cleanup_log_freeze EXIT + trap 'handle_log_freeze_signal 129 HUP "$previous_hup_trap"' HUP + trap 'handle_log_freeze_signal 130 INT "$previous_int_trap"' INT + trap 'handle_log_freeze_signal 143 TERM "$previous_term_trap"' TERM if ! podman --remote=false run --rm --privileged --pid=host --user=root:root \ -v /dev:/dev \ -v /:/target \ @@ -261,10 +366,9 @@ case ${TMT_REBOOT_COUNT:-0} in --security-opt label=type:unconfined_t \ "$target_image" \ bootc install to-existing-root --preserve-var --acknowledge-destructive; then - fsfreeze -u /var/log exit 1 fi - fsfreeze -u /var/log + thaw_log_filesystem # Find the generated stateroot by the copied bind-source sentinel. new_var= From c5c63e29fc8f03d83766ac34c57ff7efa6ff9b42 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 11:40:52 -0600 Subject: [PATCH 21/38] tests: Close TMT log freeze cleanup race A signal or failed fsfreeze could leave /var/log frozen because the cleanup handlers were installed afterward. Install traps before the freeze attempt and use one EXIT cleanup path that preserves the original status. Assisted-by: AI --- .../test-to-existing-root-preserve-var.sh | 98 +++++-------------- 1 file changed, 27 insertions(+), 71 deletions(-) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index 6bf4645d3f..570be0662f 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -17,6 +17,7 @@ tmt_plan_data_suffix=/tmt/plans/package-mode/data tmt_artifacts_parent=/var/ARTIFACTS tmt_test_data_sentinel_name=workdir-persistence-sentinel tmt_test_data_post_reboot_marker_name=post-reboot-marker +freeze_may_be_active=0 is_always_excluded_var_path() { case $1 in @@ -42,70 +43,23 @@ verify_tmt_workdir_directory() { fi } -get_trap_action() { - local signal=$1 - local trap_definition - - trap_definition=$(trap -p "$signal") - if [[ -n $trap_definition ]]; then - eval "set -- $trap_definition" - printf '%s' "$3" - fi -} - -restore_signal_trap() { - local signal=$1 - local previous_signal_trap=$2 - - if [[ -n $previous_signal_trap ]]; then - trap -- "$previous_signal_trap" "$signal" - else - trap - "$signal" - fi -} - -restore_log_freeze_signal_traps() { - restore_signal_trap HUP "${previous_hup_trap:-}" - restore_signal_trap INT "${previous_int_trap:-}" - restore_signal_trap TERM "${previous_term_trap:-}" -} - -thaw_log_filesystem() { - if (( log_fs_frozen )); then - fsfreeze -u /var/log || return 1 - log_fs_frozen=0 - restore_log_freeze_signal_traps - fi -} - cleanup_log_freeze() { local status=$? + # Prevent a second signal from interrupting or reentering the thaw attempt. + trap '' HUP INT TERM trap - EXIT - if ! thaw_log_filesystem; then - printf 'Failed to thaw /var/log during cleanup\n' >&2 - if (( status == 0 )); then - status=1 + set +e + if (( freeze_may_be_active )); then + if fsfreeze -u /var/log; then + freeze_may_be_active=0 + else + printf 'Failed to thaw /var/log during cleanup\n' >&2 + if (( status == 0 )); then + status=1 + fi fi fi - if [[ -n $previous_exit_trap ]]; then - eval "$previous_exit_trap" || : - fi - exit "$status" -} - -handle_log_freeze_signal() { - local status=$1 - local signal=$2 - local previous_signal_trap=$3 - - trap - "$signal" - if ! thaw_log_filesystem; then - printf 'Failed to thaw /var/log after %s\n' "$signal" >&2 - fi - if [[ -n $previous_signal_trap ]]; then - eval "$previous_signal_trap" || : - fi exit "$status" } @@ -348,27 +302,29 @@ case ${TMT_REBOOT_COUNT:-0} in # Keep the backing image for the separate /var/log filesystem stable # while preserve-var copies its image file from /var/lib. - fsfreeze -f /var/log - log_fs_frozen=1 - previous_exit_trap=$(get_trap_action EXIT) - previous_hup_trap=$(get_trap_action HUP) - previous_int_trap=$(get_trap_action INT) - previous_term_trap=$(get_trap_action TERM) trap cleanup_log_freeze EXIT - trap 'handle_log_freeze_signal 129 HUP "$previous_hup_trap"' HUP - trap 'handle_log_freeze_signal 130 INT "$previous_int_trap"' INT - trap 'handle_log_freeze_signal 143 TERM "$previous_term_trap"' TERM - if ! podman --remote=false run --rm --privileged --pid=host --user=root:root \ + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + # Set before invoking fsfreeze: it may freeze successfully and still + # return failure or be interrupted before the shell observes success. + freeze_may_be_active=1 + fsfreeze -f /var/log + podman --remote=false run --rm --privileged --pid=host --user=root:root \ -v /dev:/dev \ -v /:/target \ -v /boot:/target/boot \ -v /var/lib/containers:/var/lib/containers \ --security-opt label=type:unconfined_t \ "$target_image" \ - bootc install to-existing-root --preserve-var --acknowledge-destructive; then - exit 1 + bootc install to-existing-root --preserve-var --acknowledge-destructive + if fsfreeze -u /var/log; then + freeze_may_be_active=0 + else + status=$? + printf 'Failed to thaw /var/log after migration\n' >&2 + exit "$status" fi - thaw_log_filesystem # Find the generated stateroot by the copied bind-source sentinel. new_var= From f8585140c9c26e3d944a45b2f6ef8eae3ea85481 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 11:54:20 -0600 Subject: [PATCH 22/38] tests: Fix local package-mode TMT runner The package-mode runner currently asks TMT to discover directly from a large checkout, which makes TMT 1.69 traverse ignored build output. Discover from a temporary sanitized snapshot instead, and retain the local OCI archive for guest sync. Assisted-by: AI --- Justfile | 49 +++++++++++++++++++++++++++++++------- tmt/README.md | 12 ++++++---- tmt/plans/package-mode.fmf | 4 +++- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/Justfile b/Justfile index 2081f0fddc..891a71a4f3 100644 --- a/Justfile +++ b/Justfile @@ -236,16 +236,47 @@ test-tmt-package-mode image="centos-stream-10": #!/bin/bash set -euo pipefail lbi_images="{{lbi_images}}" - just build - archive_dir=$(mktemp -d -- tmt/plans/.package-mode-local.XXXXXX) - archive_path="$archive_dir/bootc.oci" - environment_file="$archive_dir/package-mode.env" + case "{{image}}" in + fedora-44) + image_base=quay.io/fedora/fedora-bootc:44 + ;; + centos-stream-10) + image_base=quay.io/centos-bootc/centos-bootc:stream10 + ;; + *) + echo "Unsupported package-mode test image: {{image}}" >&2 + exit 1 + ;; + esac + image_base=${BOOTC_base:-$image_base} + BOOTC_base="$image_base" just build + + # TMT 1.69 copies the whole git root during fmf discovery. Use a small + # snapshot so ignored build outputs and local worktrees are never copied. + test_tree= cleanup() { - rm -rf -- "$archive_dir" + if [[ -n "$test_tree" ]]; then + rm -rf -- "$test_tree" + fi } trap cleanup EXIT - trap 'cleanup; exit 130' INT - trap 'cleanup; exit 143' TERM + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + test_tree=$(mktemp -d -- "${TMPDIR:-/tmp}/bootc-tmt-package-mode.XXXXXX") + source_tree=$(git rev-parse --show-toplevel) + rsync -a --delete \ + --exclude='/.git' \ + --exclude='/target' \ + --exclude='/.worktrees' \ + --exclude='.package-mode-local.*' \ + "$source_tree/" "$test_tree/" + git -C "$test_tree" init -q + + archive_dir=$(mktemp -d -- "$test_tree/tmt/plans/.package-mode-local.XXXXXX") + archive_path="$archive_dir/bootc.oci" + archive_relative="${archive_dir#"$test_tree/"}/bootc.oci" + environment_file="$archive_dir/package-mode.env" if [[ -e "$archive_path" ]]; then echo "Refusing to overwrite existing archive: $archive_path" >&2 exit 1 @@ -259,14 +290,14 @@ test-tmt-package-mode image="centos-stream-10": # The test consumes comma-separated refs, and tmt 1.69 supports these in # a dotenv environment file without splitting the value on whitespace. printf 'PACKAGE_MODE_LBI_IMAGES=%s\n' "${lbi_images// /,}" > "$environment_file" - tmt \ + tmt --root "$test_tree" \ --context=running_env=packit \ --context=package_source=local \ --context="test_disk_image={{image}}" \ run --workdir-root "$workdir_root" \ --environment-file "$environment_file" \ --environment "PACKAGE_MODE_IMAGE_NAME={{base_img}}" \ - --environment "PACKAGE_MODE_IMAGE_ARCHIVE=$archive_path" \ + --environment "PACKAGE_MODE_IMAGE_ARCHIVE=$archive_relative" \ --environment "PACKAGE_MODE_TMT_WORKDIR_ROOT=$workdir_root" \ plans --name '^/tmt/plans/package-mode$' diff --git a/tmt/README.md b/tmt/README.md index 847bc0d0a8..7697c7f3b7 100644 --- a/tmt/README.md +++ b/tmt/README.md @@ -9,16 +9,20 @@ instructions on running them locally. Run the package-mode migration test against either supported local VM image. The command builds the configured `BOOTC_image` (default `localhost/bootc`), -transfers it as an OCI archive, and runs the TMT plan. It does not require -`sudo`: +transfers it as an OCI archive from a sanitized snapshot of the current working +tree, and runs the TMT plan without `sudo`. The Fedora test uses +`quay.io/fedora/fedora-bootc:44` and the CentOS test uses +`quay.io/centos-bootc/centos-bootc:stream10`; set `BOOTC_base` to override either +base image explicitly. ```bash just test-tmt-package-mode fedora-44 just test-tmt-package-mode centos-stream-10 ``` -The controller workdir is created under `$HOME/.cache/bootc-tmt-workdirs`. TMT -1.69 mirrors it to +The snapshot excludes `.git`, `/target`, `.worktrees`, and local package-mode +archives, while retaining uncommitted source changes. The controller workdir is +created under `$HOME/.cache/bootc-tmt-workdirs`. TMT 1.69 mirrors it to the same absolute guest path; on bootc guests `/home/...` resolves to `/var/home/...`, which is preserved by the migration. The test checks the resolved path is under preserved `/var` and rejects nested mounts, so it does diff --git a/tmt/plans/package-mode.fmf b/tmt/plans/package-mode.fmf index dac2eb149e..6418aab82e 100644 --- a/tmt/plans/package-mode.fmf +++ b/tmt/plans/package-mode.fmf @@ -200,6 +200,8 @@ execute: how: tmt discover: how: fmf - prune: true + # Keep the plan-local OCI archive in the tree synced to the guest. The + # package-mode runner uses a small sanitized snapshot, so pruning is not needed. + prune: false test: - /tmt/tests/test-50-to-existing-root-preserve-var From e3103f3133c5b3778b4bffee6d33c52a3844d30b Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 12:00:52 -0600 Subject: [PATCH 23/38] tests: Force OSTree package-mode test image The local package-mode test image must use OSTree regardless of the caller's variant. Forward the other configured build settings and override the variant only on the nested build. Assisted-by: AI --- Justfile | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/Justfile b/Justfile index 891a71a4f3..a5cb25c06d 100644 --- a/Justfile +++ b/Justfile @@ -50,7 +50,8 @@ erofs_version := env("BOOTC_erofs_version", "v1") # Baseconfigs to inject into the image for testing (e.g. "etc-transient" or "root-transient") baseconfigs := env("BOOTC_baseconfigs", "") # Base container image to build from -base := env("BOOTC_base", "quay.io/centos-bootc/centos-bootc:stream10") +_default_base := "quay.io/centos-bootc/centos-bootc:stream10" +base := env("BOOTC_base", _default_base) # Buildroot base image buildroot_base := env("BOOTC_buildroot_base", "quay.io/centos/centos:stream10") # Optional: path to extra source (e.g. composefs-rs) for local development @@ -248,8 +249,12 @@ test-tmt-package-mode image="centos-stream-10": exit 1 ;; esac - image_base=${BOOTC_base:-$image_base} - BOOTC_base="$image_base" just build + # Respect an explicit base override from the caller, while keeping the + # image-specific default selected above (e.g. Fedora for fedora-44). + if [[ "{{base}}" != "{{_default_base}}" ]]; then + image_base="{{base}}" + fi + BOOTC_base="$image_base" just {{_forward_settings}} base="$image_base" variant=ostree build # TMT 1.69 copies the whole git root during fmf discovery. Use a small # snapshot so ignored build outputs and local worktrees are never copied. @@ -301,6 +306,21 @@ test-tmt-package-mode image="centos-stream-10": --environment "PACKAGE_MODE_TMT_WORKDIR_ROOT=$workdir_root" \ plans --name '^/tmt/plans/package-mode$' +# Check that a composefs caller still gets an OSTree package-mode test image, +# without losing its configured base image or image tag. +[group('testing')] +test-tmt-package-mode-dry-run: + #!/usr/bin/env bash + set -euo pipefail + output=$(BOOTC_variant=composefs BOOTC_image=localhost/package-mode-test:latest just --dry-run base=localhost/package-mode-base:latest test-tmt-package-mode 2>&1) + build_command=$(grep 'just .* variant=ostree build' <<< "$output") + if [[ ! "$build_command" =~ base=localhost/package-mode-base:latest.*variant=composefs.*variant=ostree\ build$ ]]; then + echo "package-mode dry-run did not force OSTree while preserving settings:" >&2 + echo "$build_command" >&2 + exit 1 + fi + grep -Fq 'podman save --format oci-archive --output "$archive_path" localhost/package-mode-test:latest' <<< "$output" + # Split out from `test-container` because, unlike the container integration tests, # unit tests don't depend on variant/filesystem/bootloader/boot_type/seal_state, so # CI runs this once per OS (in the `package` job) instead of once per test-integration From 8092e18d356ecf58b36d50bfb2e8d49af1b15884 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 12:20:08 -0600 Subject: [PATCH 24/38] tests: Exclude temporary build secrets from TMT snapshot Keep root-level Podman build secrets out of the guest snapshot and clean them up if the build or test setup fails. Assisted-by: AI --- Justfile | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Justfile b/Justfile index a5cb25c06d..f20f792f21 100644 --- a/Justfile +++ b/Justfile @@ -236,6 +236,13 @@ test-ukiboot: build-ukiboot-update-test-image test-tmt-package-mode image="centos-stream-10": #!/bin/bash set -euo pipefail + source_tree=$(git rev-parse --show-toplevel) + cleanup_build_secrets() { + find "$source_tree" -maxdepth 1 \ + \( -type f -o -type l \) \ + -name 'podman-build-secret-*' -delete + } + trap cleanup_build_secrets EXIT lbi_images="{{lbi_images}}" case "{{image}}" in fedora-44) @@ -255,6 +262,7 @@ test-tmt-package-mode image="centos-stream-10": image_base="{{base}}" fi BOOTC_base="$image_base" just {{_forward_settings}} base="$image_base" variant=ostree build + cleanup_build_secrets # TMT 1.69 copies the whole git root during fmf discovery. Use a small # snapshot so ignored build outputs and local worktrees are never copied. @@ -263,17 +271,18 @@ test-tmt-package-mode image="centos-stream-10": if [[ -n "$test_tree" ]]; then rm -rf -- "$test_tree" fi + cleanup_build_secrets } trap cleanup EXIT trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM test_tree=$(mktemp -d -- "${TMPDIR:-/tmp}/bootc-tmt-package-mode.XXXXXX") - source_tree=$(git rev-parse --show-toplevel) rsync -a --delete \ --exclude='/.git' \ --exclude='/target' \ --exclude='/.worktrees' \ + --exclude='/podman-build-secret-*' \ --exclude='.package-mode-local.*' \ "$source_tree/" "$test_tree/" git -C "$test_tree" init -q @@ -320,6 +329,15 @@ test-tmt-package-mode-dry-run: exit 1 fi grep -Fq 'podman save --format oci-archive --output "$archive_path" localhost/package-mode-test:latest' <<< "$output" + grep -Fq -- "--exclude='/podman-build-secret-*'" <<< "$output" + grep -Fq -- "-name 'podman-build-secret-*' -delete" <<< "$output" + grep -Fq 'trap cleanup_build_secrets EXIT' <<< "$output" + grep -Fq 'trap cleanup EXIT' <<< "$output" + if [[ $(grep -Fc 'cleanup_build_secrets' <<< "$output") -lt 3 ]]; then + echo "package-mode dry-run did not show post-build and EXIT cleanup:" >&2 + echo "$output" >&2 + exit 1 + fi # Split out from `test-container` because, unlike the container integration tests, # unit tests don't depend on variant/filesystem/bootloader/boot_type/seal_state, so From 9fdfd4efbaba0fb6fd21fc86c660d9c41b64d1d6 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 12:24:32 -0600 Subject: [PATCH 25/38] tests: Preserve pre-existing build secret files The package-mode test currently removes every root build-secret pathname, including files that were present before the test began. Snapshot matching pathnames before installing cleanup and remove only new matches while preserving command status. Assisted-by: AI --- Justfile | 47 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/Justfile b/Justfile index f20f792f21..cbe233e217 100644 --- a/Justfile +++ b/Justfile @@ -237,12 +237,28 @@ test-tmt-package-mode image="centos-stream-10": #!/bin/bash set -euo pipefail source_tree=$(git rev-parse --show-toplevel) - cleanup_build_secrets() { + mapfile -d '' -t preexisting_build_secrets < <( find "$source_tree" -maxdepth 1 \ \( -type f -o -type l \) \ - -name 'podman-build-secret-*' -delete + -name 'podman-build-secret-*' -print0 + ) + cleanup_build_secrets() { + local path preexisting found + while IFS= read -r -d '' path; do + found=false + for preexisting in "${preexisting_build_secrets[@]}"; do + if [[ "$path" == "$preexisting" ]]; then + found=true + break + fi + done + if [[ "$found" == false && ( -f "$path" || -L "$path" ) ]]; then + rm -f -- "$path" || true + fi + done < <(find "$source_tree" -maxdepth 1 \ + \( -type f -o -type l \) \ + -name 'podman-build-secret-*' -print0) } - trap cleanup_build_secrets EXIT lbi_images="{{lbi_images}}" case "{{image}}" in fedora-44) @@ -261,19 +277,26 @@ test-tmt-package-mode image="centos-stream-10": if [[ "{{base}}" != "{{_default_base}}" ]]; then image_base="{{base}}" fi - BOOTC_base="$image_base" just {{_forward_settings}} base="$image_base" variant=ostree build - cleanup_build_secrets - - # TMT 1.69 copies the whole git root during fmf discovery. Use a small - # snapshot so ignored build outputs and local worktrees are never copied. test_tree= cleanup() { + local status=$? + trap - EXIT if [[ -n "$test_tree" ]]; then - rm -rf -- "$test_tree" + rm -rf -- "$test_tree" || true fi cleanup_build_secrets + exit "$status" } trap cleanup EXIT + build_status=0 + BOOTC_base="$image_base" just {{_forward_settings}} base="$image_base" variant=ostree build || build_status=$? + cleanup_build_secrets + if [[ "$build_status" -ne 0 ]]; then + exit "$build_status" + fi + + # TMT 1.69 copies the whole git root during fmf discovery. Use a small + # snapshot so ignored build outputs and local worktrees are never copied. trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM @@ -323,15 +346,15 @@ test-tmt-package-mode-dry-run: set -euo pipefail output=$(BOOTC_variant=composefs BOOTC_image=localhost/package-mode-test:latest just --dry-run base=localhost/package-mode-base:latest test-tmt-package-mode 2>&1) build_command=$(grep 'just .* variant=ostree build' <<< "$output") - if [[ ! "$build_command" =~ base=localhost/package-mode-base:latest.*variant=composefs.*variant=ostree\ build$ ]]; then + if [[ ! "$build_command" =~ base=localhost/package-mode-base:latest.*variant=composefs.*variant=ostree\ build ]]; then echo "package-mode dry-run did not force OSTree while preserving settings:" >&2 echo "$build_command" >&2 exit 1 fi grep -Fq 'podman save --format oci-archive --output "$archive_path" localhost/package-mode-test:latest' <<< "$output" grep -Fq -- "--exclude='/podman-build-secret-*'" <<< "$output" - grep -Fq -- "-name 'podman-build-secret-*' -delete" <<< "$output" - grep -Fq 'trap cleanup_build_secrets EXIT' <<< "$output" + grep -Fq -- "-name 'podman-build-secret-*' -print0" <<< "$output" + grep -Fq "mapfile -d '' -t preexisting_build_secrets" <<< "$output" grep -Fq 'trap cleanup EXIT' <<< "$output" if [[ $(grep -Fc 'cleanup_build_secrets' <<< "$output") -lt 3 ]]; then echo "package-mode dry-run did not show post-build and EXIT cleanup:" >&2 From 7ba91fc1c232ee057aab811e8c23dc91589bbc73 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 12:29:29 -0600 Subject: [PATCH 26/38] tests: Avoid deleting shared Podman build secrets Podman build secret paths are shared temporary paths, so cleanup from this runner could delete files owned by concurrent builds or users. Leave lifecycle management to Podman and exclude stale artifacts from the test snapshot. Assisted-by: AI --- Justfile | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/Justfile b/Justfile index cbe233e217..1e14131177 100644 --- a/Justfile +++ b/Justfile @@ -237,28 +237,6 @@ test-tmt-package-mode image="centos-stream-10": #!/bin/bash set -euo pipefail source_tree=$(git rev-parse --show-toplevel) - mapfile -d '' -t preexisting_build_secrets < <( - find "$source_tree" -maxdepth 1 \ - \( -type f -o -type l \) \ - -name 'podman-build-secret-*' -print0 - ) - cleanup_build_secrets() { - local path preexisting found - while IFS= read -r -d '' path; do - found=false - for preexisting in "${preexisting_build_secrets[@]}"; do - if [[ "$path" == "$preexisting" ]]; then - found=true - break - fi - done - if [[ "$found" == false && ( -f "$path" || -L "$path" ) ]]; then - rm -f -- "$path" || true - fi - done < <(find "$source_tree" -maxdepth 1 \ - \( -type f -o -type l \) \ - -name 'podman-build-secret-*' -print0) - } lbi_images="{{lbi_images}}" case "{{image}}" in fedora-44) @@ -284,13 +262,11 @@ test-tmt-package-mode image="centos-stream-10": if [[ -n "$test_tree" ]]; then rm -rf -- "$test_tree" || true fi - cleanup_build_secrets exit "$status" } trap cleanup EXIT build_status=0 BOOTC_base="$image_base" just {{_forward_settings}} base="$image_base" variant=ostree build || build_status=$? - cleanup_build_secrets if [[ "$build_status" -ne 0 ]]; then exit "$build_status" fi @@ -301,6 +277,8 @@ test-tmt-package-mode image="centos-stream-10": trap 'exit 130' INT trap 'exit 143' TERM test_tree=$(mktemp -d -- "${TMPDIR:-/tmp}/bootc-tmt-package-mode.XXXXXX") + # Podman-generated build secrets use shared temp paths; leave cleanup to Podman. + # Stale artifacts are not deleted here, only excluded from the test snapshot. rsync -a --delete \ --exclude='/.git' \ --exclude='/target' \ @@ -353,14 +331,7 @@ test-tmt-package-mode-dry-run: fi grep -Fq 'podman save --format oci-archive --output "$archive_path" localhost/package-mode-test:latest' <<< "$output" grep -Fq -- "--exclude='/podman-build-secret-*'" <<< "$output" - grep -Fq -- "-name 'podman-build-secret-*' -print0" <<< "$output" - grep -Fq "mapfile -d '' -t preexisting_build_secrets" <<< "$output" grep -Fq 'trap cleanup EXIT' <<< "$output" - if [[ $(grep -Fc 'cleanup_build_secrets' <<< "$output") -lt 3 ]]; then - echo "package-mode dry-run did not show post-build and EXIT cleanup:" >&2 - echo "$output" >&2 - exit 1 - fi # Split out from `test-container` because, unlike the container integration tests, # unit tests don't depend on variant/filesystem/bootloader/boot_type/seal_state, so From 999f58607cbe12b724526bb1d254044f75a50ce9 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 12:39:35 -0600 Subject: [PATCH 27/38] tmt: Exercise preserve-var relabeling from disabled source Fedora package-mode guests can keep /home outside /var, so preserve the complete lexical TMT run tree under /var/home before migration. Boot the source with SELinux disabled and verify the target relabels fresh application and PostgreSQL state when it starts enforcing. Assisted-by: AI --- Justfile | 5 +- tmt/README.md | 13 +- tmt/plans/package-mode.fmf | 8 +- .../test-to-existing-root-preserve-var.sh | 231 ++++++++++++++++-- 4 files changed, 224 insertions(+), 33 deletions(-) diff --git a/Justfile b/Justfile index 1e14131177..a9d7ca2318 100644 --- a/Justfile +++ b/Justfile @@ -297,11 +297,10 @@ test-tmt-package-mode image="centos-stream-10": exit 1 fi podman save --format oci-archive --output "$archive_path" {{base_img}} - # TMT syncs its workdir to the same absolute guest path. On bootc guests, - # /home resolves to /var/home, which preserve-var carries across migration. + # Keep TMT's lexical controller path under HOME without sudo or symlinks. + # The guest test stages its full /home run directory into preserved /var/home. workdir_root="$HOME/.cache/bootc-tmt-workdirs" mkdir -p -- "$workdir_root" - workdir_root=$(realpath -e -- "$workdir_root") # The test consumes comma-separated refs, and tmt 1.69 supports these in # a dotenv environment file without splitting the value on whitespace. printf 'PACKAGE_MODE_LBI_IMAGES=%s\n' "${lbi_images// /,}" > "$environment_file" diff --git a/tmt/README.md b/tmt/README.md index 7697c7f3b7..c5b0050fc2 100644 --- a/tmt/README.md +++ b/tmt/README.md @@ -22,9 +22,10 @@ just test-tmt-package-mode centos-stream-10 The snapshot excludes `.git`, `/target`, `.worktrees`, and local package-mode archives, while retaining uncommitted source changes. The controller workdir is -created under `$HOME/.cache/bootc-tmt-workdirs`. TMT 1.69 mirrors it to -the same absolute guest path; on bootc guests `/home/...` resolves to -`/var/home/...`, which is preserved by the migration. The test checks the -resolved path is under preserved `/var` and rejects nested mounts, so it does -not rely on `/var/tmp` or symlinks. TMT run logs and the testcloud image cache -remain under that workdir root. +created under `$HOME/.cache/bootc-tmt-workdirs` without sudo or symlinks. On a +package-mode guest where `/home` is a real directory, the test stages the full +TMT run directory into the corresponding `/var/home/...` location before +migration; after boot, the normal `/home -> /var/home` view resolves the same +lexical path. Already-persistent `/var` workdirs are not duplicated. The test +checks the persistent path and rejects excluded or nested mounts. TMT run logs +and the testcloud image cache remain under that workdir root. diff --git a/tmt/plans/package-mode.fmf b/tmt/plans/package-mode.fmf index 6418aab82e..854081ef69 100644 --- a/tmt/plans/package-mode.fmf +++ b/tmt/plans/package-mode.fmf @@ -19,7 +19,10 @@ prepare: - jq - e2fsprogs - util-linux + - grubby + - attr - bootc + - policycoreutils when: package_source == local # Packit build-only preparation needs the full integration toolchain. - how: install @@ -40,6 +43,9 @@ prepare: - dosfstools - e2fsprogs - util-linux + - grubby + - attr + - policycoreutils when: package_source == artifacts # Reject missing/unknown sources before any mode-specific setup runs. - how: shell @@ -145,7 +151,7 @@ prepare: trap 'rm -rf -- "$workdir"' EXIT cat > "$workdir/Containerfile" <&2 return 1 } - case $resolved_workdir_root in - /var/*) ;; - *) - printf 'TMT workdir root must resolve under preserved /var (got %q -> %q)\n' \ - "$workdir_root" "$resolved_workdir_root" >&2 - return 1 - ;; - esac - if is_always_excluded_var_path "$resolved_workdir_root"; then + if [[ $resolved_workdir_root == /var/* ]] && \ + is_always_excluded_var_path "$resolved_workdir_root"; then printf 'TMT workdir root is excluded from preserve-var: %s\n' \ "$resolved_workdir_root" >&2 return 1 @@ -128,6 +125,9 @@ get_tmt_run_dir() { return 1 fi verify_tmt_workdir_directory "$run_dir" "$resolved_run_dir" || return 1 + case $resolved_run_dir in + /var/*) verify_tmt_workdir_in_preserved_var "$resolved_run_dir" || return 1 ;; + esac resolved_plan_data=$(realpath -e -- "$plan_data") || { printf 'TMT_PLAN_DATA does not exist: %s\n' "$plan_data" >&2 return 1 @@ -171,7 +171,7 @@ get_tmt_run_dir() { return 1 fi case ${TMT_REBOOT_COUNT:-0} in - 0|1) ;; + 0|1|2) ;; *) printf 'Unexpected TMT_REBOOT_COUNT=%s\n' "${TMT_REBOOT_COUNT}" >&2 return 1 @@ -182,9 +182,27 @@ get_tmt_run_dir() { verify_tmt_workdir_in_preserved_var() { local run_dir=$1 + local resolved_run_dir local mount_target - if ! mount_target=$(findmnt --noheadings --output TARGET --target "$run_dir"); then + if ! resolved_run_dir=$(realpath -e -- "$run_dir"); then + printf 'TMT workdir path does not exist: %s\n' "$run_dir" >&2 + return 1 + fi + case $resolved_run_dir in + /var/*) ;; + *) + printf 'TMT workdir must resolve under preserved /var (got %q -> %q)\n' \ + "$run_dir" "$resolved_run_dir" >&2 + return 1 + ;; + esac + if is_always_excluded_var_path "$resolved_run_dir"; then + printf 'TMT workdir is excluded from preserve-var: %s\n' \ + "$resolved_run_dir" >&2 + return 1 + fi + if ! mount_target=$(findmnt --noheadings --output TARGET --target "$resolved_run_dir"); then printf 'Could not determine the filesystem containing TMT workdir %s\n' \ "$run_dir" >&2 return 1 @@ -197,6 +215,120 @@ verify_tmt_workdir_in_preserved_var() { fi } +prepare_tmt_workdir_persistence() { + local run_dir=$1 + local resolved_run_dir + local resolved_root + local run_id + local relative_run_dir + local relative_root + local stage_parent + local stage_run_dir + local resolved_plan_data + local resolved_test_data + local resolved_metadata + local plan_sentinel + local test_sentinel + + resolved_run_dir=$(realpath -e -- "$run_dir") || return 1 + run_id=${run_dir##*/} + if [[ -n ${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} ]]; then + resolved_root=$(realpath -e -- "$PACKAGE_MODE_TMT_WORKDIR_ROOT") || return 1 + fi + + # Packit already puts workdirs in /var/ARTIFACTS. Likewise, a local + # workdir that resolves under /var is already in the preserved tree. + if [[ $resolved_run_dir == /var/* ]]; then + persistent_run_dir=$resolved_run_dir + if [[ -n ${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} ]]; then + persistent_workdir_root=$resolved_root + fi + verify_tmt_workdir_in_preserved_var "$persistent_run_dir" + return + fi + + case $run_dir in + /home/*) relative_run_dir=${run_dir#/home/} ;; + *) + printf 'TMT workdir outside /var must be under /home to persist it (got %q -> %q)\n' \ + "$run_dir" "$resolved_run_dir" >&2 + return 1 + ;; + esac + case $resolved_run_dir in + /home/*) ;; + *) + printf 'TMT workdir resolves outside /home and /var (got %q -> %q)\n' \ + "$run_dir" "$resolved_run_dir" >&2 + return 1 + ;; + esac + if [[ -z ${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} || $resolved_root != /home/* ]]; then + printf 'Local TMT workdir root must resolve below /home or /var (got %q)\n' \ + "${PACKAGE_MODE_TMT_WORKDIR_ROOT:-}" >&2 + return 1 + fi + + relative_root=${resolved_root#/home/} + persistent_workdir_root=/var/home/$relative_root + persistent_run_dir=/var/home/$relative_run_dir + if is_always_excluded_var_path "$persistent_run_dir"; then + printf 'Persistent TMT workdir is excluded from preserve-var: %s\n' \ + "$persistent_run_dir" >&2 + return 1 + fi + if [[ -e $persistent_run_dir || -L $persistent_run_dir ]]; then + printf 'Refusing to overwrite existing persistent TMT workdir: %s\n' \ + "$persistent_run_dir" >&2 + return 1 + fi + + install -d -m 0755 -- "$persistent_workdir_root" + verify_tmt_workdir_in_preserved_var "$persistent_workdir_root" + stage_parent=$(mktemp -d -- "$persistent_workdir_root/.${run_id}.stage.XXXXXX") + stage_run_dir=$stage_parent/$run_id + ( + set -euo pipefail + trap 'rm -rf -- "$stage_parent"' EXIT + cp --reflink=auto -a -- "$run_dir" "$stage_run_dir" + + resolved_plan_data=$(realpath -e -- "$TMT_PLAN_DATA") + resolved_test_data=$(realpath -e -- "$TMT_TEST_DATA") + resolved_metadata=$(realpath -e -- "$TMT_TEST_METADATA") + plan_sentinel=$TMT_PLAN_DATA/$tmt_plan_data_sentinel_name + test_sentinel=$TMT_TEST_DATA/$tmt_test_data_sentinel_name + for source_path in "$resolved_plan_data" "$resolved_test_data" "$resolved_metadata"; do + case $source_path in + "$resolved_run_dir"/*) ;; + *) + printf 'TMT data path is outside its run directory: %s\n' \ + "$source_path" >&2 + exit 1 + ;; + esac + copied_path=$stage_run_dir/${source_path#"$resolved_run_dir"/} + test -e "$copied_path" + done + test -d "$stage_run_dir${tmt_plan_data_suffix}" + cmp -- "$resolved_metadata" "$stage_run_dir/${resolved_metadata#"$resolved_run_dir"/}" + cmp -- "$plan_sentinel" \ + "$stage_run_dir/${resolved_plan_data#"$resolved_run_dir"/}/$tmt_plan_data_sentinel_name" + cmp -- "$test_sentinel" \ + "$stage_run_dir/${resolved_test_data#"$resolved_run_dir"/}/$tmt_test_data_sentinel_name" + if [[ -e $persistent_run_dir || -L $persistent_run_dir ]]; then + printf 'Refusing to overwrite existing persistent TMT workdir: %s\n' \ + "$persistent_run_dir" >&2 + exit 1 + fi + mv -Tn -- "$stage_run_dir" "$persistent_run_dir" + if [[ -e $stage_run_dir || ! -d $persistent_run_dir ]]; then + printf 'Could not publish staged TMT workdir at %s\n' "$persistent_run_dir" >&2 + exit 1 + fi + ) + verify_tmt_workdir_in_preserved_var "$persistent_run_dir" +} + is_local_tmt_workdir() { local workdir_root=${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} local resolved_workdir_root @@ -211,17 +343,30 @@ is_local_tmt_workdir() { case ${TMT_REBOOT_COUNT:-0} in 0) + test ! -e /run/ostree-booted + # Always request the disabled source boot; this is harmless when the + # initial VM already has the argument and makes reboot accounting stable. + grubby --update-kernel=ALL --args=selinux=0 + tmt-reboot + ;; + 1) podman --remote=false image exists "$target_image" - # The guest is still the original package-mode system. + # This must be a real HostDisabled boot, not setenforce or a container + # label override. /proc/1/root checks the host's SELinux filesystem. test ! -e /run/ostree-booted + grep -Eq '(^|[[:space:]])selinux=0([[:space:]]|$)' /proc/cmdline + test "$(getenforce)" = Disabled + test ! -e /proc/1/root/sys/fs/selinux/enforce + tmt_run_dir=$(get_tmt_run_dir) if [[ ! -d $tmt_run_dir || -L $tmt_run_dir ]]; then printf 'TMT workdir is missing or not a real directory: %s\n' "$tmt_run_dir" >&2 exit 1 fi - verify_tmt_workdir_in_preserved_var "$tmt_run_dir" + test ! -e "$app_state" + test ! -e "$postgres_version_file" install -d -m 0755 "$var_state" "${app_state%/*}" "$bind_source" "$bind_target" printf '%s\n' preserved-application-state > "$app_state" printf '%s\n' bind-source-data > "$bind_source/sentinel" @@ -233,7 +378,7 @@ case ${TMT_REBOOT_COUNT:-0} in # Seed a database using the new deployment's own PostgreSQL version so # the test also works when the guest and bootc base use different OSes. - podman --remote=false run --rm --security-opt label=disable \ + podman --remote=false run --rm \ -v /var/lib/pgsql:/var/lib/pgsql "$target_image" \ bash -euo pipefail -c ' postgresql-setup --initdb @@ -246,6 +391,17 @@ case ${TMT_REBOOT_COUNT:-0} in INSERT INTO app_state VALUES (1, \$\$postgres-state-preserved\$\$)" runuser -u postgres -- pg_ctl -D /var/lib/pgsql/data -m fast -w stop ' + test -f "$postgres_version_file" + if getfattr --only-values -n security.selinux "$app_state" >/dev/null 2>&1; then + printf 'Application state unexpectedly has a SELinux label on the disabled source: %s\n' \ + "$app_state" >&2 + exit 1 + fi + if getfattr --only-values -n security.selinux "$postgres_version_file" >/dev/null 2>&1; then + printf 'PostgreSQL state unexpectedly has a SELinux label on the disabled source: %s\n' \ + "$postgres_version_file" >&2 + exit 1 + fi # Exercise source preflight with a nested mount whose source is a # device node. No target boot configuration may be changed on failure. @@ -287,18 +443,24 @@ case ${TMT_REBOOT_COUNT:-0} in umount "$preflight_mountpoint" rm -f "$preflight_mountpoint" "$boot_snapshot" "$boot_snapshot_after" - # TMT 1.69 rsyncs to the same absolute path on the guest. The configured - # controller path resolves under /var here (including /home -> /var/home). + # Add a sentinel before copying so the full plan/test data and metadata + # are verified in the persistent copy, not just an empty run directory. test -f "$TMT_TEST_METADATA" + tmt_plan_data_sentinel=$TMT_PLAN_DATA/$tmt_plan_data_sentinel_name + test ! -e "$tmt_plan_data_sentinel" + printf '%s\n' plan-workdir-persistence-sentinel > "$tmt_plan_data_sentinel" tmt_test_data_sentinel=$TMT_TEST_DATA/$tmt_test_data_sentinel_name test ! -e "$tmt_test_data_sentinel" printf '%s\n' workdir-persistence-sentinel > "$tmt_test_data_sentinel" + prepare_tmt_workdir_persistence "$tmt_run_dir" + if [[ -n $persistent_run_dir ]]; then + verify_tmt_workdir_in_preserved_var "$persistent_run_dir" + fi # TMT connects as root. Preserve its authorized key in /var so it can - # reconnect after the migration reboot. + # reconnect after migration; the target policy relabeler labels it. install -d -m 0700 /var/roothome/.ssh install -m 0600 /root/.ssh/authorized_keys /var/roothome/.ssh/authorized_keys - restorecon -RF /var/roothome/.ssh # Keep the backing image for the separate /var/log filesystem stable # while preserve-var copies its image file from /var/lib. @@ -338,6 +500,10 @@ case ${TMT_REBOOT_COUNT:-0} in test "$(<"$new_var${bind_source#/var}/sentinel")" = bind-source-data test "$(<"$new_var${app_state#/var}")" = preserved-application-state test -f "$new_var${log_fs_image#/var}" + test "$(getfattr --only-values -n security.selinux "$new_var${app_state#/var}")" = \ + "$(podman --remote=false run --rm "$target_image" matchpathcon -n "$app_state")" + test "$(getfattr --only-values -n security.selinux "$new_var${postgres_version_file#/var}")" = \ + "$(podman --remote=false run --rm "$target_image" matchpathcon -n "$postgres_version_file")" bind_target_in_new_var=$new_var${bind_target#/var} test -d "$bind_target_in_new_var" test -z "$(find "$bind_target_in_new_var" -mindepth 1 -print -quit)" @@ -375,6 +541,7 @@ case ${TMT_REBOOT_COUNT:-0} in install -d -m 0755 "$deployment/etc/ssh" ssh-keygen -q -t ed25519 -N "" \ -f "$deployment/etc/ssh/ssh_host_ed25519_key" + chroot "$deployment" restorecon -RF /etc/ssh if [[ $local_workdir_root == 1 ]]; then rule=$deployment/etc/tmpfiles.d/bootc-tmt-workdir.conf @@ -388,15 +555,27 @@ case ${TMT_REBOOT_COUNT:-0} in fi ' install-test-configuration \ "$(is_local_tmt_workdir "$tmt_run_dir" && printf 1 || printf 0)" \ - "$(realpath -e -- "$tmt_run_dir")" \ - "$(if [[ -n ${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} ]]; then realpath -e -- "$PACKAGE_MODE_TMT_WORKDIR_ROOT"; fi)" + "$(if [[ -n $persistent_run_dir ]]; then printf '%s' "$persistent_run_dir"; else realpath -e -- "$tmt_run_dir"; fi)" \ + "$(if [[ -n $persistent_workdir_root ]]; then printf '%s' "$persistent_workdir_root"; elif [[ -n ${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} ]]; then realpath -e -- "$PACKAGE_MODE_TMT_WORKDIR_ROOT"; fi)" + + # New target boot entries are generated by the migration; remove the + # source-only HostDisabled argument before rebooting into the target. + grubby --update-kernel=ALL --remove-args=selinux=0 + if grubby --info=ALL | grep -Eq '(^|[[:space:]"])selinux=0([[:space:]"]|$)'; then + printf 'selinux=0 remains in a target boot entry after migration\n' >&2 + exit 1 + fi bootc status # Reboot the guest itself; this avoids testcloud's ACPI soft-poweroff. tmt-reboot -c 'systemctl reboot' ;; - 1) + 2) test -e /run/ostree-booted + ! grep -Eq '(^|[[:space:]])selinux=0([[:space:]]|$)' /proc/cmdline + test "$(getenforce)" = Enforcing + test -e /proc/1/root/sys/fs/selinux/enforce + test "$( "$tmt_test_data_post_reboot_marker" test "$(<"$tmt_test_data_post_reboot_marker")" = post-reboot-workdir-writable @@ -439,6 +620,10 @@ case ${TMT_REBOOT_COUNT:-0} in -d bootc_preserve_var -c 'SELECT value FROM app_state WHERE id = 1') test "$postgres_value" = postgres-state-preserved test "$(<"$app_state")" = preserved-application-state + test "$(getfattr --only-values -n security.selinux "$app_state")" = \ + "$(matchpathcon -n "$app_state")" + test "$(getfattr --only-values -n security.selinux "$postgres_version_file")" = \ + "$(matchpathcon -n "$postgres_version_file")" test -s /etc/ssh/ssh_host_ed25519_key test -s /etc/ssh/ssh_host_ed25519_key.pub From 876b9b86dfe92d4749b893af81dcbbf75fb74a2d Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 12:49:50 -0600 Subject: [PATCH 28/38] tests: Resolve package-mode TMT workdirs canonically The package-mode VM can expose /home as a real directory, so the lexical controller workdir may not be preserved by migration. Derive the destination from the resolved TMT root and run ID, and assert test SSH key labels against target policy in both SELinux phases. Assisted-by: AI --- .../test-to-existing-root-preserve-var.sh | 263 +++++++++++------- 1 file changed, 169 insertions(+), 94 deletions(-) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index 0f7df99128..4813825f57 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -20,7 +20,7 @@ tmt_test_data_sentinel_name=workdir-persistence-sentinel tmt_test_data_post_reboot_marker_name=post-reboot-marker postgres_version_file=/var/lib/pgsql/data/PG_VERSION freeze_may_be_active=0 -persistent_workdir_root= +persistent_tmt_root= persistent_run_dir= is_always_excluded_var_path() { @@ -218,115 +218,166 @@ verify_tmt_workdir_in_preserved_var() { prepare_tmt_workdir_persistence() { local run_dir=$1 local resolved_run_dir - local resolved_root - local run_id - local relative_run_dir - local relative_root + local canonical_plan_data + local canonical_test_data + local canonical_metadata local stage_parent local stage_run_dir - local resolved_plan_data - local resolved_test_data - local resolved_metadata local plan_sentinel local test_sentinel + local data_path resolved_run_dir=$(realpath -e -- "$run_dir") || return 1 - run_id=${run_dir##*/} - if [[ -n ${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} ]]; then - resolved_root=$(realpath -e -- "$PACKAGE_MODE_TMT_WORKDIR_ROOT") || return 1 - fi - - # Packit already puts workdirs in /var/ARTIFACTS. Likewise, a local - # workdir that resolves under /var is already in the preserved tree. - if [[ $resolved_run_dir == /var/* ]]; then - persistent_run_dir=$resolved_run_dir - if [[ -n ${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} ]]; then - persistent_workdir_root=$resolved_root - fi - verify_tmt_workdir_in_preserved_var "$persistent_run_dir" - return + set_persistent_tmt_paths "$run_dir" || return 1 + + canonical_plan_data=$(realpath -e -- "$TMT_PLAN_DATA") || return 1 + canonical_test_data=$(realpath -e -- "$TMT_TEST_DATA") || return 1 + canonical_metadata=$(realpath -e -- "$TMT_TEST_METADATA") || return 1 + for data_path in "$canonical_plan_data" "$canonical_test_data" "$canonical_metadata"; do + case $data_path in + "$resolved_run_dir"/*) ;; + *) + printf 'TMT data path is outside its run directory: %s\n' \ + "$data_path" >&2 + return 1 + ;; + esac + done + if [[ ! -d $canonical_plan_data || ! -d $canonical_test_data || ! -f $canonical_metadata ]]; then + printf 'TMT plan data, test data, or metadata is missing from %s\n' \ + "$resolved_run_dir" >&2 + return 1 fi - - case $run_dir in - /home/*) relative_run_dir=${run_dir#/home/} ;; - *) - printf 'TMT workdir outside /var must be under /home to persist it (got %q -> %q)\n' \ - "$run_dir" "$resolved_run_dir" >&2 - return 1 - ;; - esac - case $resolved_run_dir in - /home/*) ;; - *) - printf 'TMT workdir resolves outside /home and /var (got %q -> %q)\n' \ - "$run_dir" "$resolved_run_dir" >&2 - return 1 - ;; - esac - if [[ -z ${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} || $resolved_root != /home/* ]]; then - printf 'Local TMT workdir root must resolve below /home or /var (got %q)\n' \ - "${PACKAGE_MODE_TMT_WORKDIR_ROOT:-}" >&2 + plan_sentinel=$canonical_plan_data/$tmt_plan_data_sentinel_name + test_sentinel=$canonical_test_data/$tmt_test_data_sentinel_name + if [[ ! -f $plan_sentinel || ! -f $test_sentinel || \ + $(<"$plan_sentinel") != plan-workdir-persistence-sentinel || \ + $(<"$test_sentinel") != workdir-persistence-sentinel ]]; then + printf 'TMT source plan/test data sentinels do not match expected values\n' >&2 return 1 fi - relative_root=${resolved_root#/home/} - persistent_workdir_root=/var/home/$relative_root - persistent_run_dir=/var/home/$relative_run_dir if is_always_excluded_var_path "$persistent_run_dir"; then printf 'Persistent TMT workdir is excluded from preserve-var: %s\n' \ "$persistent_run_dir" >&2 return 1 fi - if [[ -e $persistent_run_dir || -L $persistent_run_dir ]]; then - printf 'Refusing to overwrite existing persistent TMT workdir: %s\n' \ + if [[ $resolved_run_dir != "$persistent_run_dir" ]]; then + install -d -m 0755 -- "$persistent_tmt_root" + verify_tmt_workdir_in_preserved_var "$persistent_tmt_root" + if [[ -L $persistent_run_dir ]]; then + printf 'Refusing symlink at persistent TMT workdir destination: %s\n' \ + "$persistent_run_dir" >&2 + return 1 + elif [[ -e $persistent_run_dir ]]; then + printf 'Refusing to overwrite existing persistent TMT workdir: %s\n' \ + "$persistent_run_dir" >&2 + return 1 + fi + + stage_parent=$(mktemp -d -- "$persistent_tmt_root/.${persistent_run_dir##*/}.stage.XXXXXX") + stage_run_dir=$stage_parent/${persistent_run_dir##*/} + ( + set -euo pipefail + trap 'rm -rf -- "$stage_parent"' EXIT + cp --reflink=auto -a -- "$resolved_run_dir" "$stage_run_dir" + cmp -- "$canonical_metadata" "$stage_run_dir/${canonical_metadata#"$resolved_run_dir"/}" + cmp -- "$plan_sentinel" \ + "$stage_run_dir/${canonical_plan_data#"$resolved_run_dir"/}/$tmt_plan_data_sentinel_name" + cmp -- "$test_sentinel" \ + "$stage_run_dir/${canonical_test_data#"$resolved_run_dir"/}/$tmt_test_data_sentinel_name" + if [[ -L $persistent_run_dir ]]; then + printf 'Refusing symlink at persistent TMT workdir destination: %s\n' \ + "$persistent_run_dir" >&2 + exit 1 + elif [[ -e $persistent_run_dir ]]; then + printf 'Refusing to overwrite existing persistent TMT workdir: %s\n' \ + "$persistent_run_dir" >&2 + exit 1 + fi + mv -Tn -- "$stage_run_dir" "$persistent_run_dir" + if [[ -e $stage_run_dir || -L $persistent_run_dir || ! -d $persistent_run_dir ]]; then + printf 'Could not publish staged TMT workdir at %s\n' "$persistent_run_dir" >&2 + exit 1 + fi + ) + fi + if [[ -L $persistent_run_dir || ! -d $persistent_run_dir ]]; then + printf 'Persistent TMT workdir is missing or is a symlink: %s\n' \ "$persistent_run_dir" >&2 return 1 fi + if [[ $(realpath -e -- "$persistent_run_dir") != "$persistent_run_dir" ]]; then + printf 'Persistent TMT workdir does not resolve to its canonical destination: %s\n' \ + "$persistent_run_dir" >&2 + return 1 + fi + cmp -- "$canonical_metadata" "$persistent_run_dir/${canonical_metadata#"$resolved_run_dir"/}" + cmp -- "$plan_sentinel" \ + "$persistent_run_dir/${canonical_plan_data#"$resolved_run_dir"/}/$tmt_plan_data_sentinel_name" + cmp -- "$test_sentinel" \ + "$persistent_run_dir/${canonical_test_data#"$resolved_run_dir"/}/$tmt_test_data_sentinel_name" + verify_tmt_workdir_in_preserved_var "$persistent_run_dir" +} - install -d -m 0755 -- "$persistent_workdir_root" - verify_tmt_workdir_in_preserved_var "$persistent_workdir_root" - stage_parent=$(mktemp -d -- "$persistent_workdir_root/.${run_id}.stage.XXXXXX") - stage_run_dir=$stage_parent/$run_id - ( - set -euo pipefail - trap 'rm -rf -- "$stage_parent"' EXIT - cp --reflink=auto -a -- "$run_dir" "$stage_run_dir" - - resolved_plan_data=$(realpath -e -- "$TMT_PLAN_DATA") - resolved_test_data=$(realpath -e -- "$TMT_TEST_DATA") - resolved_metadata=$(realpath -e -- "$TMT_TEST_METADATA") - plan_sentinel=$TMT_PLAN_DATA/$tmt_plan_data_sentinel_name - test_sentinel=$TMT_TEST_DATA/$tmt_test_data_sentinel_name - for source_path in "$resolved_plan_data" "$resolved_test_data" "$resolved_metadata"; do - case $source_path in - "$resolved_run_dir"/*) ;; - *) - printf 'TMT data path is outside its run directory: %s\n' \ - "$source_path" >&2 - exit 1 - ;; - esac - copied_path=$stage_run_dir/${source_path#"$resolved_run_dir"/} - test -e "$copied_path" - done - test -d "$stage_run_dir${tmt_plan_data_suffix}" - cmp -- "$resolved_metadata" "$stage_run_dir/${resolved_metadata#"$resolved_run_dir"/}" - cmp -- "$plan_sentinel" \ - "$stage_run_dir/${resolved_plan_data#"$resolved_run_dir"/}/$tmt_plan_data_sentinel_name" - cmp -- "$test_sentinel" \ - "$stage_run_dir/${resolved_test_data#"$resolved_run_dir"/}/$tmt_test_data_sentinel_name" - if [[ -e $persistent_run_dir || -L $persistent_run_dir ]]; then - printf 'Refusing to overwrite existing persistent TMT workdir: %s\n' \ - "$persistent_run_dir" >&2 - exit 1 +set_persistent_tmt_paths() { + local run_dir=$1 + local workdir_root=${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} + local canonical_root + local canonical_run_dir + local run_id + + if [[ -n $workdir_root ]]; then + canonical_root=$(realpath -e -- "$workdir_root") || { + printf 'TMT workdir root does not exist: %s\n' "$workdir_root" >&2 + return 1 + } + case $run_dir in + "$workdir_root"/run-*) run_id=${run_dir#"$workdir_root"/} ;; + "$canonical_root"/run-*) run_id=${run_dir#"$canonical_root"/} ;; + *) + printf 'TMT run directory is not a direct run-* child of its root: %s\n' \ + "$run_dir" >&2 + return 1 + ;; + esac + if [[ ! $run_id =~ ^run-[[:alnum:]_.-]+$ ]]; then + printf 'Unexpected TMT run ID: %q\n' "$run_id" >&2 + return 1 fi - mv -Tn -- "$stage_run_dir" "$persistent_run_dir" - if [[ -e $stage_run_dir || ! -d $persistent_run_dir ]]; then - printf 'Could not publish staged TMT workdir at %s\n' "$persistent_run_dir" >&2 - exit 1 + else + canonical_root=$(realpath -e -- "$tmt_artifacts_parent") || { + printf 'TMT artifact root does not exist: %s\n' "$tmt_artifacts_parent" >&2 + return 1 + } + if [[ $canonical_root != "$tmt_artifacts_parent" || + ! $run_dir =~ ^/var/ARTIFACTS/work-[[:alnum:]_.-]+$ ]]; then + printf 'Unexpected TMT artifact root or run directory: %q\n' "$run_dir" >&2 + return 1 fi - ) - verify_tmt_workdir_in_preserved_var "$persistent_run_dir" + run_id=${run_dir##*/} + fi + + canonical_run_dir=$(realpath -e -- "$run_dir") || { + printf 'TMT run directory does not exist: %s\n' "$run_dir" >&2 + return 1 + } + if [[ $canonical_run_dir != "$canonical_root/$run_id" ]]; then + printf 'TMT run directory is not the canonical direct child of its root (%q -> %q)\n' \ + "$run_dir" "$canonical_run_dir" >&2 + return 1 + fi + + case $canonical_root in + /home/*) persistent_tmt_root=/var/home/${canonical_root#/home/} ;; + /var/home/*|/var/ARTIFACTS|/var/ARTIFACTS/*) persistent_tmt_root=$canonical_root ;; + *) + printf 'Unsupported canonical TMT root; expected /home, /var/home, or /var/ARTIFACTS: %s\n' \ + "$canonical_root" >&2 + return 1 + ;; + esac + persistent_run_dir=$persistent_tmt_root/$run_id } is_local_tmt_workdir() { @@ -542,6 +593,18 @@ case ${TMT_REBOOT_COUNT:-0} in ssh-keygen -q -t ed25519 -N "" \ -f "$deployment/etc/ssh/ssh_host_ed25519_key" chroot "$deployment" restorecon -RF /etc/ssh + for ssh_path in \ + /etc/ssh/ssh_host_ed25519_key \ + /etc/ssh/ssh_host_ed25519_key.pub; do + actual_label=$(getfattr --only-values -n security.selinux \ + "$deployment$ssh_path") + expected_label=$(chroot "$deployment" matchpathcon -n "$ssh_path") + if [[ $actual_label != "$expected_label" ]]; then + printf "Unexpected SELinux label on target SSH key %s: %s (expected %s)\n" \ + "$ssh_path" "$actual_label" "$expected_label" >&2 + exit 1 + fi + done if [[ $local_workdir_root == 1 ]]; then rule=$deployment/etc/tmpfiles.d/bootc-tmt-workdir.conf @@ -556,7 +619,7 @@ case ${TMT_REBOOT_COUNT:-0} in ' install-test-configuration \ "$(is_local_tmt_workdir "$tmt_run_dir" && printf 1 || printf 0)" \ "$(if [[ -n $persistent_run_dir ]]; then printf '%s' "$persistent_run_dir"; else realpath -e -- "$tmt_run_dir"; fi)" \ - "$(if [[ -n $persistent_workdir_root ]]; then printf '%s' "$persistent_workdir_root"; elif [[ -n ${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} ]]; then realpath -e -- "$PACKAGE_MODE_TMT_WORKDIR_ROOT"; fi)" + "$(if [[ -n $persistent_tmt_root ]]; then printf '%s' "$persistent_tmt_root"; elif [[ -n ${PACKAGE_MODE_TMT_WORKDIR_ROOT:-} ]]; then realpath -e -- "$PACKAGE_MODE_TMT_WORKDIR_ROOT"; fi)" # New target boot entries are generated by the migration; remove the # source-only HostDisabled argument before rebooting into the target. @@ -577,13 +640,20 @@ case ${TMT_REBOOT_COUNT:-0} in test -e /proc/1/root/sys/fs/selinux/enforce test "$(&2 + exit 1 + fi if is_local_tmt_workdir "$tmt_run_dir"; then test -d "$tmt_run_dir" test ! -L "$tmt_run_dir" verify_tmt_workdir_in_preserved_var "$tmt_run_dir" expected_tmpfiles_rule=( - "d $(realpath -e -- "$PACKAGE_MODE_TMT_WORKDIR_ROOT") - - - - -" - "d $(realpath -e -- "$tmt_run_dir") - - - - -" + "d $persistent_tmt_root - - - - -" + "d $persistent_run_dir - - - - -" ) printf "%s\n" "${expected_tmpfiles_rule[@]}" | \ cmp - /etc/tmpfiles.d/bootc-tmt-workdir.conf @@ -624,8 +694,13 @@ case ${TMT_REBOOT_COUNT:-0} in "$(matchpathcon -n "$app_state")" test "$(getfattr --only-values -n security.selinux "$postgres_version_file")" = \ "$(matchpathcon -n "$postgres_version_file")" - test -s /etc/ssh/ssh_host_ed25519_key - test -s /etc/ssh/ssh_host_ed25519_key.pub + for ssh_path in \ + /etc/ssh/ssh_host_ed25519_key \ + /etc/ssh/ssh_host_ed25519_key.pub; do + test -s "$ssh_path" + test "$(getfattr --only-values -n security.selinux "$ssh_path")" = \ + "$(matchpathcon -n "$ssh_path")" + done booted_image=$(bootc status --json | jq -r '.status.booted.image.image.image') test "$booted_image" = "$target_image" From 71cad733eaa5256fff52173d396e929babd6c956 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 13:02:15 -0600 Subject: [PATCH 29/38] tests: Create PostgreSQL test mountpoint The Fedora package-mode test mounts /var/lib/pgsql from the source guest, but that path is not guaranteed to exist. Create it before running the target-image PostgreSQL initializer. Assisted-by: AI --- .../booted/package-mode/test-to-existing-root-preserve-var.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index 4813825f57..3a75e8c569 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -429,6 +429,7 @@ case ${TMT_REBOOT_COUNT:-0} in # Seed a database using the new deployment's own PostgreSQL version so # the test also works when the guest and bootc base use different OSes. + install -d -m 0755 /var/lib/pgsql podman --remote=false run --rm \ -v /var/lib/pgsql:/var/lib/pgsql "$target_image" \ bash -euo pipefail -c ' From 025f6b7febbf9e3a7c3b1752dc78db2f27a8ef3d Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 13:05:42 -0600 Subject: [PATCH 30/38] tests: Set PostgreSQL test volume ownership The guest creates the PostgreSQL mountpoint as root, but initdb needs it to be writable by postgres. Set its numeric ownership from the target image and restrict its mode before database setup. Assisted-by: AI --- .../package-mode/test-to-existing-root-preserve-var.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index 3a75e8c569..56d509fc10 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -433,6 +433,12 @@ case ${TMT_REBOOT_COUNT:-0} in podman --remote=false run --rm \ -v /var/lib/pgsql:/var/lib/pgsql "$target_image" \ bash -euo pipefail -c ' + postgres_uid=$(id -u postgres) + postgres_gid=$(id -g postgres) + chown "$postgres_uid:$postgres_gid" /var/lib/pgsql + chmod 0700 /var/lib/pgsql + test "$(stat -c %u:%g /var/lib/pgsql)" = "$postgres_uid:$postgres_gid" + test "$(stat -c %a /var/lib/pgsql)" = 700 postgresql-setup --initdb runuser -u postgres -- pg_ctl -D /var/lib/pgsql/data \ -o "-c listen_addresses=localhost" -w start From f10048a4b6d194ff6849616a39a8d93bdf46223c Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 13:10:52 -0600 Subject: [PATCH 31/38] tmt: Seed PostgreSQL below bind mount The PostgreSQL data directory must be a normal directory beneath the bind mount, not the mountpoint itself. Seed it through the target image as its postgres user so the preserved data uses the target UID/GID and remains compatible with the post-migration service. Assisted-by: AI --- .../test-to-existing-root-preserve-var.sh | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index 56d509fc10..4a3f80a50e 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -431,23 +431,28 @@ case ${TMT_REBOOT_COUNT:-0} in # the test also works when the guest and bootc base use different OSes. install -d -m 0755 /var/lib/pgsql podman --remote=false run --rm \ - -v /var/lib/pgsql:/var/lib/pgsql "$target_image" \ + -v /var/lib/pgsql:/mnt/pgsql "$target_image" \ bash -euo pipefail -c ' postgres_uid=$(id -u postgres) postgres_gid=$(id -g postgres) - chown "$postgres_uid:$postgres_gid" /var/lib/pgsql - chmod 0700 /var/lib/pgsql - test "$(stat -c %u:%g /var/lib/pgsql)" = "$postgres_uid:$postgres_gid" - test "$(stat -c %a /var/lib/pgsql)" = 700 - postgresql-setup --initdb - runuser -u postgres -- pg_ctl -D /var/lib/pgsql/data \ + chown "$postgres_uid:$postgres_gid" /mnt/pgsql + chmod 0700 /mnt/pgsql + test "$(stat -c %u:%g /mnt/pgsql)" = "$postgres_uid:$postgres_gid" + test "$(stat -c %a /mnt/pgsql)" = 700 + runuser -u postgres -- initdb -D /mnt/pgsql/data + if ! runuser -u postgres -- pg_ctl -D /mnt/pgsql/data \ + -l /mnt/pgsql/server.log \ -o "-c listen_addresses=localhost" -w start + then + cat /mnt/pgsql/server.log >&2 + exit 1 + fi runuser -u postgres -- psql -v ON_ERROR_STOP=1 \ -c "CREATE DATABASE bootc_preserve_var" runuser -u postgres -- psql -v ON_ERROR_STOP=1 -d bootc_preserve_var \ -c "CREATE TABLE app_state (id integer PRIMARY KEY, value text NOT NULL); \ INSERT INTO app_state VALUES (1, \$\$postgres-state-preserved\$\$)" - runuser -u postgres -- pg_ctl -D /var/lib/pgsql/data -m fast -w stop + runuser -u postgres -- pg_ctl -D /mnt/pgsql/data -m fast -w stop ' test -f "$postgres_version_file" if getfattr --only-values -n security.selinux "$app_state" >/dev/null 2>&1; then From 0e2fac6ef1f40626308f36ab1a2a836a002b5bf3 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 13:14:01 -0600 Subject: [PATCH 32/38] tmt: Use explicit PostgreSQL seed endpoints The one-shot target-image container does not start systemd, so PostgreSQL's default socket directory may be unavailable. Configure a /tmp socket and connect the seed queries over loopback TCP. Assisted-by: AI --- .../package-mode/test-to-existing-root-preserve-var.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index 4a3f80a50e..388499b87a 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -442,14 +442,14 @@ case ${TMT_REBOOT_COUNT:-0} in runuser -u postgres -- initdb -D /mnt/pgsql/data if ! runuser -u postgres -- pg_ctl -D /mnt/pgsql/data \ -l /mnt/pgsql/server.log \ - -o "-c listen_addresses=localhost" -w start + -o "-c listen_addresses=localhost -c unix_socket_directories=/tmp" -w start then cat /mnt/pgsql/server.log >&2 exit 1 fi - runuser -u postgres -- psql -v ON_ERROR_STOP=1 \ + runuser -u postgres -- psql -h 127.0.0.1 -v ON_ERROR_STOP=1 \ -c "CREATE DATABASE bootc_preserve_var" - runuser -u postgres -- psql -v ON_ERROR_STOP=1 -d bootc_preserve_var \ + runuser -u postgres -- psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -d bootc_preserve_var \ -c "CREATE TABLE app_state (id integer PRIMARY KEY, value text NOT NULL); \ INSERT INTO app_state VALUES (1, \$\$postgres-state-preserved\$\$)" runuser -u postgres -- pg_ctl -D /mnt/pgsql/data -m fast -w stop From ea7b30512c64732cacc69ce3584cfda4bf15818a Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 13:15:10 -0600 Subject: [PATCH 33/38] tmt: Mount PostgreSQL seed at target data path The target image already provides /var/lib/pgsql, while /mnt may not exist. Seed in the mounted data directory so the state survives preserve-var and is available to the target service. Assisted-by: AI --- .../test-to-existing-root-preserve-var.sh | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index 388499b87a..e369920076 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -431,20 +431,20 @@ case ${TMT_REBOOT_COUNT:-0} in # the test also works when the guest and bootc base use different OSes. install -d -m 0755 /var/lib/pgsql podman --remote=false run --rm \ - -v /var/lib/pgsql:/mnt/pgsql "$target_image" \ + -v /var/lib/pgsql:/var/lib/pgsql "$target_image" \ bash -euo pipefail -c ' postgres_uid=$(id -u postgres) postgres_gid=$(id -g postgres) - chown "$postgres_uid:$postgres_gid" /mnt/pgsql - chmod 0700 /mnt/pgsql - test "$(stat -c %u:%g /mnt/pgsql)" = "$postgres_uid:$postgres_gid" - test "$(stat -c %a /mnt/pgsql)" = 700 - runuser -u postgres -- initdb -D /mnt/pgsql/data - if ! runuser -u postgres -- pg_ctl -D /mnt/pgsql/data \ - -l /mnt/pgsql/server.log \ + chown "$postgres_uid:$postgres_gid" /var/lib/pgsql + chmod 0700 /var/lib/pgsql + test "$(stat -c %u:%g /var/lib/pgsql)" = "$postgres_uid:$postgres_gid" + test "$(stat -c %a /var/lib/pgsql)" = 700 + runuser -u postgres -- initdb -D /var/lib/pgsql/data + if ! runuser -u postgres -- pg_ctl -D /var/lib/pgsql/data \ + -l /var/lib/pgsql/server.log \ -o "-c listen_addresses=localhost -c unix_socket_directories=/tmp" -w start then - cat /mnt/pgsql/server.log >&2 + cat /var/lib/pgsql/server.log >&2 exit 1 fi runuser -u postgres -- psql -h 127.0.0.1 -v ON_ERROR_STOP=1 \ @@ -452,7 +452,7 @@ case ${TMT_REBOOT_COUNT:-0} in runuser -u postgres -- psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -d bootc_preserve_var \ -c "CREATE TABLE app_state (id integer PRIMARY KEY, value text NOT NULL); \ INSERT INTO app_state VALUES (1, \$\$postgres-state-preserved\$\$)" - runuser -u postgres -- pg_ctl -D /mnt/pgsql/data -m fast -w stop + runuser -u postgres -- pg_ctl -D /var/lib/pgsql/data -m fast -w stop ' test -f "$postgres_version_file" if getfattr --only-values -n security.selinux "$app_state" >/dev/null 2>&1; then From abd747f3bfd730c6c35240dd397ba39929dc7ac3 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 13:17:14 -0600 Subject: [PATCH 34/38] tmt: Verify PostgreSQL data path in target image Check that the target image already contains /var/lib/pgsql before attaching a bind mount there. This ensures the seed run cannot pass by relying on Podman's bind-mount behavior to create a missing destination. Assisted-by: AI --- .../booted/package-mode/test-to-existing-root-preserve-var.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index e369920076..37afc531ca 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -429,6 +429,7 @@ case ${TMT_REBOOT_COUNT:-0} in # Seed a database using the new deployment's own PostgreSQL version so # the test also works when the guest and bootc base use different OSes. + podman --remote=false run --rm "$target_image" test -d /var/lib/pgsql install -d -m 0755 /var/lib/pgsql podman --remote=false run --rm \ -v /var/lib/pgsql:/var/lib/pgsql "$target_image" \ From 05ae3deedb698f09940a95542e16dfc3626a0c35 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 13:24:08 -0600 Subject: [PATCH 35/38] tmt: Compare boot manifests around rejected preflight The full /boot tar snapshot is sensitive to incidental metadata changes, which can make a rejected preflight appear to mutate boot configuration. Compare only the paths and contents of boot-critical files instead. Assisted-by: AI --- .../test-to-existing-root-preserve-var.sh | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index 37afc531ca..c8ba43b35a 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -23,6 +23,34 @@ freeze_may_be_active=0 persistent_tmt_root= persistent_run_dir= +boot_config_manifest() { + local relative_path + local digest + + while IFS= read -r -d '' relative_path; do + digest=$(sha256sum -- "/boot/$relative_path") + digest=${digest%% *} + printf '%s %s\n' "$digest" "$relative_path" + done < <( + { + find /boot/loader/entries -maxdepth 1 \ + \( -type f -o -type l \) -name '*.conf' \ + -printf 'loader/entries/%f\0' + for relative_path in grub2/grub.cfg grub2/grubenv; do + if [[ -f /boot/$relative_path ]]; then + printf '%s\0' "$relative_path" + fi + done + if [[ -d /boot/efi/EFI/fedora ]]; then + find /boot/efi/EFI/fedora -maxdepth 1 \ + \( -type f -o -type l \) \ + \( -name grub.cfg -o -iname '*.efi' \) \ + -printf 'efi/EFI/fedora/%f\0' + fi + } | LC_ALL=C sort -z -u + ) +} + is_always_excluded_var_path() { case $1 in /var/tmp|/var/tmp/*|/var/cache|/var/cache/*|/var/log/journal|/var/log/journal/*|/var/lib/containers|/var/lib/containers/*) @@ -485,9 +513,20 @@ case ${TMT_REBOOT_COUNT:-0} in : > "$preflight_mountpoint" mount --bind /dev/null "$preflight_mountpoint" + expected_bls_entry=$(find /boot/loader/entries -maxdepth 1 -type f \ + -name 'ostree-*.conf' -print -quit) + if [[ -z $expected_bls_entry ]]; then + printf 'No expected OSTree BLS entry found under /boot/loader/entries\n' >&2 + exit 1 + fi + if [[ ! -f /boot/grub2/grub.cfg ]]; then + printf 'Expected GRUB configuration is missing: /boot/grub2/grub.cfg\n' >&2 + exit 1 + fi + boot_snapshot=$(mktemp /run/boot-config-before.XXXXXX) boot_snapshot_after=$(mktemp /run/boot-config-after.XXXXXX) - tar --sort=name --numeric-owner --xattrs --acls -cf "$boot_snapshot" -C /boot . + boot_config_manifest > "$boot_snapshot" if podman --remote=false run --rm --privileged --pid=host --user=root:root \ -v /dev:/dev \ -v /:/target \ @@ -500,10 +539,14 @@ case ${TMT_REBOOT_COUNT:-0} in printf 'preserve-var unexpectedly accepted the special-file mountpoint\n' >&2 exit 1 fi + boot_config_manifest > "$boot_snapshot_after" + if ! diff -u --label 'boot manifest before preflight' \ + --label 'boot manifest after preflight' "$boot_snapshot" "$boot_snapshot_after"; then + printf 'Boot-critical file contents or paths changed during rejected preflight\n' >&2 + exit 1 + fi grep -F "$preflight_mountpoint is a mountpoint that is neither a directory nor a file" \ /run/preserve-var-preflight.log - tar --sort=name --numeric-owner --xattrs --acls -cf "$boot_snapshot_after" -C /boot . - cmp "$boot_snapshot" "$boot_snapshot_after" umount "$preflight_mountpoint" rm -f "$preflight_mountpoint" "$boot_snapshot" "$boot_snapshot_after" From bdcec68813d2cc0ada28f22fd350fd0d9dfad0ce Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 13:26:49 -0600 Subject: [PATCH 36/38] tmt: Cover EFI fallback boot entries The rejected preflight check should detect changes to all active EFI loader entries, including the architecture-independent fallback path. Record symlink targets alongside referent content digests so retargeting links is visible. Assisted-by: AI --- .../test-to-existing-root-preserve-var.sh | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index c8ba43b35a..c7dcd5fcbf 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -25,12 +25,25 @@ persistent_run_dir= boot_config_manifest() { local relative_path + local absolute_path local digest + local link_target while IFS= read -r -d '' relative_path; do - digest=$(sha256sum -- "/boot/$relative_path") - digest=${digest%% *} - printf '%s %s\n' "$digest" "$relative_path" + absolute_path=/boot/$relative_path + if [[ -L $absolute_path ]]; then + link_target=$(readlink -- "$absolute_path") + digest=- + if [[ -f $absolute_path ]]; then + digest=$(sha256sum -- "$absolute_path") + digest=${digest%% *} + fi + printf 'symlink %s %q -> %q\n' "$digest" "$relative_path" "$link_target" + else + digest=$(sha256sum -- "$absolute_path") + digest=${digest%% *} + printf 'file %s %q\n' "$digest" "$relative_path" + fi done < <( { find /boot/loader/entries -maxdepth 1 \ @@ -47,6 +60,12 @@ boot_config_manifest() { \( -name grub.cfg -o -iname '*.efi' \) \ -printf 'efi/EFI/fedora/%f\0' fi + if [[ -d /boot/efi/EFI/BOOT ]]; then + find /boot/efi/EFI/BOOT -maxdepth 1 \ + \( -type f -o -type l \) \ + \( -name grub.cfg -o -iname '*.efi' \) \ + -printf 'efi/EFI/BOOT/%f\0' + fi } | LC_ALL=C sort -z -u ) } From d620ac219b1235707a4d3be347e00d0fcda7f998 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 13:30:52 -0600 Subject: [PATCH 37/38] tests: Snapshot package-mode BLS entries for preflight The package-mode guest has a normal BLS entry before migration, so the preflight snapshot must not require an OSTree-prefixed name. Assisted-by: AI --- .../package-mode/test-to-existing-root-preserve-var.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index c7dcd5fcbf..5e06596e64 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -532,10 +532,12 @@ case ${TMT_REBOOT_COUNT:-0} in : > "$preflight_mountpoint" mount --bind /dev/null "$preflight_mountpoint" - expected_bls_entry=$(find /boot/loader/entries -maxdepth 1 -type f \ - -name 'ostree-*.conf' -print -quit) - if [[ -z $expected_bls_entry ]]; then - printf 'No expected OSTree BLS entry found under /boot/loader/entries\n' >&2 + # The package-mode baseline has a normal BLS entry; migration creates + # the OSTree-prefixed entries later. Snapshot all existing boot paths. + baseline_bls_entry=$(find /boot/loader/entries -maxdepth 1 -type f \ + -name '*.conf' -print -quit) + if [[ -z $baseline_bls_entry ]]; then + printf 'No BLS entry found under /boot/loader/entries before migration\n' >&2 exit 1 fi if [[ ! -f /boot/grub2/grub.cfg ]]; then From 79113f2173b0474708aee0e8e60b208e57b21122 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Thu, 12 Nov 2026 13:32:35 -0600 Subject: [PATCH 38/38] tmt: Exclude OSTree entries from baseline BLS lookup A leftover OSTree BLS entry must not satisfy the package-mode baseline check. Assisted-by: AI --- .../booted/package-mode/test-to-existing-root-preserve-var.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh index 5e06596e64..909ad6498f 100755 --- a/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh +++ b/tmt/tests/booted/package-mode/test-to-existing-root-preserve-var.sh @@ -535,7 +535,7 @@ case ${TMT_REBOOT_COUNT:-0} in # The package-mode baseline has a normal BLS entry; migration creates # the OSTree-prefixed entries later. Snapshot all existing boot paths. baseline_bls_entry=$(find /boot/loader/entries -maxdepth 1 -type f \ - -name '*.conf' -print -quit) + -name '*.conf' ! -name 'ostree-*.conf' -print -quit) if [[ -z $baseline_bls_entry ]]; then printf 'No BLS entry found under /boot/loader/entries before migration\n' >&2 exit 1