From eaf9696718549526aeeadc5213195b953922af2f Mon Sep 17 00:00:00 2001 From: Joseph Marrero Corchado Date: Sat, 12 Sep 2026 12:40:51 -0400 Subject: [PATCH 1/6] boundimage: Expose quadlet path and contents from bound image query Prep for `bootc apply-live bound-images`, which needs to know not just which image a bound image definition references, but where the quadlet file that defines it lives (so it can be related to podman's quadlet search directories) and its contents (so definitions can be compared across deployments and materialized elsewhere). Add `query_bound_image_specs` returning that alongside the parsed `BoundImage`, and make `query_bound_images` a thin wrapper over it. Symlink targets are resolved lexically to a root-relative path with the same semantics as the `RESOLVE_IN_ROOT` read of the file (excess `..` stays at the root); no further symlinks are followed. Assisted-by: AI --- crates/lib/src/boundimage.rs | 112 +++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 4 deletions(-) diff --git a/crates/lib/src/boundimage.rs b/crates/lib/src/boundimage.rs index 31b4e5fb3c..0f5f24c9c2 100644 --- a/crates/lib/src/boundimage.rs +++ b/crates/lib/src/boundimage.rs @@ -6,7 +6,7 @@ //! is considered ready. use anyhow::{Context, Result}; -use camino::Utf8Path; +use camino::{Utf8Path, Utf8PathBuf}; use cap_std_ext::cap_std::fs::Dir; use cap_std_ext::dirext::CapStdExtDirExt; use fn_error_context::context; @@ -18,19 +18,30 @@ use crate::store::Storage; /// The path in a root for bound images; this directory should only contain /// symbolic links to `.container` or `.image` files. -const BOUND_IMAGE_DIR: &str = "usr/lib/bootc/bound-images.d"; +pub(crate) const BOUND_IMAGE_DIR: &str = "usr/lib/bootc/bound-images.d"; /// A subset of data parsed from a `.image` or `.container` file with /// the minimal information necessary to fetch the image. /// /// In the future this may be extended to include e.g. certificates or /// other pull options. -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct BoundImage { pub(crate) image: String, pub(crate) auth_file: Option, } +/// A bound image definition together with the quadlet file that defines it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct BoundImageSpec { + pub(crate) image: BoundImage, + /// Path of the quadlet file relative to the root, e.g. + /// `usr/share/containers/systemd/foo.container`. + pub(crate) path: Utf8PathBuf, + /// The raw contents of the quadlet file. + pub(crate) contents: String, +} + #[derive(Debug, PartialEq, Eq)] pub(crate) struct ResolvedBoundImage { pub(crate) image: String, @@ -70,6 +81,39 @@ pub(crate) fn query_bound_images_for_deployment( #[context("Querying bound images")] pub(crate) fn query_bound_images(root: &Dir) -> Result> { + let specs = query_bound_image_specs(root)?; + Ok(specs.into_iter().map(|s| s.image).collect()) +} + +/// Lexically resolve the target of a symlink in `dir` (relative to the root) +/// to a root-relative path, without following any further symlinks. Like +/// `RESOLVE_IN_ROOT` (which is how the file is actually read), `..` at the +/// root stays at the root. +fn resolve_link_target(dir: &Utf8Path, target: &Utf8Path) -> Result { + let mut ret = Utf8PathBuf::new(); + let base = if target.is_absolute() { + Utf8Path::new("") + } else { + dir + }; + for component in base.components().chain(target.components()) { + use camino::Utf8Component::*; + match component { + RootDir | CurDir => {} + ParentDir => { + ret.pop(); + } + Normal(n) => ret.push(n), + Prefix(_) => anyhow::bail!("Unexpected path prefix: {target}"), + } + } + Ok(ret) +} + +/// Like [`query_bound_images`], but also returns the path and contents +/// of the quadlet file defining each image. +#[context("Querying bound image specs")] +pub(crate) fn query_bound_image_specs(root: &Dir) -> Result> { let spec_dir = BOUND_IMAGE_DIR; let Some(bound_images_dir) = root.open_dir_optional(spec_dir)? else { tracing::debug!("Missing {spec_dir}"); @@ -112,7 +156,19 @@ pub(crate) fn query_bound_images(root: &Dir) -> Result> { _ => anyhow::bail!("Invalid file extension: {file_name}"), }?; - bound_images.push(bound_image); + // Record where the quadlet actually lives, so callers can relate it + // to podman's quadlet search directories. + let target = bound_images_dir + .read_link_contents(file_name) + .with_context(|| format!("Reading link {path}"))?; + let target = Utf8PathBuf::try_from(target).context("Non-UTF8 symlink target")?; + let target = resolve_link_target(Utf8Path::new(spec_dir), &target)?; + + bound_images.push(BoundImageSpec { + image: bound_image, + path: target, + contents: file_contents, + }); } Ok(bound_images) @@ -294,6 +350,32 @@ mod tests { assert_eq!(images[0].image, "quay.io/bar/bar:latest"); assert_eq!(images[1].image, "quay.io/foo/foo:latest"); + // The specs should record the resolved quadlet path and contents + let mut specs = query_bound_image_specs(td).unwrap(); + specs.sort_by(|a, b| a.path.cmp(&b.path)); + assert_eq!(specs.len(), 2); + assert_eq!(specs[0].path, format!("{CONTAINER_IMAGE_DIR}/bar.image")); + assert_eq!(specs[0].image.image, "quay.io/bar/bar:latest"); + assert!(specs[0].contents.contains("quay.io/bar/bar:latest")); + assert_eq!(specs[1].path, format!("{CONTAINER_IMAGE_DIR}/foo.image")); + + // Relative symlinks are resolved relative to the bound images directory + td.symlink( + "../../../share/containers/systemd/bar.image", + format!("{BOUND_IMAGE_DIR}/relative.image"), + ) + .unwrap(); + let specs = query_bound_image_specs(td).unwrap(); + assert_eq!(specs.len(), 3); + assert!( + specs + .iter() + .all(|s| s.path == format!("{CONTAINER_IMAGE_DIR}/bar.image") + || s.path == format!("{CONTAINER_IMAGE_DIR}/foo.image")) + ); + td.remove_file(format!("{BOUND_IMAGE_DIR}/relative.image")) + .unwrap(); + // Invalid symlink should return an error td.symlink("./blah", format!("{BOUND_IMAGE_DIR}/blah.image")) .unwrap(); @@ -308,6 +390,28 @@ mod tests { Ok(()) } + #[test] + fn test_resolve_link_target() { + let d = Utf8Path::new("usr/lib/bootc/bound-images.d"); + assert_eq!( + resolve_link_target(d, "/usr/share/containers/systemd/foo.image".into()).unwrap(), + "usr/share/containers/systemd/foo.image" + ); + assert_eq!( + resolve_link_target(d, "../../../share/containers/systemd/foo.image".into()).unwrap(), + "usr/share/containers/systemd/foo.image" + ); + assert_eq!( + resolve_link_target(d, "./foo.image".into()).unwrap(), + "usr/lib/bootc/bound-images.d/foo.image" + ); + // Excess `..` is clamped at the root, matching RESOLVE_IN_ROOT + assert_eq!( + resolve_link_target(d, "../../../../../../foo.image".into()).unwrap(), + "foo.image" + ); + } + #[test] fn test_parse_spec_value() -> Result<()> { //should parse string with no % characters From e8d213ba0bb96237970bdfa70361c94a3a069afe Mon Sep 17 00:00:00 2001 From: Joseph Marrero Corchado Date: Sat, 12 Sep 2026 12:40:51 -0400 Subject: [PATCH 2/6] Add `bootc apply-live bound-images` A customer using logically bound images wants to update them out of band without rebooting, while bootc keeps owning their lifecycle (storage, garbage collection, rollback). This is a scoped first step toward the general `bootc apply-live` in #76. The key observation is that the image bytes are already live: there is a single bootc-owned containers-storage shared by all deployments, and quadlets read it via `additionalimagestore`. What is tied to a deployment is only the definition (the `bound-images.d` symlink and the quadlet it points to), which the running system reads from its own `/usr`. So no overmounting is needed: changed quadlets are written to `/run/containers/systemd/`, which podman gives precedence over `/etc` and `/usr`, systemd is reloaded, and the affected `.container` units restarted. Because `/run` is transient the override disappears on reboot, at which point the staged deployment's content applies and the system converges on its own. A soft reboot preserves `/run`, so the override is cleared when one is prepared. The command refuses to run unless the ostree diff between the booted and staged commits is entirely accounted for by the bound image changes it will apply, so it never silently applies half an upgrade; since the diff doesn't enumerate added/removed directories, those are walked in the deployment roots. The applied state is recorded in `/run/bootc/apply-live/bound-images.json` before any unit is touched, with each unit marked pending until its restart succeeds, so a failed restart (or `--no-restart`) is retried by simply re-running. The state is surfaced in `bootc status`, and its image names become extra GC roots so a `bootc rollback` that discards the staged deployment doesn't prune an image a running container uses. A later staged deployment reverts any overrides from a previous apply that it no longer needs. The verb is a subcommand group (`apply-live bound-images`, alias `lbi`) so that the bare `apply-live` stays reserved for the general feature and further scopes can be added without a new top-level verb. It is hidden and experimental, and ostree-only for now; composefs needs a tree diff API before the scope check can be implemented there. This has unit coverage for the planning, scope check and state logic. The /run write, SELinux labeling, systemctl sequencing, status output, GC root handling, out-of-scope refusal and post-reboot convergence were verified on a CentOS Stream 10 VM by the tmt test in the following commit. Assisted-by: AI --- crates/lib/src/applylive.rs | 855 +++++++++++++++++++++++++++++ crates/lib/src/cli.rs | 57 ++ crates/lib/src/deploy.rs | 12 + crates/lib/src/lib.rs | 2 + crates/lib/src/spec.rs | 42 ++ crates/lib/src/status.rs | 25 + docs/src/host-v1.schema.json | 70 +++ docs/src/logically-bound-images.md | 42 ++ 8 files changed, 1105 insertions(+) create mode 100644 crates/lib/src/applylive.rs diff --git a/crates/lib/src/applylive.rs b/crates/lib/src/applylive.rs new file mode 100644 index 0000000000..7509c9e758 --- /dev/null +++ b/crates/lib/src/applylive.rs @@ -0,0 +1,855 @@ +//! # Live application of staged content +//! +//! This module implements `bootc apply-live`, which makes content from the +//! staged deployment visible on the running system without a reboot. +//! See . +//! +//! The only scope supported today is logically bound images +//! (`bootc apply-live bound-images`). The image bytes are already shared +//! between deployments via the bootc-owned container storage; what is +//! pinned to a deployment is only the *definition*: the symlink in +//! `/usr/lib/bootc/bound-images.d` and the quadlet file it references. +//! +//! Applying these live therefore does not require mounting anything. +//! Changed quadlet files are written to `/run/containers/systemd/`, which +//! podman gives precedence over `/etc` and `/usr`, and the affected units +//! are restarted. Because `/run` is transient, the override disappears on +//! the next boot, at which point the staged deployment's own `/usr` content +//! applies and the system converges without any further action. (A soft +//! reboot preserves `/run`, so the override is explicitly cleared when one +//! is prepared.) +//! +//! The state file is written *before* any units are touched: it describes +//! what is on disk, and each unit is marked `pending` until it has been +//! restarted successfully. Re-running the command retries pending units. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::Write; +use std::process::Command; + +use anyhow::{Context, Result, ensure}; +use bootc_utils::CommandRunExt; +use camino::{Utf8Path, Utf8PathBuf}; +use cap_std_ext::cap_std::{self, fs::Dir}; +use cap_std_ext::dirext::CapStdExtDirExt; +use fn_error_context::context; +use ostree_ext::diff::FileTreeDiff; + +use crate::boundimage::{BOUND_IMAGE_DIR, BoundImageSpec}; +use crate::cli::ApplyLiveBoundImagesOpts; +use crate::spec::{LiveBoundImage, LiveBoundImages}; +use crate::store::{BootedOstree, Storage}; + +/// Directory (relative to `/run`) holding apply-live state. +const STATE_DIR: &str = "bootc/apply-live"; +/// State file (relative to `/run`) describing live-applied bound images. +const BOUND_IMAGES_STATE: &str = "bootc/apply-live/bound-images.json"; +/// Podman's highest-precedence quadlet search directory, relative to `/`. +/// Paths are relative to the root (not `/run`) so that SELinux labels are +/// computed for the real absolute path. +const QUADLET_RUN_DIR: &str = "run/containers/systemd"; +/// Quadlet search directories relative to a deployment root, in decreasing +/// precedence (excluding `/run`, which is what we write to). +const QUADLET_SEARCH_DIRS: &[&str] = &["etc/containers/systemd", "usr/share/containers/systemd"]; +/// Directories which may be added/removed/changed in an ostree commit diff +/// while still being in scope for `apply-live bound-images`. Note that +/// `/etc` is stored as `/usr/etc` in ostree commits. The contents of an +/// added or removed directory are checked separately, since the diff does +/// not enumerate them. +const SCOPE_DIRS: &[&str] = &[ + "/usr/lib/bootc", + "/usr/lib/bootc/bound-images.d", + "/usr/share/containers", + "/usr/share/containers/systemd", + "/usr/etc/containers", + "/usr/etc/containers/systemd", +]; +/// Maximum number of out-of-scope paths to include in an error message. +const MAX_REPORTED_PATHS: usize = 10; +/// Journal message ID for apply-live operations. +const APPLY_LIVE_JOURNAL_ID: &str = "4c9e2b7d1f0a4e8b9c6d3a2f5e7b1c0d"; + +/// How a bound image definition differs between the booted and staged deployments. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ChangeKind { + /// Present in the staged deployment only + Added, + /// Present in both, with different quadlet contents + Updated, + /// Present in the booted deployment only, and the quadlet file + /// no longer exists in the staged deployment + Removed, +} + +impl std::fmt::Display for ChangeKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + ChangeKind::Added => "added", + ChangeKind::Updated => "updated", + ChangeKind::Removed => "removed", + }; + f.write_str(s) + } +} + +/// A single bound image definition change to apply. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct QuadletChange { + pub(crate) kind: ChangeKind, + /// The staged spec for `Added`/`Updated`, the booted spec for `Removed`. + pub(crate) spec: BoundImageSpec, + /// Path of the quadlet relative to its search directory, if it lives in + /// one podman will find. Definitions outside a search directory are only + /// used as pull specs and have nothing to materialize. + pub(crate) quadlet: Option, + /// The systemd unit to restart, if any. + pub(crate) unit: Option, +} + +/// If `path` (relative to a root) is inside a quadlet search directory, +/// return the remainder relative to that directory. +pub(crate) fn quadlet_relpath(path: &Utf8Path) -> Option<&Utf8Path> { + QUADLET_SEARCH_DIRS + .iter() + .find_map(|d| path.strip_prefix(d).ok()) + .filter(|p| !p.as_str().is_empty()) +} + +/// Compute the systemd service unit that quadlet generates for a `.container` +/// file. Returns `None` for other quadlet types; a `.image` unit is a oneshot +/// pull which we don't need to run since bootc pulls the image itself. +pub(crate) fn quadlet_unit_name(relpath: &Utf8Path, contents: &str) -> Result> { + if relpath.extension() != Some("container") { + return Ok(None); + } + let ini = tini::Ini::from_string(contents).context("Parse to ini")?; + let name = ini + .get::("Container", "ServiceName") + .or_else(|| relpath.file_stem().map(ToOwned::to_owned)) + .ok_or_else(|| anyhow::anyhow!("Invalid quadlet name: {relpath}"))?; + Ok(Some(format!("{name}.service"))) +} + +/// Convert a deployment-relative path to the form used in an ostree commit +/// (and hence [`FileTreeDiff`]): absolute, with `/etc` stored as `/usr/etc`. +fn commit_path(path: &Utf8Path) -> Utf8PathBuf { + if let Ok(rest) = path.strip_prefix("etc") { + Utf8Path::new("/usr/etc").join(rest) + } else { + Utf8Path::new("/").join(path) + } +} + +/// Compute the set of definition changes between the booted and staged +/// bound image specs. +pub(crate) fn compute_changes( + booted: &[BoundImageSpec], + staged: &[BoundImageSpec], + staged_root: &Dir, +) -> Result> { + let booted: BTreeMap<_, _> = booted.iter().map(|s| (s.path.as_path(), s)).collect(); + let staged: BTreeMap<_, _> = staged.iter().map(|s| (s.path.as_path(), s)).collect(); + let mut changes = Vec::new(); + + let mk = |kind, spec: &BoundImageSpec| -> Result { + let quadlet = quadlet_relpath(&spec.path).map(ToOwned::to_owned); + let unit = quadlet + .as_deref() + .map(|q| quadlet_unit_name(q, &spec.contents)) + .transpose()? + .flatten(); + Ok(QuadletChange { + kind, + spec: spec.clone(), + quadlet, + unit, + }) + }; + + for (path, spec) in staged.iter() { + match booted.get(path) { + Some(b) if b.contents == spec.contents => {} + Some(_) => changes.push(mk(ChangeKind::Updated, spec)?), + None => changes.push(mk(ChangeKind::Added, spec)?), + } + } + for (path, spec) in booted.iter() { + if staged.contains_key(path) { + continue; + } + if staged_root.try_exists(path)? { + // The quadlet still exists, it's just no longer a bound image; + // there's nothing to change on the running system. + tracing::debug!("No longer bound, but still present: {path}"); + continue; + } + changes.push(mk(ChangeKind::Removed, spec)?); + } + Ok(changes) +} + +/// Recursively collect the files under `dir` (a commit-form absolute path) +/// in the given root, as commit-form absolute paths. +fn walk_files(root: &Dir, dir: &str, out: &mut Vec) -> Result<()> { + let rel = dir.trim_start_matches('/'); + let Some(d) = root.open_dir_optional(rel)? else { + return Ok(()); + }; + for entry in d.entries()? { + let entry = entry?; + let name = entry.file_name(); + let name = name + .to_str() + .ok_or_else(|| anyhow::anyhow!("Invalid non-UTF8 filename: {name:?} in {dir}"))?; + let path = format!("{dir}/{name}"); + if entry.file_type()?.is_dir() { + walk_files(root, &path, out)?; + } else { + out.push(path); + } + } + Ok(()) +} + +/// Verify that every path which differs between the booted and staged commits +/// is accounted for by the bound image changes we are going to apply. Any other +/// change means the staged deployment is not just a bound image update, and a +/// reboot is required. +/// +/// The ostree diff does not enumerate the contents of added or removed +/// directories, so those are walked in the staged (respectively booted) +/// deployment root and checked file by file. +pub(crate) fn check_diff_scope( + diff: &FileTreeDiff, + changes: &[QuadletChange], + booted_root: &Dir, + staged_root: &Dir, +) -> Result<()> { + let bound_dir = format!("/{BOUND_IMAGE_DIR}/"); + let allowed_files: BTreeSet<_> = changes.iter().map(|c| commit_path(&c.spec.path)).collect(); + let file_in_scope = + |p: &str| -> bool { p.starts_with(&bound_dir) || allowed_files.contains(Utf8Path::new(p)) }; + let dir_in_scope = |p: &str| -> bool { SCOPE_DIRS.contains(&p) }; + + let mut out_of_scope: Vec = diff + .added_files + .iter() + .chain(&diff.removed_files) + .chain(&diff.changed_files) + .filter(|p| !file_in_scope(p)) + .chain(diff.changed_dirs.iter().filter(|p| !dir_in_scope(p))) + .cloned() + .collect(); + for (dirs, root) in [ + (&diff.added_dirs, staged_root), + (&diff.removed_dirs, booted_root), + ] { + for d in dirs { + if !dir_in_scope(d) { + out_of_scope.push(d.clone()); + continue; + } + let mut files = Vec::new(); + walk_files(root, d, &mut files)?; + out_of_scope.extend(files.into_iter().filter(|p| !file_in_scope(p))); + } + } + if out_of_scope.is_empty() { + return Ok(()); + } + out_of_scope.sort(); + let n = out_of_scope.len(); + let mut msg = String::from( + "Staged deployment contains changes outside of bound image definitions; a reboot is required to apply it:\n", + ); + for p in out_of_scope.iter().take(MAX_REPORTED_PATHS) { + msg.push_str(" "); + msg.push_str(p); + msg.push('\n'); + } + if n > MAX_REPORTED_PATHS { + msg.push_str(&format!(" ...and {} more\n", n - MAX_REPORTED_PATHS)); + } + anyhow::bail!("{}", msg.trim_end()) +} + +/// Read the live bound image state from `/run`, if any. +pub(crate) fn read_state(run: &Dir) -> Result> { + let Some(f) = run.open_optional(BOUND_IMAGES_STATE)? else { + return Ok(None); + }; + let r = serde_json::from_reader(std::io::BufReader::new(f)) + .with_context(|| format!("Parsing /run/{BOUND_IMAGES_STATE}"))?; + Ok(Some(r)) +} + +/// Read the live bound image state from the host `/run`, if any. +pub(crate) fn read_state_from_host() -> Result> { + let run = + Dir::open_ambient_dir("/run", cap_std::ambient_authority()).context("Opening /run")?; + read_state(&run) +} + +fn write_state(run: &Dir, state: &LiveBoundImages) -> Result<()> { + run.create_dir_all(STATE_DIR)?; + run.atomic_replace_with(BOUND_IMAGES_STATE, |w| { + serde_json::to_writer_pretty(w, state).map_err(anyhow::Error::from) + }) + .with_context(|| format!("Writing /run/{BOUND_IMAGES_STATE}")) +} + +/// Remove any live-applied bound image overrides and state. This is used when +/// preparing a soft reboot, which preserves `/run` but must boot into the +/// target deployment's own definitions. +#[context("Clearing live bound image state")] +pub(crate) fn clear_state_from_host() -> Result<()> { + let run = + Dir::open_ambient_dir("/run", cap_std::ambient_authority()).context("Opening /run")?; + let Some(state) = read_state(&run)? else { + return Ok(()); + }; + let rootfs = Dir::open_ambient_dir("/", cap_std::ambient_authority()).context("Opening /")?; + for q in state.images.iter().filter_map(|i| i.quadlet.as_deref()) { + rootfs.remove_file_optional(Utf8Path::new(QUADLET_RUN_DIR).join(q))?; + } + run.remove_file_optional(BOUND_IMAGES_STATE)?; + println!("Cleared live-applied bound image definitions"); + Ok(()) +} + +#[context("Running systemctl {}", args.join(" "))] +fn systemctl(args: &[&str]) -> Result<()> { + Command::new("systemctl").args(args).run_capture_stderr() +} + +/// Create `path` and any missing ancestors below `/run`, labeling only the +/// directories we create; existing ones (e.g. podman's `/run/containers`) +/// are left alone. +fn ensure_quadlet_dir( + rootfs: &Dir, + path: &Utf8Path, + sepolicy: Option<&ostree_ext::ostree::SePolicy>, +) -> Result<()> { + let mode = rustix::fs::Mode::from_raw_mode(0o755); + let ancestors: Vec<_> = path + .ancestors() + .take_while(|p| p.as_str().len() > "run".len()) + .collect(); + for dir in ancestors.into_iter().rev() { + if rootfs.try_exists(dir)? { + continue; + } + crate::lsm::ensure_dir_labeled(rootfs, dir, None, mode, sepolicy)?; + } + Ok(()) +} + +/// Restart every unit still marked pending in `state`, updating the state +/// file as each one succeeds so a failure part way through can be retried. +fn restart_pending(run: &Dir, state: &mut LiveBoundImages) -> Result<()> { + let mut r = Ok(()); + for img in state.images.iter_mut().filter(|i| i.pending) { + let Some(unit) = img.unit.as_deref() else { + img.pending = false; + continue; + }; + // `restart` also starts a unit which is not running (the added case). + if let Err(e) = systemctl(&["restart", unit]) { + r = Err(e); + break; + } + img.pending = false; + } + write_state(run, state)?; + r +} + +/// Implementation of `bootc apply-live bound-images`. +#[context("Applying bound images live")] +pub(crate) async fn apply_bound_images( + storage: &Storage, + booted: &BootedOstree<'_>, + opts: &ApplyLiveBoundImagesOpts, +) -> Result<()> { + let sysroot = booted.sysroot; + let booted_deployment = &booted.deployment; + let staged = sysroot.staged_deployment().ok_or_else(|| { + anyhow::anyhow!("No staged deployment; run `bootc upgrade` or `bootc switch` first") + })?; + ensure!( + staged.osname() == booted_deployment.osname(), + "Staged deployment is in a different stateroot" + ); + let booted_csum = booted_deployment.csum(); + let staged_csum = staged.csum(); + + let run = + &Dir::open_ambient_dir("/run", cap_std::ambient_authority()).context("Opening /run")?; + let rootfs = &Dir::open_ambient_dir("/", cap_std::ambient_authority()).context("Opening /")?; + let previous = read_state(run)?; + + // The definitions from this staged deployment are already on disk; all + // that may be left is restarting units (after `--no-restart`, or a failure). + if let Some(mut previous) = previous + .as_ref() + .filter(|p| p.checksum == staged_csum) + .cloned() + { + let pending: Vec<_> = previous + .images + .iter() + .filter(|i| i.pending) + .filter_map(|i| i.unit.as_deref()) + .collect(); + if pending.is_empty() { + println!("Bound images from staged deployment are already applied"); + return Ok(()); + } + for unit in pending { + println!("restart pending: {unit}"); + } + if opts.dry_run || opts.no_restart { + return Ok(()); + } + systemctl(&["daemon-reload"])?; + return restart_pending(run, &mut previous); + } + + let booted_root = crate::utils::deployment_fd(sysroot, booted_deployment)?; + let staged_root = crate::utils::deployment_fd(sysroot, &staged)?; + let booted_specs = crate::boundimage::query_bound_image_specs(&booted_root)?; + let staged_specs = crate::boundimage::query_bound_image_specs(&staged_root)?; + let changes = compute_changes(&booted_specs, &staged_specs, &staged_root)?; + + if booted_csum != staged_csum { + let diff = + ostree_ext::diff::diff(&sysroot.repo(), &booted_csum, &staged_csum, None::<&str>)?; + tracing::debug!("Diff booted -> staged: {diff}"); + check_diff_scope(&diff, &changes, &booted_root, &staged_root)?; + } + + // Overrides written by a previous apply-live which are not part of this + // one need to be removed so the running system converges on the staged + // definitions rather than a mix of the two. + let previous_quadlets: BTreeMap<&Utf8Path, &LiveBoundImage> = previous + .iter() + .flat_map(|p| &p.images) + .filter_map(|i| i.quadlet.as_deref().map(|q| (Utf8Path::new(q), i))) + .collect(); + let current_quadlets: BTreeSet<&Utf8Path> = changes + .iter() + .filter_map(|c| c.quadlet.as_deref()) + .collect(); + let stale: Vec<(&Utf8Path, &LiveBoundImage)> = previous_quadlets + .iter() + .filter(|(q, _)| !current_quadlets.contains(*q)) + .map(|(q, i)| (*q, *i)) + .collect(); + + if changes.is_empty() && stale.is_empty() { + println!("No bound image changes to apply"); + return Ok(()); + } + + for c in changes.iter() { + let unit = c.unit.as_deref().unwrap_or("(no unit)"); + println!("{}: {} ({unit})", c.kind, c.spec.image.image); + } + for (q, _) in stale.iter() { + println!("reverting stale override: {q}"); + } + if opts.dry_run { + return Ok(()); + } + + tracing::info!( + message_id = APPLY_LIVE_JOURNAL_ID, + bootc.deployment.checksum = staged_csum.as_str(), + bootc.bound_images_changes = changes.len(), + "Applying bound image definitions live from staged deployment" + ); + + // Ensure all images referenced by the staged deployment are present. They + // normally were pulled when the deployment was staged, but this also acts + // as a retry if that failed. + let staged_images = staged_specs.iter().map(|s| s.image.clone()).collect(); + crate::boundimage::pull_images(storage, staged_images).await?; + + // Materialize the definitions into /run. A definition may already be live + // from a previous apply, in which case its unit doesn't need a restart. + let quadlet_run_dir = Utf8Path::new(QUADLET_RUN_DIR); + let sepolicy = crate::lsm::new_sepolicy_at(&booted_root)?; + let filemode = rustix::fs::Mode::from_raw_mode(0o644); + let mut written = BTreeMap::new(); + for c in changes.iter() { + let Some(quadlet) = c.quadlet.as_deref() else { + continue; + }; + let dest = quadlet_run_dir.join(quadlet); + match c.kind { + ChangeKind::Added | ChangeKind::Updated => { + let already_live = previous_quadlets + .get(quadlet) + .is_some_and(|prev| !prev.pending) + && rootfs.read_to_string_optional(&dest)?.as_deref() + == Some(c.spec.contents.as_str()); + if !already_live { + // SAFETY: We know there's a parent + ensure_quadlet_dir(rootfs, dest.parent().unwrap(), sepolicy.as_ref())?; + crate::lsm::atomic_replace_labeled( + rootfs, + &dest, + filemode, + sepolicy.as_ref(), + |w| w.write_all(c.spec.contents.as_bytes()).map_err(Into::into), + )?; + } + written.insert(quadlet, !already_live); + } + ChangeKind::Removed => { + rootfs.remove_file_optional(&dest)?; + } + } + } + for (q, _) in stale.iter() { + rootfs.remove_file_optional(quadlet_run_dir.join(q))?; + } + + // Record all bound images of the staged deployment, not just the changed + // ones: the state file protects them from garbage collection even if the + // staged deployment is later discarded (e.g. by `bootc rollback`). + let images = staged_specs + .iter() + .map(|s| { + let quadlet = quadlet_relpath(&s.path).filter(|q| written.contains_key(q)); + let unit = quadlet + .map(|q| quadlet_unit_name(q, &s.contents)) + .transpose()? + .flatten(); + let pending = unit.is_some() && quadlet.is_some_and(|q| written[q]); + Ok(LiveBoundImage { + image: s.image.image.clone(), + quadlet: quadlet.map(|q| q.to_string()), + unit, + pending, + }) + }) + .collect::>>()?; + let mut state = LiveBoundImages { + checksum: staged_csum.to_string(), + deploy_serial: staged.deployserial() as u32, + images, + }; + write_state(run, &state)?; + + if opts.no_restart { + println!("Skipping unit restart; re-run without --no-restart to restart pending units"); + return Ok(()); + } + + // Units for removed definitions must be stopped before the daemon reload + // which makes them disappear from systemd's view. + let removed_units = changes + .iter() + .filter(|c| c.kind == ChangeKind::Removed) + .filter_map(|c| c.unit.as_deref()) + .chain(stale.iter().filter_map(|(_, img)| img.unit.as_deref())); + for unit in removed_units { + systemctl(&["stop", unit])?; + } + systemctl(&["daemon-reload"])?; + // A stale override reverts to the booted deployment's definition if + // there is one, otherwise the unit is simply gone. + for (q, img) in stale.iter() { + let Some(unit) = img.unit.as_deref() else { + continue; + }; + let booted_has = booted_specs + .iter() + .any(|s| quadlet_relpath(&s.path) == Some(q)); + if booted_has { + systemctl(&["start", unit])?; + } + } + restart_pending(run, &mut state)?; + + println!( + "Applied {} bound image definition change(s) from staged deployment", + changes.len() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::boundimage::BoundImage; + + fn spec(path: &str, image: &str, contents: &str) -> BoundImageSpec { + BoundImageSpec { + image: BoundImage { + image: image.into(), + auth_file: None, + }, + path: path.into(), + contents: contents.into(), + } + } + + fn tempdir() -> Result { + Ok(cap_std_ext::cap_tempfile::TempDir::new( + cap_std::ambient_authority(), + )?) + } + + #[test] + fn test_quadlet_relpath() { + assert_eq!( + quadlet_relpath("usr/share/containers/systemd/foo.container".into()), + Some("foo.container".into()) + ); + assert_eq!( + quadlet_relpath("etc/containers/systemd/sub/foo.image".into()), + Some("sub/foo.image".into()) + ); + assert_eq!(quadlet_relpath("usr/lib/foo/foo.container".into()), None); + assert_eq!(quadlet_relpath("usr/share/containers/systemd".into()), None); + } + + #[test] + fn test_quadlet_unit_name() -> Result<()> { + let c = "[Container]\nImage=quay.io/foo/foo:latest\n"; + assert_eq!( + quadlet_unit_name("foo.container".into(), c)?.as_deref(), + Some("foo.service") + ); + assert_eq!( + quadlet_unit_name("sub/foo.container".into(), c)?.as_deref(), + Some("foo.service") + ); + let c = "[Container]\nImage=quay.io/foo/foo:latest\nServiceName=bar\n"; + assert_eq!( + quadlet_unit_name("foo.container".into(), c)?.as_deref(), + Some("bar.service") + ); + let c = "[Image]\nImage=quay.io/foo/foo:latest\n"; + assert_eq!(quadlet_unit_name("foo.image".into(), c)?, None); + Ok(()) + } + + #[test] + fn test_commit_path() { + assert_eq!( + commit_path("etc/containers/systemd/a.container".into()), + Utf8PathBuf::from("/usr/etc/containers/systemd/a.container") + ); + assert_eq!( + commit_path("usr/share/containers/systemd/a.container".into()), + Utf8PathBuf::from("/usr/share/containers/systemd/a.container") + ); + } + + #[test] + fn test_compute_changes() -> Result<()> { + let staged_root = &tempdir()?; + const Q: &str = "usr/share/containers/systemd"; + let v1 = "[Container]\nImage=quay.io/foo/foo:v1\n"; + let v2 = "[Container]\nImage=quay.io/foo/foo:v2\n"; + let bar = "[Image]\nImage=quay.io/foo/bar:latest\n"; + let unbound = "[Container]\nImage=quay.io/foo/unbound:latest\n"; + let gone = "[Container]\nImage=quay.io/foo/gone:latest\n"; + + let booted = [ + spec(&format!("{Q}/foo.container"), "quay.io/foo/foo:v1", v1), + spec(&format!("{Q}/bar.image"), "quay.io/foo/bar:latest", bar), + spec( + &format!("{Q}/unbound.container"), + "quay.io/foo/unbound:latest", + unbound, + ), + spec( + &format!("{Q}/gone.container"), + "quay.io/foo/gone:latest", + gone, + ), + ]; + let staged = [ + spec(&format!("{Q}/foo.container"), "quay.io/foo/foo:v2", v2), + spec(&format!("{Q}/bar.image"), "quay.io/foo/bar:latest", bar), + spec("usr/lib/misc/new.container", "quay.io/foo/new:latest", v1), + ]; + // The unbound quadlet still exists in the staged root + staged_root.create_dir_all(Q)?; + staged_root.write(format!("{Q}/unbound.container"), unbound)?; + + let changes = compute_changes(&booted, &staged, staged_root)?; + let summary: Vec<_> = changes + .iter() + .map(|c| { + ( + c.kind, + c.spec.path.as_str(), + c.quadlet.as_deref().map(|q| q.as_str()), + c.unit.as_deref(), + ) + }) + .collect(); + assert_eq!( + summary, + [ + (ChangeKind::Added, "usr/lib/misc/new.container", None, None), + ( + ChangeKind::Updated, + "usr/share/containers/systemd/foo.container", + Some("foo.container"), + Some("foo.service") + ), + ( + ChangeKind::Removed, + "usr/share/containers/systemd/gone.container", + Some("gone.container"), + Some("gone.service") + ), + ] + ); + Ok(()) + } + + #[test] + fn test_check_diff_scope() -> Result<()> { + let booted_root = &tempdir()?; + let staged_root = &tempdir()?; + let changes = vec![ + QuadletChange { + kind: ChangeKind::Updated, + spec: spec( + "usr/share/containers/systemd/foo.container", + "quay.io/foo/foo:v2", + "", + ), + quadlet: Some("foo.container".into()), + unit: Some("foo.service".into()), + }, + QuadletChange { + kind: ChangeKind::Added, + spec: spec("etc/containers/systemd/bar.image", "quay.io/foo/bar:v1", ""), + quadlet: Some("bar.image".into()), + unit: None, + }, + ]; + let check = + |diff: &FileTreeDiff| check_diff_scope(diff, &changes, booted_root, staged_root); + + let mut diff = FileTreeDiff::default(); + diff.changed_files + .insert("/usr/share/containers/systemd/foo.container".into()); + diff.added_files + .insert("/usr/etc/containers/systemd/bar.image".into()); + diff.added_files + .insert("/usr/lib/bootc/bound-images.d/bar.image".into()); + diff.added_dirs + .insert("/usr/lib/bootc/bound-images.d".into()); + check(&diff).unwrap(); + + // A bound-images.d symlink target changing isn't visible in the diff + // as a file change, but the removal of the old link is. + diff.removed_files + .insert("/usr/lib/bootc/bound-images.d/old.image".into()); + check(&diff).unwrap(); + + // Anything else is out of scope + diff.changed_files.insert("/usr/bin/bash".into()); + let e = check(&diff).unwrap_err().to_string(); + assert!(e.contains("/usr/bin/bash"), "{e}"); + assert!(!e.contains("foo.container"), "{e}"); + diff.changed_files.remove("/usr/bin/bash"); + + // A quadlet we're not going to apply (not bound) is also out of scope + diff.changed_files + .insert("/usr/share/containers/systemd/other.container".into()); + assert!(check(&diff).is_err()); + diff.changed_files + .remove("/usr/share/containers/systemd/other.container"); + + // As is a new subdirectory of quadlets, since the diff doesn't recurse into it + diff.added_dirs + .insert("/usr/share/containers/systemd/sub".into()); + assert!(check(&diff).is_err()); + diff.added_dirs.remove("/usr/share/containers/systemd/sub"); + + // The quadlet dir itself being added is fine when it only contains + // what we apply (the "first bound image" case)... + const Q: &str = "usr/etc/containers/systemd"; + staged_root.create_dir_all(Q)?; + staged_root.write(format!("{Q}/bar.image"), "")?; + diff.added_dirs.insert("/usr/etc/containers".into()); + diff.added_dirs.insert(format!("/{Q}")); + check(&diff).unwrap(); + // ...but not when it contains anything else, even nested + staged_root.create_dir_all(format!("{Q}/sub"))?; + staged_root.write(format!("{Q}/sub/other.container"), "")?; + let e = check(&diff).unwrap_err().to_string(); + assert!(e.contains(&format!("/{Q}/sub/other.container")), "{e}"); + staged_root.remove_dir_all(format!("{Q}/sub"))?; + check(&diff).unwrap(); + + // Likewise for a removed directory, which is checked in the booted root + booted_root.create_dir_all("usr/lib/bootc/bound-images.d")?; + booted_root.write("usr/lib/bootc/bound-images.d/old.image", "")?; + diff.removed_dirs.insert("/usr/lib/bootc".into()); + check(&diff).unwrap(); + booted_root.write("usr/lib/bootc/other", "")?; + let e = check(&diff).unwrap_err().to_string(); + assert!(e.contains("/usr/lib/bootc/other"), "{e}"); + booted_root.remove_file("usr/lib/bootc/other")?; + + // Error message is truncated + for i in 0..20 { + diff.added_files.insert(format!("/usr/bin/tool{i}")); + } + let e = check(&diff).unwrap_err().to_string(); + assert!(e.contains("and 10 more"), "{e}"); + Ok(()) + } + + #[test] + fn test_state_roundtrip() -> Result<()> { + let run = &tempdir()?; + assert_eq!(read_state(run)?, None); + let state = LiveBoundImages { + checksum: "abc".into(), + deploy_serial: 1, + images: vec![ + LiveBoundImage { + image: "quay.io/foo/foo:v2".into(), + quadlet: Some("foo.container".into()), + unit: Some("foo.service".into()), + pending: true, + }, + LiveBoundImage { + image: "quay.io/foo/bar:v2".into(), + quadlet: None, + unit: None, + pending: false, + }, + ], + }; + write_state(run, &state)?; + assert_eq!(read_state(run)?.as_ref(), Some(&state)); + let raw = run.read_to_string(BOUND_IMAGES_STATE)?; + assert!(raw.contains("\"pending\": true"), "{raw}"); + assert_eq!(raw.matches("pending").count(), 1, "{raw}"); + Ok(()) + } + + #[test] + fn test_ensure_quadlet_dir() -> Result<()> { + let rootfs = &tempdir()?; + rootfs.create_dir("run")?; + ensure_quadlet_dir(rootfs, "run/containers/systemd/sub/deeper".into(), None)?; + assert!(rootfs.is_dir("run/containers/systemd/sub/deeper")); + // Idempotent + ensure_quadlet_dir(rootfs, "run/containers/systemd/sub/deeper".into(), None)?; + Ok(()) + } +} diff --git a/crates/lib/src/cli.rs b/crates/lib/src/cli.rs index 97b9a9e778..1dfe266a56 100644 --- a/crates/lib/src/cli.rs +++ b/crates/lib/src/cli.rs @@ -261,6 +261,42 @@ pub(crate) enum SoftRebootMode { Auto, } +/// Options for `bootc apply-live bound-images` +#[derive(Debug, Parser, PartialEq, Eq)] +pub(crate) struct ApplyLiveBoundImagesOpts { + /// Write the definitions and ensure the images are present, but don't + /// reload systemd or restart any units. + #[clap(long)] + pub(crate) no_restart: bool, + + /// Print what would be done without changing the running system. + #[clap(long)] + pub(crate) dry_run: bool, +} + +/// Subcommands for applying staged content to the running system. +#[derive(Debug, clap::Subcommand, PartialEq, Eq)] +pub(crate) enum ApplyLiveOpts { + /// Apply the staged deployment's logically bound image definitions to the running system. + /// + /// The staged deployment must differ from the booted deployment only in + /// logically bound image definitions: the `/usr/lib/bootc/bound-images.d` + /// symlinks and the `.container` or `.image` files they reference. If + /// anything else changed, this command fails and a reboot is required. + /// + /// Changed quadlet files are written to `/run/containers/systemd/`, which + /// podman gives precedence over `/etc` and `/usr`, and the corresponding + /// units are restarted (added units are started, removed units stopped). + /// The images themselves are already present in the bootc-owned storage + /// shared by all deployments. + /// + /// Because `/run` is transient this state is discarded on reboot, at which + /// point the staged deployment's own content applies. The applied state is + /// visible in `bootc status`. + #[clap(alias = "lbi")] + BoundImages(ApplyLiveBoundImagesOpts), +} + /// Perform an status operation #[derive(Debug, Parser, PartialEq, Eq)] pub(crate) struct StatusOpts { @@ -977,6 +1013,11 @@ pub(crate) enum Opt { /// Allows temporary package installation that will be discarded on reboot. #[clap(alias = "usroverlay")] UsrOverlay(UsrOverlayOpts), + /// Apply content from the staged deployment to the running system without a reboot. + /// + /// Stability: This interface is experimental and may change in the future. + #[clap(subcommand, hide = true)] + ApplyLive(ApplyLiveOpts), /// Install the running container to a target. /// /// Takes a container image and installs it to disk in a bootable format. @@ -1117,6 +1158,9 @@ fn prepare_soft_reboot(sysroot: &SysrootLock, deployment: &ostree::Deployment) - sysroot .deployment_set_soft_reboot(deployment, false, cancellable) .context("Failed to prepare soft-reboot")?; + // A soft reboot preserves /run, so anything applied live must not leak + // into the target deployment. + crate::applylive::clear_state_from_host()?; Ok(()) } @@ -1941,6 +1985,19 @@ async fn run_from_opt(opt: Opt) -> Result { Ok(()) } Opt::Edit(opts) => edit(opts).await, + Opt::ApplyLive(opts) => { + let storage = &get_storage().await?; + match storage.kind()? { + BootedStorageKind::Ostree(booted_ostree) => match opts { + ApplyLiveOpts::BoundImages(opts) => { + crate::applylive::apply_bound_images(storage, &booted_ostree, &opts).await + } + }, + BootedStorageKind::Composefs(_) => { + anyhow::bail!("apply-live is not yet supported on composefs systems") + } + } + } Opt::UsrOverlay(opts) => { use crate::store::Environment; let env = Environment::detect()?; diff --git a/crates/lib/src/deploy.rs b/crates/lib/src/deploy.rs index b361a3b79f..f9202ed88e 100644 --- a/crates/lib/src/deploy.rs +++ b/crates/lib/src/deploy.rs @@ -415,6 +415,18 @@ pub(crate) async fn prune_container_store(sysroot: &Storage) -> Result<()> { }); } } + // Images applied live via `bootc apply-live bound-images` must survive + // even if the deployment they came from is discarded before reboot. + if let Some(live) = crate::applylive::read_state_from_host()? { + all_bound_images.extend( + live.images + .into_iter() + .map(|img| crate::boundimage::BoundImage { + image: img.image, + auth_file: None, + }), + ); + } // Convert to a hashset of just the image names let image_names = HashSet::from_iter(all_bound_images.iter().map(|img| img.image.as_str())); let pruned = sysroot diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index d9eccc0c81..3cb4c7be43 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -34,6 +34,7 @@ //! //! - [`image`] - Image operations and queries //! - [`boundimage`] - Logically Bound Images (LBIs) +//! - [`applylive`] - Applying staged content to the running system (`bootc apply-live`) //! - [`podstorage`] - bootc-owned container storage (`/usr/lib/bootc/storage`) //! - [`podman`] - Podman command helpers //! @@ -63,6 +64,7 @@ //! - [`linux-kernel-cmdline`](../linux_kernel_cmdline/index.html) - Cmdline parsing //! - [`etc-merge`](../etc_merge/index.html) - `/etc` three-way merge +mod applylive; mod bootc_composefs; pub(crate) mod bootc_kargs; mod bootloader; diff --git a/crates/lib/src/spec.rs b/crates/lib/src/spec.rs index 15ad1014fe..90c1b74ade 100644 --- a/crates/lib/src/spec.rs +++ b/crates/lib/src/spec.rs @@ -426,6 +426,42 @@ impl Display for FilesystemOverlay { } } +/// A logically bound image definition that was applied to the running +/// system by `bootc apply-live bound-images`. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct LiveBoundImage { + /// The container image reference + pub image: String, + /// The quadlet file written under `/run/containers/systemd`, if any + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub quadlet: Option, + /// The systemd unit generated from the quadlet, if any + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub unit: Option, + /// True if the unit has not yet been restarted with this definition + /// (e.g. `--no-restart` was used, or the restart failed). Re-running + /// `bootc apply-live bound-images` will retry it. + #[serde(skip_serializing_if = "std::ops::Not::not")] + #[serde(default)] + pub pending: bool, +} + +/// Logically bound image definitions applied to the running system without +/// a reboot. This state lives in `/run` and is discarded on reboot. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct LiveBoundImages { + /// The ostree commit checksum of the deployment the definitions came from + pub checksum: String, + /// The deployment serial + pub deploy_serial: u32, + /// The applied definitions + pub images: Vec, +} + /// The status of the host system #[derive(Debug, Clone, Serialize, Default, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(rename_all = "camelCase")] @@ -451,6 +487,12 @@ pub struct HostStatus { /// The state of the overlay mounted on /usr pub usr_overlay: Option, + /// Logically bound image definitions applied live to the booted + /// deployment via `bootc apply-live bound-images`. + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub live_bound_images: Option, + /// Set to true if the physical root (`/sysroot`) is on a read-only medium /// (e.g. a live ISO) and so cannot be mutated; commands that would change /// the system (upgrade, switch, etc.) are not available. diff --git a/crates/lib/src/status.rs b/crates/lib/src/status.rs index 52ae08458d..232859e26b 100644 --- a/crates/lib/src/status.rs +++ b/crates/lib/src/status.rs @@ -428,6 +428,8 @@ pub(crate) fn get_status( .map(|d| d.unlocked()) .and_then(crate::spec::deployment_unlocked_state_to_usr_overlay); + let live_bound_images = crate::applylive::read_state_from_host()?; + let mut host = Host::new(spec); host.status = HostStatus { staged, @@ -437,6 +439,7 @@ pub(crate) fn get_status( rollback_queued, ty, usr_overlay, + live_bound_images, // Set by callers that have storage context (e.g. get_host). read_only: false, }; @@ -731,6 +734,7 @@ fn human_render_slot( // Show /usr overlay status write_usr_overlay(&mut out, slot, host_status, prefix_len)?; + write_live_bound_images(&mut out, slot, host_status, prefix_len)?; if verbose { // Show additional information in verbose mode similar to rpm-ostree @@ -790,6 +794,26 @@ fn write_usr_overlay( Ok(()) } +/// Helper function to render live-applied bound images +fn write_live_bound_images( + mut out: impl Write, + slot: Option, + host_status: &crate::spec::HostStatus, + prefix_len: usize, +) -> Result<()> { + // Only the booted deployment can have live-applied content + if !matches!(slot, Some(Slot::Booted)) { + return Ok(()); + } + if let Some(live) = host_status.live_bound_images.as_ref() { + write_row_name(&mut out, "Live bound images", prefix_len)?; + let n = live.images.iter().filter(|i| i.quadlet.is_some()).count(); + let short = live.checksum.get(..12).unwrap_or(&live.checksum); + writeln!(out, "{n} definition(s) applied from {short}")?; + } + Ok(()) +} + /// Output a rendering of a non-container boot entry. fn human_render_slot_ostree( mut out: impl Write, @@ -818,6 +842,7 @@ fn human_render_slot_ostree( // Show /usr overlay status write_usr_overlay(&mut out, slot, host_status, prefix_len)?; + write_live_bound_images(&mut out, slot, host_status, prefix_len)?; if verbose { // Show additional information in verbose mode similar to rpm-ostree diff --git a/docs/src/host-v1.schema.json b/docs/src/host-v1.schema.json index 686b71e046..cef7d6f15b 100644 --- a/docs/src/host-v1.schema.json +++ b/docs/src/host-v1.schema.json @@ -317,6 +317,17 @@ } ] }, + "liveBoundImages": { + "description": "Logically bound image definitions applied live to the booted\ndeployment via `bootc apply-live bound-images`.", + "anyOf": [ + { + "$ref": "#/$defs/LiveBoundImages" + }, + { + "type": "null" + } + ] + }, "otherDeployments": { "description": "Other deployments (i.e. pinned)", "type": "array", @@ -485,6 +496,65 @@ "architecture" ] }, + "LiveBoundImage": { + "description": "A logically bound image definition that was applied to the running\nsystem by `bootc apply-live bound-images`.", + "type": "object", + "properties": { + "image": { + "description": "The container image reference", + "type": "string" + }, + "pending": { + "description": "True if the unit has not yet been restarted with this definition\n(e.g. `--no-restart` was used, or the restart failed). Re-running\n`bootc apply-live bound-images` will retry it.", + "type": "boolean" + }, + "quadlet": { + "description": "The quadlet file written under `/run/containers/systemd`, if any", + "type": [ + "string", + "null" + ] + }, + "unit": { + "description": "The systemd unit generated from the quadlet, if any", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "image" + ] + }, + "LiveBoundImages": { + "description": "Logically bound image definitions applied to the running system without\na reboot. This state lives in `/run` and is discarded on reboot.", + "type": "object", + "properties": { + "checksum": { + "description": "The ostree commit checksum of the deployment the definitions came from", + "type": "string" + }, + "deploySerial": { + "description": "The deployment serial", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "images": { + "description": "The applied definitions", + "type": "array", + "items": { + "$ref": "#/$defs/LiveBoundImage" + } + } + }, + "required": [ + "checksum", + "deploySerial", + "images" + ] + }, "ObjectMeta": { "type": "object", "properties": { diff --git a/docs/src/logically-bound-images.md b/docs/src/logically-bound-images.md index 6be695aef3..37d39e3cfd 100644 --- a/docs/src/logically-bound-images.md +++ b/docs/src/logically-bound-images.md @@ -56,6 +56,48 @@ Images are fetched using the global bootc pull secret by default (`/etc/ostree/a The bootc image store is owned by bootc; images will be garbage collected when they are no longer referenced by a file in `/usr/lib/bootc/bound-images.d`. +## Applying changes without a reboot (experimental) + +The container images for logically bound images live in a single bootc-owned +storage shared by all deployments, so after a `bootc upgrade` or `bootc switch` +the new images are already on disk. What is tied to a deployment is only the +*definition*: the symlink in `/usr/lib/bootc/bound-images.d` and the quadlet +file it references, which the running system reads from its own `/usr`. + +If the staged deployment differs from the booted one *only* in bound image +definitions, they can be applied to the running system without a reboot: + +``` +bootc apply-live bound-images +``` + +This writes the changed quadlet files to `/run/containers/systemd/`, which +podman gives precedence over `/etc` and `/usr`, reloads systemd and restarts +the affected `.container` units (removed units are stopped). Pass `--dry-run` +to see what would change, or `--no-restart` to write the definitions without +touching any units; a later plain invocation restarts the units still +pending, as does re-running after a failed restart. + +If the staged deployment contains any other change, the command fails and +lists the out-of-scope paths; a reboot is required to apply it. Note that a +locally modified quadlet in `/etc/containers/systemd` that also changed in +the image is reported this way, since the local copy wins in `/etc`. + +Because `/run` is transient, the override disappears on the next boot, at +which point the staged deployment's own content applies. (A soft reboot +preserves `/run`; the override is cleared when one is prepared.) The applied +state is shown in `bootc status` and recorded in `/run/bootc/apply-live/`; +images recorded there are protected from garbage collection until reboot, +even if the staged deployment is discarded (for example by `bootc rollback`). + +Limitations: only the `.container` or `.image` file referenced from +`bound-images.d` is applied. Quadlet drop-ins (`*.container.d/`) and other +referenced quadlet files (`.volume`, `.network`, `.pod`) are not copied, and +changes to them are treated as out of scope. A `.container` unit that +references a changed `.image` is not restarted unless its own file changed. +Removed units are stopped but not masked, so a dependency could start them +again until reboot. Only supported on ostree-backed systems. + ## Installation Logically bound images must be present in the default container store (`/var/lib/containers`) when invoking From 78607038b971559ef13a1d1a5e5fd8663c77f69b Mon Sep 17 00:00:00 2001 From: Joseph Marrero Corchado Date: Sat, 12 Sep 2026 12:48:41 -0400 Subject: [PATCH 3/6] tests: Add booted test for `bootc apply-live bound-images` The unit tests cover the planning, scope check and state file logic, but not the parts that only exist on a real host: writing to `/run/containers/systemd` with the right SELinux label, the systemctl sequencing, and the interaction with `bootc status`, garbage collection and reboot. Switch to an image with a bound `.container` that just sleeps, then upgrade to one that only changes the bound image, and walk through `--dry-run`, `--no-restart` (unit still on the old image, state marked pending), a plain re-run (unit restarted onto the new image), and the idempotent no-op. Discarding the staged deployment via rollback and running cleanup checks the live state acts as a GC root. An upgrade with an unrelated file change checks the out-of-scope refusal. After the reboot, the `/run` override and state must be gone and the deployment's own definition in effect. Assisted-by: AI --- tmt/plans/integration.fmf | 8 + .../booted/test-logically-bound-apply-live.nu | 191 ++++++++++++++++++ tmt/tests/tests.fmf | 5 + 3 files changed, 204 insertions(+) create mode 100644 tmt/tests/booted/test-logically-bound-apply-live.nu diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index 125ad46975..2c074b1dee 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -314,4 +314,12 @@ execute: test: - /tmt/tests/tests/test-48-composefs-uki-dumpfile extra-skip_if_ostree: true + +/plan-49-logically-bound-apply-live: + summary: Execute logically bound images tests for bootc apply-live bound-images + discover: + how: fmf + test: + - /tmt/tests/tests/test-49-logically-bound-apply-live + extra-fixme_skip_if_composefs: true # END GENERATED PLANS diff --git a/tmt/tests/booted/test-logically-bound-apply-live.nu b/tmt/tests/booted/test-logically-bound-apply-live.nu new file mode 100644 index 0000000000..b1b60a0f5f --- /dev/null +++ b/tmt/tests/booted/test-logically-bound-apply-live.nu @@ -0,0 +1,191 @@ +# number: 49 +# tmt: +# summary: Execute logically bound images tests for bootc apply-live bound-images +# duration: 30m +# extra: +# fixme_skip_if_composefs: true +# +# This test does: +# bootc switch to an image with a bound .container (v1) +# +# +# bootc upgrade to an image whose only change is the bound .container (v2) +# bootc apply-live bound-images --dry-run / --no-restart / (plain) +# +# bootc upgrade to an image with an unrelated change +# +# +# + +use std assert +use tap.nu + +# This code runs on *each* boot. +bootc status +let st = bootc status --json | from json +let booted = $st.status.booted.image + +# Two distinct tags of the same small image so we can tell which one a +# container was started from. +const image_v1 = "registry.access.redhat.com/ubi9/ubi-minimal:9.4" +const image_v2 = "registry.access.redhat.com/ubi9/ubi-minimal:9.3" +const quadlet = "/usr/share/containers/systemd/lbi-sleeper.container" +const run_quadlet = "/run/containers/systemd/lbi-sleeper.container" +const unit = "lbi-sleeper.service" +const state_file = "/run/bootc/apply-live/bound-images.json" + +def initial_setup [] { + bootc image copy-to-storage + podman images + podman image inspect localhost/bootc | from json +} + +# Build a bootc image on top of the booted one with a single bound +# .container that just sleeps, referencing $image. If $extra is set, also +# add an unrelated file so the diff is out of scope for apply-live. +def build_image [name image extra] { + let td = mktemp -d + cd $td + mkdir usr/share/containers/systemd + $"[Container] +Image=($image) +Exec=sleep infinity +GlobalArgs=--storage-opt=additionalimagestore=/usr/lib/bootc/storage + +[Install] +WantedBy=multi-user.target +" | save usr/share/containers/systemd/lbi-sleeper.container + + mut dockerfile = "FROM localhost/bootc +COPY usr/ /usr/ +RUN ln -s /usr/share/containers/systemd/lbi-sleeper.container /usr/lib/bootc/bound-images.d/lbi-sleeper.container +" + if $extra { + $dockerfile = $dockerfile + "RUN echo unrelated > /usr/share/apply-live-out-of-scope.txt\n" + } + $dockerfile | save Dockerfile + podman build -t $name . +} + +# Return the image a running container was created from. The image lives in +# the bootc storage, so podman needs the same additional store the quadlet uses. +def running_image [] { + podman --storage-opt=additionalimagestore=/usr/lib/bootc/storage inspect systemd-lbi-sleeper --format '{{.ImageName}}' | str trim +} + +def first_boot [] { + tap begin "bootc apply-live bound-images" + initial_setup + build_image localhost/bootc-lbi-live $image_v1 false + bootc switch --transport containers-storage localhost/bootc-lbi-live + tmt-reboot +} + +def second_boot [] { + print "verifying second boot after switch" + assert equal $booted.image.image localhost/bootc-lbi-live + systemctl is-active $unit + assert equal (running_image) $image_v1 + assert not ($run_quadlet | path exists) + assert equal $st.status.liveBoundImages? null + + # Without a staged deployment there's nothing to apply + let r = do { bootc apply-live bound-images } | complete + assert not equal $r.exit_code 0 + assert ($r.stderr | str contains "No staged deployment") + + # Stage an image whose only change is the bound container's image + print "bootc upgrade to v2 of the bound image" + build_image localhost/bootc-lbi-live $image_v2 false + bootc upgrade + let st = bootc status --json | from json + assert not equal $st.status.staged null + let staged_checksum = $st.status.staged.ostree.checksum + + # Dry run changes nothing + let out = bootc apply-live bound-images --dry-run + print $out + assert ($out | str contains $"updated: ($image_v2) \(($unit)\)") + assert not ($run_quadlet | path exists) + assert equal (running_image) $image_v1 + + # --no-restart writes the definition but leaves the unit alone + bootc apply-live lbi --no-restart + assert ($run_quadlet | path exists) + assert (open $run_quadlet | str contains $image_v2) + assert equal (getfattr --only-values -n security.selinux $run_quadlet | split row ':' | get 2) container_var_run_t + assert equal (running_image) $image_v1 + let state = open $state_file + assert equal $state.checksum $staged_checksum + let entry = $state.images | where unit? == $unit | first + assert equal $entry.pending true + let live = bootc status --json | from json | get status.liveBoundImages + assert equal $live.checksum $staged_checksum + + # A plain re-run restarts the pending unit + let out = bootc apply-live bound-images + print $out + assert ($out | str contains $"restart pending: ($unit)") + systemctl is-active $unit + assert equal (running_image) $image_v2 + let state = open $state_file + let entry = $state.images | where unit? == $unit | first + assert equal ($entry.pending? | default false) false + let human = bootc status --format humanreadable + print $human + assert ($human | str contains "Live bound images") + + # And is idempotent afterwards + let out = bootc apply-live bound-images + assert ($out | str contains "already applied") + assert equal (running_image) $image_v2 + + # The live-applied images survive garbage collection even without the + # staged deployment: rollback discards it (then revert the queued rollback + # so we still boot the default), and cleanup prunes the store. + bootc rollback + bootc rollback + let st = bootc status --json | from json + assert equal $st.status.staged null + assert equal $st.status.rollbackQueued false + bootc internals cleanup + let names = podman --storage-opt=additionalimagestore=/usr/lib/bootc/storage images --format '{{.Repository}}:{{.Tag}}' | lines + assert ($image_v2 in $names) + assert equal (running_image) $image_v2 + + # A staged deployment with an unrelated change is refused + print "bootc upgrade with an out-of-scope change" + build_image localhost/bootc-lbi-live $image_v2 true + bootc upgrade + let r = do { bootc apply-live bound-images } | complete + print $r.stderr + assert not equal $r.exit_code 0 + assert ($r.stderr | str contains "reboot is required") + assert ($r.stderr | str contains "/usr/share/apply-live-out-of-scope.txt") + # The previously applied definition is still live + assert equal (running_image) $image_v2 + + tmt-reboot +} + +def third_boot [] { + print "verifying third boot after upgrade" + # /run is fresh: no override, no state, and the deployment's own + # definition (v2) is what runs. + assert not ($run_quadlet | path exists) + assert not ($state_file | path exists) + assert equal $st.status.liveBoundImages? null + assert (open $quadlet | str contains $image_v2) + systemctl is-active $unit + assert equal (running_image) $image_v2 + tap ok +} + +def main [] { + match $env.TMT_REBOOT_COUNT? { + null | "0" => first_boot, + "1" => second_boot, + "2" => third_boot, + $o => { error make { msg: $"Invalid TMT_REBOOT_COUNT ($o)" } }, + } +} diff --git a/tmt/tests/tests.fmf b/tmt/tests/tests.fmf index 93364f28d1..af57ee6591 100644 --- a/tmt/tests/tests.fmf +++ b/tmt/tests/tests.fmf @@ -193,3 +193,8 @@ check: summary: Test composefs UKI dumpfile diff print duration: 30m test: nu booted/test-composefs-uki-dumpfile.nu + +/test-49-logically-bound-apply-live: + summary: Execute logically bound images tests for bootc apply-live bound-images + duration: 30m + test: nu booted/test-logically-bound-apply-live.nu From 5ba73ae852869583dc367e2035bae6ba7c8c28a9 Mon Sep 17 00:00:00 2001 From: ckyrouac Date: Fri, 16 Oct 2026 10:22:17 -0500 Subject: [PATCH 4/6] ci: Fetch grub-cc from Fedora Koji The ELN latest image no longer provides a reliable source for grub-cc. Fetch the architecture-specific RPM from a pinned Fedora Koji build instead. This is temporary until bootupd supports installing grub-cc into the ESP. Assisted-by: AI Signed-off-by: ckyrouac --- Dockerfile | 45 ++++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index bd8c289a89..f84095c034 100644 --- a/Dockerfile +++ b/Dockerfile @@ -102,32 +102,8 @@ RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp <&2 - exit 1 - ;; -esac - -mv ./*.rpm grub-cc.rpm - -EOF - FROM scratch as fetch COPY --from=target-base /target-rootfs/ / -COPY --from=grub-cc-download /grub-cc.rpm /var/grub-cc.rpm # SKIP_CONFIGS=1 skips LBIs, test kargs, and install configs (for FCOS testing) ARG SKIP_CONFIGS ARG boot_type @@ -164,6 +140,24 @@ RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp \ dnf install -y "${pkgs_to_install[@]}" fi + if [[ "$bootloader" == "grub-cc" ]]; then + case "$(uname -m)" in + x86_64) + grub_cc_url=https://kojipkgs.fedoraproject.org/packages/grub2/2.12/60.eln156/x86_64/grub2-efi-x64-cc-2.12-60.eln156.x86_64.rpm + ;; + aarch64) + grub_cc_url=https://kojipkgs.fedoraproject.org/packages/grub2/2.12/60.eln156/aarch64/grub2-efi-aa64-cc-2.12-60.eln156.aarch64.rpm + ;; + *) + echo "Unsupported architecture: $(uname -m)" >&2 + exit 1 + ;; + esac + + curl --fail --location --retry 5 --retry-all-errors --retry-delay 1 \ + --output /var/grub-cc.rpm "$grub_cc_url" + fi + # Currently dnf installs grub-cc at /usr/lib/efi/grub2/1:2.12-60.eln156/EFI/eln/cc/grubx64-cc.efi # which is less than ideal because: # - the "cc" subdirectory @@ -173,7 +167,8 @@ RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp \ # we can copy the EFI binary to the ESP if [[ "$bootloader" == "grub-cc" ]]; then mkdir /var/grub-cc - rpm2archive /var/grub-cc.rpm | tar -xvz -C /var/grub-cc + rpm2archive /var/grub-cc.rpm > /tmp/grub-cc.tar.gz + tar -xvz -f /tmp/grub-cc.tar.gz -C /var/grub-cc file=$(find /var/grub-cc -name '*.efi') mkdir -p /usr/lib/grub-cc cp "$file" /usr/lib/grub-cc/grub-cc.efi From bd5345b5c02e06e6861f144eefd342657027370c Mon Sep 17 00:00:00 2001 From: Yasmin de Souza Date: Wed, 28 Oct 2026 17:26:12 -0300 Subject: [PATCH 5/6] applylive: Address review feedback on docs and state tracking Apply two pieces of feedback from cgwalters's review of #2458: - Scope the module doc's "doesn't require mounting anything" claim to the ostree backend, since the composefs backend lays out image storage differently. - Replace the `/run/bootc/apply-live/bound-images.json` state file with state embedded directly in the quadlet files written to `/run/containers/systemd/`: each one now carries a leading marker comment recording its staged checksum, image reference, and whether its unit still needs a restart. That directory is now the source of truth instead of a bespoke JSON file. A small flat-text list of protected image references is still written separately, since a bound image with an unchanged definition has no quadlet override to carry that state, and this list is only used to protect such images from garbage collection. Also address several correctness issues GitHub Copilot's automated review found in that state-tracking code: - Track the previous unit name across a `ServiceName=` change so the old unit is actually stopped instead of left running alongside the new one. - Merge newly staged protected images into the existing `protected-images.list` instead of replacing it, so an earlier deployment's still-running bound image isn't GC'd out from under it. - Make the fast path compare the full set of expected quadlets, not just checksums, so a retry after a partially-failed apply doesn't mistake incomplete state for complete. - Refuse to overwrite a `/run` quadlet override bootc didn't write itself. - Guard against an Added/Removed pair that target the same `/run` destination (e.g. a quadlet moving between search dirs) from deleting the replacement that was just written. - Always refresh a quadlet's marker checksum, even when its contents are unchanged, so status and the fast path stay accurate across later deployments. - Skip symlinks and other non-regular files when scanning `/run/containers/systemd/`, so a unit masked via `-> /dev/null` doesn't break `bootc status`. - Update the booted integration test to read state from `bootc status --json`'s `liveBoundImages` and `protected-images.list` instead of the now-removed JSON state file. Still under review; not yet addressing the separate comment about whether `liveBoundImages` belongs in `bootc status`/the schema, nor the deeper `/etc`-vs-vendor quadlet precedence question Copilot also raised. Assisted-by: AI Co-authored-by: Cursor --- crates/lib/src/applylive.rs | 649 +++++++++++++----- crates/lib/src/deploy.rs | 18 +- crates/lib/src/status.rs | 16 +- .../booted/test-logically-bound-apply-live.nu | 15 +- 4 files changed, 507 insertions(+), 191 deletions(-) diff --git a/crates/lib/src/applylive.rs b/crates/lib/src/applylive.rs index 7509c9e758..9e19cedbc4 100644 --- a/crates/lib/src/applylive.rs +++ b/crates/lib/src/applylive.rs @@ -10,18 +10,28 @@ //! pinned to a deployment is only the *definition*: the symlink in //! `/usr/lib/bootc/bound-images.d` and the quadlet file it references. //! -//! Applying these live therefore does not require mounting anything. -//! Changed quadlet files are written to `/run/containers/systemd/`, which -//! podman gives precedence over `/etc` and `/usr`, and the affected units -//! are restarted. Because `/run` is transient, the override disappears on -//! the next boot, at which point the staged deployment's own `/usr` content -//! applies and the system converges without any further action. (A soft -//! reboot preserves `/run`, so the override is explicitly cleared when one -//! is prepared.) +//! On the ostree backend, that shared container storage is just a plain +//! directory, so applying a definition change live does not require +//! mounting anything new -- podman already reads the same storage the +//! booted and staged deployments share. (The composefs backend lays out +//! storage differently and is not handled here yet.) Changed quadlet files +//! are written to `/run/containers/systemd/`, which podman gives +//! precedence over `/etc` and `/usr`, and the affected units are restarted. +//! Because `/run` is transient, the override disappears on the next boot, +//! at which point the staged deployment's own `/usr` content applies and +//! the system converges without any further action. (A soft reboot +//! preserves `/run`, so the override is explicitly cleared when one is +//! prepared.) //! -//! The state file is written *before* any units are touched: it describes -//! what is on disk, and each unit is marked `pending` until it has been -//! restarted successfully. Re-running the command retries pending units. +//! Rather than a separate state file, each quadlet written to +//! `/run/containers/systemd/` carries its own state as a leading comment +//! (see [`marker_line`]): the staged checksum, the image reference, and +//! whether its unit still needs restarting ("pending"). Re-running the +//! command re-derives everything by scanning that directory (see +//! [`scan_live_quadlets`]), so a failure part way through is safe to +//! resume from. The one exception is garbage-collection protection for +//! bound images with no quadlet override (unchanged definitions); see +//! [`write_protected_images`]. use std::collections::{BTreeMap, BTreeSet}; use std::io::Write; @@ -42,8 +52,14 @@ use crate::store::{BootedOstree, Storage}; /// Directory (relative to `/run`) holding apply-live state. const STATE_DIR: &str = "bootc/apply-live"; -/// State file (relative to `/run`) describing live-applied bound images. -const BOUND_IMAGES_STATE: &str = "bootc/apply-live/bound-images.json"; +/// Flat, one-image-reference-per-line list (relative to `/run`) protecting +/// every bound image of an in-progress (or completed) apply-live from +/// garbage collection; see [`write_protected_images`]. +const PROTECTED_IMAGES_LIST: &str = "bootc/apply-live/protected-images.list"; +/// Prefix of the leading marker comment line bootc writes at the top of +/// each quadlet file it materializes under [`QUADLET_RUN_DIR`]; see +/// [`marker_line`] and [`parse_marker`]. +const MARKER_PREFIX: &str = "# X-Bootc-Apply-Live:"; /// Podman's highest-precedence quadlet search directory, relative to `/`. /// Paths are relative to the root (not `/run`) so that SELinux labels are /// computed for the real absolute path. @@ -104,6 +120,29 @@ pub(crate) struct QuadletChange { pub(crate) quadlet: Option, /// The systemd unit to restart, if any. pub(crate) unit: Option, + /// For `Updated`, the previous (booted) unit name, if it differs from + /// `unit` (e.g. `ServiceName=` changed): this needs to be stopped + /// separately, since restarting `unit` won't touch it. + pub(crate) old_unit: Option, +} + +/// A quadlet file bootc has written to [`QUADLET_RUN_DIR`], parsed back +/// from its marker comment line (see [`marker_line`]/[`parse_marker`]). +/// Recovered by scanning, not persisted elsewhere; see [`scan_live_quadlets`]. +#[derive(Debug, Clone, PartialEq, Eq)] +struct LiveQuadlet { + /// Path relative to [`QUADLET_RUN_DIR`]. + relpath: Utf8PathBuf, + /// The staged deployment checksum this definition came from. + checksum: String, + /// True if the unit has not yet been restarted with this definition. + pending: bool, + /// The container image reference, for display purposes. + image: String, + /// The quadlet contents, with the marker comment stripped. + contents: String, + /// The systemd unit generated from the quadlet, if any. + unit: Option, } /// If `path` (relative to a root) is inside a quadlet search directory, @@ -151,26 +190,38 @@ pub(crate) fn compute_changes( let staged: BTreeMap<_, _> = staged.iter().map(|s| (s.path.as_path(), s)).collect(); let mut changes = Vec::new(); - let mk = |kind, spec: &BoundImageSpec| -> Result { + let unit_for = |quadlet: Option<&Utf8Path>, contents: &str| -> Result> { + quadlet + .map(|q| quadlet_unit_name(q, contents)) + .transpose() + .map(Option::flatten) + }; + let mk = |kind, spec: &BoundImageSpec, old_unit: Option| -> Result { let quadlet = quadlet_relpath(&spec.path).map(ToOwned::to_owned); - let unit = quadlet - .as_deref() - .map(|q| quadlet_unit_name(q, &spec.contents)) - .transpose()? - .flatten(); + let unit = unit_for(quadlet.as_deref(), &spec.contents)?; Ok(QuadletChange { kind, spec: spec.clone(), quadlet, unit, + old_unit, }) }; for (path, spec) in staged.iter() { match booted.get(path) { Some(b) if b.contents == spec.contents => {} - Some(_) => changes.push(mk(ChangeKind::Updated, spec)?), - None => changes.push(mk(ChangeKind::Added, spec)?), + Some(b) => { + // The unit name quadlet generates can change (e.g. via + // `ServiceName=`) even though the file's path didn't; the old + // unit needs to be stopped separately in that case. + let quadlet = quadlet_relpath(&spec.path); + let staged_unit = unit_for(quadlet, &spec.contents)?; + let booted_unit = unit_for(quadlet, &b.contents)?; + let old_unit = booted_unit.filter(|u| Some(u) != staged_unit.as_ref()); + changes.push(mk(ChangeKind::Updated, spec, old_unit)?); + } + None => changes.push(mk(ChangeKind::Added, spec, None)?), } } for (path, spec) in booted.iter() { @@ -183,7 +234,7 @@ pub(crate) fn compute_changes( tracing::debug!("No longer bound, but still present: {path}"); continue; } - changes.push(mk(ChangeKind::Removed, spec)?); + changes.push(mk(ChangeKind::Removed, spec, None)?); } Ok(changes) } @@ -202,11 +253,16 @@ fn walk_files(root: &Dir, dir: &str, out: &mut Vec) -> Result<()> { .to_str() .ok_or_else(|| anyhow::anyhow!("Invalid non-UTF8 filename: {name:?} in {dir}"))?; let path = format!("{dir}/{name}"); - if entry.file_type()?.is_dir() { + let ft = entry.file_type()?; + if ft.is_dir() { walk_files(root, &path, out)?; - } else { + } else if ft.is_file() { out.push(path); } + // Symlinks (e.g. a unit masked via `-> /dev/null`) and other + // non-regular entries are neither traversed nor reported: they + // aren't files bootc itself would have written, and following them + // can fail cap-std's path resolution for absolute targets. } Ok(()) } @@ -273,29 +329,127 @@ pub(crate) fn check_diff_scope( anyhow::bail!("{}", msg.trim_end()) } -/// Read the live bound image state from `/run`, if any. -pub(crate) fn read_state(run: &Dir) -> Result> { - let Some(f) = run.open_optional(BOUND_IMAGES_STATE)? else { - return Ok(None); +/// Build the leading marker comment for a quadlet file; see [`parse_marker`]. +fn marker_line(checksum: &str, pending: bool, image: &str) -> String { + format!("{MARKER_PREFIX} checksum={checksum} pending={pending} image={image}\n") +} + +/// Parse a [`marker_line`] from the start of `contents`, returning +/// `(checksum, pending, image, rest)` where `rest` is the original quadlet +/// contents with the marker removed. Returns `None` if there's no marker, +/// e.g. a quadlet bootc didn't write. +fn parse_marker(contents: &str) -> Option<(String, bool, String, &str)> { + let (first, rest) = contents.split_once('\n')?; + let kvs = first.strip_prefix(MARKER_PREFIX)?; + let (mut checksum, mut pending, mut image) = (None, None, None); + for kv in kvs.split_whitespace() { + let (k, v) = kv.split_once('=')?; + match k { + "checksum" => checksum = Some(v.to_owned()), + "pending" => pending = Some(v.parse().ok()?), + "image" => image = Some(v.to_owned()), + _ => {} + } + } + Some((checksum?, pending?, image?, rest)) +} + +/// Scan [`QUADLET_RUN_DIR`] for quadlet files bootc has written, parsing +/// their marker comments back into [`LiveQuadlet`]s. Files with no +/// recognized marker are skipped. +fn scan_live_quadlets(rootfs: &Dir) -> Result> { + let mut paths = Vec::new(); + walk_files(rootfs, &format!("/{QUADLET_RUN_DIR}"), &mut paths)?; + let prefix = format!("/{QUADLET_RUN_DIR}/"); + let mut out = Vec::new(); + for p in paths { + let Some(relpath) = p.strip_prefix(prefix.as_str()) else { + continue; + }; + let Some(contents) = rootfs.read_to_string_optional(p.trim_start_matches('/'))? else { + continue; + }; + let Some((checksum, pending, image, body)) = parse_marker(&contents) else { + continue; + }; + let relpath = Utf8PathBuf::from(relpath); + let unit = quadlet_unit_name(&relpath, body)?; + out.push(LiveQuadlet { + relpath, + checksum, + pending, + image, + contents: body.to_owned(), + unit, + }); + } + Ok(out) +} + +/// Add the staged deployment's image references to the protected list (one +/// reference per line), protecting them from garbage collection even if the +/// staged deployment is later discarded (e.g. by `bootc rollback`). +/// Unrelated to the restart state in [`scan_live_quadlets`]: it covers +/// bound images with no quadlet override to carry that state. +/// +/// This merges with whatever is already there rather than replacing it: an +/// earlier live-applied deployment's images may still be in use by units +/// that haven't transitioned yet, so they must stay protected too until a +/// reboot (or [`clear_state_from_host`]) resets `/run`. +fn write_protected_images<'a>(run: &Dir, images: impl Iterator) -> Result<()> { + let mut all: BTreeSet = protected_images_from_host_at(run)?.into_iter().collect(); + all.extend(images.map(ToOwned::to_owned)); + run.create_dir_all(STATE_DIR)?; + run.atomic_replace_with(PROTECTED_IMAGES_LIST, |w| -> Result<()> { + for image in &all { + writeln!(w, "{image}")?; + } + Ok(()) + }) + .with_context(|| format!("Writing /run/{PROTECTED_IMAGES_LIST}")) +} + +/// Read the image references written by [`write_protected_images`] from an +/// arbitrary directory, if any. +fn protected_images_from_host_at(run: &Dir) -> Result> { + let Some(contents) = run.read_to_string_optional(PROTECTED_IMAGES_LIST)? else { + return Ok(Vec::new()); }; - let r = serde_json::from_reader(std::io::BufReader::new(f)) - .with_context(|| format!("Parsing /run/{BOUND_IMAGES_STATE}"))?; - Ok(Some(r)) + Ok(contents.lines().map(ToOwned::to_owned).collect()) } -/// Read the live bound image state from the host `/run`, if any. -pub(crate) fn read_state_from_host() -> Result> { +/// Read back the image references written by [`write_protected_images`], if +/// any, from the host `/run`. +pub(crate) fn protected_images_from_host() -> Result> { let run = Dir::open_ambient_dir("/run", cap_std::ambient_authority()).context("Opening /run")?; - read_state(&run) + protected_images_from_host_at(&run) } -fn write_state(run: &Dir, state: &LiveBoundImages) -> Result<()> { - run.create_dir_all(STATE_DIR)?; - run.atomic_replace_with(BOUND_IMAGES_STATE, |w| { - serde_json::to_writer_pretty(w, state).map_err(anyhow::Error::from) - }) - .with_context(|| format!("Writing /run/{BOUND_IMAGES_STATE}")) +/// Build the `bootc status` view of live-applied bound images by scanning +/// [`QUADLET_RUN_DIR`]. Returns `None` if nothing is applied live. +/// `deploy_serial` isn't recorded in the markers, so it's left as `0` for +/// callers to fill in by matching `checksum` against a deployment. +pub(crate) fn status_from_host() -> Result> { + let rootfs = Dir::open_ambient_dir("/", cap_std::ambient_authority()).context("Opening /")?; + let live = scan_live_quadlets(&rootfs)?; + let Some(checksum) = live.first().map(|q| q.checksum.clone()) else { + return Ok(None); + }; + let images = live + .into_iter() + .map(|q| LiveBoundImage { + image: q.image, + quadlet: Some(q.relpath.to_string()), + unit: q.unit, + pending: q.pending, + }) + .collect(); + Ok(Some(LiveBoundImages { + checksum, + deploy_serial: 0, + images, + })) } /// Remove any live-applied bound image overrides and state. This is used when @@ -303,16 +457,17 @@ fn write_state(run: &Dir, state: &LiveBoundImages) -> Result<()> { /// target deployment's own definitions. #[context("Clearing live bound image state")] pub(crate) fn clear_state_from_host() -> Result<()> { + let rootfs = Dir::open_ambient_dir("/", cap_std::ambient_authority()).context("Opening /")?; + let live = scan_live_quadlets(&rootfs)?; + for q in &live { + rootfs.remove_file_optional(Utf8Path::new(QUADLET_RUN_DIR).join(&q.relpath))?; + } let run = Dir::open_ambient_dir("/run", cap_std::ambient_authority()).context("Opening /run")?; - let Some(state) = read_state(&run)? else { + let had_protected = run.remove_file_optional(PROTECTED_IMAGES_LIST)?; + if live.is_empty() && !had_protected { return Ok(()); - }; - let rootfs = Dir::open_ambient_dir("/", cap_std::ambient_authority()).context("Opening /")?; - for q in state.images.iter().filter_map(|i| i.quadlet.as_deref()) { - rootfs.remove_file_optional(Utf8Path::new(QUADLET_RUN_DIR).join(q))?; } - run.remove_file_optional(BOUND_IMAGES_STATE)?; println!("Cleared live-applied bound image definitions"); Ok(()) } @@ -344,24 +499,51 @@ fn ensure_quadlet_dir( Ok(()) } -/// Restart every unit still marked pending in `state`, updating the state -/// file as each one succeeds so a failure part way through can be retried. -fn restart_pending(run: &Dir, state: &mut LiveBoundImages) -> Result<()> { - let mut r = Ok(()); - for img in state.images.iter_mut().filter(|i| i.pending) { - let Some(unit) = img.unit.as_deref() else { - img.pending = false; - continue; - }; - // `restart` also starts a unit which is not running (the added case). - if let Err(e) = systemctl(&["restart", unit]) { - r = Err(e); - break; +/// Rewrite the marker on an already-materialized quadlet file, leaving its +/// contents otherwise untouched. Used to flip `pending` to `false` once a +/// unit has been restarted. +fn set_quadlet_pending( + rootfs: &Dir, + relpath: &Utf8Path, + checksum: &str, + pending: bool, + image: &str, + body: &str, + sepolicy: Option<&ostree_ext::ostree::SePolicy>, +) -> Result<()> { + let dest = Utf8Path::new(QUADLET_RUN_DIR).join(relpath); + let filemode = rustix::fs::Mode::from_raw_mode(0o644); + crate::lsm::atomic_replace_labeled(rootfs, &dest, filemode, sepolicy, |w| { + w.write_all(marker_line(checksum, pending, image).as_bytes())?; + w.write_all(body.as_bytes())?; + Ok(()) + }) +} + +/// Restart every unit still marked pending among `live`, updating each +/// quadlet's marker as it succeeds so a failure part way through can be +/// retried on the next run. +fn restart_pending( + rootfs: &Dir, + live: &[LiveQuadlet], + sepolicy: Option<&ostree_ext::ostree::SePolicy>, +) -> Result<()> { + for q in live.iter().filter(|q| q.pending) { + if let Some(unit) = q.unit.as_deref() { + // `restart` also starts a unit which is not running (the added case). + systemctl(&["restart", unit])?; } - img.pending = false; + set_quadlet_pending( + rootfs, + &q.relpath, + &q.checksum, + false, + &q.image, + &q.contents, + sepolicy, + )?; } - write_state(run, state)?; - r + Ok(()) } /// Implementation of `bootc apply-live bound-images`. @@ -383,23 +565,30 @@ pub(crate) async fn apply_bound_images( let booted_csum = booted_deployment.csum(); let staged_csum = staged.csum(); - let run = - &Dir::open_ambient_dir("/run", cap_std::ambient_authority()).context("Opening /run")?; let rootfs = &Dir::open_ambient_dir("/", cap_std::ambient_authority()).context("Opening /")?; - let previous = read_state(run)?; + let booted_root = crate::utils::deployment_fd(sysroot, booted_deployment)?; + let staged_root = crate::utils::deployment_fd(sysroot, &staged)?; + let sepolicy = crate::lsm::new_sepolicy_at(&booted_root)?; + + let live = scan_live_quadlets(rootfs)?; + let staged_specs = crate::boundimage::query_bound_image_specs(&staged_root)?; // The definitions from this staged deployment are already on disk; all - // that may be left is restarting units (after `--no-restart`, or a failure). - if let Some(mut previous) = previous - .as_ref() - .filter(|p| p.checksum == staged_csum) - .cloned() - { - let pending: Vec<_> = previous - .images + // that may be left is restarting units (after `--no-restart`, or a + // failure). Checking checksums alone isn't enough: if a previous run + // only got partway through materializing new overrides, the live set + // may simply be missing some of them. Require every quadlet-bearing + // staged spec to already have a live entry for this checksum. + let expected_quadlets: BTreeSet<&Utf8Path> = staged_specs + .iter() + .filter_map(|s| quadlet_relpath(&s.path)) + .collect(); + let live_quadlets: BTreeSet<&Utf8Path> = live.iter().map(|q| q.relpath.as_path()).collect(); + if live_quadlets == expected_quadlets && live.iter().all(|q| q.checksum == staged_csum) { + let pending: Vec<_> = live .iter() - .filter(|i| i.pending) - .filter_map(|i| i.unit.as_deref()) + .filter(|q| q.pending) + .filter_map(|q| q.unit.as_deref()) .collect(); if pending.is_empty() { println!("Bound images from staged deployment are already applied"); @@ -412,13 +601,10 @@ pub(crate) async fn apply_bound_images( return Ok(()); } systemctl(&["daemon-reload"])?; - return restart_pending(run, &mut previous); + return restart_pending(rootfs, &live, sepolicy.as_ref()); } - let booted_root = crate::utils::deployment_fd(sysroot, booted_deployment)?; - let staged_root = crate::utils::deployment_fd(sysroot, &staged)?; let booted_specs = crate::boundimage::query_bound_image_specs(&booted_root)?; - let staged_specs = crate::boundimage::query_bound_image_specs(&staged_root)?; let changes = compute_changes(&booted_specs, &staged_specs, &staged_root)?; if booted_csum != staged_csum { @@ -428,22 +614,18 @@ pub(crate) async fn apply_bound_images( check_diff_scope(&diff, &changes, &booted_root, &staged_root)?; } - // Overrides written by a previous apply-live which are not part of this - // one need to be removed so the running system converges on the staged + // Overrides from a previous apply-live that aren't part of this one + // need to be removed so the system converges on the current staged // definitions rather than a mix of the two. - let previous_quadlets: BTreeMap<&Utf8Path, &LiveBoundImage> = previous - .iter() - .flat_map(|p| &p.images) - .filter_map(|i| i.quadlet.as_deref().map(|q| (Utf8Path::new(q), i))) - .collect(); + let live_by_path: BTreeMap<&Utf8Path, &LiveQuadlet> = + live.iter().map(|q| (q.relpath.as_path(), q)).collect(); let current_quadlets: BTreeSet<&Utf8Path> = changes .iter() .filter_map(|c| c.quadlet.as_deref()) .collect(); - let stale: Vec<(&Utf8Path, &LiveBoundImage)> = previous_quadlets + let stale: Vec<&LiveQuadlet> = live .iter() - .filter(|(q, _)| !current_quadlets.contains(*q)) - .map(|(q, i)| (*q, *i)) + .filter(|q| !current_quadlets.contains(q.relpath.as_path())) .collect(); if changes.is_empty() && stale.is_empty() { @@ -455,8 +637,8 @@ pub(crate) async fn apply_bound_images( let unit = c.unit.as_deref().unwrap_or("(no unit)"); println!("{}: {} ({unit})", c.kind, c.spec.image.image); } - for (q, _) in stale.iter() { - println!("reverting stale override: {q}"); + for q in stale.iter() { + println!("reverting stale override: {}", q.relpath); } if opts.dry_run { return Ok(()); @@ -475,12 +657,30 @@ pub(crate) async fn apply_bound_images( let staged_images = staged_specs.iter().map(|s| s.image.clone()).collect(); crate::boundimage::pull_images(storage, staged_images).await?; - // Materialize the definitions into /run. A definition may already be live - // from a previous apply, in which case its unit doesn't need a restart. + // Protect every bound image of the staged deployment from GC, not just + // the changed ones. Written first since it's the one piece of state + // that can't be recovered later by scanning `/run/containers/systemd/`. + let run = + &Dir::open_ambient_dir("/run", cap_std::ambient_authority()).context("Opening /run")?; + write_protected_images(run, staged_specs.iter().map(|s| s.image.image.as_str()))?; + + // A quadlet moving between search dirs (e.g. vendor -> /etc override) + // produces an Added and a Removed change for the same `/run` destination; + // the Removed side must not delete what the Added side just wrote. + let materialized: BTreeSet<&Utf8Path> = changes + .iter() + .filter(|c| matches!(c.kind, ChangeKind::Added | ChangeKind::Updated)) + .filter_map(|c| c.quadlet.as_deref()) + .collect(); + + // Materialize the definitions into /run, each carrying a marker. A + // definition may already be live from a previous apply, in which case + // its unit doesn't need a restart -- but the marker is still refreshed + // to the current checksum so status and the fast path above stay + // accurate. let quadlet_run_dir = Utf8Path::new(QUADLET_RUN_DIR); - let sepolicy = crate::lsm::new_sepolicy_at(&booted_root)?; let filemode = rustix::fs::Mode::from_raw_mode(0o644); - let mut written = BTreeMap::new(); + let mut written: BTreeMap<&Utf8Path, bool> = BTreeMap::new(); for c in changes.iter() { let Some(quadlet) = c.quadlet.as_deref() else { continue; @@ -488,90 +688,98 @@ pub(crate) async fn apply_bound_images( let dest = quadlet_run_dir.join(quadlet); match c.kind { ChangeKind::Added | ChangeKind::Updated => { - let already_live = previous_quadlets + let already_live = live_by_path .get(quadlet) - .is_some_and(|prev| !prev.pending) - && rootfs.read_to_string_optional(&dest)?.as_deref() - == Some(c.spec.contents.as_str()); - if !already_live { - // SAFETY: We know there's a parent - ensure_quadlet_dir(rootfs, dest.parent().unwrap(), sepolicy.as_ref())?; - crate::lsm::atomic_replace_labeled( - rootfs, - &dest, - filemode, - sepolicy.as_ref(), - |w| w.write_all(c.spec.contents.as_bytes()).map_err(Into::into), - )?; + .is_some_and(|prev| !prev.pending && prev.contents == c.spec.contents); + // Refuse to clobber a `/run` quadlet we didn't write. + if live_by_path.get(quadlet).is_none() && rootfs.try_exists(&dest)? { + anyhow::bail!("Refusing to overwrite non-bootc quadlet override at /{dest}"); } - written.insert(quadlet, !already_live); + // SAFETY: We know there's a parent + ensure_quadlet_dir(rootfs, dest.parent().unwrap(), sepolicy.as_ref())?; + let pending = !already_live && c.unit.is_some(); + crate::lsm::atomic_replace_labeled( + rootfs, + &dest, + filemode, + sepolicy.as_ref(), + |w| { + let marker = + marker_line(staged_csum.as_str(), pending, c.spec.image.image.as_str()); + w.write_all(marker.as_bytes())?; + w.write_all(c.spec.contents.as_bytes())?; + Ok(()) + }, + )?; + written.insert(quadlet, pending); } ChangeKind::Removed => { - rootfs.remove_file_optional(&dest)?; + if !materialized.contains(quadlet) { + rootfs.remove_file_optional(&dest)?; + } } } } - for (q, _) in stale.iter() { - rootfs.remove_file_optional(quadlet_run_dir.join(q))?; + for q in stale.iter() { + // `stale` is disjoint from `materialized` by construction (it's + // exactly the live quadlets not touched by any change). + rootfs.remove_file_optional(quadlet_run_dir.join(&q.relpath))?; } - // Record all bound images of the staged deployment, not just the changed - // ones: the state file protects them from garbage collection even if the - // staged deployment is later discarded (e.g. by `bootc rollback`). - let images = staged_specs - .iter() - .map(|s| { - let quadlet = quadlet_relpath(&s.path).filter(|q| written.contains_key(q)); - let unit = quadlet - .map(|q| quadlet_unit_name(q, &s.contents)) - .transpose()? - .flatten(); - let pending = unit.is_some() && quadlet.is_some_and(|q| written[q]); - Ok(LiveBoundImage { - image: s.image.image.clone(), - quadlet: quadlet.map(|q| q.to_string()), - unit, - pending, - }) - }) - .collect::>>()?; - let mut state = LiveBoundImages { - checksum: staged_csum.to_string(), - deploy_serial: staged.deployserial() as u32, - images, - }; - write_state(run, &state)?; - if opts.no_restart { println!("Skipping unit restart; re-run without --no-restart to restart pending units"); return Ok(()); } - // Units for removed definitions must be stopped before the daemon reload - // which makes them disappear from systemd's view. + // Units for removed definitions, and old units left behind by a rename + // (e.g. `ServiceName=` changing), must be stopped before the daemon + // reload which makes them disappear from systemd's view. let removed_units = changes .iter() .filter(|c| c.kind == ChangeKind::Removed) .filter_map(|c| c.unit.as_deref()) - .chain(stale.iter().filter_map(|(_, img)| img.unit.as_deref())); + .chain(changes.iter().filter_map(|c| c.old_unit.as_deref())) + .chain(stale.iter().filter_map(|q| q.unit.as_deref())); for unit in removed_units { systemctl(&["stop", unit])?; } systemctl(&["daemon-reload"])?; // A stale override reverts to the booted deployment's definition if // there is one, otherwise the unit is simply gone. - for (q, img) in stale.iter() { - let Some(unit) = img.unit.as_deref() else { + for q in stale.iter() { + let Some(unit) = q.unit.as_deref() else { continue; }; let booted_has = booted_specs .iter() - .any(|s| quadlet_relpath(&s.path) == Some(q)); + .any(|s| quadlet_relpath(&s.path) == Some(q.relpath.as_path())); if booted_has { systemctl(&["start", unit])?; } } - restart_pending(run, &mut state)?; + + // Restart units for the definitions we just materialized, flipping + // their markers from pending to applied as each one succeeds. + let newly_written: Vec = changes + .iter() + .filter(|c| matches!(c.kind, ChangeKind::Added | ChangeKind::Updated)) + .filter_map(|c| { + let quadlet = c.quadlet.as_deref()?; + written + .get(quadlet) + .copied() + .unwrap_or(false) + .then(|| LiveQuadlet { + relpath: quadlet.to_owned(), + checksum: staged_csum.to_string(), + pending: true, + image: c.spec.image.image.clone(), + contents: c.spec.contents.clone(), + unit: c.unit.clone(), + }) + }) + .collect(); + restart_pending(rootfs, &newly_written, sepolicy.as_ref())?; println!( "Applied {} bound image definition change(s) from staged deployment", @@ -715,6 +923,43 @@ mod tests { Ok(()) } + #[test] + fn test_compute_changes_unit_rename() -> Result<()> { + // Same quadlet path in both, but `ServiceName=` changed: the old + // unit must still be reported so it gets stopped. + const Q: &str = "usr/share/containers/systemd"; + let staged_root = &tempdir()?; + let old = "[Container]\nImage=quay.io/foo/foo:v1\nServiceName=old\n"; + let new = "[Container]\nImage=quay.io/foo/foo:v2\nServiceName=new\n"; + let booted = [spec( + &format!("{Q}/foo.container"), + "quay.io/foo/foo:v1", + old, + )]; + let staged = [spec( + &format!("{Q}/foo.container"), + "quay.io/foo/foo:v2", + new, + )]; + let changes = compute_changes(&booted, &staged, staged_root)?; + assert_eq!(changes.len(), 1); + let c = &changes[0]; + assert_eq!(c.kind, ChangeKind::Updated); + assert_eq!(c.unit.as_deref(), Some("new.service")); + assert_eq!(c.old_unit.as_deref(), Some("old.service")); + + // Same scenario, but the unit name doesn't actually change + let new_same_name = "[Container]\nImage=quay.io/foo/foo:v2\nServiceName=old\n"; + let staged = [spec( + &format!("{Q}/foo.container"), + "quay.io/foo/foo:v2", + new_same_name, + )]; + let changes = compute_changes(&booted, &staged, staged_root)?; + assert_eq!(changes[0].old_unit, None); + Ok(()) + } + #[test] fn test_check_diff_scope() -> Result<()> { let booted_root = &tempdir()?; @@ -729,12 +974,14 @@ mod tests { ), quadlet: Some("foo.container".into()), unit: Some("foo.service".into()), + old_unit: None, }, QuadletChange { kind: ChangeKind::Added, spec: spec("etc/containers/systemd/bar.image", "quay.io/foo/bar:v1", ""), quadlet: Some("bar.image".into()), unit: None, + old_unit: None, }, ]; let check = @@ -813,32 +1060,90 @@ mod tests { } #[test] - fn test_state_roundtrip() -> Result<()> { + fn test_marker_roundtrip() { + let line = marker_line("abc123", true, "quay.io/foo/foo:v2"); + assert_eq!( + line, + "# X-Bootc-Apply-Live: checksum=abc123 pending=true image=quay.io/foo/foo:v2\n" + ); + let contents = format!("{line}[Container]\nImage=quay.io/foo/foo:v2\n"); + let (checksum, pending, image, rest) = parse_marker(&contents).unwrap(); + assert_eq!(checksum, "abc123"); + assert!(pending); + assert_eq!(image, "quay.io/foo/foo:v2"); + assert_eq!(rest, "[Container]\nImage=quay.io/foo/foo:v2\n"); + + // A quadlet bootc didn't write has no marker + assert_eq!( + parse_marker("[Container]\nImage=quay.io/foo/foo:v2\n"), + None + ); + // Likewise an empty file + assert_eq!(parse_marker(""), None); + } + + #[test] + fn test_scan_live_quadlets() -> Result<()> { + let rootfs = &tempdir()?; + rootfs.create_dir_all(QUADLET_RUN_DIR)?; + let body = "[Container]\nImage=quay.io/foo/foo:v2\n"; + rootfs.write( + format!("{QUADLET_RUN_DIR}/foo.container"), + format!( + "{}{body}", + marker_line("abc123", true, "quay.io/foo/foo:v2") + ), + )?; + // A sibling file bootc didn't write is ignored + rootfs.write( + format!("{QUADLET_RUN_DIR}/unrelated.container"), + "[Container]\nImage=quay.io/other/other:latest\n", + )?; + // A pull-only bound image (no search-dir quadlet) has nothing to + // scan here; that's covered by `write_protected_images` instead. + + // A unit masked by symlinking it to /dev/null must not break the + // scan (cap-std rejects resolving absolute symlink targets). + rootfs.symlink_contents("/dev/null", format!("{QUADLET_RUN_DIR}/masked.container"))?; + + let live = scan_live_quadlets(rootfs)?; + assert_eq!(live.len(), 1); + let q = &live[0]; + assert_eq!(q.relpath, Utf8PathBuf::from("foo.container")); + assert_eq!(q.checksum, "abc123"); + assert!(q.pending); + assert_eq!(q.image, "quay.io/foo/foo:v2"); + assert_eq!(q.contents, body); + assert_eq!(q.unit.as_deref(), Some("foo.service")); + Ok(()) + } + + #[test] + fn test_protected_images_roundtrip() -> Result<()> { let run = &tempdir()?; - assert_eq!(read_state(run)?, None); - let state = LiveBoundImages { - checksum: "abc".into(), - deploy_serial: 1, - images: vec![ - LiveBoundImage { - image: "quay.io/foo/foo:v2".into(), - quadlet: Some("foo.container".into()), - unit: Some("foo.service".into()), - pending: true, - }, - LiveBoundImage { - image: "quay.io/foo/bar:v2".into(), - quadlet: None, - unit: None, - pending: false, - }, - ], - }; - write_state(run, &state)?; - assert_eq!(read_state(run)?.as_ref(), Some(&state)); - let raw = run.read_to_string(BOUND_IMAGES_STATE)?; - assert!(raw.contains("\"pending\": true"), "{raw}"); - assert_eq!(raw.matches("pending").count(), 1, "{raw}"); + assert_eq!(protected_images_from_host_at(run)?, Vec::::new()); + write_protected_images( + run, + ["quay.io/foo/foo:v2", "quay.io/foo/bar:latest"].into_iter(), + )?; + assert_eq!( + protected_images_from_host_at(run)?, + vec![ + "quay.io/foo/bar:latest".to_string(), + "quay.io/foo/foo:v2".to_string(), + ] + ); + // A later apply (e.g. a subsequent staged deployment) must not drop + // images still protected from an earlier one. + write_protected_images(run, ["quay.io/foo/baz:latest"].into_iter())?; + assert_eq!( + protected_images_from_host_at(run)?, + vec![ + "quay.io/foo/bar:latest".to_string(), + "quay.io/foo/baz:latest".to_string(), + "quay.io/foo/foo:v2".to_string(), + ] + ); Ok(()) } diff --git a/crates/lib/src/deploy.rs b/crates/lib/src/deploy.rs index f9202ed88e..09ced460d4 100644 --- a/crates/lib/src/deploy.rs +++ b/crates/lib/src/deploy.rs @@ -417,16 +417,14 @@ pub(crate) async fn prune_container_store(sysroot: &Storage) -> Result<()> { } // Images applied live via `bootc apply-live bound-images` must survive // even if the deployment they came from is discarded before reboot. - if let Some(live) = crate::applylive::read_state_from_host()? { - all_bound_images.extend( - live.images - .into_iter() - .map(|img| crate::boundimage::BoundImage { - image: img.image, - auth_file: None, - }), - ); - } + all_bound_images.extend( + crate::applylive::protected_images_from_host()? + .into_iter() + .map(|image| crate::boundimage::BoundImage { + image, + auth_file: None, + }), + ); // Convert to a hashset of just the image names let image_names = HashSet::from_iter(all_bound_images.iter().map(|img| img.image.as_str())); let pruned = sysroot diff --git a/crates/lib/src/status.rs b/crates/lib/src/status.rs index 232859e26b..faab12e8cf 100644 --- a/crates/lib/src/status.rs +++ b/crates/lib/src/status.rs @@ -428,7 +428,21 @@ pub(crate) fn get_status( .map(|d| d.unlocked()) .and_then(crate::spec::deployment_unlocked_state_to_usr_overlay); - let live_bound_images = crate::applylive::read_state_from_host()?; + // `deploy_serial` isn't recoverable from the quadlet markers alone + // (they only record the checksum), so fill it in here from whichever + // deployment the live-applied definitions came from. + let mut live_bound_images = crate::applylive::status_from_host()?; + if let Some(live) = live_bound_images.as_mut() { + let matching = deployments + .staged + .iter() + .chain(deployments.rollback.iter()) + .chain(deployments.other.iter()) + .find(|d| d.csum() == live.checksum); + if let Some(d) = matching { + live.deploy_serial = d.deployserial() as u32; + } + } let mut host = Host::new(spec); host.status = HostStatus { diff --git a/tmt/tests/booted/test-logically-bound-apply-live.nu b/tmt/tests/booted/test-logically-bound-apply-live.nu index b1b60a0f5f..5d24895c03 100644 --- a/tmt/tests/booted/test-logically-bound-apply-live.nu +++ b/tmt/tests/booted/test-logically-bound-apply-live.nu @@ -32,7 +32,7 @@ const image_v2 = "registry.access.redhat.com/ubi9/ubi-minimal:9.3" const quadlet = "/usr/share/containers/systemd/lbi-sleeper.container" const run_quadlet = "/run/containers/systemd/lbi-sleeper.container" const unit = "lbi-sleeper.service" -const state_file = "/run/bootc/apply-live/bound-images.json" +const protected_images_list = "/run/bootc/apply-live/protected-images.list" def initial_setup [] { bootc image copy-to-storage @@ -115,12 +115,11 @@ def second_boot [] { assert (open $run_quadlet | str contains $image_v2) assert equal (getfattr --only-values -n security.selinux $run_quadlet | split row ':' | get 2) container_var_run_t assert equal (running_image) $image_v1 - let state = open $state_file - assert equal $state.checksum $staged_checksum - let entry = $state.images | where unit? == $unit | first - assert equal $entry.pending true let live = bootc status --json | from json | get status.liveBoundImages assert equal $live.checksum $staged_checksum + let entry = $live.images | where unit? == $unit | first + assert equal $entry.pending true + assert (open $protected_images_list | str contains $image_v2) # A plain re-run restarts the pending unit let out = bootc apply-live bound-images @@ -128,8 +127,8 @@ def second_boot [] { assert ($out | str contains $"restart pending: ($unit)") systemctl is-active $unit assert equal (running_image) $image_v2 - let state = open $state_file - let entry = $state.images | where unit? == $unit | first + let live = bootc status --json | from json | get status.liveBoundImages + let entry = $live.images | where unit? == $unit | first assert equal ($entry.pending? | default false) false let human = bootc status --format humanreadable print $human @@ -173,7 +172,7 @@ def third_boot [] { # /run is fresh: no override, no state, and the deployment's own # definition (v2) is what runs. assert not ($run_quadlet | path exists) - assert not ($state_file | path exists) + assert not ($protected_images_list | path exists) assert equal $st.status.liveBoundImages? null assert (open $quadlet | str contains $image_v2) systemctl is-active $unit From 7dbc38dfa6b6556bcc4586948d2553244bab9e40 Mon Sep 17 00:00:00 2001 From: Yasmin de Souza Date: Wed, 4 Nov 2026 12:39:33 -0300 Subject: [PATCH 6/6] applylive: add code review suggestions Add new code to solve problems that was found in review by CoPilot and other AI review tools. Assisted-by: AI Signed-off-by: Yasmin de Souza --- crates/lib/src/applylive.rs | 810 +++++++++++++++++---- crates/lib/src/cli.rs | 25 +- docs/src/bootc-logically-bound-images.7.md | 13 +- 3 files changed, 713 insertions(+), 135 deletions(-) diff --git a/crates/lib/src/applylive.rs b/crates/lib/src/applylive.rs index 9e19cedbc4..0b59c263e5 100644 --- a/crates/lib/src/applylive.rs +++ b/crates/lib/src/applylive.rs @@ -25,13 +25,17 @@ //! //! Rather than a separate state file, each quadlet written to //! `/run/containers/systemd/` carries its own state as a leading comment -//! (see [`marker_line`]): the staged checksum, the image reference, and -//! whether its unit still needs restarting ("pending"). Re-running the -//! command re-derives everything by scanning that directory (see -//! [`scan_live_quadlets`]), so a failure part way through is safe to -//! resume from. The one exception is garbage-collection protection for -//! bound images with no quadlet override (unchanged definitions); see -//! [`write_protected_images`]. +//! (see [`marker_line`]): the staged checksum, the image reference, +//! whether its unit still needs restarting ("pending"), and optionally a +//! unit that still needs stopping (e.g. after a `ServiceName=` rename). +//! Pure removals and stale-override reverts record their pending stops in +//! a small flat list under `/run/bootc/apply-live/` (see +//! [`PENDING_STOPS_LIST`]). Re-running the command re-derives everything +//! from that directory (see [`scan_live_quadlets`]) plus the pending-stops +//! list, so a failure part way through -- including a failed stop -- is +//! safe to resume from. Garbage-collection protection for bound images +//! with no quadlet override (unchanged definitions) lives in the same +//! directory; see [`write_protected_images`]. use std::collections::{BTreeMap, BTreeSet}; use std::io::Write; @@ -56,6 +60,16 @@ const STATE_DIR: &str = "bootc/apply-live"; /// every bound image of an in-progress (or completed) apply-live from /// garbage collection; see [`write_protected_images`]. const PROTECTED_IMAGES_LIST: &str = "bootc/apply-live/protected-images.list"; +/// Flat, one-unit-per-line list (relative to `/run`) of systemd units that +/// still need to be stopped (pure removals and stale-override reverts). +/// Rename stops are recorded on the replacement's marker instead; see +/// [`marker_line`]. Cleared entry-by-entry only after each `systemctl stop` +/// succeeds, so `--no-restart` and failed stops remain retryable. +const PENDING_STOPS_LIST: &str = "bootc/apply-live/pending-stops.list"; +/// Records the staged deployment checksum for which materialization (and +/// image protection) has completed. Needed so removal-only and pull-only +/// changes (no `/run` override) don't look "already applied" on first run. +const APPLIED_CHECKSUM: &str = "bootc/apply-live/applied-checksum"; /// Prefix of the leading marker comment line bootc writes at the top of /// each quadlet file it materializes under [`QUADLET_RUN_DIR`]; see /// [`marker_line`] and [`parse_marker`]. @@ -64,6 +78,10 @@ const MARKER_PREFIX: &str = "# X-Bootc-Apply-Live:"; /// Paths are relative to the root (not `/run`) so that SELinux labels are /// computed for the real absolute path. const QUADLET_RUN_DIR: &str = "run/containers/systemd"; +/// Host `/etc` quadlet search directory (relative to `/`), which takes +/// precedence over vendor (`usr/share`) definitions. Used to refuse +/// materializing a vendor-bound quadlet that would clobber an admin override. +const QUADLET_ETC_DIR: &str = "etc/containers/systemd"; /// Quadlet search directories relative to a deployment root, in decreasing /// precedence (excluding `/run`, which is what we write to). const QUADLET_SEARCH_DIRS: &[&str] = &["etc/containers/systemd", "usr/share/containers/systemd"]; @@ -143,6 +161,10 @@ struct LiveQuadlet { contents: String, /// The systemd unit generated from the quadlet, if any. unit: Option, + /// A previous unit that still needs stopping (e.g. after a + /// `ServiceName=` rename), recorded in the marker until the stop + /// succeeds. + stop: Option, } /// If `path` (relative to a root) is inside a quadlet search directory, @@ -239,8 +261,12 @@ pub(crate) fn compute_changes( Ok(changes) } -/// Recursively collect the files under `dir` (a commit-form absolute path) -/// in the given root, as commit-form absolute paths. +/// Recursively collect every non-directory entry under `dir` (a commit-form +/// absolute path) in the given root, as commit-form absolute paths. +/// Symlinks are included: [`check_diff_scope`] needs them so a symlink +/// inside an added in-scope directory is still reported as out of scope. +/// Callers that only want regular files (e.g. [`scan_live_quadlets`]) must +/// filter themselves. fn walk_files(root: &Dir, dir: &str, out: &mut Vec) -> Result<()> { let rel = dir.trim_start_matches('/'); let Some(d) = root.open_dir_optional(rel)? else { @@ -256,13 +282,9 @@ fn walk_files(root: &Dir, dir: &str, out: &mut Vec) -> Result<()> { let ft = entry.file_type()?; if ft.is_dir() { walk_files(root, &path, out)?; - } else if ft.is_file() { + } else { out.push(path); } - // Symlinks (e.g. a unit masked via `-> /dev/null`) and other - // non-regular entries are neither traversed nor reported: they - // aren't files bootc itself would have written, and following them - // can fail cap-std's path resolution for absolute targets. } Ok(()) } @@ -330,33 +352,43 @@ pub(crate) fn check_diff_scope( } /// Build the leading marker comment for a quadlet file; see [`parse_marker`]. -fn marker_line(checksum: &str, pending: bool, image: &str) -> String { - format!("{MARKER_PREFIX} checksum={checksum} pending={pending} image={image}\n") +/// `stop` records a unit that still needs stopping (e.g. the old name after +/// a `ServiceName=` change); omit it once the stop has succeeded. +fn marker_line(checksum: &str, pending: bool, image: &str, stop: Option<&str>) -> String { + let mut line = format!("{MARKER_PREFIX} checksum={checksum} pending={pending} image={image}"); + if let Some(stop) = stop { + line.push_str(&format!(" stop={stop}")); + } + line.push('\n'); + line } /// Parse a [`marker_line`] from the start of `contents`, returning -/// `(checksum, pending, image, rest)` where `rest` is the original quadlet -/// contents with the marker removed. Returns `None` if there's no marker, -/// e.g. a quadlet bootc didn't write. -fn parse_marker(contents: &str) -> Option<(String, bool, String, &str)> { +/// `(checksum, pending, image, stop, rest)` where `rest` is the original +/// quadlet contents with the marker removed. Returns `None` if there's no +/// marker, e.g. a quadlet bootc didn't write. +fn parse_marker(contents: &str) -> Option<(String, bool, String, Option, &str)> { let (first, rest) = contents.split_once('\n')?; let kvs = first.strip_prefix(MARKER_PREFIX)?; - let (mut checksum, mut pending, mut image) = (None, None, None); + let (mut checksum, mut pending, mut image, mut stop) = (None, None, None, None); for kv in kvs.split_whitespace() { let (k, v) = kv.split_once('=')?; match k { "checksum" => checksum = Some(v.to_owned()), "pending" => pending = Some(v.parse().ok()?), "image" => image = Some(v.to_owned()), + "stop" => stop = Some(v.to_owned()), _ => {} } } - Some((checksum?, pending?, image?, rest)) + Some((checksum?, pending?, image?, stop, rest)) } /// Scan [`QUADLET_RUN_DIR`] for quadlet files bootc has written, parsing /// their marker comments back into [`LiveQuadlet`]s. Files with no -/// recognized marker are skipped. +/// recognized marker are skipped. Symlinks and other non-regular entries +/// (e.g. a unit masked via `-> /dev/null`) are skipped before reading, so +/// cap-std's absolute-symlink resolution can't break `bootc status`. fn scan_live_quadlets(rootfs: &Dir) -> Result> { let mut paths = Vec::new(); walk_files(rootfs, &format!("/{QUADLET_RUN_DIR}"), &mut paths)?; @@ -366,10 +398,19 @@ fn scan_live_quadlets(rootfs: &Dir) -> Result> { let Some(relpath) = p.strip_prefix(prefix.as_str()) else { continue; }; - let Some(contents) = rootfs.read_to_string_optional(p.trim_start_matches('/'))? else { + let rel = p.trim_start_matches('/'); + // Don't follow symlinks: an absolute target like `/dev/null` fails + // cap-std path resolution, and bootc only writes regular files. + let Some(meta) = rootfs.symlink_metadata_optional(rel)? else { + continue; + }; + if !meta.is_file() { + continue; + } + let Some(contents) = rootfs.read_to_string_optional(rel)? else { continue; }; - let Some((checksum, pending, image, body)) = parse_marker(&contents) else { + let Some((checksum, pending, image, stop, body)) = parse_marker(&contents) else { continue; }; let relpath = Utf8PathBuf::from(relpath); @@ -381,6 +422,7 @@ fn scan_live_quadlets(rootfs: &Dir) -> Result> { image, contents: body.to_owned(), unit, + stop, }); } Ok(out) @@ -426,8 +468,71 @@ pub(crate) fn protected_images_from_host() -> Result> { protected_images_from_host_at(&run) } +/// Read the pending-stop unit names from [`PENDING_STOPS_LIST`], if any. +fn pending_stops_from(run: &Dir) -> Result> { + let Some(contents) = run.read_to_string_optional(PENDING_STOPS_LIST)? else { + return Ok(BTreeSet::new()); + }; + Ok(contents.lines().map(ToOwned::to_owned).collect()) +} + +/// Replace [`PENDING_STOPS_LIST`] with `units` (creating the state dir if +/// needed). An empty set removes the file. +fn write_pending_stops(run: &Dir, units: &BTreeSet) -> Result<()> { + if units.is_empty() { + run.remove_file_optional(PENDING_STOPS_LIST)?; + return Ok(()); + } + run.create_dir_all(STATE_DIR)?; + run.atomic_replace_with(PENDING_STOPS_LIST, |w| -> Result<()> { + for unit in units { + writeln!(w, "{unit}")?; + } + Ok(()) + }) + .with_context(|| format!("Writing /run/{PENDING_STOPS_LIST}")) +} + +/// Units that still need stopping: the flat list plus any `stop=` recorded +/// on live markers (renames). +fn collect_pending_stops(run: &Dir, live: &[LiveQuadlet]) -> Result> { + let mut stops = pending_stops_from(run)?; + for q in live { + if let Some(unit) = q.stop.as_deref() { + stops.insert(unit.to_owned()); + } + } + Ok(stops) +} + +/// Read the staged checksum recorded by [`write_applied_checksum`], if any. +fn applied_checksum_from(run: &Dir) -> Result> { + let Some(contents) = run.read_to_string_optional(APPLIED_CHECKSUM)? else { + return Ok(None); + }; + let csum = contents.trim(); + if csum.is_empty() { + return Ok(None); + } + Ok(Some(csum.to_owned())) +} + +/// Record that materialization for `checksum` has finished (protected images +/// written, `/run` overrides updated). Cleared with the rest of live state. +fn write_applied_checksum(run: &Dir, checksum: &str) -> Result<()> { + run.create_dir_all(STATE_DIR)?; + run.atomic_replace_with(APPLIED_CHECKSUM, |w| -> Result<()> { + writeln!(w, "{checksum}")?; + Ok(()) + }) + .with_context(|| format!("Writing /run/{APPLIED_CHECKSUM}")) +} + /// Build the `bootc status` view of live-applied bound images by scanning -/// [`QUADLET_RUN_DIR`]. Returns `None` if nothing is applied live. +/// [`QUADLET_RUN_DIR`]. Returns `None` if nothing is applied live, or if +/// the markers disagree on the staged checksum (e.g. a partial apply left +/// a mix of deployments) -- status then shows no live state rather than +/// attributing every image to whichever file the scan returned first. /// `deploy_serial` isn't recorded in the markers, so it's left as `0` for /// callers to fill in by matching `checksum` against a deployment. pub(crate) fn status_from_host() -> Result> { @@ -436,6 +541,12 @@ pub(crate) fn status_from_host() -> Result> { let Some(checksum) = live.first().map(|q| q.checksum.clone()) else { return Ok(None); }; + if live.iter().any(|q| q.checksum != checksum) { + tracing::warn!( + "Live bound-image markers disagree on staged checksum; omitting liveBoundImages from status" + ); + return Ok(None); + } let images = live .into_iter() .map(|q| LiveBoundImage { @@ -452,9 +563,10 @@ pub(crate) fn status_from_host() -> Result> { })) } -/// Remove any live-applied bound image overrides and state. This is used when -/// preparing a soft reboot, which preserves `/run` but must boot into the -/// target deployment's own definitions. +/// Remove any live-applied bound image overrides and state. Call this at the +/// soft-reboot / reboot handoff (which preserves `/run`), not when merely +/// preparing a soft reboot without `--apply`, so live containers keep their +/// overrides and GC protection until the system actually transitions. #[context("Clearing live bound image state")] pub(crate) fn clear_state_from_host() -> Result<()> { let rootfs = Dir::open_ambient_dir("/", cap_std::ambient_authority()).context("Opening /")?; @@ -465,7 +577,9 @@ pub(crate) fn clear_state_from_host() -> Result<()> { let run = Dir::open_ambient_dir("/run", cap_std::ambient_authority()).context("Opening /run")?; let had_protected = run.remove_file_optional(PROTECTED_IMAGES_LIST)?; - if live.is_empty() && !had_protected { + let had_stops = run.remove_file_optional(PENDING_STOPS_LIST)?; + let had_applied = run.remove_file_optional(APPLIED_CHECKSUM)?; + if live.is_empty() && !had_protected && !had_stops && !had_applied { return Ok(()); } println!("Cleared live-applied bound image definitions"); @@ -486,9 +600,10 @@ fn ensure_quadlet_dir( sepolicy: Option<&ostree_ext::ostree::SePolicy>, ) -> Result<()> { let mode = rustix::fs::Mode::from_raw_mode(0o755); + // Walk parents under `/run` but not `/run` itself (podman owns that). let ancestors: Vec<_> = path .ancestors() - .take_while(|p| p.as_str().len() > "run".len()) + .take_while(|p| p.components().count() > 1) .collect(); for dir in ancestors.into_iter().rev() { if rootfs.try_exists(dir)? { @@ -499,30 +614,48 @@ fn ensure_quadlet_dir( Ok(()) } -/// Rewrite the marker on an already-materialized quadlet file, leaving its -/// contents otherwise untouched. Used to flip `pending` to `false` once a -/// unit has been restarted. -fn set_quadlet_pending( +/// Rewrite the marker on an already-materialized quadlet, leaving its +/// body untouched. `pending`/`stop` override the values on `q`. +fn set_quadlet_marker( rootfs: &Dir, - relpath: &Utf8Path, - checksum: &str, + q: &LiveQuadlet, pending: bool, - image: &str, - body: &str, + stop: Option<&str>, sepolicy: Option<&ostree_ext::ostree::SePolicy>, ) -> Result<()> { - let dest = Utf8Path::new(QUADLET_RUN_DIR).join(relpath); + let dest = Utf8Path::new(QUADLET_RUN_DIR).join(&q.relpath); let filemode = rustix::fs::Mode::from_raw_mode(0o644); crate::lsm::atomic_replace_labeled(rootfs, &dest, filemode, sepolicy, |w| { - w.write_all(marker_line(checksum, pending, image).as_bytes())?; - w.write_all(body.as_bytes())?; + w.write_all(marker_line(&q.checksum, pending, &q.image, stop).as_bytes())?; + w.write_all(q.contents.as_bytes())?; Ok(()) }) } +/// Stop every unit in `units`, then clear matching `stop=` markers and the +/// pending-stops list. Each stop is attempted before clearing, so a failure +/// leaves the remaining entries for the next run. +fn stop_pending( + rootfs: &Dir, + run: &Dir, + live: &[LiveQuadlet], + units: &BTreeSet, + sepolicy: Option<&ostree_ext::ostree::SePolicy>, +) -> Result<()> { + for unit in units { + systemctl(&["stop", unit])?; + } + for q in live.iter().filter(|q| q.stop.is_some()) { + set_quadlet_marker(rootfs, q, q.pending, None, sepolicy)?; + } + write_pending_stops(run, &BTreeSet::new())?; + Ok(()) +} + /// Restart every unit still marked pending among `live`, updating each /// quadlet's marker as it succeeds so a failure part way through can be -/// retried on the next run. +/// retried on the next run. `stop=` has already been cleared by +/// [`stop_pending`] when applicable. fn restart_pending( rootfs: &Dir, live: &[LiveQuadlet], @@ -533,15 +666,138 @@ fn restart_pending( // `restart` also starts a unit which is not running (the added case). systemctl(&["restart", unit])?; } - set_quadlet_pending( - rootfs, - &q.relpath, - &q.checksum, - false, - &q.image, - &q.contents, - sepolicy, - )?; + set_quadlet_marker(rootfs, q, false, None, sepolicy)?; + } + Ok(()) +} + +/// True when every change has already been materialized for `staged_csum`. +/// Added/Updated with a quadlet need a matching `/run` override; Removed +/// destinations that aren't being replaced must be gone; pull-only and +/// removal-only cases require [`APPLIED_CHECKSUM`] so a first run doesn't +/// treat "nothing on disk" as already applied. A removal with its unit still +/// listed in `pending_stops` (after `--no-restart`) also counts as complete. +fn materialization_complete( + changes: &[QuadletChange], + live_by_path: &BTreeMap<&Utf8Path, &LiveQuadlet>, + staged_csum: &str, + rootfs: &Dir, + applied_csum: Option<&str>, + pending_stops: &BTreeSet, +) -> Result { + let already_applied = applied_csum == Some(staged_csum); + let materialized: BTreeSet<&Utf8Path> = changes + .iter() + .filter(|c| matches!(c.kind, ChangeKind::Added | ChangeKind::Updated)) + .filter_map(|c| c.quadlet.as_deref()) + .collect(); + for c in changes { + match c.kind { + ChangeKind::Added | ChangeKind::Updated => { + let Some(quadlet) = c.quadlet.as_deref() else { + // Pull-only: no `/run` override to inspect; require the + // applied-checksum record (written with protected images). + if !already_applied { + return Ok(false); + } + continue; + }; + let Some(prev) = live_by_path.get(quadlet) else { + return Ok(false); + }; + if prev.checksum != staged_csum || prev.contents != c.spec.contents { + return Ok(false); + } + } + ChangeKind::Removed => { + let Some(quadlet) = c.quadlet.as_deref() else { + if !already_applied { + return Ok(false); + } + continue; + }; + if materialized.contains(quadlet) { + continue; + } + if rootfs.try_exists(Utf8Path::new(QUADLET_RUN_DIR).join(quadlet))? { + return Ok(false); + } + // Absent `/run` override is only proof after a prior successful + // materialization, or when a pending stop remains from + // `--no-restart` (file deleted, stop not yet run). + if already_applied { + continue; + } + if c.unit.as_deref().is_some_and(|u| pending_stops.contains(u)) { + continue; + } + return Ok(false); + } + } + } + Ok(true) +} + +/// Refuse to clobber or delete a `/run` quadlet bootc didn't write. Runs +/// before any pull or write so a conflict can't leave a half-applied `/run` +/// state. Removals are included so an admin override isn't deleted when the +/// staged image drops the bound definition. +fn check_run_destinations( + rootfs: &Dir, + changes: &[QuadletChange], + live_by_path: &BTreeMap<&Utf8Path, &LiveQuadlet>, +) -> Result<()> { + let quadlet_run_dir = Utf8Path::new(QUADLET_RUN_DIR); + for c in changes { + if !matches!( + c.kind, + ChangeKind::Added | ChangeKind::Updated | ChangeKind::Removed + ) { + continue; + } + let Some(quadlet) = c.quadlet.as_deref() else { + continue; + }; + let dest = quadlet_run_dir.join(quadlet); + if !live_by_path.contains_key(quadlet) && rootfs.try_exists(&dest)? { + anyhow::bail!( + "Refusing to {} non-bootc quadlet override at /{dest}", + match c.kind { + ChangeKind::Removed => "delete", + _ => "overwrite", + } + ); + } + } + Ok(()) +} + +/// Refuse to materialize a vendor-bound quadlet that is shadowed by a higher- +/// precedence `/etc/containers/systemd/` definition on the running system. +/// Writing into `/run` would override the administrator's configuration. +fn check_host_etc_shadowing(rootfs: &Dir, changes: &[QuadletChange]) -> Result<()> { + for c in changes { + if !matches!(c.kind, ChangeKind::Added | ChangeKind::Updated) { + continue; + } + let Some(quadlet) = c.quadlet.as_deref() else { + continue; + }; + if !c + .spec + .path + .as_str() + .starts_with("usr/share/containers/systemd/") + { + continue; + } + let etc_path = Utf8Path::new(QUADLET_ETC_DIR).join(quadlet); + if rootfs.try_exists(&etc_path)? { + anyhow::bail!( + "Refusing to apply-live bound quadlet /{}: shadowed by /{etc_path} on the running system", + c.spec.path + ); + } } Ok(()) } @@ -566,68 +822,89 @@ pub(crate) async fn apply_bound_images( let staged_csum = staged.csum(); let rootfs = &Dir::open_ambient_dir("/", cap_std::ambient_authority()).context("Opening /")?; + let run = + &Dir::open_ambient_dir("/run", cap_std::ambient_authority()).context("Opening /run")?; let booted_root = crate::utils::deployment_fd(sysroot, booted_deployment)?; let staged_root = crate::utils::deployment_fd(sysroot, &staged)?; let sepolicy = crate::lsm::new_sepolicy_at(&booted_root)?; let live = scan_live_quadlets(rootfs)?; + let live_by_path: BTreeMap<&Utf8Path, &LiveQuadlet> = + live.iter().map(|q| (q.relpath.as_path(), q)).collect(); let staged_specs = crate::boundimage::query_bound_image_specs(&staged_root)?; + let booted_specs = crate::boundimage::query_bound_image_specs(&booted_root)?; + let changes = compute_changes(&booted_specs, &staged_specs, &staged_root)?; - // The definitions from this staged deployment are already on disk; all - // that may be left is restarting units (after `--no-restart`, or a - // failure). Checking checksums alone isn't enough: if a previous run - // only got partway through materializing new overrides, the live set - // may simply be missing some of them. Require every quadlet-bearing - // staged spec to already have a live entry for this checksum. - let expected_quadlets: BTreeSet<&Utf8Path> = staged_specs + // Overrides from a previous apply-live that aren't part of this one + // need to be removed so the system converges on the current staged + // definitions rather than a mix of the two. + let current_quadlets: BTreeSet<&Utf8Path> = changes .iter() - .filter_map(|s| quadlet_relpath(&s.path)) + .filter_map(|c| c.quadlet.as_deref()) .collect(); - let live_quadlets: BTreeSet<&Utf8Path> = live.iter().map(|q| q.relpath.as_path()).collect(); - if live_quadlets == expected_quadlets && live.iter().all(|q| q.checksum == staged_csum) { - let pending: Vec<_> = live + let stale: Vec<&LiveQuadlet> = live + .iter() + .filter(|q| !current_quadlets.contains(q.relpath.as_path())) + .collect(); + + // Scope check before the fast path so dry-run (and empty change sets) + // still reject staged deployments with unrelated file changes. + if booted_csum != staged_csum { + let diff = + ostree_ext::diff::diff(&sysroot.repo(), &booted_csum, &staged_csum, None::<&str>)?; + tracing::debug!("Diff booted -> staged: {diff}"); + check_diff_scope(&diff, &changes, &booted_root, &staged_root)?; + } + + let pending_stops = collect_pending_stops(run, &live)?; + let applied_csum = applied_checksum_from(run)?; + let materialize_done = stale.is_empty() + && materialization_complete( + &changes, + &live_by_path, + staged_csum.as_str(), + rootfs, + applied_csum.as_deref(), + &pending_stops, + )?; + + // Fast path: definitions are already on disk for this staged checksum. + // All that may be left is stopping units (after `--no-restart` or a + // failed stop) and/or restarting pending ones. Derived from + // `compute_changes` so unchanged bound quadlets (which never get a + // `/run` override) don't block it, and an empty change set with + // nothing pending doesn't spuriously report "already applied". + if materialize_done { + let pending_restarts: Vec<&LiveQuadlet> = live .iter() - .filter(|q| q.pending) - .filter_map(|q| q.unit.as_deref()) + .filter(|q| q.pending && q.checksum == staged_csum) .collect(); - if pending.is_empty() { + if changes.is_empty() && pending_stops.is_empty() && pending_restarts.is_empty() { + println!("No bound image changes to apply"); + return Ok(()); + } + if pending_stops.is_empty() && pending_restarts.is_empty() { println!("Bound images from staged deployment are already applied"); return Ok(()); } - for unit in pending { - println!("restart pending: {unit}"); + for unit in &pending_stops { + println!("stop pending: {unit}"); + } + for q in &pending_restarts { + if let Some(unit) = q.unit.as_deref() { + println!("restart pending: {unit}"); + } } if opts.dry_run || opts.no_restart { return Ok(()); } + if !pending_stops.is_empty() { + stop_pending(rootfs, run, &live, &pending_stops, sepolicy.as_ref())?; + } systemctl(&["daemon-reload"])?; return restart_pending(rootfs, &live, sepolicy.as_ref()); } - let booted_specs = crate::boundimage::query_bound_image_specs(&booted_root)?; - let changes = compute_changes(&booted_specs, &staged_specs, &staged_root)?; - - if booted_csum != staged_csum { - let diff = - ostree_ext::diff::diff(&sysroot.repo(), &booted_csum, &staged_csum, None::<&str>)?; - tracing::debug!("Diff booted -> staged: {diff}"); - check_diff_scope(&diff, &changes, &booted_root, &staged_root)?; - } - - // Overrides from a previous apply-live that aren't part of this one - // need to be removed so the system converges on the current staged - // definitions rather than a mix of the two. - let live_by_path: BTreeMap<&Utf8Path, &LiveQuadlet> = - live.iter().map(|q| (q.relpath.as_path(), q)).collect(); - let current_quadlets: BTreeSet<&Utf8Path> = changes - .iter() - .filter_map(|c| c.quadlet.as_deref()) - .collect(); - let stale: Vec<&LiveQuadlet> = live - .iter() - .filter(|q| !current_quadlets.contains(q.relpath.as_path())) - .collect(); - if changes.is_empty() && stale.is_empty() { println!("No bound image changes to apply"); return Ok(()); @@ -640,6 +917,12 @@ pub(crate) async fn apply_bound_images( for q in stale.iter() { println!("reverting stale override: {}", q.relpath); } + + // Ownership / shadowing before any pull/write (and before dry-run returns) + // so conflicts are visible without mutating `/run`. + check_run_destinations(rootfs, &changes, &live_by_path)?; + check_host_etc_shadowing(rootfs, &changes)?; + if opts.dry_run { return Ok(()); } @@ -660,8 +943,6 @@ pub(crate) async fn apply_bound_images( // Protect every bound image of the staged deployment from GC, not just // the changed ones. Written first since it's the one piece of state // that can't be recovered later by scanning `/run/containers/systemd/`. - let run = - &Dir::open_ambient_dir("/run", cap_std::ambient_authority()).context("Opening /run")?; write_protected_images(run, staged_specs.iter().map(|s| s.image.image.as_str()))?; // A quadlet moving between search dirs (e.g. vendor -> /etc override) @@ -673,14 +954,31 @@ pub(crate) async fn apply_bound_images( .filter_map(|c| c.quadlet.as_deref()) .collect(); + // Units that a Removed change (or its live override) would stop. When the + // same destination is also Added with matching already-live contents, + // those units must still be marked pending so a stop doesn't leave them down. + let units_stopped_by_removal: BTreeSet = changes + .iter() + .filter(|c| c.kind == ChangeKind::Removed) + .filter_map(|c| { + let quadlet = c.quadlet.as_deref()?; + live_by_path + .get(quadlet) + .and_then(|p| p.unit.clone()) + .or_else(|| c.unit.clone()) + }) + .collect(); + // Materialize the definitions into /run, each carrying a marker. A // definition may already be live from a previous apply, in which case // its unit doesn't need a restart -- but the marker is still refreshed // to the current checksum so status and the fast path above stay - // accurate. + // accurate. Rename stops (`old_unit` / previous live unit) are recorded + // on the marker so a later re-run still stops them after `--no-restart`. let quadlet_run_dir = Utf8Path::new(QUADLET_RUN_DIR); let filemode = rustix::fs::Mode::from_raw_mode(0o644); let mut written: BTreeMap<&Utf8Path, bool> = BTreeMap::new(); + let mut stops = pending_stops; for c in changes.iter() { let Some(quadlet) = c.quadlet.as_deref() else { continue; @@ -691,21 +989,48 @@ pub(crate) async fn apply_bound_images( let already_live = live_by_path .get(quadlet) .is_some_and(|prev| !prev.pending && prev.contents == c.spec.contents); - // Refuse to clobber a `/run` quadlet we didn't write. - if live_by_path.get(quadlet).is_none() && rootfs.try_exists(&dest)? { - anyhow::bail!("Refusing to overwrite non-bootc quadlet override at /{dest}"); - } // SAFETY: We know there's a parent ensure_quadlet_dir(rootfs, dest.parent().unwrap(), sepolicy.as_ref())?; - let pending = !already_live && c.unit.is_some(); + let pending = c + .unit + .as_ref() + .is_some_and(|u| !already_live || units_stopped_by_removal.contains(u)); + // Prefer the previous *live* unit when it differs from the + // staged unit (a→b→c: stop b, not the booted a). Fall back to + // `old_unit` from compute_changes, then any recorded `stop=`. + let stop = live_by_path + .get(quadlet) + .and_then(|prev| { + let live_unit = prev.unit.as_ref()?; + if Some(live_unit) != c.unit.as_ref() { + Some(live_unit.clone()) + } else { + prev.stop.clone() + } + }) + .or_else(|| c.old_unit.clone()); + if let Some(unit) = stop.as_deref() { + stops.insert(unit.to_owned()); + } + // Also stop the booted-side rename target when it differs from + // both the live and staged units (covers a still running after b→c). + if let Some(old) = c.old_unit.as_deref() { + if Some(old) != stop.as_deref() && Some(old) != c.unit.as_deref() { + stops.insert(old.to_owned()); + } + } crate::lsm::atomic_replace_labeled( rootfs, &dest, filemode, sepolicy.as_ref(), |w| { - let marker = - marker_line(staged_csum.as_str(), pending, c.spec.image.image.as_str()); + let marker = marker_line( + staged_csum.as_str(), + pending, + c.spec.image.image.as_str(), + stop.as_deref(), + ); w.write_all(marker.as_bytes())?; w.write_all(c.spec.contents.as_bytes())?; Ok(()) @@ -714,6 +1039,15 @@ pub(crate) async fn apply_bound_images( written.insert(quadlet, pending); } ChangeKind::Removed => { + // Prefer the live unit (may differ after ServiceName= renames) + // over the booted definition's unit name. + let unit = live_by_path + .get(quadlet) + .and_then(|p| p.unit.as_deref()) + .or(c.unit.as_deref()); + if let Some(unit) = unit { + stops.insert(unit.to_owned()); + } if !materialized.contains(quadlet) { rootfs.remove_file_optional(&dest)?; } @@ -723,39 +1057,41 @@ pub(crate) async fn apply_bound_images( for q in stale.iter() { // `stale` is disjoint from `materialized` by construction (it's // exactly the live quadlets not touched by any change). + if let Some(unit) = q.unit.as_deref() { + stops.insert(unit.to_owned()); + } rootfs.remove_file_optional(quadlet_run_dir.join(&q.relpath))?; } + // Persist stops and applied-checksum before attempting stops so + // `--no-restart` and a failed `systemctl stop` remain retryable. + write_pending_stops(run, &stops)?; + write_applied_checksum(run, staged_csum.as_str())?; if opts.no_restart { - println!("Skipping unit restart; re-run without --no-restart to restart pending units"); + println!( + "Skipping unit stop/restart; re-run without --no-restart to apply pending unit changes" + ); return Ok(()); } - // Units for removed definitions, and old units left behind by a rename - // (e.g. `ServiceName=` changing), must be stopped before the daemon - // reload which makes them disappear from systemd's view. - let removed_units = changes - .iter() - .filter(|c| c.kind == ChangeKind::Removed) - .filter_map(|c| c.unit.as_deref()) - .chain(changes.iter().filter_map(|c| c.old_unit.as_deref())) - .chain(stale.iter().filter_map(|q| q.unit.as_deref())); - for unit in removed_units { - systemctl(&["stop", unit])?; - } + // Re-scan so stop_pending can clear `stop=` from the markers we just wrote. + let live_after = scan_live_quadlets(rootfs)?; + stop_pending(rootfs, run, &live_after, &stops, sepolicy.as_ref())?; systemctl(&["daemon-reload"])?; // A stale override reverts to the booted deployment's definition if - // there is one, otherwise the unit is simply gone. + // there is one. Start the *booted* unit name (ServiceName= may differ + // from the live override we just removed). for q in stale.iter() { - let Some(unit) = q.unit.as_deref() else { + let Some(booted_spec) = booted_specs + .iter() + .find(|s| quadlet_relpath(&s.path) == Some(q.relpath.as_path())) + else { continue; }; - let booted_has = booted_specs - .iter() - .any(|s| quadlet_relpath(&s.path) == Some(q.relpath.as_path())); - if booted_has { - systemctl(&["start", unit])?; - } + let Some(unit) = quadlet_unit_name(&q.relpath, &booted_spec.contents)? else { + continue; + }; + systemctl(&["start", &unit])?; } // Restart units for the definitions we just materialized, flipping @@ -776,6 +1112,7 @@ pub(crate) async fn apply_bound_images( image: c.spec.image.image.clone(), contents: c.spec.contents.clone(), unit: c.unit.clone(), + stop: None, }) }) .collect(); @@ -1040,6 +1377,15 @@ mod tests { staged_root.remove_dir_all(format!("{Q}/sub"))?; check(&diff).unwrap(); + // A symlink inside an added in-scope dir is also out of scope + // (walk_files must report non-directory entries, not only regular + // files -- otherwise a masked/unbound symlink would slip through). + staged_root.symlink_contents("/dev/null", format!("{Q}/masked.container"))?; + let e = check(&diff).unwrap_err().to_string(); + assert!(e.contains(&format!("/{Q}/masked.container")), "{e}"); + staged_root.remove_file(format!("{Q}/masked.container"))?; + check(&diff).unwrap(); + // Likewise for a removed directory, which is checked in the booted root booted_root.create_dir_all("usr/lib/bootc/bound-images.d")?; booted_root.write("usr/lib/bootc/bound-images.d/old.image", "")?; @@ -1061,18 +1407,25 @@ mod tests { #[test] fn test_marker_roundtrip() { - let line = marker_line("abc123", true, "quay.io/foo/foo:v2"); + let line = marker_line("abc123", true, "quay.io/foo/foo:v2", None); assert_eq!( line, "# X-Bootc-Apply-Live: checksum=abc123 pending=true image=quay.io/foo/foo:v2\n" ); let contents = format!("{line}[Container]\nImage=quay.io/foo/foo:v2\n"); - let (checksum, pending, image, rest) = parse_marker(&contents).unwrap(); + let (checksum, pending, image, stop, rest) = parse_marker(&contents).unwrap(); assert_eq!(checksum, "abc123"); assert!(pending); assert_eq!(image, "quay.io/foo/foo:v2"); + assert_eq!(stop, None); assert_eq!(rest, "[Container]\nImage=quay.io/foo/foo:v2\n"); + let with_stop = marker_line("abc123", true, "quay.io/foo/foo:v2", Some("old.service")); + assert!(with_stop.contains("stop=old.service")); + let contents = format!("{with_stop}[Container]\nImage=quay.io/foo/foo:v2\n"); + let (_, _, _, stop, _) = parse_marker(&contents).unwrap(); + assert_eq!(stop.as_deref(), Some("old.service")); + // A quadlet bootc didn't write has no marker assert_eq!( parse_marker("[Container]\nImage=quay.io/foo/foo:v2\n"), @@ -1091,7 +1444,7 @@ mod tests { format!("{QUADLET_RUN_DIR}/foo.container"), format!( "{}{body}", - marker_line("abc123", true, "quay.io/foo/foo:v2") + marker_line("abc123", true, "quay.io/foo/foo:v2", Some("old.service")) ), )?; // A sibling file bootc didn't write is ignored @@ -1115,6 +1468,211 @@ mod tests { assert_eq!(q.image, "quay.io/foo/foo:v2"); assert_eq!(q.contents, body); assert_eq!(q.unit.as_deref(), Some("foo.service")); + assert_eq!(q.stop.as_deref(), Some("old.service")); + Ok(()) + } + + #[test] + fn test_pending_stops_roundtrip() -> Result<()> { + let run = &tempdir()?; + assert!(pending_stops_from(run)?.is_empty()); + let units = BTreeSet::from(["foo.service".to_string(), "bar.service".to_string()]); + write_pending_stops(run, &units)?; + assert_eq!(pending_stops_from(run)?, units); + write_pending_stops(run, &BTreeSet::new())?; + assert!(pending_stops_from(run)?.is_empty()); + assert!(!run.try_exists(PENDING_STOPS_LIST)?); + Ok(()) + } + + #[test] + fn test_materialization_complete() -> Result<()> { + let rootfs = &tempdir()?; + rootfs.create_dir_all(QUADLET_RUN_DIR)?; + const Q: &str = "usr/share/containers/systemd"; + let body = "[Container]\nImage=quay.io/foo/foo:v2\n"; + let changes = vec![QuadletChange { + kind: ChangeKind::Updated, + spec: spec(&format!("{Q}/foo.container"), "quay.io/foo/foo:v2", body), + quadlet: Some("foo.container".into()), + unit: Some("foo.service".into()), + old_unit: None, + }]; + let live_by_path = BTreeMap::new(); + let empty_stops = BTreeSet::new(); + assert!(!materialization_complete( + &changes, + &live_by_path, + "abc123", + rootfs, + None, + &empty_stops, + )?); + + rootfs.write( + format!("{QUADLET_RUN_DIR}/foo.container"), + format!( + "{}{body}", + marker_line("abc123", true, "quay.io/foo/foo:v2", None) + ), + )?; + let live = scan_live_quadlets(rootfs)?; + let live_by_path: BTreeMap<&Utf8Path, &LiveQuadlet> = + live.iter().map(|q| (q.relpath.as_path(), q)).collect(); + assert!(materialization_complete( + &changes, + &live_by_path, + "abc123", + rootfs, + None, + &empty_stops, + )?); + // Wrong checksum => not complete + assert!(!materialization_complete( + &changes, + &live_by_path, + "other", + rootfs, + None, + &empty_stops, + )?); + + // Removal-only: absent /run must not look complete without applied-checksum + // or a pending stop. + let removed = vec![QuadletChange { + kind: ChangeKind::Removed, + spec: spec(&format!("{Q}/gone.container"), "quay.io/foo/gone:v1", body), + quadlet: Some("gone.container".into()), + unit: Some("gone.service".into()), + old_unit: None, + }]; + let live_by_path = BTreeMap::new(); + assert!(!materialization_complete( + &removed, + &live_by_path, + "abc123", + rootfs, + None, + &empty_stops, + )?); + assert!(materialization_complete( + &removed, + &live_by_path, + "abc123", + rootfs, + Some("abc123"), + &empty_stops, + )?); + let pending = BTreeSet::from(["gone.service".to_string()]); + assert!(materialization_complete( + &removed, + &live_by_path, + "abc123", + rootfs, + None, + &pending, + )?); + + // Pull-only (no quadlet) requires applied-checksum + let pull_only = vec![QuadletChange { + kind: ChangeKind::Added, + spec: spec("usr/lib/misc/pull.container", "quay.io/foo/pull:v1", body), + quadlet: None, + unit: None, + old_unit: None, + }]; + assert!(!materialization_complete( + &pull_only, + &live_by_path, + "abc123", + rootfs, + None, + &empty_stops, + )?); + assert!(materialization_complete( + &pull_only, + &live_by_path, + "abc123", + rootfs, + Some("abc123"), + &empty_stops, + )?); + Ok(()) + } + + #[test] + fn test_check_run_destinations_removed() -> Result<()> { + let rootfs = &tempdir()?; + rootfs.create_dir_all(QUADLET_RUN_DIR)?; + rootfs.write( + format!("{QUADLET_RUN_DIR}/foo.container"), + "[Container]\nImage=admin\n", + )?; + let changes = vec![QuadletChange { + kind: ChangeKind::Removed, + spec: spec( + "usr/share/containers/systemd/foo.container", + "quay.io/foo/foo:v1", + "", + ), + quadlet: Some("foo.container".into()), + unit: Some("foo.service".into()), + old_unit: None, + }]; + let live_by_path = BTreeMap::new(); + let e = check_run_destinations(rootfs, &changes, &live_by_path) + .unwrap_err() + .to_string(); + assert!(e.contains("delete"), "{e}"); + assert!(e.contains("non-bootc"), "{e}"); + Ok(()) + } + + #[test] + fn test_check_host_etc_shadowing() -> Result<()> { + let rootfs = &tempdir()?; + rootfs.create_dir_all(QUADLET_ETC_DIR)?; + rootfs.write( + format!("{QUADLET_ETC_DIR}/foo.container"), + "[Container]\nImage=admin\n", + )?; + let changes = vec![QuadletChange { + kind: ChangeKind::Updated, + spec: spec( + "usr/share/containers/systemd/foo.container", + "quay.io/foo/foo:v2", + "[Container]\nImage=vendor\n", + ), + quadlet: Some("foo.container".into()), + unit: Some("foo.service".into()), + old_unit: None, + }]; + let e = check_host_etc_shadowing(rootfs, &changes) + .unwrap_err() + .to_string(); + assert!(e.contains("shadowed"), "{e}"); + // /etc-bound definitions are fine + let etc_bound = vec![QuadletChange { + kind: ChangeKind::Updated, + spec: spec( + "etc/containers/systemd/foo.container", + "quay.io/foo/foo:v2", + "[Container]\nImage=etc\n", + ), + quadlet: Some("foo.container".into()), + unit: Some("foo.service".into()), + old_unit: None, + }]; + check_host_etc_shadowing(rootfs, &etc_bound)?; + Ok(()) + } + + #[test] + fn test_applied_checksum_roundtrip() -> Result<()> { + let run = &tempdir()?; + assert_eq!(applied_checksum_from(run)?, None); + write_applied_checksum(run, "deadbeef")?; + assert_eq!(applied_checksum_from(run)?.as_deref(), Some("deadbeef")); Ok(()) } diff --git a/crates/lib/src/cli.rs b/crates/lib/src/cli.rs index b49e4ae5d1..4353ce91b1 100644 --- a/crates/lib/src/cli.rs +++ b/crates/lib/src/cli.rs @@ -1340,16 +1340,28 @@ fn has_soft_reboot_capability(deployment: Option<&crate::spec::BootEntry>) -> bo deployment.map(|d| d.soft_reboot_capable).unwrap_or(false) } -/// Prepare a soft reboot for the given deployment +/// Prepare a soft reboot for the given deployment. +/// +/// Does not clear live-applied `/run` state: soft-reboot without `--apply` +/// leaves containers running, and clearing protection early would allow GC +/// to prune in-use images. Clear at reboot handoff via +/// [`clear_live_state_before_reboot`] instead. #[context("Preparing soft reboot")] fn prepare_soft_reboot(sysroot: &SysrootLock, deployment: &ostree::Deployment) -> Result<()> { let cancellable = ostree::gio::Cancellable::NONE; sysroot .deployment_set_soft_reboot(deployment, false, cancellable) .context("Failed to prepare soft-reboot")?; - // A soft reboot preserves /run, so anything applied live must not leak - // into the target deployment. - crate::applylive::clear_state_from_host()?; + Ok(()) +} + +/// Clear live-applied bound-image state immediately before reboot when a +/// soft reboot was prepared. Soft reboot preserves `/run`, so overrides and +/// image protection must not leak into the target deployment. +fn clear_live_state_before_reboot(soft_reboot: Option) -> Result<()> { + if soft_reboot.is_some() { + crate::applylive::clear_state_from_host()?; + } Ok(()) } @@ -1486,6 +1498,7 @@ async fn apply_from_downloaded_ostree( handle_staged_soft_reboot(booted_ostree, opts.soft_reboot, &host)?; if opts.apply { + clear_live_state_before_reboot(opts.soft_reboot)?; crate::reboot::reboot()?; } return Ok(()); @@ -1671,6 +1684,7 @@ async fn upgrade( handle_staged_soft_reboot(booted_ostree, opts.soft_reboot, &host)?; if opts.apply { + clear_live_state_before_reboot(opts.soft_reboot)?; crate::reboot::reboot()?; } } else if booted_unchanged { @@ -1708,6 +1722,7 @@ async fn upgrade( } if opts.apply { + clear_live_state_before_reboot(opts.soft_reboot)?; crate::reboot::reboot()?; } } else { @@ -1886,6 +1901,7 @@ async fn switch_ostree( // `--apply` cannot be passed along with `--download-only` (handled by clap) // but for sanity nonetheless if opts.apply { + clear_live_state_before_reboot(opts.soft_reboot)?; crate::reboot::reboot()?; } @@ -2294,6 +2310,7 @@ async fn run_from_opt(opt: Opt) -> Result { Opt::Rollback(opts) => { rollback(&opts).await?; if opts.apply { + clear_live_state_before_reboot(opts.soft_reboot)?; crate::reboot::reboot()?; } Ok(()) diff --git a/docs/src/bootc-logically-bound-images.7.md b/docs/src/bootc-logically-bound-images.7.md index 58d775bdb5..f5c7666790 100644 --- a/docs/src/bootc-logically-bound-images.7.md +++ b/docs/src/bootc-logically-bound-images.7.md @@ -77,8 +77,8 @@ This writes the changed quadlet files to `/run/containers/systemd/`, which podman gives precedence over `/etc` and `/usr`, reloads systemd and restarts the affected `.container` units (removed units are stopped). Pass `--dry-run` to see what would change, or `--no-restart` to write the definitions without -touching any units; a later plain invocation restarts the units still -pending, as does re-running after a failed restart. +touching any units; a later plain invocation stops and restarts the units +still pending, as does re-running after a failed stop or restart. If the staged deployment contains any other change, the command fails and lists the out-of-scope paths; a reboot is required to apply it. Note that a @@ -88,9 +88,12 @@ the image is reported this way, since the local copy wins in `/etc`. Because `/run` is transient, the override disappears on the next boot, at which point the staged deployment's own content applies. (A soft reboot preserves `/run`; the override is cleared when one is prepared.) The applied -state is shown in `bootc status` and recorded in `/run/bootc/apply-live/`; -images recorded there are protected from garbage collection until reboot, -even if the staged deployment is discarded (for example by `bootc rollback`). +state is shown in `bootc status`. Each quadlet written to +`/run/containers/systemd/` carries a leading marker comment (checksum, +pending restart, and any unit still needing a stop); a small flat list under +`/run/bootc/apply-live/` protects the staged images from garbage collection +until reboot, and records pending stops for pure removals, even if the staged +deployment is discarded (for example by `bootc rollback`). Limitations: only the `.container` or `.image` file referenced from `bound-images.d` is applied. Quadlet drop-ins (`*.container.d/`) and other