diff --git a/crates/lib/src/applylive.rs b/crates/lib/src/applylive.rs new file mode 100644 index 0000000000..0b59c263e5 --- /dev/null +++ b/crates/lib/src/applylive.rs @@ -0,0 +1,1718 @@ +//! # Live application of staged content +//! +//! This module implements `bootc apply-live`, which makes content from the +//! staged deployment visible on the running system without a reboot. +//! See . +//! +//! The only scope supported today is logically bound images +//! (`bootc apply-live bound-images`). The image bytes are already shared +//! between deployments via the bootc-owned container storage; what is +//! pinned to a deployment is only the *definition*: the symlink in +//! `/usr/lib/bootc/bound-images.d` and the quadlet file it references. +//! +//! On the ostree backend, that shared container storage is just a plain +//! directory, so applying a definition change live does not require +//! mounting anything new -- podman already reads the same storage the +//! booted and staged deployments share. (The composefs backend lays out +//! storage differently and is not handled here yet.) Changed quadlet files +//! are written to `/run/containers/systemd/`, which podman gives +//! precedence over `/etc` and `/usr`, and the affected units are restarted. +//! Because `/run` is transient, the override disappears on the next boot, +//! at which point the staged deployment's own `/usr` content applies and +//! the system converges without any further action. (A soft reboot +//! preserves `/run`, so the override is explicitly cleared when one is +//! prepared.) +//! +//! Rather than a separate state file, each quadlet written to +//! `/run/containers/systemd/` carries its own state as a leading comment +//! (see [`marker_line`]): the staged checksum, the image reference, +//! whether its unit still needs restarting ("pending"), and optionally a +//! unit that still needs stopping (e.g. after a `ServiceName=` rename). +//! Pure removals and stale-override reverts record their pending stops in +//! a small flat list under `/run/bootc/apply-live/` (see +//! [`PENDING_STOPS_LIST`]). Re-running the command re-derives everything +//! from that directory (see [`scan_live_quadlets`]) plus the pending-stops +//! list, so a failure part way through -- including a failed stop -- is +//! safe to resume from. Garbage-collection protection for bound images +//! with no quadlet override (unchanged definitions) lives in the same +//! directory; see [`write_protected_images`]. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::Write; +use std::process::Command; + +use anyhow::{Context, Result, ensure}; +use bootc_utils::CommandRunExt; +use camino::{Utf8Path, Utf8PathBuf}; +use cap_std_ext::cap_std::{self, fs::Dir}; +use cap_std_ext::dirext::CapStdExtDirExt; +use fn_error_context::context; +use ostree_ext::diff::FileTreeDiff; + +use crate::boundimage::{BOUND_IMAGE_DIR, BoundImageSpec}; +use crate::cli::ApplyLiveBoundImagesOpts; +use crate::spec::{LiveBoundImage, LiveBoundImages}; +use crate::store::{BootedOstree, Storage}; + +/// Directory (relative to `/run`) holding apply-live state. +const STATE_DIR: &str = "bootc/apply-live"; +/// Flat, one-image-reference-per-line list (relative to `/run`) protecting +/// every bound image of an in-progress (or completed) apply-live from +/// garbage collection; see [`write_protected_images`]. +const PROTECTED_IMAGES_LIST: &str = "bootc/apply-live/protected-images.list"; +/// Flat, one-unit-per-line list (relative to `/run`) of systemd units that +/// still need to be stopped (pure removals and stale-override reverts). +/// Rename stops are recorded on the replacement's marker instead; see +/// [`marker_line`]. Cleared entry-by-entry only after each `systemctl stop` +/// succeeds, so `--no-restart` and failed stops remain retryable. +const PENDING_STOPS_LIST: &str = "bootc/apply-live/pending-stops.list"; +/// Records the staged deployment checksum for which materialization (and +/// image protection) has completed. Needed so removal-only and pull-only +/// changes (no `/run` override) don't look "already applied" on first run. +const APPLIED_CHECKSUM: &str = "bootc/apply-live/applied-checksum"; +/// Prefix of the leading marker comment line bootc writes at the top of +/// each quadlet file it materializes under [`QUADLET_RUN_DIR`]; see +/// [`marker_line`] and [`parse_marker`]. +const MARKER_PREFIX: &str = "# X-Bootc-Apply-Live:"; +/// Podman's highest-precedence quadlet search directory, relative to `/`. +/// Paths are relative to the root (not `/run`) so that SELinux labels are +/// computed for the real absolute path. +const QUADLET_RUN_DIR: &str = "run/containers/systemd"; +/// Host `/etc` quadlet search directory (relative to `/`), which takes +/// precedence over vendor (`usr/share`) definitions. Used to refuse +/// materializing a vendor-bound quadlet that would clobber an admin override. +const QUADLET_ETC_DIR: &str = "etc/containers/systemd"; +/// Quadlet search directories relative to a deployment root, in decreasing +/// precedence (excluding `/run`, which is what we write to). +const QUADLET_SEARCH_DIRS: &[&str] = &["etc/containers/systemd", "usr/share/containers/systemd"]; +/// Directories which may be added/removed/changed in an ostree commit diff +/// while still being in scope for `apply-live bound-images`. Note that +/// `/etc` is stored as `/usr/etc` in ostree commits. The contents of an +/// added or removed directory are checked separately, since the diff does +/// not enumerate them. +const SCOPE_DIRS: &[&str] = &[ + "/usr/lib/bootc", + "/usr/lib/bootc/bound-images.d", + "/usr/share/containers", + "/usr/share/containers/systemd", + "/usr/etc/containers", + "/usr/etc/containers/systemd", +]; +/// Maximum number of out-of-scope paths to include in an error message. +const MAX_REPORTED_PATHS: usize = 10; +/// Journal message ID for apply-live operations. +const APPLY_LIVE_JOURNAL_ID: &str = "4c9e2b7d1f0a4e8b9c6d3a2f5e7b1c0d"; + +/// How a bound image definition differs between the booted and staged deployments. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ChangeKind { + /// Present in the staged deployment only + Added, + /// Present in both, with different quadlet contents + Updated, + /// Present in the booted deployment only, and the quadlet file + /// no longer exists in the staged deployment + Removed, +} + +impl std::fmt::Display for ChangeKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + ChangeKind::Added => "added", + ChangeKind::Updated => "updated", + ChangeKind::Removed => "removed", + }; + f.write_str(s) + } +} + +/// A single bound image definition change to apply. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct QuadletChange { + pub(crate) kind: ChangeKind, + /// The staged spec for `Added`/`Updated`, the booted spec for `Removed`. + pub(crate) spec: BoundImageSpec, + /// Path of the quadlet relative to its search directory, if it lives in + /// one podman will find. Definitions outside a search directory are only + /// used as pull specs and have nothing to materialize. + pub(crate) quadlet: Option, + /// The systemd unit to restart, if any. + pub(crate) unit: Option, + /// For `Updated`, the previous (booted) unit name, if it differs from + /// `unit` (e.g. `ServiceName=` changed): this needs to be stopped + /// separately, since restarting `unit` won't touch it. + pub(crate) old_unit: Option, +} + +/// A quadlet file bootc has written to [`QUADLET_RUN_DIR`], parsed back +/// from its marker comment line (see [`marker_line`]/[`parse_marker`]). +/// Recovered by scanning, not persisted elsewhere; see [`scan_live_quadlets`]. +#[derive(Debug, Clone, PartialEq, Eq)] +struct LiveQuadlet { + /// Path relative to [`QUADLET_RUN_DIR`]. + relpath: Utf8PathBuf, + /// The staged deployment checksum this definition came from. + checksum: String, + /// True if the unit has not yet been restarted with this definition. + pending: bool, + /// The container image reference, for display purposes. + image: String, + /// The quadlet contents, with the marker comment stripped. + contents: String, + /// The systemd unit generated from the quadlet, if any. + unit: Option, + /// A previous unit that still needs stopping (e.g. after a + /// `ServiceName=` rename), recorded in the marker until the stop + /// succeeds. + stop: Option, +} + +/// If `path` (relative to a root) is inside a quadlet search directory, +/// return the remainder relative to that directory. +pub(crate) fn quadlet_relpath(path: &Utf8Path) -> Option<&Utf8Path> { + QUADLET_SEARCH_DIRS + .iter() + .find_map(|d| path.strip_prefix(d).ok()) + .filter(|p| !p.as_str().is_empty()) +} + +/// Compute the systemd service unit that quadlet generates for a `.container` +/// file. Returns `None` for other quadlet types; a `.image` unit is a oneshot +/// pull which we don't need to run since bootc pulls the image itself. +pub(crate) fn quadlet_unit_name(relpath: &Utf8Path, contents: &str) -> Result> { + if relpath.extension() != Some("container") { + return Ok(None); + } + let ini = tini::Ini::from_string(contents).context("Parse to ini")?; + let name = ini + .get::("Container", "ServiceName") + .or_else(|| relpath.file_stem().map(ToOwned::to_owned)) + .ok_or_else(|| anyhow::anyhow!("Invalid quadlet name: {relpath}"))?; + Ok(Some(format!("{name}.service"))) +} + +/// Convert a deployment-relative path to the form used in an ostree commit +/// (and hence [`FileTreeDiff`]): absolute, with `/etc` stored as `/usr/etc`. +fn commit_path(path: &Utf8Path) -> Utf8PathBuf { + if let Ok(rest) = path.strip_prefix("etc") { + Utf8Path::new("/usr/etc").join(rest) + } else { + Utf8Path::new("/").join(path) + } +} + +/// Compute the set of definition changes between the booted and staged +/// bound image specs. +pub(crate) fn compute_changes( + booted: &[BoundImageSpec], + staged: &[BoundImageSpec], + staged_root: &Dir, +) -> Result> { + let booted: BTreeMap<_, _> = booted.iter().map(|s| (s.path.as_path(), s)).collect(); + let staged: BTreeMap<_, _> = staged.iter().map(|s| (s.path.as_path(), s)).collect(); + let mut changes = Vec::new(); + + let unit_for = |quadlet: Option<&Utf8Path>, contents: &str| -> Result> { + quadlet + .map(|q| quadlet_unit_name(q, contents)) + .transpose() + .map(Option::flatten) + }; + let mk = |kind, spec: &BoundImageSpec, old_unit: Option| -> Result { + let quadlet = quadlet_relpath(&spec.path).map(ToOwned::to_owned); + let unit = unit_for(quadlet.as_deref(), &spec.contents)?; + Ok(QuadletChange { + kind, + spec: spec.clone(), + quadlet, + unit, + old_unit, + }) + }; + + for (path, spec) in staged.iter() { + match booted.get(path) { + Some(b) if b.contents == spec.contents => {} + Some(b) => { + // The unit name quadlet generates can change (e.g. via + // `ServiceName=`) even though the file's path didn't; the old + // unit needs to be stopped separately in that case. + let quadlet = quadlet_relpath(&spec.path); + let staged_unit = unit_for(quadlet, &spec.contents)?; + let booted_unit = unit_for(quadlet, &b.contents)?; + let old_unit = booted_unit.filter(|u| Some(u) != staged_unit.as_ref()); + changes.push(mk(ChangeKind::Updated, spec, old_unit)?); + } + None => changes.push(mk(ChangeKind::Added, spec, None)?), + } + } + for (path, spec) in booted.iter() { + if staged.contains_key(path) { + continue; + } + if staged_root.try_exists(path)? { + // The quadlet still exists, it's just no longer a bound image; + // there's nothing to change on the running system. + tracing::debug!("No longer bound, but still present: {path}"); + continue; + } + changes.push(mk(ChangeKind::Removed, spec, None)?); + } + Ok(changes) +} + +/// Recursively collect every non-directory entry under `dir` (a commit-form +/// absolute path) in the given root, as commit-form absolute paths. +/// Symlinks are included: [`check_diff_scope`] needs them so a symlink +/// inside an added in-scope directory is still reported as out of scope. +/// Callers that only want regular files (e.g. [`scan_live_quadlets`]) must +/// filter themselves. +fn walk_files(root: &Dir, dir: &str, out: &mut Vec) -> Result<()> { + let rel = dir.trim_start_matches('/'); + let Some(d) = root.open_dir_optional(rel)? else { + return Ok(()); + }; + for entry in d.entries()? { + let entry = entry?; + let name = entry.file_name(); + let name = name + .to_str() + .ok_or_else(|| anyhow::anyhow!("Invalid non-UTF8 filename: {name:?} in {dir}"))?; + let path = format!("{dir}/{name}"); + let ft = entry.file_type()?; + if ft.is_dir() { + walk_files(root, &path, out)?; + } else { + out.push(path); + } + } + Ok(()) +} + +/// Verify that every path which differs between the booted and staged commits +/// is accounted for by the bound image changes we are going to apply. Any other +/// change means the staged deployment is not just a bound image update, and a +/// reboot is required. +/// +/// The ostree diff does not enumerate the contents of added or removed +/// directories, so those are walked in the staged (respectively booted) +/// deployment root and checked file by file. +pub(crate) fn check_diff_scope( + diff: &FileTreeDiff, + changes: &[QuadletChange], + booted_root: &Dir, + staged_root: &Dir, +) -> Result<()> { + let bound_dir = format!("/{BOUND_IMAGE_DIR}/"); + let allowed_files: BTreeSet<_> = changes.iter().map(|c| commit_path(&c.spec.path)).collect(); + let file_in_scope = + |p: &str| -> bool { p.starts_with(&bound_dir) || allowed_files.contains(Utf8Path::new(p)) }; + let dir_in_scope = |p: &str| -> bool { SCOPE_DIRS.contains(&p) }; + + let mut out_of_scope: Vec = diff + .added_files + .iter() + .chain(&diff.removed_files) + .chain(&diff.changed_files) + .filter(|p| !file_in_scope(p)) + .chain(diff.changed_dirs.iter().filter(|p| !dir_in_scope(p))) + .cloned() + .collect(); + for (dirs, root) in [ + (&diff.added_dirs, staged_root), + (&diff.removed_dirs, booted_root), + ] { + for d in dirs { + if !dir_in_scope(d) { + out_of_scope.push(d.clone()); + continue; + } + let mut files = Vec::new(); + walk_files(root, d, &mut files)?; + out_of_scope.extend(files.into_iter().filter(|p| !file_in_scope(p))); + } + } + if out_of_scope.is_empty() { + return Ok(()); + } + out_of_scope.sort(); + let n = out_of_scope.len(); + let mut msg = String::from( + "Staged deployment contains changes outside of bound image definitions; a reboot is required to apply it:\n", + ); + for p in out_of_scope.iter().take(MAX_REPORTED_PATHS) { + msg.push_str(" "); + msg.push_str(p); + msg.push('\n'); + } + if n > MAX_REPORTED_PATHS { + msg.push_str(&format!(" ...and {} more\n", n - MAX_REPORTED_PATHS)); + } + anyhow::bail!("{}", msg.trim_end()) +} + +/// Build the leading marker comment for a quadlet file; see [`parse_marker`]. +/// `stop` records a unit that still needs stopping (e.g. the old name after +/// a `ServiceName=` change); omit it once the stop has succeeded. +fn marker_line(checksum: &str, pending: bool, image: &str, stop: Option<&str>) -> String { + let mut line = format!("{MARKER_PREFIX} checksum={checksum} pending={pending} image={image}"); + if let Some(stop) = stop { + line.push_str(&format!(" stop={stop}")); + } + line.push('\n'); + line +} + +/// Parse a [`marker_line`] from the start of `contents`, returning +/// `(checksum, pending, image, stop, rest)` where `rest` is the original +/// quadlet contents with the marker removed. Returns `None` if there's no +/// marker, e.g. a quadlet bootc didn't write. +fn parse_marker(contents: &str) -> Option<(String, bool, String, Option, &str)> { + let (first, rest) = contents.split_once('\n')?; + let kvs = first.strip_prefix(MARKER_PREFIX)?; + let (mut checksum, mut pending, mut image, mut stop) = (None, None, None, None); + for kv in kvs.split_whitespace() { + let (k, v) = kv.split_once('=')?; + match k { + "checksum" => checksum = Some(v.to_owned()), + "pending" => pending = Some(v.parse().ok()?), + "image" => image = Some(v.to_owned()), + "stop" => stop = Some(v.to_owned()), + _ => {} + } + } + Some((checksum?, pending?, image?, stop, rest)) +} + +/// Scan [`QUADLET_RUN_DIR`] for quadlet files bootc has written, parsing +/// their marker comments back into [`LiveQuadlet`]s. Files with no +/// recognized marker are skipped. Symlinks and other non-regular entries +/// (e.g. a unit masked via `-> /dev/null`) are skipped before reading, so +/// cap-std's absolute-symlink resolution can't break `bootc status`. +fn scan_live_quadlets(rootfs: &Dir) -> Result> { + let mut paths = Vec::new(); + walk_files(rootfs, &format!("/{QUADLET_RUN_DIR}"), &mut paths)?; + let prefix = format!("/{QUADLET_RUN_DIR}/"); + let mut out = Vec::new(); + for p in paths { + let Some(relpath) = p.strip_prefix(prefix.as_str()) else { + continue; + }; + let rel = p.trim_start_matches('/'); + // Don't follow symlinks: an absolute target like `/dev/null` fails + // cap-std path resolution, and bootc only writes regular files. + let Some(meta) = rootfs.symlink_metadata_optional(rel)? else { + continue; + }; + if !meta.is_file() { + continue; + } + let Some(contents) = rootfs.read_to_string_optional(rel)? else { + continue; + }; + let Some((checksum, pending, image, stop, body)) = parse_marker(&contents) else { + continue; + }; + let relpath = Utf8PathBuf::from(relpath); + let unit = quadlet_unit_name(&relpath, body)?; + out.push(LiveQuadlet { + relpath, + checksum, + pending, + image, + contents: body.to_owned(), + unit, + stop, + }); + } + Ok(out) +} + +/// Add the staged deployment's image references to the protected list (one +/// reference per line), protecting them from garbage collection even if the +/// staged deployment is later discarded (e.g. by `bootc rollback`). +/// Unrelated to the restart state in [`scan_live_quadlets`]: it covers +/// bound images with no quadlet override to carry that state. +/// +/// This merges with whatever is already there rather than replacing it: an +/// earlier live-applied deployment's images may still be in use by units +/// that haven't transitioned yet, so they must stay protected too until a +/// reboot (or [`clear_state_from_host`]) resets `/run`. +fn write_protected_images<'a>(run: &Dir, images: impl Iterator) -> Result<()> { + let mut all: BTreeSet = protected_images_from_host_at(run)?.into_iter().collect(); + all.extend(images.map(ToOwned::to_owned)); + run.create_dir_all(STATE_DIR)?; + run.atomic_replace_with(PROTECTED_IMAGES_LIST, |w| -> Result<()> { + for image in &all { + writeln!(w, "{image}")?; + } + Ok(()) + }) + .with_context(|| format!("Writing /run/{PROTECTED_IMAGES_LIST}")) +} + +/// Read the image references written by [`write_protected_images`] from an +/// arbitrary directory, if any. +fn protected_images_from_host_at(run: &Dir) -> Result> { + let Some(contents) = run.read_to_string_optional(PROTECTED_IMAGES_LIST)? else { + return Ok(Vec::new()); + }; + Ok(contents.lines().map(ToOwned::to_owned).collect()) +} + +/// Read back the image references written by [`write_protected_images`], if +/// any, from the host `/run`. +pub(crate) fn protected_images_from_host() -> Result> { + let run = + Dir::open_ambient_dir("/run", cap_std::ambient_authority()).context("Opening /run")?; + protected_images_from_host_at(&run) +} + +/// Read the pending-stop unit names from [`PENDING_STOPS_LIST`], if any. +fn pending_stops_from(run: &Dir) -> Result> { + let Some(contents) = run.read_to_string_optional(PENDING_STOPS_LIST)? else { + return Ok(BTreeSet::new()); + }; + Ok(contents.lines().map(ToOwned::to_owned).collect()) +} + +/// Replace [`PENDING_STOPS_LIST`] with `units` (creating the state dir if +/// needed). An empty set removes the file. +fn write_pending_stops(run: &Dir, units: &BTreeSet) -> Result<()> { + if units.is_empty() { + run.remove_file_optional(PENDING_STOPS_LIST)?; + return Ok(()); + } + run.create_dir_all(STATE_DIR)?; + run.atomic_replace_with(PENDING_STOPS_LIST, |w| -> Result<()> { + for unit in units { + writeln!(w, "{unit}")?; + } + Ok(()) + }) + .with_context(|| format!("Writing /run/{PENDING_STOPS_LIST}")) +} + +/// Units that still need stopping: the flat list plus any `stop=` recorded +/// on live markers (renames). +fn collect_pending_stops(run: &Dir, live: &[LiveQuadlet]) -> Result> { + let mut stops = pending_stops_from(run)?; + for q in live { + if let Some(unit) = q.stop.as_deref() { + stops.insert(unit.to_owned()); + } + } + Ok(stops) +} + +/// Read the staged checksum recorded by [`write_applied_checksum`], if any. +fn applied_checksum_from(run: &Dir) -> Result> { + let Some(contents) = run.read_to_string_optional(APPLIED_CHECKSUM)? else { + return Ok(None); + }; + let csum = contents.trim(); + if csum.is_empty() { + return Ok(None); + } + Ok(Some(csum.to_owned())) +} + +/// Record that materialization for `checksum` has finished (protected images +/// written, `/run` overrides updated). Cleared with the rest of live state. +fn write_applied_checksum(run: &Dir, checksum: &str) -> Result<()> { + run.create_dir_all(STATE_DIR)?; + run.atomic_replace_with(APPLIED_CHECKSUM, |w| -> Result<()> { + writeln!(w, "{checksum}")?; + Ok(()) + }) + .with_context(|| format!("Writing /run/{APPLIED_CHECKSUM}")) +} + +/// Build the `bootc status` view of live-applied bound images by scanning +/// [`QUADLET_RUN_DIR`]. Returns `None` if nothing is applied live, or if +/// the markers disagree on the staged checksum (e.g. a partial apply left +/// a mix of deployments) -- status then shows no live state rather than +/// attributing every image to whichever file the scan returned first. +/// `deploy_serial` isn't recorded in the markers, so it's left as `0` for +/// callers to fill in by matching `checksum` against a deployment. +pub(crate) fn status_from_host() -> Result> { + let rootfs = Dir::open_ambient_dir("/", cap_std::ambient_authority()).context("Opening /")?; + let live = scan_live_quadlets(&rootfs)?; + let Some(checksum) = live.first().map(|q| q.checksum.clone()) else { + return Ok(None); + }; + if live.iter().any(|q| q.checksum != checksum) { + tracing::warn!( + "Live bound-image markers disagree on staged checksum; omitting liveBoundImages from status" + ); + return Ok(None); + } + let images = live + .into_iter() + .map(|q| LiveBoundImage { + image: q.image, + quadlet: Some(q.relpath.to_string()), + unit: q.unit, + pending: q.pending, + }) + .collect(); + Ok(Some(LiveBoundImages { + checksum, + deploy_serial: 0, + images, + })) +} + +/// Remove any live-applied bound image overrides and state. Call this at the +/// soft-reboot / reboot handoff (which preserves `/run`), not when merely +/// preparing a soft reboot without `--apply`, so live containers keep their +/// overrides and GC protection until the system actually transitions. +#[context("Clearing live bound image state")] +pub(crate) fn clear_state_from_host() -> Result<()> { + let rootfs = Dir::open_ambient_dir("/", cap_std::ambient_authority()).context("Opening /")?; + let live = scan_live_quadlets(&rootfs)?; + for q in &live { + rootfs.remove_file_optional(Utf8Path::new(QUADLET_RUN_DIR).join(&q.relpath))?; + } + let run = + Dir::open_ambient_dir("/run", cap_std::ambient_authority()).context("Opening /run")?; + let had_protected = run.remove_file_optional(PROTECTED_IMAGES_LIST)?; + let had_stops = run.remove_file_optional(PENDING_STOPS_LIST)?; + let had_applied = run.remove_file_optional(APPLIED_CHECKSUM)?; + if live.is_empty() && !had_protected && !had_stops && !had_applied { + return Ok(()); + } + println!("Cleared live-applied bound image definitions"); + Ok(()) +} + +#[context("Running systemctl {}", args.join(" "))] +fn systemctl(args: &[&str]) -> Result<()> { + Command::new("systemctl").args(args).run_capture_stderr() +} + +/// Create `path` and any missing ancestors below `/run`, labeling only the +/// directories we create; existing ones (e.g. podman's `/run/containers`) +/// are left alone. +fn ensure_quadlet_dir( + rootfs: &Dir, + path: &Utf8Path, + sepolicy: Option<&ostree_ext::ostree::SePolicy>, +) -> Result<()> { + let mode = rustix::fs::Mode::from_raw_mode(0o755); + // Walk parents under `/run` but not `/run` itself (podman owns that). + let ancestors: Vec<_> = path + .ancestors() + .take_while(|p| p.components().count() > 1) + .collect(); + for dir in ancestors.into_iter().rev() { + if rootfs.try_exists(dir)? { + continue; + } + crate::lsm::ensure_dir_labeled(rootfs, dir, None, mode, sepolicy)?; + } + Ok(()) +} + +/// Rewrite the marker on an already-materialized quadlet, leaving its +/// body untouched. `pending`/`stop` override the values on `q`. +fn set_quadlet_marker( + rootfs: &Dir, + q: &LiveQuadlet, + pending: bool, + stop: Option<&str>, + sepolicy: Option<&ostree_ext::ostree::SePolicy>, +) -> Result<()> { + let dest = Utf8Path::new(QUADLET_RUN_DIR).join(&q.relpath); + let filemode = rustix::fs::Mode::from_raw_mode(0o644); + crate::lsm::atomic_replace_labeled(rootfs, &dest, filemode, sepolicy, |w| { + w.write_all(marker_line(&q.checksum, pending, &q.image, stop).as_bytes())?; + w.write_all(q.contents.as_bytes())?; + Ok(()) + }) +} + +/// Stop every unit in `units`, then clear matching `stop=` markers and the +/// pending-stops list. Each stop is attempted before clearing, so a failure +/// leaves the remaining entries for the next run. +fn stop_pending( + rootfs: &Dir, + run: &Dir, + live: &[LiveQuadlet], + units: &BTreeSet, + sepolicy: Option<&ostree_ext::ostree::SePolicy>, +) -> Result<()> { + for unit in units { + systemctl(&["stop", unit])?; + } + for q in live.iter().filter(|q| q.stop.is_some()) { + set_quadlet_marker(rootfs, q, q.pending, None, sepolicy)?; + } + write_pending_stops(run, &BTreeSet::new())?; + Ok(()) +} + +/// Restart every unit still marked pending among `live`, updating each +/// quadlet's marker as it succeeds so a failure part way through can be +/// retried on the next run. `stop=` has already been cleared by +/// [`stop_pending`] when applicable. +fn restart_pending( + rootfs: &Dir, + live: &[LiveQuadlet], + sepolicy: Option<&ostree_ext::ostree::SePolicy>, +) -> Result<()> { + for q in live.iter().filter(|q| q.pending) { + if let Some(unit) = q.unit.as_deref() { + // `restart` also starts a unit which is not running (the added case). + systemctl(&["restart", unit])?; + } + set_quadlet_marker(rootfs, q, false, None, sepolicy)?; + } + Ok(()) +} + +/// True when every change has already been materialized for `staged_csum`. +/// Added/Updated with a quadlet need a matching `/run` override; Removed +/// destinations that aren't being replaced must be gone; pull-only and +/// removal-only cases require [`APPLIED_CHECKSUM`] so a first run doesn't +/// treat "nothing on disk" as already applied. A removal with its unit still +/// listed in `pending_stops` (after `--no-restart`) also counts as complete. +fn materialization_complete( + changes: &[QuadletChange], + live_by_path: &BTreeMap<&Utf8Path, &LiveQuadlet>, + staged_csum: &str, + rootfs: &Dir, + applied_csum: Option<&str>, + pending_stops: &BTreeSet, +) -> Result { + let already_applied = applied_csum == Some(staged_csum); + let materialized: BTreeSet<&Utf8Path> = changes + .iter() + .filter(|c| matches!(c.kind, ChangeKind::Added | ChangeKind::Updated)) + .filter_map(|c| c.quadlet.as_deref()) + .collect(); + for c in changes { + match c.kind { + ChangeKind::Added | ChangeKind::Updated => { + let Some(quadlet) = c.quadlet.as_deref() else { + // Pull-only: no `/run` override to inspect; require the + // applied-checksum record (written with protected images). + if !already_applied { + return Ok(false); + } + continue; + }; + let Some(prev) = live_by_path.get(quadlet) else { + return Ok(false); + }; + if prev.checksum != staged_csum || prev.contents != c.spec.contents { + return Ok(false); + } + } + ChangeKind::Removed => { + let Some(quadlet) = c.quadlet.as_deref() else { + if !already_applied { + return Ok(false); + } + continue; + }; + if materialized.contains(quadlet) { + continue; + } + if rootfs.try_exists(Utf8Path::new(QUADLET_RUN_DIR).join(quadlet))? { + return Ok(false); + } + // Absent `/run` override is only proof after a prior successful + // materialization, or when a pending stop remains from + // `--no-restart` (file deleted, stop not yet run). + if already_applied { + continue; + } + if c.unit.as_deref().is_some_and(|u| pending_stops.contains(u)) { + continue; + } + return Ok(false); + } + } + } + Ok(true) +} + +/// Refuse to clobber or delete a `/run` quadlet bootc didn't write. Runs +/// before any pull or write so a conflict can't leave a half-applied `/run` +/// state. Removals are included so an admin override isn't deleted when the +/// staged image drops the bound definition. +fn check_run_destinations( + rootfs: &Dir, + changes: &[QuadletChange], + live_by_path: &BTreeMap<&Utf8Path, &LiveQuadlet>, +) -> Result<()> { + let quadlet_run_dir = Utf8Path::new(QUADLET_RUN_DIR); + for c in changes { + if !matches!( + c.kind, + ChangeKind::Added | ChangeKind::Updated | ChangeKind::Removed + ) { + continue; + } + let Some(quadlet) = c.quadlet.as_deref() else { + continue; + }; + let dest = quadlet_run_dir.join(quadlet); + if !live_by_path.contains_key(quadlet) && rootfs.try_exists(&dest)? { + anyhow::bail!( + "Refusing to {} non-bootc quadlet override at /{dest}", + match c.kind { + ChangeKind::Removed => "delete", + _ => "overwrite", + } + ); + } + } + Ok(()) +} + +/// Refuse to materialize a vendor-bound quadlet that is shadowed by a higher- +/// precedence `/etc/containers/systemd/` definition on the running system. +/// Writing into `/run` would override the administrator's configuration. +fn check_host_etc_shadowing(rootfs: &Dir, changes: &[QuadletChange]) -> Result<()> { + for c in changes { + if !matches!(c.kind, ChangeKind::Added | ChangeKind::Updated) { + continue; + } + let Some(quadlet) = c.quadlet.as_deref() else { + continue; + }; + if !c + .spec + .path + .as_str() + .starts_with("usr/share/containers/systemd/") + { + continue; + } + let etc_path = Utf8Path::new(QUADLET_ETC_DIR).join(quadlet); + if rootfs.try_exists(&etc_path)? { + anyhow::bail!( + "Refusing to apply-live bound quadlet /{}: shadowed by /{etc_path} on the running system", + c.spec.path + ); + } + } + Ok(()) +} + +/// Implementation of `bootc apply-live bound-images`. +#[context("Applying bound images live")] +pub(crate) async fn apply_bound_images( + storage: &Storage, + booted: &BootedOstree<'_>, + opts: &ApplyLiveBoundImagesOpts, +) -> Result<()> { + let sysroot = booted.sysroot; + let booted_deployment = &booted.deployment; + let staged = sysroot.staged_deployment().ok_or_else(|| { + anyhow::anyhow!("No staged deployment; run `bootc upgrade` or `bootc switch` first") + })?; + ensure!( + staged.osname() == booted_deployment.osname(), + "Staged deployment is in a different stateroot" + ); + let booted_csum = booted_deployment.csum(); + let staged_csum = staged.csum(); + + let rootfs = &Dir::open_ambient_dir("/", cap_std::ambient_authority()).context("Opening /")?; + let run = + &Dir::open_ambient_dir("/run", cap_std::ambient_authority()).context("Opening /run")?; + let booted_root = crate::utils::deployment_fd(sysroot, booted_deployment)?; + let staged_root = crate::utils::deployment_fd(sysroot, &staged)?; + let sepolicy = crate::lsm::new_sepolicy_at(&booted_root)?; + + let live = scan_live_quadlets(rootfs)?; + let live_by_path: BTreeMap<&Utf8Path, &LiveQuadlet> = + live.iter().map(|q| (q.relpath.as_path(), q)).collect(); + let staged_specs = crate::boundimage::query_bound_image_specs(&staged_root)?; + let booted_specs = crate::boundimage::query_bound_image_specs(&booted_root)?; + let changes = compute_changes(&booted_specs, &staged_specs, &staged_root)?; + + // Overrides from a previous apply-live that aren't part of this one + // need to be removed so the system converges on the current staged + // definitions rather than a mix of the two. + let current_quadlets: BTreeSet<&Utf8Path> = changes + .iter() + .filter_map(|c| c.quadlet.as_deref()) + .collect(); + let stale: Vec<&LiveQuadlet> = live + .iter() + .filter(|q| !current_quadlets.contains(q.relpath.as_path())) + .collect(); + + // Scope check before the fast path so dry-run (and empty change sets) + // still reject staged deployments with unrelated file changes. + if booted_csum != staged_csum { + let diff = + ostree_ext::diff::diff(&sysroot.repo(), &booted_csum, &staged_csum, None::<&str>)?; + tracing::debug!("Diff booted -> staged: {diff}"); + check_diff_scope(&diff, &changes, &booted_root, &staged_root)?; + } + + let pending_stops = collect_pending_stops(run, &live)?; + let applied_csum = applied_checksum_from(run)?; + let materialize_done = stale.is_empty() + && materialization_complete( + &changes, + &live_by_path, + staged_csum.as_str(), + rootfs, + applied_csum.as_deref(), + &pending_stops, + )?; + + // Fast path: definitions are already on disk for this staged checksum. + // All that may be left is stopping units (after `--no-restart` or a + // failed stop) and/or restarting pending ones. Derived from + // `compute_changes` so unchanged bound quadlets (which never get a + // `/run` override) don't block it, and an empty change set with + // nothing pending doesn't spuriously report "already applied". + if materialize_done { + let pending_restarts: Vec<&LiveQuadlet> = live + .iter() + .filter(|q| q.pending && q.checksum == staged_csum) + .collect(); + if changes.is_empty() && pending_stops.is_empty() && pending_restarts.is_empty() { + println!("No bound image changes to apply"); + return Ok(()); + } + if pending_stops.is_empty() && pending_restarts.is_empty() { + println!("Bound images from staged deployment are already applied"); + return Ok(()); + } + for unit in &pending_stops { + println!("stop pending: {unit}"); + } + for q in &pending_restarts { + if let Some(unit) = q.unit.as_deref() { + println!("restart pending: {unit}"); + } + } + if opts.dry_run || opts.no_restart { + return Ok(()); + } + if !pending_stops.is_empty() { + stop_pending(rootfs, run, &live, &pending_stops, sepolicy.as_ref())?; + } + systemctl(&["daemon-reload"])?; + return restart_pending(rootfs, &live, sepolicy.as_ref()); + } + + if changes.is_empty() && stale.is_empty() { + println!("No bound image changes to apply"); + return Ok(()); + } + + for c in changes.iter() { + let unit = c.unit.as_deref().unwrap_or("(no unit)"); + println!("{}: {} ({unit})", c.kind, c.spec.image.image); + } + for q in stale.iter() { + println!("reverting stale override: {}", q.relpath); + } + + // Ownership / shadowing before any pull/write (and before dry-run returns) + // so conflicts are visible without mutating `/run`. + check_run_destinations(rootfs, &changes, &live_by_path)?; + check_host_etc_shadowing(rootfs, &changes)?; + + if opts.dry_run { + return Ok(()); + } + + tracing::info!( + message_id = APPLY_LIVE_JOURNAL_ID, + bootc.deployment.checksum = staged_csum.as_str(), + bootc.bound_images_changes = changes.len(), + "Applying bound image definitions live from staged deployment" + ); + + // Ensure all images referenced by the staged deployment are present. They + // normally were pulled when the deployment was staged, but this also acts + // as a retry if that failed. + let staged_images = staged_specs.iter().map(|s| s.image.clone()).collect(); + crate::boundimage::pull_images(storage, staged_images).await?; + + // Protect every bound image of the staged deployment from GC, not just + // the changed ones. Written first since it's the one piece of state + // that can't be recovered later by scanning `/run/containers/systemd/`. + write_protected_images(run, staged_specs.iter().map(|s| s.image.image.as_str()))?; + + // A quadlet moving between search dirs (e.g. vendor -> /etc override) + // produces an Added and a Removed change for the same `/run` destination; + // the Removed side must not delete what the Added side just wrote. + let materialized: BTreeSet<&Utf8Path> = changes + .iter() + .filter(|c| matches!(c.kind, ChangeKind::Added | ChangeKind::Updated)) + .filter_map(|c| c.quadlet.as_deref()) + .collect(); + + // Units that a Removed change (or its live override) would stop. When the + // same destination is also Added with matching already-live contents, + // those units must still be marked pending so a stop doesn't leave them down. + let units_stopped_by_removal: BTreeSet = changes + .iter() + .filter(|c| c.kind == ChangeKind::Removed) + .filter_map(|c| { + let quadlet = c.quadlet.as_deref()?; + live_by_path + .get(quadlet) + .and_then(|p| p.unit.clone()) + .or_else(|| c.unit.clone()) + }) + .collect(); + + // Materialize the definitions into /run, each carrying a marker. A + // definition may already be live from a previous apply, in which case + // its unit doesn't need a restart -- but the marker is still refreshed + // to the current checksum so status and the fast path above stay + // accurate. Rename stops (`old_unit` / previous live unit) are recorded + // on the marker so a later re-run still stops them after `--no-restart`. + let quadlet_run_dir = Utf8Path::new(QUADLET_RUN_DIR); + let filemode = rustix::fs::Mode::from_raw_mode(0o644); + let mut written: BTreeMap<&Utf8Path, bool> = BTreeMap::new(); + let mut stops = pending_stops; + for c in changes.iter() { + let Some(quadlet) = c.quadlet.as_deref() else { + continue; + }; + let dest = quadlet_run_dir.join(quadlet); + match c.kind { + ChangeKind::Added | ChangeKind::Updated => { + let already_live = live_by_path + .get(quadlet) + .is_some_and(|prev| !prev.pending && prev.contents == c.spec.contents); + // SAFETY: We know there's a parent + ensure_quadlet_dir(rootfs, dest.parent().unwrap(), sepolicy.as_ref())?; + let pending = c + .unit + .as_ref() + .is_some_and(|u| !already_live || units_stopped_by_removal.contains(u)); + // Prefer the previous *live* unit when it differs from the + // staged unit (a→b→c: stop b, not the booted a). Fall back to + // `old_unit` from compute_changes, then any recorded `stop=`. + let stop = live_by_path + .get(quadlet) + .and_then(|prev| { + let live_unit = prev.unit.as_ref()?; + if Some(live_unit) != c.unit.as_ref() { + Some(live_unit.clone()) + } else { + prev.stop.clone() + } + }) + .or_else(|| c.old_unit.clone()); + if let Some(unit) = stop.as_deref() { + stops.insert(unit.to_owned()); + } + // Also stop the booted-side rename target when it differs from + // both the live and staged units (covers a still running after b→c). + if let Some(old) = c.old_unit.as_deref() { + if Some(old) != stop.as_deref() && Some(old) != c.unit.as_deref() { + stops.insert(old.to_owned()); + } + } + crate::lsm::atomic_replace_labeled( + rootfs, + &dest, + filemode, + sepolicy.as_ref(), + |w| { + let marker = marker_line( + staged_csum.as_str(), + pending, + c.spec.image.image.as_str(), + stop.as_deref(), + ); + w.write_all(marker.as_bytes())?; + w.write_all(c.spec.contents.as_bytes())?; + Ok(()) + }, + )?; + written.insert(quadlet, pending); + } + ChangeKind::Removed => { + // Prefer the live unit (may differ after ServiceName= renames) + // over the booted definition's unit name. + let unit = live_by_path + .get(quadlet) + .and_then(|p| p.unit.as_deref()) + .or(c.unit.as_deref()); + if let Some(unit) = unit { + stops.insert(unit.to_owned()); + } + if !materialized.contains(quadlet) { + rootfs.remove_file_optional(&dest)?; + } + } + } + } + for q in stale.iter() { + // `stale` is disjoint from `materialized` by construction (it's + // exactly the live quadlets not touched by any change). + if let Some(unit) = q.unit.as_deref() { + stops.insert(unit.to_owned()); + } + rootfs.remove_file_optional(quadlet_run_dir.join(&q.relpath))?; + } + // Persist stops and applied-checksum before attempting stops so + // `--no-restart` and a failed `systemctl stop` remain retryable. + write_pending_stops(run, &stops)?; + write_applied_checksum(run, staged_csum.as_str())?; + + if opts.no_restart { + println!( + "Skipping unit stop/restart; re-run without --no-restart to apply pending unit changes" + ); + return Ok(()); + } + + // Re-scan so stop_pending can clear `stop=` from the markers we just wrote. + let live_after = scan_live_quadlets(rootfs)?; + stop_pending(rootfs, run, &live_after, &stops, sepolicy.as_ref())?; + systemctl(&["daemon-reload"])?; + // A stale override reverts to the booted deployment's definition if + // there is one. Start the *booted* unit name (ServiceName= may differ + // from the live override we just removed). + for q in stale.iter() { + let Some(booted_spec) = booted_specs + .iter() + .find(|s| quadlet_relpath(&s.path) == Some(q.relpath.as_path())) + else { + continue; + }; + let Some(unit) = quadlet_unit_name(&q.relpath, &booted_spec.contents)? else { + continue; + }; + systemctl(&["start", &unit])?; + } + + // Restart units for the definitions we just materialized, flipping + // their markers from pending to applied as each one succeeds. + let newly_written: Vec = changes + .iter() + .filter(|c| matches!(c.kind, ChangeKind::Added | ChangeKind::Updated)) + .filter_map(|c| { + let quadlet = c.quadlet.as_deref()?; + written + .get(quadlet) + .copied() + .unwrap_or(false) + .then(|| LiveQuadlet { + relpath: quadlet.to_owned(), + checksum: staged_csum.to_string(), + pending: true, + image: c.spec.image.image.clone(), + contents: c.spec.contents.clone(), + unit: c.unit.clone(), + stop: None, + }) + }) + .collect(); + restart_pending(rootfs, &newly_written, sepolicy.as_ref())?; + + println!( + "Applied {} bound image definition change(s) from staged deployment", + changes.len() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::boundimage::BoundImage; + + fn spec(path: &str, image: &str, contents: &str) -> BoundImageSpec { + BoundImageSpec { + image: BoundImage { + image: image.into(), + auth_file: None, + }, + path: path.into(), + contents: contents.into(), + } + } + + fn tempdir() -> Result { + Ok(cap_std_ext::cap_tempfile::TempDir::new( + cap_std::ambient_authority(), + )?) + } + + #[test] + fn test_quadlet_relpath() { + assert_eq!( + quadlet_relpath("usr/share/containers/systemd/foo.container".into()), + Some("foo.container".into()) + ); + assert_eq!( + quadlet_relpath("etc/containers/systemd/sub/foo.image".into()), + Some("sub/foo.image".into()) + ); + assert_eq!(quadlet_relpath("usr/lib/foo/foo.container".into()), None); + assert_eq!(quadlet_relpath("usr/share/containers/systemd".into()), None); + } + + #[test] + fn test_quadlet_unit_name() -> Result<()> { + let c = "[Container]\nImage=quay.io/foo/foo:latest\n"; + assert_eq!( + quadlet_unit_name("foo.container".into(), c)?.as_deref(), + Some("foo.service") + ); + assert_eq!( + quadlet_unit_name("sub/foo.container".into(), c)?.as_deref(), + Some("foo.service") + ); + let c = "[Container]\nImage=quay.io/foo/foo:latest\nServiceName=bar\n"; + assert_eq!( + quadlet_unit_name("foo.container".into(), c)?.as_deref(), + Some("bar.service") + ); + let c = "[Image]\nImage=quay.io/foo/foo:latest\n"; + assert_eq!(quadlet_unit_name("foo.image".into(), c)?, None); + Ok(()) + } + + #[test] + fn test_commit_path() { + assert_eq!( + commit_path("etc/containers/systemd/a.container".into()), + Utf8PathBuf::from("/usr/etc/containers/systemd/a.container") + ); + assert_eq!( + commit_path("usr/share/containers/systemd/a.container".into()), + Utf8PathBuf::from("/usr/share/containers/systemd/a.container") + ); + } + + #[test] + fn test_compute_changes() -> Result<()> { + let staged_root = &tempdir()?; + const Q: &str = "usr/share/containers/systemd"; + let v1 = "[Container]\nImage=quay.io/foo/foo:v1\n"; + let v2 = "[Container]\nImage=quay.io/foo/foo:v2\n"; + let bar = "[Image]\nImage=quay.io/foo/bar:latest\n"; + let unbound = "[Container]\nImage=quay.io/foo/unbound:latest\n"; + let gone = "[Container]\nImage=quay.io/foo/gone:latest\n"; + + let booted = [ + spec(&format!("{Q}/foo.container"), "quay.io/foo/foo:v1", v1), + spec(&format!("{Q}/bar.image"), "quay.io/foo/bar:latest", bar), + spec( + &format!("{Q}/unbound.container"), + "quay.io/foo/unbound:latest", + unbound, + ), + spec( + &format!("{Q}/gone.container"), + "quay.io/foo/gone:latest", + gone, + ), + ]; + let staged = [ + spec(&format!("{Q}/foo.container"), "quay.io/foo/foo:v2", v2), + spec(&format!("{Q}/bar.image"), "quay.io/foo/bar:latest", bar), + spec("usr/lib/misc/new.container", "quay.io/foo/new:latest", v1), + ]; + // The unbound quadlet still exists in the staged root + staged_root.create_dir_all(Q)?; + staged_root.write(format!("{Q}/unbound.container"), unbound)?; + + let changes = compute_changes(&booted, &staged, staged_root)?; + let summary: Vec<_> = changes + .iter() + .map(|c| { + ( + c.kind, + c.spec.path.as_str(), + c.quadlet.as_deref().map(|q| q.as_str()), + c.unit.as_deref(), + ) + }) + .collect(); + assert_eq!( + summary, + [ + (ChangeKind::Added, "usr/lib/misc/new.container", None, None), + ( + ChangeKind::Updated, + "usr/share/containers/systemd/foo.container", + Some("foo.container"), + Some("foo.service") + ), + ( + ChangeKind::Removed, + "usr/share/containers/systemd/gone.container", + Some("gone.container"), + Some("gone.service") + ), + ] + ); + Ok(()) + } + + #[test] + fn test_compute_changes_unit_rename() -> Result<()> { + // Same quadlet path in both, but `ServiceName=` changed: the old + // unit must still be reported so it gets stopped. + const Q: &str = "usr/share/containers/systemd"; + let staged_root = &tempdir()?; + let old = "[Container]\nImage=quay.io/foo/foo:v1\nServiceName=old\n"; + let new = "[Container]\nImage=quay.io/foo/foo:v2\nServiceName=new\n"; + let booted = [spec( + &format!("{Q}/foo.container"), + "quay.io/foo/foo:v1", + old, + )]; + let staged = [spec( + &format!("{Q}/foo.container"), + "quay.io/foo/foo:v2", + new, + )]; + let changes = compute_changes(&booted, &staged, staged_root)?; + assert_eq!(changes.len(), 1); + let c = &changes[0]; + assert_eq!(c.kind, ChangeKind::Updated); + assert_eq!(c.unit.as_deref(), Some("new.service")); + assert_eq!(c.old_unit.as_deref(), Some("old.service")); + + // Same scenario, but the unit name doesn't actually change + let new_same_name = "[Container]\nImage=quay.io/foo/foo:v2\nServiceName=old\n"; + let staged = [spec( + &format!("{Q}/foo.container"), + "quay.io/foo/foo:v2", + new_same_name, + )]; + let changes = compute_changes(&booted, &staged, staged_root)?; + assert_eq!(changes[0].old_unit, None); + Ok(()) + } + + #[test] + fn test_check_diff_scope() -> Result<()> { + let booted_root = &tempdir()?; + let staged_root = &tempdir()?; + let changes = vec![ + QuadletChange { + kind: ChangeKind::Updated, + spec: spec( + "usr/share/containers/systemd/foo.container", + "quay.io/foo/foo:v2", + "", + ), + quadlet: Some("foo.container".into()), + unit: Some("foo.service".into()), + old_unit: None, + }, + QuadletChange { + kind: ChangeKind::Added, + spec: spec("etc/containers/systemd/bar.image", "quay.io/foo/bar:v1", ""), + quadlet: Some("bar.image".into()), + unit: None, + old_unit: None, + }, + ]; + let check = + |diff: &FileTreeDiff| check_diff_scope(diff, &changes, booted_root, staged_root); + + let mut diff = FileTreeDiff::default(); + diff.changed_files + .insert("/usr/share/containers/systemd/foo.container".into()); + diff.added_files + .insert("/usr/etc/containers/systemd/bar.image".into()); + diff.added_files + .insert("/usr/lib/bootc/bound-images.d/bar.image".into()); + diff.added_dirs + .insert("/usr/lib/bootc/bound-images.d".into()); + check(&diff).unwrap(); + + // A bound-images.d symlink target changing isn't visible in the diff + // as a file change, but the removal of the old link is. + diff.removed_files + .insert("/usr/lib/bootc/bound-images.d/old.image".into()); + check(&diff).unwrap(); + + // Anything else is out of scope + diff.changed_files.insert("/usr/bin/bash".into()); + let e = check(&diff).unwrap_err().to_string(); + assert!(e.contains("/usr/bin/bash"), "{e}"); + assert!(!e.contains("foo.container"), "{e}"); + diff.changed_files.remove("/usr/bin/bash"); + + // A quadlet we're not going to apply (not bound) is also out of scope + diff.changed_files + .insert("/usr/share/containers/systemd/other.container".into()); + assert!(check(&diff).is_err()); + diff.changed_files + .remove("/usr/share/containers/systemd/other.container"); + + // As is a new subdirectory of quadlets, since the diff doesn't recurse into it + diff.added_dirs + .insert("/usr/share/containers/systemd/sub".into()); + assert!(check(&diff).is_err()); + diff.added_dirs.remove("/usr/share/containers/systemd/sub"); + + // The quadlet dir itself being added is fine when it only contains + // what we apply (the "first bound image" case)... + const Q: &str = "usr/etc/containers/systemd"; + staged_root.create_dir_all(Q)?; + staged_root.write(format!("{Q}/bar.image"), "")?; + diff.added_dirs.insert("/usr/etc/containers".into()); + diff.added_dirs.insert(format!("/{Q}")); + check(&diff).unwrap(); + // ...but not when it contains anything else, even nested + staged_root.create_dir_all(format!("{Q}/sub"))?; + staged_root.write(format!("{Q}/sub/other.container"), "")?; + let e = check(&diff).unwrap_err().to_string(); + assert!(e.contains(&format!("/{Q}/sub/other.container")), "{e}"); + staged_root.remove_dir_all(format!("{Q}/sub"))?; + check(&diff).unwrap(); + + // A symlink inside an added in-scope dir is also out of scope + // (walk_files must report non-directory entries, not only regular + // files -- otherwise a masked/unbound symlink would slip through). + staged_root.symlink_contents("/dev/null", format!("{Q}/masked.container"))?; + let e = check(&diff).unwrap_err().to_string(); + assert!(e.contains(&format!("/{Q}/masked.container")), "{e}"); + staged_root.remove_file(format!("{Q}/masked.container"))?; + check(&diff).unwrap(); + + // Likewise for a removed directory, which is checked in the booted root + booted_root.create_dir_all("usr/lib/bootc/bound-images.d")?; + booted_root.write("usr/lib/bootc/bound-images.d/old.image", "")?; + diff.removed_dirs.insert("/usr/lib/bootc".into()); + check(&diff).unwrap(); + booted_root.write("usr/lib/bootc/other", "")?; + let e = check(&diff).unwrap_err().to_string(); + assert!(e.contains("/usr/lib/bootc/other"), "{e}"); + booted_root.remove_file("usr/lib/bootc/other")?; + + // Error message is truncated + for i in 0..20 { + diff.added_files.insert(format!("/usr/bin/tool{i}")); + } + let e = check(&diff).unwrap_err().to_string(); + assert!(e.contains("and 10 more"), "{e}"); + Ok(()) + } + + #[test] + fn test_marker_roundtrip() { + let line = marker_line("abc123", true, "quay.io/foo/foo:v2", None); + assert_eq!( + line, + "# X-Bootc-Apply-Live: checksum=abc123 pending=true image=quay.io/foo/foo:v2\n" + ); + let contents = format!("{line}[Container]\nImage=quay.io/foo/foo:v2\n"); + let (checksum, pending, image, stop, rest) = parse_marker(&contents).unwrap(); + assert_eq!(checksum, "abc123"); + assert!(pending); + assert_eq!(image, "quay.io/foo/foo:v2"); + assert_eq!(stop, None); + assert_eq!(rest, "[Container]\nImage=quay.io/foo/foo:v2\n"); + + let with_stop = marker_line("abc123", true, "quay.io/foo/foo:v2", Some("old.service")); + assert!(with_stop.contains("stop=old.service")); + let contents = format!("{with_stop}[Container]\nImage=quay.io/foo/foo:v2\n"); + let (_, _, _, stop, _) = parse_marker(&contents).unwrap(); + assert_eq!(stop.as_deref(), Some("old.service")); + + // A quadlet bootc didn't write has no marker + assert_eq!( + parse_marker("[Container]\nImage=quay.io/foo/foo:v2\n"), + None + ); + // Likewise an empty file + assert_eq!(parse_marker(""), None); + } + + #[test] + fn test_scan_live_quadlets() -> Result<()> { + let rootfs = &tempdir()?; + rootfs.create_dir_all(QUADLET_RUN_DIR)?; + let body = "[Container]\nImage=quay.io/foo/foo:v2\n"; + rootfs.write( + format!("{QUADLET_RUN_DIR}/foo.container"), + format!( + "{}{body}", + marker_line("abc123", true, "quay.io/foo/foo:v2", Some("old.service")) + ), + )?; + // A sibling file bootc didn't write is ignored + rootfs.write( + format!("{QUADLET_RUN_DIR}/unrelated.container"), + "[Container]\nImage=quay.io/other/other:latest\n", + )?; + // A pull-only bound image (no search-dir quadlet) has nothing to + // scan here; that's covered by `write_protected_images` instead. + + // A unit masked by symlinking it to /dev/null must not break the + // scan (cap-std rejects resolving absolute symlink targets). + rootfs.symlink_contents("/dev/null", format!("{QUADLET_RUN_DIR}/masked.container"))?; + + let live = scan_live_quadlets(rootfs)?; + assert_eq!(live.len(), 1); + let q = &live[0]; + assert_eq!(q.relpath, Utf8PathBuf::from("foo.container")); + assert_eq!(q.checksum, "abc123"); + assert!(q.pending); + assert_eq!(q.image, "quay.io/foo/foo:v2"); + assert_eq!(q.contents, body); + assert_eq!(q.unit.as_deref(), Some("foo.service")); + assert_eq!(q.stop.as_deref(), Some("old.service")); + Ok(()) + } + + #[test] + fn test_pending_stops_roundtrip() -> Result<()> { + let run = &tempdir()?; + assert!(pending_stops_from(run)?.is_empty()); + let units = BTreeSet::from(["foo.service".to_string(), "bar.service".to_string()]); + write_pending_stops(run, &units)?; + assert_eq!(pending_stops_from(run)?, units); + write_pending_stops(run, &BTreeSet::new())?; + assert!(pending_stops_from(run)?.is_empty()); + assert!(!run.try_exists(PENDING_STOPS_LIST)?); + Ok(()) + } + + #[test] + fn test_materialization_complete() -> Result<()> { + let rootfs = &tempdir()?; + rootfs.create_dir_all(QUADLET_RUN_DIR)?; + const Q: &str = "usr/share/containers/systemd"; + let body = "[Container]\nImage=quay.io/foo/foo:v2\n"; + let changes = vec![QuadletChange { + kind: ChangeKind::Updated, + spec: spec(&format!("{Q}/foo.container"), "quay.io/foo/foo:v2", body), + quadlet: Some("foo.container".into()), + unit: Some("foo.service".into()), + old_unit: None, + }]; + let live_by_path = BTreeMap::new(); + let empty_stops = BTreeSet::new(); + assert!(!materialization_complete( + &changes, + &live_by_path, + "abc123", + rootfs, + None, + &empty_stops, + )?); + + rootfs.write( + format!("{QUADLET_RUN_DIR}/foo.container"), + format!( + "{}{body}", + marker_line("abc123", true, "quay.io/foo/foo:v2", None) + ), + )?; + let live = scan_live_quadlets(rootfs)?; + let live_by_path: BTreeMap<&Utf8Path, &LiveQuadlet> = + live.iter().map(|q| (q.relpath.as_path(), q)).collect(); + assert!(materialization_complete( + &changes, + &live_by_path, + "abc123", + rootfs, + None, + &empty_stops, + )?); + // Wrong checksum => not complete + assert!(!materialization_complete( + &changes, + &live_by_path, + "other", + rootfs, + None, + &empty_stops, + )?); + + // Removal-only: absent /run must not look complete without applied-checksum + // or a pending stop. + let removed = vec![QuadletChange { + kind: ChangeKind::Removed, + spec: spec(&format!("{Q}/gone.container"), "quay.io/foo/gone:v1", body), + quadlet: Some("gone.container".into()), + unit: Some("gone.service".into()), + old_unit: None, + }]; + let live_by_path = BTreeMap::new(); + assert!(!materialization_complete( + &removed, + &live_by_path, + "abc123", + rootfs, + None, + &empty_stops, + )?); + assert!(materialization_complete( + &removed, + &live_by_path, + "abc123", + rootfs, + Some("abc123"), + &empty_stops, + )?); + let pending = BTreeSet::from(["gone.service".to_string()]); + assert!(materialization_complete( + &removed, + &live_by_path, + "abc123", + rootfs, + None, + &pending, + )?); + + // Pull-only (no quadlet) requires applied-checksum + let pull_only = vec![QuadletChange { + kind: ChangeKind::Added, + spec: spec("usr/lib/misc/pull.container", "quay.io/foo/pull:v1", body), + quadlet: None, + unit: None, + old_unit: None, + }]; + assert!(!materialization_complete( + &pull_only, + &live_by_path, + "abc123", + rootfs, + None, + &empty_stops, + )?); + assert!(materialization_complete( + &pull_only, + &live_by_path, + "abc123", + rootfs, + Some("abc123"), + &empty_stops, + )?); + Ok(()) + } + + #[test] + fn test_check_run_destinations_removed() -> Result<()> { + let rootfs = &tempdir()?; + rootfs.create_dir_all(QUADLET_RUN_DIR)?; + rootfs.write( + format!("{QUADLET_RUN_DIR}/foo.container"), + "[Container]\nImage=admin\n", + )?; + let changes = vec![QuadletChange { + kind: ChangeKind::Removed, + spec: spec( + "usr/share/containers/systemd/foo.container", + "quay.io/foo/foo:v1", + "", + ), + quadlet: Some("foo.container".into()), + unit: Some("foo.service".into()), + old_unit: None, + }]; + let live_by_path = BTreeMap::new(); + let e = check_run_destinations(rootfs, &changes, &live_by_path) + .unwrap_err() + .to_string(); + assert!(e.contains("delete"), "{e}"); + assert!(e.contains("non-bootc"), "{e}"); + Ok(()) + } + + #[test] + fn test_check_host_etc_shadowing() -> Result<()> { + let rootfs = &tempdir()?; + rootfs.create_dir_all(QUADLET_ETC_DIR)?; + rootfs.write( + format!("{QUADLET_ETC_DIR}/foo.container"), + "[Container]\nImage=admin\n", + )?; + let changes = vec![QuadletChange { + kind: ChangeKind::Updated, + spec: spec( + "usr/share/containers/systemd/foo.container", + "quay.io/foo/foo:v2", + "[Container]\nImage=vendor\n", + ), + quadlet: Some("foo.container".into()), + unit: Some("foo.service".into()), + old_unit: None, + }]; + let e = check_host_etc_shadowing(rootfs, &changes) + .unwrap_err() + .to_string(); + assert!(e.contains("shadowed"), "{e}"); + // /etc-bound definitions are fine + let etc_bound = vec![QuadletChange { + kind: ChangeKind::Updated, + spec: spec( + "etc/containers/systemd/foo.container", + "quay.io/foo/foo:v2", + "[Container]\nImage=etc\n", + ), + quadlet: Some("foo.container".into()), + unit: Some("foo.service".into()), + old_unit: None, + }]; + check_host_etc_shadowing(rootfs, &etc_bound)?; + Ok(()) + } + + #[test] + fn test_applied_checksum_roundtrip() -> Result<()> { + let run = &tempdir()?; + assert_eq!(applied_checksum_from(run)?, None); + write_applied_checksum(run, "deadbeef")?; + assert_eq!(applied_checksum_from(run)?.as_deref(), Some("deadbeef")); + Ok(()) + } + + #[test] + fn test_protected_images_roundtrip() -> Result<()> { + let run = &tempdir()?; + assert_eq!(protected_images_from_host_at(run)?, Vec::::new()); + write_protected_images( + run, + ["quay.io/foo/foo:v2", "quay.io/foo/bar:latest"].into_iter(), + )?; + assert_eq!( + protected_images_from_host_at(run)?, + vec![ + "quay.io/foo/bar:latest".to_string(), + "quay.io/foo/foo:v2".to_string(), + ] + ); + // A later apply (e.g. a subsequent staged deployment) must not drop + // images still protected from an earlier one. + write_protected_images(run, ["quay.io/foo/baz:latest"].into_iter())?; + assert_eq!( + protected_images_from_host_at(run)?, + vec![ + "quay.io/foo/bar:latest".to_string(), + "quay.io/foo/baz:latest".to_string(), + "quay.io/foo/foo:v2".to_string(), + ] + ); + Ok(()) + } + + #[test] + fn test_ensure_quadlet_dir() -> Result<()> { + let rootfs = &tempdir()?; + rootfs.create_dir("run")?; + ensure_quadlet_dir(rootfs, "run/containers/systemd/sub/deeper".into(), None)?; + assert!(rootfs.is_dir("run/containers/systemd/sub/deeper")); + // Idempotent + ensure_quadlet_dir(rootfs, "run/containers/systemd/sub/deeper".into(), None)?; + Ok(()) + } +} diff --git a/crates/lib/src/boundimage.rs b/crates/lib/src/boundimage.rs index c99aeb6fbf..0f5f24c9c2 100644 --- a/crates/lib/src/boundimage.rs +++ b/crates/lib/src/boundimage.rs @@ -6,7 +6,7 @@ //! is considered ready. use anyhow::{Context, Result}; -use camino::Utf8Path; +use camino::{Utf8Path, Utf8PathBuf}; use cap_std_ext::cap_std::fs::Dir; use cap_std_ext::dirext::CapStdExtDirExt; use fn_error_context::context; @@ -25,12 +25,23 @@ pub(crate) const BOUND_IMAGE_DIR: &str = "usr/lib/bootc/bound-images.d"; /// /// In the future this may be extended to include e.g. certificates or /// other pull options. -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct BoundImage { pub(crate) image: String, pub(crate) auth_file: Option, } +/// A bound image definition together with the quadlet file that defines it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct BoundImageSpec { + pub(crate) image: BoundImage, + /// Path of the quadlet file relative to the root, e.g. + /// `usr/share/containers/systemd/foo.container`. + pub(crate) path: Utf8PathBuf, + /// The raw contents of the quadlet file. + pub(crate) contents: String, +} + #[derive(Debug, PartialEq, Eq)] pub(crate) struct ResolvedBoundImage { pub(crate) image: String, @@ -70,6 +81,39 @@ pub(crate) fn query_bound_images_for_deployment( #[context("Querying bound images")] pub(crate) fn query_bound_images(root: &Dir) -> Result> { + let specs = query_bound_image_specs(root)?; + Ok(specs.into_iter().map(|s| s.image).collect()) +} + +/// Lexically resolve the target of a symlink in `dir` (relative to the root) +/// to a root-relative path, without following any further symlinks. Like +/// `RESOLVE_IN_ROOT` (which is how the file is actually read), `..` at the +/// root stays at the root. +fn resolve_link_target(dir: &Utf8Path, target: &Utf8Path) -> Result { + let mut ret = Utf8PathBuf::new(); + let base = if target.is_absolute() { + Utf8Path::new("") + } else { + dir + }; + for component in base.components().chain(target.components()) { + use camino::Utf8Component::*; + match component { + RootDir | CurDir => {} + ParentDir => { + ret.pop(); + } + Normal(n) => ret.push(n), + Prefix(_) => anyhow::bail!("Unexpected path prefix: {target}"), + } + } + Ok(ret) +} + +/// Like [`query_bound_images`], but also returns the path and contents +/// of the quadlet file defining each image. +#[context("Querying bound image specs")] +pub(crate) fn query_bound_image_specs(root: &Dir) -> Result> { let spec_dir = BOUND_IMAGE_DIR; let Some(bound_images_dir) = root.open_dir_optional(spec_dir)? else { tracing::debug!("Missing {spec_dir}"); @@ -112,7 +156,19 @@ pub(crate) fn query_bound_images(root: &Dir) -> Result> { _ => anyhow::bail!("Invalid file extension: {file_name}"), }?; - bound_images.push(bound_image); + // Record where the quadlet actually lives, so callers can relate it + // to podman's quadlet search directories. + let target = bound_images_dir + .read_link_contents(file_name) + .with_context(|| format!("Reading link {path}"))?; + let target = Utf8PathBuf::try_from(target).context("Non-UTF8 symlink target")?; + let target = resolve_link_target(Utf8Path::new(spec_dir), &target)?; + + bound_images.push(BoundImageSpec { + image: bound_image, + path: target, + contents: file_contents, + }); } Ok(bound_images) @@ -294,6 +350,32 @@ mod tests { assert_eq!(images[0].image, "quay.io/bar/bar:latest"); assert_eq!(images[1].image, "quay.io/foo/foo:latest"); + // The specs should record the resolved quadlet path and contents + let mut specs = query_bound_image_specs(td).unwrap(); + specs.sort_by(|a, b| a.path.cmp(&b.path)); + assert_eq!(specs.len(), 2); + assert_eq!(specs[0].path, format!("{CONTAINER_IMAGE_DIR}/bar.image")); + assert_eq!(specs[0].image.image, "quay.io/bar/bar:latest"); + assert!(specs[0].contents.contains("quay.io/bar/bar:latest")); + assert_eq!(specs[1].path, format!("{CONTAINER_IMAGE_DIR}/foo.image")); + + // Relative symlinks are resolved relative to the bound images directory + td.symlink( + "../../../share/containers/systemd/bar.image", + format!("{BOUND_IMAGE_DIR}/relative.image"), + ) + .unwrap(); + let specs = query_bound_image_specs(td).unwrap(); + assert_eq!(specs.len(), 3); + assert!( + specs + .iter() + .all(|s| s.path == format!("{CONTAINER_IMAGE_DIR}/bar.image") + || s.path == format!("{CONTAINER_IMAGE_DIR}/foo.image")) + ); + td.remove_file(format!("{BOUND_IMAGE_DIR}/relative.image")) + .unwrap(); + // Invalid symlink should return an error td.symlink("./blah", format!("{BOUND_IMAGE_DIR}/blah.image")) .unwrap(); @@ -308,6 +390,28 @@ mod tests { Ok(()) } + #[test] + fn test_resolve_link_target() { + let d = Utf8Path::new("usr/lib/bootc/bound-images.d"); + assert_eq!( + resolve_link_target(d, "/usr/share/containers/systemd/foo.image".into()).unwrap(), + "usr/share/containers/systemd/foo.image" + ); + assert_eq!( + resolve_link_target(d, "../../../share/containers/systemd/foo.image".into()).unwrap(), + "usr/share/containers/systemd/foo.image" + ); + assert_eq!( + resolve_link_target(d, "./foo.image".into()).unwrap(), + "usr/lib/bootc/bound-images.d/foo.image" + ); + // Excess `..` is clamped at the root, matching RESOLVE_IN_ROOT + assert_eq!( + resolve_link_target(d, "../../../../../../foo.image".into()).unwrap(), + "foo.image" + ); + } + #[test] fn test_parse_spec_value() -> Result<()> { //should parse string with no % characters diff --git a/crates/lib/src/cli.rs b/crates/lib/src/cli.rs index ed784c4e59..4353ce91b1 100644 --- a/crates/lib/src/cli.rs +++ b/crates/lib/src/cli.rs @@ -312,6 +312,42 @@ pub(crate) enum SoftRebootMode { Auto, } +/// Options for `bootc apply-live bound-images` +#[derive(Debug, Parser, PartialEq, Eq)] +pub(crate) struct ApplyLiveBoundImagesOpts { + /// Write the definitions and ensure the images are present, but don't + /// reload systemd or restart any units. + #[clap(long)] + pub(crate) no_restart: bool, + + /// Print what would be done without changing the running system. + #[clap(long)] + pub(crate) dry_run: bool, +} + +/// Subcommands for applying staged content to the running system. +#[derive(Debug, clap::Subcommand, PartialEq, Eq)] +pub(crate) enum ApplyLiveOpts { + /// Apply the staged deployment's logically bound image definitions to the running system. + /// + /// The staged deployment must differ from the booted deployment only in + /// logically bound image definitions: the `/usr/lib/bootc/bound-images.d` + /// symlinks and the `.container` or `.image` files they reference. If + /// anything else changed, this command fails and a reboot is required. + /// + /// Changed quadlet files are written to `/run/containers/systemd/`, which + /// podman gives precedence over `/etc` and `/usr`, and the corresponding + /// units are restarted (added units are started, removed units stopped). + /// The images themselves are already present in the bootc-owned storage + /// shared by all deployments. + /// + /// Because `/run` is transient this state is discarded on reboot, at which + /// point the staged deployment's own content applies. The applied state is + /// visible in `bootc status`. + #[clap(alias = "lbi")] + BoundImages(ApplyLiveBoundImagesOpts), +} + /// Perform an status operation #[derive(Debug, Parser, PartialEq, Eq)] pub(crate) struct StatusOpts { @@ -1159,6 +1195,11 @@ pub(crate) enum Opt { /// Allows temporary package installation that will be discarded on reboot. #[clap(alias = "usroverlay")] UsrOverlay(UsrOverlayOpts), + /// Apply content from the staged deployment to the running system without a reboot. + /// + /// Stability: This interface is experimental and may change in the future. + #[clap(subcommand, hide = true)] + ApplyLive(ApplyLiveOpts), /// Install the running container to a target. /// /// Takes a container image and installs it to disk in a bootable format. @@ -1299,7 +1340,12 @@ fn has_soft_reboot_capability(deployment: Option<&crate::spec::BootEntry>) -> bo deployment.map(|d| d.soft_reboot_capable).unwrap_or(false) } -/// Prepare a soft reboot for the given deployment +/// Prepare a soft reboot for the given deployment. +/// +/// Does not clear live-applied `/run` state: soft-reboot without `--apply` +/// leaves containers running, and clearing protection early would allow GC +/// to prune in-use images. Clear at reboot handoff via +/// [`clear_live_state_before_reboot`] instead. #[context("Preparing soft reboot")] fn prepare_soft_reboot(sysroot: &SysrootLock, deployment: &ostree::Deployment) -> Result<()> { let cancellable = ostree::gio::Cancellable::NONE; @@ -1309,6 +1355,16 @@ fn prepare_soft_reboot(sysroot: &SysrootLock, deployment: &ostree::Deployment) - Ok(()) } +/// Clear live-applied bound-image state immediately before reboot when a +/// soft reboot was prepared. Soft reboot preserves `/run`, so overrides and +/// image protection must not leak into the target deployment. +fn clear_live_state_before_reboot(soft_reboot: Option) -> Result<()> { + if soft_reboot.is_some() { + crate::applylive::clear_state_from_host()?; + } + Ok(()) +} + /// Handle soft reboot based on the configured mode #[context("Handling soft reboot")] fn handle_soft_reboot( @@ -1442,6 +1498,7 @@ async fn apply_from_downloaded_ostree( handle_staged_soft_reboot(booted_ostree, opts.soft_reboot, &host)?; if opts.apply { + clear_live_state_before_reboot(opts.soft_reboot)?; crate::reboot::reboot()?; } return Ok(()); @@ -1627,6 +1684,7 @@ async fn upgrade( handle_staged_soft_reboot(booted_ostree, opts.soft_reboot, &host)?; if opts.apply { + clear_live_state_before_reboot(opts.soft_reboot)?; crate::reboot::reboot()?; } } else if booted_unchanged { @@ -1664,6 +1722,7 @@ async fn upgrade( } if opts.apply { + clear_live_state_before_reboot(opts.soft_reboot)?; crate::reboot::reboot()?; } } else { @@ -1842,6 +1901,7 @@ async fn switch_ostree( // `--apply` cannot be passed along with `--download-only` (handled by clap) // but for sanity nonetheless if opts.apply { + clear_live_state_before_reboot(opts.soft_reboot)?; crate::reboot::reboot()?; } @@ -2250,11 +2310,25 @@ async fn run_from_opt(opt: Opt) -> Result { Opt::Rollback(opts) => { rollback(&opts).await?; if opts.apply { + clear_live_state_before_reboot(opts.soft_reboot)?; crate::reboot::reboot()?; } Ok(()) } Opt::Edit(opts) => edit(opts).await, + Opt::ApplyLive(opts) => { + let storage = &get_storage().await?; + match storage.kind()? { + BootedStorageKind::Ostree(booted_ostree) => match opts { + ApplyLiveOpts::BoundImages(opts) => { + crate::applylive::apply_bound_images(storage, &booted_ostree, &opts).await + } + }, + BootedStorageKind::Composefs(_) => { + anyhow::bail!("apply-live is not yet supported on composefs systems") + } + } + } Opt::UsrOverlay(opts) => { use crate::store::Environment; let env = Environment::detect()?; diff --git a/crates/lib/src/deploy.rs b/crates/lib/src/deploy.rs index 1f19afc622..92dc27ec32 100644 --- a/crates/lib/src/deploy.rs +++ b/crates/lib/src/deploy.rs @@ -457,6 +457,16 @@ pub(crate) async fn prune_container_store(sysroot: &Storage) -> Result<()> { }); } } + // Images applied live via `bootc apply-live bound-images` must survive + // even if the deployment they came from is discarded before reboot. + all_bound_images.extend( + crate::applylive::protected_images_from_host()? + .into_iter() + .map(|image| crate::boundimage::BoundImage { + image, + auth_file: None, + }), + ); // Convert to a hashset of just the image names let image_names = HashSet::from_iter(all_bound_images.iter().map(|img| img.image.as_str())); let pruned = sysroot diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index 428ba21a21..73e0622bbd 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -35,6 +35,7 @@ //! - [`image`] - Image operations and queries //! - [`delta`] - oci-delta artifacts (`--from-delta`) //! - [`boundimage`] - Logically Bound Images (LBIs) +//! - [`applylive`] - Applying staged content to the running system (`bootc apply-live`) //! - [`podstorage`] - bootc-owned container storage (`/usr/lib/bootc/storage`) //! - [`podman`] - Podman command helpers //! @@ -65,6 +66,7 @@ //! - [`etc-merge`](../etc_merge/index.html) - `/etc` three-way merge mod aboot; +mod applylive; mod boot_artifact; mod bootc_composefs; pub(crate) mod bootc_kargs; diff --git a/crates/lib/src/spec.rs b/crates/lib/src/spec.rs index ab3428d312..4758d536a2 100644 --- a/crates/lib/src/spec.rs +++ b/crates/lib/src/spec.rs @@ -432,6 +432,42 @@ impl Display for FilesystemOverlay { } } +/// A logically bound image definition that was applied to the running +/// system by `bootc apply-live bound-images`. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct LiveBoundImage { + /// The container image reference + pub image: String, + /// The quadlet file written under `/run/containers/systemd`, if any + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub quadlet: Option, + /// The systemd unit generated from the quadlet, if any + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub unit: Option, + /// True if the unit has not yet been restarted with this definition + /// (e.g. `--no-restart` was used, or the restart failed). Re-running + /// `bootc apply-live bound-images` will retry it. + #[serde(skip_serializing_if = "std::ops::Not::not")] + #[serde(default)] + pub pending: bool, +} + +/// Logically bound image definitions applied to the running system without +/// a reboot. This state lives in `/run` and is discarded on reboot. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct LiveBoundImages { + /// The ostree commit checksum of the deployment the definitions came from + pub checksum: String, + /// The deployment serial + pub deploy_serial: u32, + /// The applied definitions + pub images: Vec, +} + /// The status of the host system #[derive(Debug, Clone, Serialize, Default, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(rename_all = "camelCase")] @@ -457,6 +493,12 @@ pub struct HostStatus { /// The state of the overlay mounted on /usr pub usr_overlay: Option, + /// Logically bound image definitions applied live to the booted + /// deployment via `bootc apply-live bound-images`. + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub live_bound_images: Option, + /// Set to true if the physical root (`/sysroot`) is on a read-only medium /// (e.g. a live ISO) and so cannot be mutated; commands that would change /// the system (upgrade, switch, etc.) are not available. diff --git a/crates/lib/src/status.rs b/crates/lib/src/status.rs index cae628a50d..d3477b7a50 100644 --- a/crates/lib/src/status.rs +++ b/crates/lib/src/status.rs @@ -428,6 +428,22 @@ pub(crate) fn get_status( .map(|d| d.unlocked()) .and_then(crate::spec::deployment_unlocked_state_to_usr_overlay); + // `deploy_serial` isn't recoverable from the quadlet markers alone + // (they only record the checksum), so fill it in here from whichever + // deployment the live-applied definitions came from. + let mut live_bound_images = crate::applylive::status_from_host()?; + if let Some(live) = live_bound_images.as_mut() { + let matching = deployments + .staged + .iter() + .chain(deployments.rollback.iter()) + .chain(deployments.other.iter()) + .find(|d| d.csum() == live.checksum); + if let Some(d) = matching { + live.deploy_serial = d.deployserial() as u32; + } + } + let mut host = Host::new(spec); host.status = HostStatus { staged, @@ -437,6 +453,7 @@ pub(crate) fn get_status( rollback_queued, ty, usr_overlay, + live_bound_images, // Set by callers that have storage context (e.g. get_host). read_only: false, }; @@ -732,6 +749,7 @@ fn human_render_slot( // Show /usr overlay status write_usr_overlay(&mut out, slot, host_status, prefix_len)?; + write_live_bound_images(&mut out, slot, host_status, prefix_len)?; if verbose { // Show additional information in verbose mode similar to rpm-ostree @@ -791,6 +809,26 @@ fn write_usr_overlay( Ok(()) } +/// Helper function to render live-applied bound images +fn write_live_bound_images( + mut out: impl Write, + slot: Option, + host_status: &crate::spec::HostStatus, + prefix_len: usize, +) -> Result<()> { + // Only the booted deployment can have live-applied content + if !matches!(slot, Some(Slot::Booted)) { + return Ok(()); + } + if let Some(live) = host_status.live_bound_images.as_ref() { + write_row_name(&mut out, "Live bound images", prefix_len)?; + let n = live.images.iter().filter(|i| i.quadlet.is_some()).count(); + let short = live.checksum.get(..12).unwrap_or(&live.checksum); + writeln!(out, "{n} definition(s) applied from {short}")?; + } + Ok(()) +} + /// Output a rendering of a non-container boot entry. fn human_render_slot_ostree( mut out: impl Write, @@ -820,6 +858,7 @@ fn human_render_slot_ostree( // Show /usr overlay status write_usr_overlay(&mut out, slot, host_status, prefix_len)?; + write_live_bound_images(&mut out, slot, host_status, prefix_len)?; if verbose { // Show additional information in verbose mode similar to rpm-ostree diff --git a/docs/src/bootc-logically-bound-images.7.md b/docs/src/bootc-logically-bound-images.7.md index b6d41e7a49..f5c7666790 100644 --- a/docs/src/bootc-logically-bound-images.7.md +++ b/docs/src/bootc-logically-bound-images.7.md @@ -58,6 +58,51 @@ Images are fetched using the global bootc pull secret by default (`/etc/ostree/a The bootc image store is owned by bootc; images will be garbage collected when they are no longer referenced by a file in `/usr/lib/bootc/bound-images.d`. +## Applying changes without a reboot (experimental) + +The container images for logically bound images live in a single bootc-owned +storage shared by all deployments, so after a `bootc upgrade` or `bootc switch` +the new images are already on disk. What is tied to a deployment is only the +*definition*: the symlink in `/usr/lib/bootc/bound-images.d` and the quadlet +file it references, which the running system reads from its own `/usr`. + +If the staged deployment differs from the booted one *only* in bound image +definitions, they can be applied to the running system without a reboot: + +``` +bootc apply-live bound-images +``` + +This writes the changed quadlet files to `/run/containers/systemd/`, which +podman gives precedence over `/etc` and `/usr`, reloads systemd and restarts +the affected `.container` units (removed units are stopped). Pass `--dry-run` +to see what would change, or `--no-restart` to write the definitions without +touching any units; a later plain invocation stops and restarts the units +still pending, as does re-running after a failed stop or restart. + +If the staged deployment contains any other change, the command fails and +lists the out-of-scope paths; a reboot is required to apply it. Note that a +locally modified quadlet in `/etc/containers/systemd` that also changed in +the image is reported this way, since the local copy wins in `/etc`. + +Because `/run` is transient, the override disappears on the next boot, at +which point the staged deployment's own content applies. (A soft reboot +preserves `/run`; the override is cleared when one is prepared.) The applied +state is shown in `bootc status`. Each quadlet written to +`/run/containers/systemd/` carries a leading marker comment (checksum, +pending restart, and any unit still needing a stop); a small flat list under +`/run/bootc/apply-live/` protects the staged images from garbage collection +until reboot, and records pending stops for pure removals, even if the staged +deployment is discarded (for example by `bootc rollback`). + +Limitations: only the `.container` or `.image` file referenced from +`bound-images.d` is applied. Quadlet drop-ins (`*.container.d/`) and other +referenced quadlet files (`.volume`, `.network`, `.pod`) are not copied, and +changes to them are treated as out of scope. A `.container` unit that +references a changed `.image` is not restarted unless its own file changed. +Removed units are stopped but not masked, so a dependency could start them +again until reboot. Only supported on ostree-backed systems. + ## Installation Logically bound images must be present in the default container store (`/var/lib/containers`) when invoking diff --git a/docs/src/host-v1.schema.json b/docs/src/host-v1.schema.json index 80bda0e8f6..efc546f422 100644 --- a/docs/src/host-v1.schema.json +++ b/docs/src/host-v1.schema.json @@ -323,6 +323,17 @@ } ] }, + "liveBoundImages": { + "description": "Logically bound image definitions applied live to the booted\ndeployment via `bootc apply-live bound-images`.", + "anyOf": [ + { + "$ref": "#/$defs/LiveBoundImages" + }, + { + "type": "null" + } + ] + }, "otherDeployments": { "description": "Other deployments (i.e. pinned)", "type": "array", @@ -491,6 +502,65 @@ "architecture" ] }, + "LiveBoundImage": { + "description": "A logically bound image definition that was applied to the running\nsystem by `bootc apply-live bound-images`.", + "type": "object", + "properties": { + "image": { + "description": "The container image reference", + "type": "string" + }, + "pending": { + "description": "True if the unit has not yet been restarted with this definition\n(e.g. `--no-restart` was used, or the restart failed). Re-running\n`bootc apply-live bound-images` will retry it.", + "type": "boolean" + }, + "quadlet": { + "description": "The quadlet file written under `/run/containers/systemd`, if any", + "type": [ + "string", + "null" + ] + }, + "unit": { + "description": "The systemd unit generated from the quadlet, if any", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "image" + ] + }, + "LiveBoundImages": { + "description": "Logically bound image definitions applied to the running system without\na reboot. This state lives in `/run` and is discarded on reboot.", + "type": "object", + "properties": { + "checksum": { + "description": "The ostree commit checksum of the deployment the definitions came from", + "type": "string" + }, + "deploySerial": { + "description": "The deployment serial", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "images": { + "description": "The applied definitions", + "type": "array", + "items": { + "$ref": "#/$defs/LiveBoundImage" + } + } + }, + "required": [ + "checksum", + "deploySerial", + "images" + ] + }, "ObjectMeta": { "type": "object", "properties": { diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index 1ce612e6ad..596f1d7c2d 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -332,6 +332,14 @@ execute: extra-try_bind_storage: true extra-skip_if_ostree: true +/plan-49-logically-bound-apply-live: + summary: Execute logically bound images tests for bootc apply-live bound-images + discover: + how: fmf + test: + - /tmt/tests/tests/test-49-logically-bound-apply-live + extra-fixme_skip_if_composefs: true + /plan-50-switch-zstd-chunked: summary: Switch to an image with zstd:chunked compressed layers discover: diff --git a/tmt/tests/booted/test-logically-bound-apply-live.nu b/tmt/tests/booted/test-logically-bound-apply-live.nu new file mode 100644 index 0000000000..5d24895c03 --- /dev/null +++ b/tmt/tests/booted/test-logically-bound-apply-live.nu @@ -0,0 +1,190 @@ +# number: 49 +# tmt: +# summary: Execute logically bound images tests for bootc apply-live bound-images +# duration: 30m +# extra: +# fixme_skip_if_composefs: true +# +# This test does: +# bootc switch to an image with a bound .container (v1) +# +# +# bootc upgrade to an image whose only change is the bound .container (v2) +# bootc apply-live bound-images --dry-run / --no-restart / (plain) +# +# bootc upgrade to an image with an unrelated change +# +# +# + +use std assert +use tap.nu + +# This code runs on *each* boot. +bootc status +let st = bootc status --json | from json +let booted = $st.status.booted.image + +# Two distinct tags of the same small image so we can tell which one a +# container was started from. +const image_v1 = "registry.access.redhat.com/ubi9/ubi-minimal:9.4" +const image_v2 = "registry.access.redhat.com/ubi9/ubi-minimal:9.3" +const quadlet = "/usr/share/containers/systemd/lbi-sleeper.container" +const run_quadlet = "/run/containers/systemd/lbi-sleeper.container" +const unit = "lbi-sleeper.service" +const protected_images_list = "/run/bootc/apply-live/protected-images.list" + +def initial_setup [] { + bootc image copy-to-storage + podman images + podman image inspect localhost/bootc | from json +} + +# Build a bootc image on top of the booted one with a single bound +# .container that just sleeps, referencing $image. If $extra is set, also +# add an unrelated file so the diff is out of scope for apply-live. +def build_image [name image extra] { + let td = mktemp -d + cd $td + mkdir usr/share/containers/systemd + $"[Container] +Image=($image) +Exec=sleep infinity +GlobalArgs=--storage-opt=additionalimagestore=/usr/lib/bootc/storage + +[Install] +WantedBy=multi-user.target +" | save usr/share/containers/systemd/lbi-sleeper.container + + mut dockerfile = "FROM localhost/bootc +COPY usr/ /usr/ +RUN ln -s /usr/share/containers/systemd/lbi-sleeper.container /usr/lib/bootc/bound-images.d/lbi-sleeper.container +" + if $extra { + $dockerfile = $dockerfile + "RUN echo unrelated > /usr/share/apply-live-out-of-scope.txt\n" + } + $dockerfile | save Dockerfile + podman build -t $name . +} + +# Return the image a running container was created from. The image lives in +# the bootc storage, so podman needs the same additional store the quadlet uses. +def running_image [] { + podman --storage-opt=additionalimagestore=/usr/lib/bootc/storage inspect systemd-lbi-sleeper --format '{{.ImageName}}' | str trim +} + +def first_boot [] { + tap begin "bootc apply-live bound-images" + initial_setup + build_image localhost/bootc-lbi-live $image_v1 false + bootc switch --transport containers-storage localhost/bootc-lbi-live + tmt-reboot +} + +def second_boot [] { + print "verifying second boot after switch" + assert equal $booted.image.image localhost/bootc-lbi-live + systemctl is-active $unit + assert equal (running_image) $image_v1 + assert not ($run_quadlet | path exists) + assert equal $st.status.liveBoundImages? null + + # Without a staged deployment there's nothing to apply + let r = do { bootc apply-live bound-images } | complete + assert not equal $r.exit_code 0 + assert ($r.stderr | str contains "No staged deployment") + + # Stage an image whose only change is the bound container's image + print "bootc upgrade to v2 of the bound image" + build_image localhost/bootc-lbi-live $image_v2 false + bootc upgrade + let st = bootc status --json | from json + assert not equal $st.status.staged null + let staged_checksum = $st.status.staged.ostree.checksum + + # Dry run changes nothing + let out = bootc apply-live bound-images --dry-run + print $out + assert ($out | str contains $"updated: ($image_v2) \(($unit)\)") + assert not ($run_quadlet | path exists) + assert equal (running_image) $image_v1 + + # --no-restart writes the definition but leaves the unit alone + bootc apply-live lbi --no-restart + assert ($run_quadlet | path exists) + assert (open $run_quadlet | str contains $image_v2) + assert equal (getfattr --only-values -n security.selinux $run_quadlet | split row ':' | get 2) container_var_run_t + assert equal (running_image) $image_v1 + let live = bootc status --json | from json | get status.liveBoundImages + assert equal $live.checksum $staged_checksum + let entry = $live.images | where unit? == $unit | first + assert equal $entry.pending true + assert (open $protected_images_list | str contains $image_v2) + + # A plain re-run restarts the pending unit + let out = bootc apply-live bound-images + print $out + assert ($out | str contains $"restart pending: ($unit)") + systemctl is-active $unit + assert equal (running_image) $image_v2 + let live = bootc status --json | from json | get status.liveBoundImages + let entry = $live.images | where unit? == $unit | first + assert equal ($entry.pending? | default false) false + let human = bootc status --format humanreadable + print $human + assert ($human | str contains "Live bound images") + + # And is idempotent afterwards + let out = bootc apply-live bound-images + assert ($out | str contains "already applied") + assert equal (running_image) $image_v2 + + # The live-applied images survive garbage collection even without the + # staged deployment: rollback discards it (then revert the queued rollback + # so we still boot the default), and cleanup prunes the store. + bootc rollback + bootc rollback + let st = bootc status --json | from json + assert equal $st.status.staged null + assert equal $st.status.rollbackQueued false + bootc internals cleanup + let names = podman --storage-opt=additionalimagestore=/usr/lib/bootc/storage images --format '{{.Repository}}:{{.Tag}}' | lines + assert ($image_v2 in $names) + assert equal (running_image) $image_v2 + + # A staged deployment with an unrelated change is refused + print "bootc upgrade with an out-of-scope change" + build_image localhost/bootc-lbi-live $image_v2 true + bootc upgrade + let r = do { bootc apply-live bound-images } | complete + print $r.stderr + assert not equal $r.exit_code 0 + assert ($r.stderr | str contains "reboot is required") + assert ($r.stderr | str contains "/usr/share/apply-live-out-of-scope.txt") + # The previously applied definition is still live + assert equal (running_image) $image_v2 + + tmt-reboot +} + +def third_boot [] { + print "verifying third boot after upgrade" + # /run is fresh: no override, no state, and the deployment's own + # definition (v2) is what runs. + assert not ($run_quadlet | path exists) + assert not ($protected_images_list | path exists) + assert equal $st.status.liveBoundImages? null + assert (open $quadlet | str contains $image_v2) + systemctl is-active $unit + assert equal (running_image) $image_v2 + tap ok +} + +def main [] { + match $env.TMT_REBOOT_COUNT? { + null | "0" => first_boot, + "1" => second_boot, + "2" => third_boot, + $o => { error make { msg: $"Invalid TMT_REBOOT_COUNT ($o)" } }, + } +} diff --git a/tmt/tests/tests.fmf b/tmt/tests/tests.fmf index c22858d18d..81015dd63e 100644 --- a/tmt/tests/tests.fmf +++ b/tmt/tests/tests.fmf @@ -207,6 +207,11 @@ check: enabled: true test: nu booted/test-49-composefs-1-16-bridge.nu +/test-49-logically-bound-apply-live: + summary: Execute logically bound images tests for bootc apply-live bound-images + duration: 30m + test: nu booted/test-logically-bound-apply-live.nu + /test-50-switch-zstd-chunked: summary: Switch to an image with zstd:chunked compressed layers duration: 30m