Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 59 additions & 31 deletions crates/lib/src/generator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ pub(crate) const BOOTC_EDITED_STAMP: &str = "Updated by bootc-fstab-edit.service
const TRANSIENT_RELABEL_UNIT: &str = "bootc-early-overlay-relabel.service";
const SYSINIT_TARGET: &str = "sysinit.target";
const SHADOW_SYNC_UNIT: &str = "bootc-sysusers-shadow-sync.service";
const EARLY_TMPFILES_UNIT: &str = "bootc-early-tmpfiles.service";

/// Called when the root is read-only composefs to reconcile /etc/fstab
#[context("bootc generator")]
Expand Down Expand Up @@ -168,7 +169,7 @@ pub(crate) fn generator(root: &Dir, unit_dir: &Dir) -> Result<()> {
tracing::trace!("Enabled shadow sync: {updated}");
}

// === tmpfiles ordering fix: native composefs boots only ===
// === Early /var/lib/systemd: native composefs boots only ===
// On ostree boots, ostree-remount.service provides
// Before=systemd-tmpfiles-setup.service
// Before=systemd-random-seed.service
Expand All @@ -181,10 +182,13 @@ pub(crate) fn generator(root: &Dir, unit_dir: &Dir) -> Result<()> {
// lint), /var/lib/systemd may not exist yet when random-seed starts,
// causing an SELinux denial or ENOENT.
//
// Fix: generate a drop-in that orders tmpfiles-setup before the services
// that need its output.
// Fix: create /var/lib/systemd from a unit with random-seed's own early
// ordering. Ordering all of systemd-tmpfiles-setup (which runs after
// local-fs.target) before random-seed instead creates an ordering cycle
// whenever local-fs.target needs random-seed, as with a random-key
// encrypted swap and tmp.mount (After=swap.target).
if is_composefs && !is_ostree {
generate_tmpfiles_ordering(unit_dir)?;
generate_early_tmpfiles(unit_dir)?;
}

aboot_generator_impl(root, unit_dir)?;
Expand Down Expand Up @@ -285,8 +289,8 @@ ExecStart=bootc internals fixup-etc-fstab\n\
Ok(())
}

/// On native composefs boots (no ostree-remount), ensure
/// systemd-tmpfiles-setup runs before services that need /var/lib/systemd.
/// On native composefs boots (no ostree-remount), ensure /var/lib/systemd
/// exists before services that need it.
///
/// Without this, systemd-tmpfiles-setup and systemd-random-seed race: both
/// are wanted by sysinit.target with no ordering between them. If the
Expand All @@ -296,19 +300,22 @@ ExecStart=bootc internals fixup-etc-fstab\n\
/// /var/lib with an unlabeled_t SELinux context before policy loads, causing
/// an AVC denial when random-seed tries to mkdir /var/lib/systemd.
///
/// The generated unit runs `systemd-tmpfiles` for just /var/lib/systemd,
/// ordered only after what random-seed itself is ordered after, so it can't
/// add an ordering cycle; see bootc-dev/bootc#2624.
///
/// On ostree boots, ostree-remount.service already provides this ordering.
fn generate_tmpfiles_ordering(unit_dir: &Dir) -> Result<()> {
let dropin_dir = "systemd-tmpfiles-setup.service.d";
unit_dir.create_dir_all(dropin_dir)?;
fn generate_early_tmpfiles(unit_dir: &Dir) -> Result<()> {
unit_dir.atomic_write(
&format!("{dropin_dir}/bootc-ordering.conf"),
"# Generated by bootc generator for native composefs boots.\n\
# Ensure /var/lib/systemd and other tmpfiles-managed directories\n\
# exist before services that depend on them.\n\
[Unit]\n\
Before=systemd-random-seed.service systemd-tpm2-setup.service\n",
EARLY_TMPFILES_UNIT,
include_str!("units/bootc-early-tmpfiles.service"),
)?;
tracing::debug!("Generated tmpfiles-setup ordering drop-in for composefs boot");
let wants = format!("{SYSINIT_TARGET}.wants");
unit_dir.create_dir_all(&wants)?;
let link = format!("{wants}/{EARLY_TMPFILES_UNIT}");
unit_dir.remove_file_optional(&link)?;
unit_dir.symlink(&format!("../{EARLY_TMPFILES_UNIT}"), &link)?;
tracing::debug!("Generated {EARLY_TMPFILES_UNIT} for composefs boot");
Ok(())
}

Expand Down Expand Up @@ -637,30 +644,51 @@ UUID=715be2b7-c458-49f2-acec-b2fdb53d9089 / xfs defaults,ro 0 0
Ok(())
}

/// Values of every `key=` line in the unit, split on whitespace.
fn unit_values<'a>(unit: &'a str, key: &str) -> Vec<&'a str> {
unit.lines()
.filter_map(|l| l.strip_prefix(key)?.strip_prefix('='))
.flat_map(str::split_ascii_whitespace)
.collect()
}

#[test]
fn test_tmpfiles_ordering_dropin() -> Result<()> {
fn test_early_tmpfiles_unit() -> Result<()> {
let tempdir = fixture()?;
let unit_dir = &tempdir.open_dir("run/systemd/system")?;

generate_tmpfiles_ordering(unit_dir)?;
// Twice, to check it is idempotent
for _ in 0..2 {
generate_early_tmpfiles(unit_dir)?;
}

let dropin_dir = unit_dir.open_dir("systemd-tmpfiles-setup.service.d")?;
let content = dropin_dir.read_to_string("bootc-ordering.conf")?;
assert!(
content.contains("\n[Unit]\n"),
"drop-in [Unit] header must be left-aligned: {content}"
let unit = unit_dir.read_to_string(EARLY_TMPFILES_UNIT)?;
assert_eq!(
unit_values(&unit, "ExecStart"),
["systemd-tmpfiles", "--create", "--prefix=/var/lib/systemd"]
);
assert!(
content.contains("\nBefore=systemd-random-seed.service"),
"drop-in must order tmpfiles before random-seed: {content}"
assert_eq!(
unit_values(&unit, "Before"),
[
"systemd-random-seed.service",
"systemd-tpm2-setup.service",
"sysinit.target",
"shutdown.target"
]
);
assert!(
content.contains("systemd-tpm2-setup.service"),
"drop-in must order tmpfiles before tpm2-setup: {content}"
// Nothing beyond what systemd-random-seed.service is ordered
// after itself: in particular not local-fs.target, which
// random-seed may have to precede (bootc-dev/bootc#2624).
assert_eq!(unit_values(&unit, "DefaultDependencies"), ["no"]);
assert_eq!(unit_values(&unit, "After"), ["systemd-remount-fs.service"]);
assert_eq!(
unit_values(&unit, "RequiresMountsFor"),
["/var/lib/systemd"]
);

// Calling again must succeed (idempotent)
generate_tmpfiles_ordering(unit_dir)?;
let link = unit_dir
.read_link_contents(format!("{SYSINIT_TARGET}.wants/{EARLY_TMPFILES_UNIT}"))?;
assert_eq!(link, std::path::Path::new("..").join(EARLY_TMPFILES_UNIT));

Ok(())
}
Expand Down
21 changes: 21 additions & 0 deletions crates/lib/src/units/bootc-early-tmpfiles.service
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
[Unit]
Description=Create /var/lib/systemd for early boot services
Documentation=man:bootc(1)
DefaultDependencies=no
Conflicts=shutdown.target
# Only the ordering systemd-random-seed.service itself has, so this can
# run wherever it can. Notably not after local-fs.target, which
# random-seed may have to precede (e.g. for a random-key encrypted swap).
After=systemd-remount-fs.service
RequiresMountsFor=/var/lib/systemd
Before=systemd-random-seed.service systemd-tpm2-setup.service
Before=sysinit.target shutdown.target

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=systemd-tmpfiles --create --prefix=/var/lib/systemd
# As in systemd-tmpfiles-setup.service, so that an unrelated line under
# the prefix can't fail this unit; a failure to create /var/lib/systemd
# itself still shows up as systemd-random-seed.service failing.
SuccessExitStatus=DATAERR CANTCREAT
7 changes: 7 additions & 0 deletions tmt/plans/integration.fmf
Original file line number Diff line number Diff line change
Expand Up @@ -428,4 +428,11 @@ execute:
test:
- /tmt/tests/tests/test-63-install-shared-propagation
extra-fixme_skip_if_composefs: true

/plan-66-encrypted-swap-ordering:
summary: Verify a random-key encrypted swap boots without an ordering cycle
discover:
how: fmf
test:
- /tmt/tests/tests/test-66-encrypted-swap-ordering
# END GENERATED PLANS
2 changes: 1 addition & 1 deletion tmt/tests/booted/readonly/014-test-no-failed-units.nu
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# This catches first-boot ordering issues such as the race between
# systemd-tmpfiles-setup and systemd-random-seed/systemd-tpm2-setup
# when /var/lib/systemd does not yet exist (the bootc generator
# emits a drop-in to prevent this).
# emits bootc-early-tmpfiles.service to prevent this).
use std assert
use tap.nu

Expand Down
75 changes: 75 additions & 0 deletions tmt/tests/booted/test-encrypted-swap-ordering.nu
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# number: 66
# tmt:
# summary: Verify a random-key encrypted swap boots without an ordering cycle
# duration: 30m
#
# Regression test for https://github.com/bootc-dev/bootc/issues/2624.
# A crypttab entry keyed from /dev/urandom orders its systemd-cryptsetup@
# unit after systemd-random-seed.service, and tmp.mount is
# After=swap.target, so local-fs.target ends up after random-seed.
# Anything bootc orders before random-seed must therefore not wait for
# local-fs.target itself: bootc 1.17 ordered systemd-tmpfiles-setup.service
# (After=local-fs.target) before it on native composefs boots, and systemd
# broke the resulting cycle by dropping a job, which could leave /tmp
# mounted over what tmpfiles had just created.
#
# Boot 0 sets up such a swap on a file in /var and reboots; boot 1 checks
# that there was no ordering cycle and that the swap and /tmp came up in
# order.
use std assert
use tap.nu
use bootc_testlib.nu

const SWAP_NAME = "bootc-test-swap"
const SWAP_FILE = "/var/bootc-test-swap.img"

def initial_setup [] {
tap begin "random-key encrypted swap ordering"

^fallocate -l 64M $SWAP_FILE
$"($SWAP_NAME) ($SWAP_FILE) /dev/urandom swap,cipher=aes-xts-plain64,size=256\n" | save --append /etc/crypttab
$"/dev/mapper/($SWAP_NAME) none swap defaults 0 0\n" | save --append /etc/fstab
# tmp.mount (After=swap.target) closes the loop; most images enable it
# already, but make sure.
systemctl enable tmp.mount

bootc_testlib reboot
}

# A monotonic timestamp property of a unit, in microseconds
def timestamp [unit: string, prop: string] {
^systemctl show -P $prop $unit | str trim | into int
}

def second_boot [] {
let cycles = (^journalctl -b 0 -o cat --no-pager
| lines
| where { |l| $l | str contains "ordering cycle" })
assert equal ($cycles | length) 0 $"Ordering cycles at boot:\n($cycles | str join "\n")"

let swap_unit = (^systemd-escape --path --suffix=swap $"/dev/mapper/($SWAP_NAME)" | str trim)
let cryptsetup_unit = (^systemd-escape --template=systemd-cryptsetup@.service $SWAP_NAME | str trim)
mut units = [$cryptsetup_unit $swap_unit "tmp.mount" "systemd-random-seed.service" "systemd-tmpfiles-setup.service"]
if (tap is_composefs) {
$units = ($units | append "bootc-early-tmpfiles.service")
}
for unit in $units {
let state = (^systemctl show -P ActiveState $unit | str trim)
assert equal $state "active" $"($unit) is ($state)"
}

# The symptom from the issue: tmpfiles must only start once /tmp is mounted
let tmp_mounted = (timestamp tmp.mount ActiveEnterTimestampMonotonic)
let tmpfiles_started = (timestamp systemd-tmpfiles-setup.service InactiveExitTimestampMonotonic)
assert ($tmp_mounted <= $tmpfiles_started) $"systemd-tmpfiles-setup started at ($tmpfiles_started), before tmp.mount at ($tmp_mounted)"

tap ok
}

def main [] {
match $env.TMT_REBOOT_COUNT? {
null | "0" => initial_setup,
"1" => second_boot,
$o => { error make { msg: $"Invalid TMT_REBOOT_COUNT ($o)" } },
}
}
5 changes: 5 additions & 0 deletions tmt/tests/tests.fmf
Original file line number Diff line number Diff line change
Expand Up @@ -276,3 +276,8 @@ check:
summary: Install from a shared mount namespace without leaking the storage bind mount
duration: 30m
test: nu booted/test-install-shared-propagation.nu

/test-66-encrypted-swap-ordering:
summary: Verify a random-key encrypted swap boots without an ordering cycle
duration: 30m
test: nu booted/test-encrypted-swap-ordering.nu
Loading