diff --git a/crates/lib/src/generator.rs b/crates/lib/src/generator.rs index dba8485b25..08cbc958ab 100644 --- a/crates/lib/src/generator.rs +++ b/crates/lib/src/generator.rs @@ -24,6 +24,7 @@ pub(crate) const BOOTC_EDITED_STAMP: &str = "Updated by bootc-fstab-edit.service const TRANSIENT_RELABEL_UNIT: &str = "bootc-early-overlay-relabel.service"; const SYSINIT_TARGET: &str = "sysinit.target"; const SHADOW_SYNC_UNIT: &str = "bootc-sysusers-shadow-sync.service"; +const EARLY_TMPFILES_UNIT: &str = "bootc-early-tmpfiles.service"; /// Called when the root is read-only composefs to reconcile /etc/fstab #[context("bootc generator")] @@ -168,7 +169,7 @@ pub(crate) fn generator(root: &Dir, unit_dir: &Dir) -> Result<()> { tracing::trace!("Enabled shadow sync: {updated}"); } - // === tmpfiles ordering fix: native composefs boots only === + // === Early /var/lib/systemd: native composefs boots only === // On ostree boots, ostree-remount.service provides // Before=systemd-tmpfiles-setup.service // Before=systemd-random-seed.service @@ -181,10 +182,13 @@ pub(crate) fn generator(root: &Dir, unit_dir: &Dir) -> Result<()> { // lint), /var/lib/systemd may not exist yet when random-seed starts, // causing an SELinux denial or ENOENT. // - // Fix: generate a drop-in that orders tmpfiles-setup before the services - // that need its output. + // Fix: create /var/lib/systemd from a unit with random-seed's own early + // ordering. Ordering all of systemd-tmpfiles-setup (which runs after + // local-fs.target) before random-seed instead creates an ordering cycle + // whenever local-fs.target needs random-seed, as with a random-key + // encrypted swap and tmp.mount (After=swap.target). if is_composefs && !is_ostree { - generate_tmpfiles_ordering(unit_dir)?; + generate_early_tmpfiles(unit_dir)?; } aboot_generator_impl(root, unit_dir)?; @@ -285,8 +289,8 @@ ExecStart=bootc internals fixup-etc-fstab\n\ Ok(()) } -/// On native composefs boots (no ostree-remount), ensure -/// systemd-tmpfiles-setup runs before services that need /var/lib/systemd. +/// On native composefs boots (no ostree-remount), ensure /var/lib/systemd +/// exists before services that need it. /// /// Without this, systemd-tmpfiles-setup and systemd-random-seed race: both /// are wanted by sysinit.target with no ordering between them. If the @@ -296,19 +300,22 @@ ExecStart=bootc internals fixup-etc-fstab\n\ /// /var/lib with an unlabeled_t SELinux context before policy loads, causing /// an AVC denial when random-seed tries to mkdir /var/lib/systemd. /// +/// The generated unit runs `systemd-tmpfiles` for just /var/lib/systemd, +/// ordered only after what random-seed itself is ordered after, so it can't +/// add an ordering cycle; see bootc-dev/bootc#2624. +/// /// On ostree boots, ostree-remount.service already provides this ordering. -fn generate_tmpfiles_ordering(unit_dir: &Dir) -> Result<()> { - let dropin_dir = "systemd-tmpfiles-setup.service.d"; - unit_dir.create_dir_all(dropin_dir)?; +fn generate_early_tmpfiles(unit_dir: &Dir) -> Result<()> { unit_dir.atomic_write( - &format!("{dropin_dir}/bootc-ordering.conf"), - "# Generated by bootc generator for native composefs boots.\n\ -# Ensure /var/lib/systemd and other tmpfiles-managed directories\n\ -# exist before services that depend on them.\n\ -[Unit]\n\ -Before=systemd-random-seed.service systemd-tpm2-setup.service\n", + EARLY_TMPFILES_UNIT, + include_str!("units/bootc-early-tmpfiles.service"), )?; - tracing::debug!("Generated tmpfiles-setup ordering drop-in for composefs boot"); + let wants = format!("{SYSINIT_TARGET}.wants"); + unit_dir.create_dir_all(&wants)?; + let link = format!("{wants}/{EARLY_TMPFILES_UNIT}"); + unit_dir.remove_file_optional(&link)?; + unit_dir.symlink(&format!("../{EARLY_TMPFILES_UNIT}"), &link)?; + tracing::debug!("Generated {EARLY_TMPFILES_UNIT} for composefs boot"); Ok(()) } @@ -637,30 +644,51 @@ UUID=715be2b7-c458-49f2-acec-b2fdb53d9089 / xfs defaults,ro 0 0 Ok(()) } + /// Values of every `key=` line in the unit, split on whitespace. + fn unit_values<'a>(unit: &'a str, key: &str) -> Vec<&'a str> { + unit.lines() + .filter_map(|l| l.strip_prefix(key)?.strip_prefix('=')) + .flat_map(str::split_ascii_whitespace) + .collect() + } + #[test] - fn test_tmpfiles_ordering_dropin() -> Result<()> { + fn test_early_tmpfiles_unit() -> Result<()> { let tempdir = fixture()?; let unit_dir = &tempdir.open_dir("run/systemd/system")?; - generate_tmpfiles_ordering(unit_dir)?; + // Twice, to check it is idempotent + for _ in 0..2 { + generate_early_tmpfiles(unit_dir)?; + } - let dropin_dir = unit_dir.open_dir("systemd-tmpfiles-setup.service.d")?; - let content = dropin_dir.read_to_string("bootc-ordering.conf")?; - assert!( - content.contains("\n[Unit]\n"), - "drop-in [Unit] header must be left-aligned: {content}" + let unit = unit_dir.read_to_string(EARLY_TMPFILES_UNIT)?; + assert_eq!( + unit_values(&unit, "ExecStart"), + ["systemd-tmpfiles", "--create", "--prefix=/var/lib/systemd"] ); - assert!( - content.contains("\nBefore=systemd-random-seed.service"), - "drop-in must order tmpfiles before random-seed: {content}" + assert_eq!( + unit_values(&unit, "Before"), + [ + "systemd-random-seed.service", + "systemd-tpm2-setup.service", + "sysinit.target", + "shutdown.target" + ] ); - assert!( - content.contains("systemd-tpm2-setup.service"), - "drop-in must order tmpfiles before tpm2-setup: {content}" + // Nothing beyond what systemd-random-seed.service is ordered + // after itself: in particular not local-fs.target, which + // random-seed may have to precede (bootc-dev/bootc#2624). + assert_eq!(unit_values(&unit, "DefaultDependencies"), ["no"]); + assert_eq!(unit_values(&unit, "After"), ["systemd-remount-fs.service"]); + assert_eq!( + unit_values(&unit, "RequiresMountsFor"), + ["/var/lib/systemd"] ); - // Calling again must succeed (idempotent) - generate_tmpfiles_ordering(unit_dir)?; + let link = unit_dir + .read_link_contents(format!("{SYSINIT_TARGET}.wants/{EARLY_TMPFILES_UNIT}"))?; + assert_eq!(link, std::path::Path::new("..").join(EARLY_TMPFILES_UNIT)); Ok(()) } diff --git a/crates/lib/src/units/bootc-early-tmpfiles.service b/crates/lib/src/units/bootc-early-tmpfiles.service new file mode 100644 index 0000000000..072a04bcde --- /dev/null +++ b/crates/lib/src/units/bootc-early-tmpfiles.service @@ -0,0 +1,21 @@ +[Unit] +Description=Create /var/lib/systemd for early boot services +Documentation=man:bootc(1) +DefaultDependencies=no +Conflicts=shutdown.target +# Only the ordering systemd-random-seed.service itself has, so this can +# run wherever it can. Notably not after local-fs.target, which +# random-seed may have to precede (e.g. for a random-key encrypted swap). +After=systemd-remount-fs.service +RequiresMountsFor=/var/lib/systemd +Before=systemd-random-seed.service systemd-tpm2-setup.service +Before=sysinit.target shutdown.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=systemd-tmpfiles --create --prefix=/var/lib/systemd +# As in systemd-tmpfiles-setup.service, so that an unrelated line under +# the prefix can't fail this unit; a failure to create /var/lib/systemd +# itself still shows up as systemd-random-seed.service failing. +SuccessExitStatus=DATAERR CANTCREAT diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index bc150a0b70..efeacf5063 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -428,4 +428,11 @@ execute: test: - /tmt/tests/tests/test-63-install-shared-propagation extra-fixme_skip_if_composefs: true + +/plan-66-encrypted-swap-ordering: + summary: Verify a random-key encrypted swap boots without an ordering cycle + discover: + how: fmf + test: + - /tmt/tests/tests/test-66-encrypted-swap-ordering # END GENERATED PLANS diff --git a/tmt/tests/booted/readonly/014-test-no-failed-units.nu b/tmt/tests/booted/readonly/014-test-no-failed-units.nu index 8a4ca04ae7..32f9ddcbb8 100644 --- a/tmt/tests/booted/readonly/014-test-no-failed-units.nu +++ b/tmt/tests/booted/readonly/014-test-no-failed-units.nu @@ -3,7 +3,7 @@ # This catches first-boot ordering issues such as the race between # systemd-tmpfiles-setup and systemd-random-seed/systemd-tpm2-setup # when /var/lib/systemd does not yet exist (the bootc generator -# emits a drop-in to prevent this). +# emits bootc-early-tmpfiles.service to prevent this). use std assert use tap.nu diff --git a/tmt/tests/booted/test-encrypted-swap-ordering.nu b/tmt/tests/booted/test-encrypted-swap-ordering.nu new file mode 100644 index 0000000000..611ecbcaf7 --- /dev/null +++ b/tmt/tests/booted/test-encrypted-swap-ordering.nu @@ -0,0 +1,75 @@ +# number: 66 +# tmt: +# summary: Verify a random-key encrypted swap boots without an ordering cycle +# duration: 30m +# +# Regression test for https://github.com/bootc-dev/bootc/issues/2624. +# A crypttab entry keyed from /dev/urandom orders its systemd-cryptsetup@ +# unit after systemd-random-seed.service, and tmp.mount is +# After=swap.target, so local-fs.target ends up after random-seed. +# Anything bootc orders before random-seed must therefore not wait for +# local-fs.target itself: bootc 1.17 ordered systemd-tmpfiles-setup.service +# (After=local-fs.target) before it on native composefs boots, and systemd +# broke the resulting cycle by dropping a job, which could leave /tmp +# mounted over what tmpfiles had just created. +# +# Boot 0 sets up such a swap on a file in /var and reboots; boot 1 checks +# that there was no ordering cycle and that the swap and /tmp came up in +# order. +use std assert +use tap.nu +use bootc_testlib.nu + +const SWAP_NAME = "bootc-test-swap" +const SWAP_FILE = "/var/bootc-test-swap.img" + +def initial_setup [] { + tap begin "random-key encrypted swap ordering" + + ^fallocate -l 64M $SWAP_FILE + $"($SWAP_NAME) ($SWAP_FILE) /dev/urandom swap,cipher=aes-xts-plain64,size=256\n" | save --append /etc/crypttab + $"/dev/mapper/($SWAP_NAME) none swap defaults 0 0\n" | save --append /etc/fstab + # tmp.mount (After=swap.target) closes the loop; most images enable it + # already, but make sure. + systemctl enable tmp.mount + + bootc_testlib reboot +} + +# A monotonic timestamp property of a unit, in microseconds +def timestamp [unit: string, prop: string] { + ^systemctl show -P $prop $unit | str trim | into int +} + +def second_boot [] { + let cycles = (^journalctl -b 0 -o cat --no-pager + | lines + | where { |l| $l | str contains "ordering cycle" }) + assert equal ($cycles | length) 0 $"Ordering cycles at boot:\n($cycles | str join "\n")" + + let swap_unit = (^systemd-escape --path --suffix=swap $"/dev/mapper/($SWAP_NAME)" | str trim) + let cryptsetup_unit = (^systemd-escape --template=systemd-cryptsetup@.service $SWAP_NAME | str trim) + mut units = [$cryptsetup_unit $swap_unit "tmp.mount" "systemd-random-seed.service" "systemd-tmpfiles-setup.service"] + if (tap is_composefs) { + $units = ($units | append "bootc-early-tmpfiles.service") + } + for unit in $units { + let state = (^systemctl show -P ActiveState $unit | str trim) + assert equal $state "active" $"($unit) is ($state)" + } + + # The symptom from the issue: tmpfiles must only start once /tmp is mounted + let tmp_mounted = (timestamp tmp.mount ActiveEnterTimestampMonotonic) + let tmpfiles_started = (timestamp systemd-tmpfiles-setup.service InactiveExitTimestampMonotonic) + assert ($tmp_mounted <= $tmpfiles_started) $"systemd-tmpfiles-setup started at ($tmpfiles_started), before tmp.mount at ($tmp_mounted)" + + tap ok +} + +def main [] { + match $env.TMT_REBOOT_COUNT? { + null | "0" => initial_setup, + "1" => second_boot, + $o => { error make { msg: $"Invalid TMT_REBOOT_COUNT ($o)" } }, + } +} diff --git a/tmt/tests/tests.fmf b/tmt/tests/tests.fmf index 7b7c804992..38094cd514 100644 --- a/tmt/tests/tests.fmf +++ b/tmt/tests/tests.fmf @@ -276,3 +276,8 @@ check: summary: Install from a shared mount namespace without leaking the storage bind mount duration: 30m test: nu booted/test-install-shared-propagation.nu + +/test-66-encrypted-swap-ordering: + summary: Verify a random-key encrypted swap boots without an ordering cycle + duration: 30m + test: nu booted/test-encrypted-swap-ordering.nu