From 3a1dc46da3d44eb6cc9b750d9142981d7cb64fb2 Mon Sep 17 00:00:00 2001 From: Joseph Marrero Date: Thu, 5 Nov 2026 10:44:04 +0000 Subject: [PATCH 1/2] generator: Create /var/lib/systemd early instead of reordering tmpfiles On native composefs boots, 41049cca ordered all of systemd-tmpfiles-setup.service before systemd-random-seed.service so that /var/lib/systemd exists when random-seed needs it. But tmpfiles-setup runs after local-fs.target, while random-seed is an early unit that local-fs.target may itself need: a crypttab swap keyed from /dev/urandom orders its systemd-cryptsetup@ unit after random-seed, and tmp.mount is After=swap.target. That closes an ordering cycle, which systemd breaks by deleting one of its jobs: depending on the boot, random-seed doesn't run at all, or (as in the report) tmpfiles runs before /tmp is mounted and GNOME can't start Xwayland. Instead, generate a small unit that runs systemd-tmpfiles for just /var/lib/systemd, ordered after only what random-seed is ordered after itself (systemd-remount-fs and the mounts for /var/lib/systemd), so it can't add a cycle that random-seed doesn't already have. Every tmpfiles.d line under /var/lib/systemd in Fedora and CentOS is root-owned, so it doesn't need systemd-sysusers first. Fixes: 41049ccada2b ("generator: Order tmpfiles before random seed and TPM setup") Closes: https://github.com/bootc-dev/bootc/issues/2624 Generated-by: AI Signed-off-by: Joseph Marrero Corchado --- crates/lib/src/generator.rs | 90 ++++++++++++------- .../src/units/bootc-early-tmpfiles.service | 21 +++++ .../readonly/014-test-no-failed-units.nu | 2 +- 3 files changed, 81 insertions(+), 32 deletions(-) create mode 100644 crates/lib/src/units/bootc-early-tmpfiles.service diff --git a/crates/lib/src/generator.rs b/crates/lib/src/generator.rs index dba8485b25..08cbc958ab 100644 --- a/crates/lib/src/generator.rs +++ b/crates/lib/src/generator.rs @@ -24,6 +24,7 @@ pub(crate) const BOOTC_EDITED_STAMP: &str = "Updated by bootc-fstab-edit.service const TRANSIENT_RELABEL_UNIT: &str = "bootc-early-overlay-relabel.service"; const SYSINIT_TARGET: &str = "sysinit.target"; const SHADOW_SYNC_UNIT: &str = "bootc-sysusers-shadow-sync.service"; +const EARLY_TMPFILES_UNIT: &str = "bootc-early-tmpfiles.service"; /// Called when the root is read-only composefs to reconcile /etc/fstab #[context("bootc generator")] @@ -168,7 +169,7 @@ pub(crate) fn generator(root: &Dir, unit_dir: &Dir) -> Result<()> { tracing::trace!("Enabled shadow sync: {updated}"); } - // === tmpfiles ordering fix: native composefs boots only === + // === Early /var/lib/systemd: native composefs boots only === // On ostree boots, ostree-remount.service provides // Before=systemd-tmpfiles-setup.service // Before=systemd-random-seed.service @@ -181,10 +182,13 @@ pub(crate) fn generator(root: &Dir, unit_dir: &Dir) -> Result<()> { // lint), /var/lib/systemd may not exist yet when random-seed starts, // causing an SELinux denial or ENOENT. // - // Fix: generate a drop-in that orders tmpfiles-setup before the services - // that need its output. + // Fix: create /var/lib/systemd from a unit with random-seed's own early + // ordering. Ordering all of systemd-tmpfiles-setup (which runs after + // local-fs.target) before random-seed instead creates an ordering cycle + // whenever local-fs.target needs random-seed, as with a random-key + // encrypted swap and tmp.mount (After=swap.target). if is_composefs && !is_ostree { - generate_tmpfiles_ordering(unit_dir)?; + generate_early_tmpfiles(unit_dir)?; } aboot_generator_impl(root, unit_dir)?; @@ -285,8 +289,8 @@ ExecStart=bootc internals fixup-etc-fstab\n\ Ok(()) } -/// On native composefs boots (no ostree-remount), ensure -/// systemd-tmpfiles-setup runs before services that need /var/lib/systemd. +/// On native composefs boots (no ostree-remount), ensure /var/lib/systemd +/// exists before services that need it. /// /// Without this, systemd-tmpfiles-setup and systemd-random-seed race: both /// are wanted by sysinit.target with no ordering between them. If the @@ -296,19 +300,22 @@ ExecStart=bootc internals fixup-etc-fstab\n\ /// /var/lib with an unlabeled_t SELinux context before policy loads, causing /// an AVC denial when random-seed tries to mkdir /var/lib/systemd. /// +/// The generated unit runs `systemd-tmpfiles` for just /var/lib/systemd, +/// ordered only after what random-seed itself is ordered after, so it can't +/// add an ordering cycle; see bootc-dev/bootc#2624. +/// /// On ostree boots, ostree-remount.service already provides this ordering. -fn generate_tmpfiles_ordering(unit_dir: &Dir) -> Result<()> { - let dropin_dir = "systemd-tmpfiles-setup.service.d"; - unit_dir.create_dir_all(dropin_dir)?; +fn generate_early_tmpfiles(unit_dir: &Dir) -> Result<()> { unit_dir.atomic_write( - &format!("{dropin_dir}/bootc-ordering.conf"), - "# Generated by bootc generator for native composefs boots.\n\ -# Ensure /var/lib/systemd and other tmpfiles-managed directories\n\ -# exist before services that depend on them.\n\ -[Unit]\n\ -Before=systemd-random-seed.service systemd-tpm2-setup.service\n", + EARLY_TMPFILES_UNIT, + include_str!("units/bootc-early-tmpfiles.service"), )?; - tracing::debug!("Generated tmpfiles-setup ordering drop-in for composefs boot"); + let wants = format!("{SYSINIT_TARGET}.wants"); + unit_dir.create_dir_all(&wants)?; + let link = format!("{wants}/{EARLY_TMPFILES_UNIT}"); + unit_dir.remove_file_optional(&link)?; + unit_dir.symlink(&format!("../{EARLY_TMPFILES_UNIT}"), &link)?; + tracing::debug!("Generated {EARLY_TMPFILES_UNIT} for composefs boot"); Ok(()) } @@ -637,30 +644,51 @@ UUID=715be2b7-c458-49f2-acec-b2fdb53d9089 / xfs defaults,ro 0 0 Ok(()) } + /// Values of every `key=` line in the unit, split on whitespace. + fn unit_values<'a>(unit: &'a str, key: &str) -> Vec<&'a str> { + unit.lines() + .filter_map(|l| l.strip_prefix(key)?.strip_prefix('=')) + .flat_map(str::split_ascii_whitespace) + .collect() + } + #[test] - fn test_tmpfiles_ordering_dropin() -> Result<()> { + fn test_early_tmpfiles_unit() -> Result<()> { let tempdir = fixture()?; let unit_dir = &tempdir.open_dir("run/systemd/system")?; - generate_tmpfiles_ordering(unit_dir)?; + // Twice, to check it is idempotent + for _ in 0..2 { + generate_early_tmpfiles(unit_dir)?; + } - let dropin_dir = unit_dir.open_dir("systemd-tmpfiles-setup.service.d")?; - let content = dropin_dir.read_to_string("bootc-ordering.conf")?; - assert!( - content.contains("\n[Unit]\n"), - "drop-in [Unit] header must be left-aligned: {content}" + let unit = unit_dir.read_to_string(EARLY_TMPFILES_UNIT)?; + assert_eq!( + unit_values(&unit, "ExecStart"), + ["systemd-tmpfiles", "--create", "--prefix=/var/lib/systemd"] ); - assert!( - content.contains("\nBefore=systemd-random-seed.service"), - "drop-in must order tmpfiles before random-seed: {content}" + assert_eq!( + unit_values(&unit, "Before"), + [ + "systemd-random-seed.service", + "systemd-tpm2-setup.service", + "sysinit.target", + "shutdown.target" + ] ); - assert!( - content.contains("systemd-tpm2-setup.service"), - "drop-in must order tmpfiles before tpm2-setup: {content}" + // Nothing beyond what systemd-random-seed.service is ordered + // after itself: in particular not local-fs.target, which + // random-seed may have to precede (bootc-dev/bootc#2624). + assert_eq!(unit_values(&unit, "DefaultDependencies"), ["no"]); + assert_eq!(unit_values(&unit, "After"), ["systemd-remount-fs.service"]); + assert_eq!( + unit_values(&unit, "RequiresMountsFor"), + ["/var/lib/systemd"] ); - // Calling again must succeed (idempotent) - generate_tmpfiles_ordering(unit_dir)?; + let link = unit_dir + .read_link_contents(format!("{SYSINIT_TARGET}.wants/{EARLY_TMPFILES_UNIT}"))?; + assert_eq!(link, std::path::Path::new("..").join(EARLY_TMPFILES_UNIT)); Ok(()) } diff --git a/crates/lib/src/units/bootc-early-tmpfiles.service b/crates/lib/src/units/bootc-early-tmpfiles.service new file mode 100644 index 0000000000..072a04bcde --- /dev/null +++ b/crates/lib/src/units/bootc-early-tmpfiles.service @@ -0,0 +1,21 @@ +[Unit] +Description=Create /var/lib/systemd for early boot services +Documentation=man:bootc(1) +DefaultDependencies=no +Conflicts=shutdown.target +# Only the ordering systemd-random-seed.service itself has, so this can +# run wherever it can. Notably not after local-fs.target, which +# random-seed may have to precede (e.g. for a random-key encrypted swap). +After=systemd-remount-fs.service +RequiresMountsFor=/var/lib/systemd +Before=systemd-random-seed.service systemd-tpm2-setup.service +Before=sysinit.target shutdown.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=systemd-tmpfiles --create --prefix=/var/lib/systemd +# As in systemd-tmpfiles-setup.service, so that an unrelated line under +# the prefix can't fail this unit; a failure to create /var/lib/systemd +# itself still shows up as systemd-random-seed.service failing. +SuccessExitStatus=DATAERR CANTCREAT diff --git a/tmt/tests/booted/readonly/014-test-no-failed-units.nu b/tmt/tests/booted/readonly/014-test-no-failed-units.nu index 8a4ca04ae7..32f9ddcbb8 100644 --- a/tmt/tests/booted/readonly/014-test-no-failed-units.nu +++ b/tmt/tests/booted/readonly/014-test-no-failed-units.nu @@ -3,7 +3,7 @@ # This catches first-boot ordering issues such as the race between # systemd-tmpfiles-setup and systemd-random-seed/systemd-tpm2-setup # when /var/lib/systemd does not yet exist (the bootc generator -# emits a drop-in to prevent this). +# emits bootc-early-tmpfiles.service to prevent this). use std assert use tap.nu From 7960a5d3fd1036b09de03a784666621b74a69dd1 Mon Sep 17 00:00:00 2001 From: Joseph Marrero Date: Thu, 5 Nov 2026 10:44:04 +0000 Subject: [PATCH 2/2] tests: Boot with a random-key encrypted swap Regression test for the ordering cycle in bootc-dev/bootc#2624: set up a crypttab swap keyed from /dev/urandom with tmp.mount enabled, reboot, and check that systemd found no ordering cycle and mounted /tmp before systemd-tmpfiles-setup started. Generated-by: AI Signed-off-by: Joseph Marrero Corchado --- tmt/plans/integration.fmf | 7 ++ .../booted/test-encrypted-swap-ordering.nu | 75 +++++++++++++++++++ tmt/tests/tests.fmf | 5 ++ 3 files changed, 87 insertions(+) create mode 100644 tmt/tests/booted/test-encrypted-swap-ordering.nu diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index bc150a0b70..efeacf5063 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -428,4 +428,11 @@ execute: test: - /tmt/tests/tests/test-63-install-shared-propagation extra-fixme_skip_if_composefs: true + +/plan-66-encrypted-swap-ordering: + summary: Verify a random-key encrypted swap boots without an ordering cycle + discover: + how: fmf + test: + - /tmt/tests/tests/test-66-encrypted-swap-ordering # END GENERATED PLANS diff --git a/tmt/tests/booted/test-encrypted-swap-ordering.nu b/tmt/tests/booted/test-encrypted-swap-ordering.nu new file mode 100644 index 0000000000..611ecbcaf7 --- /dev/null +++ b/tmt/tests/booted/test-encrypted-swap-ordering.nu @@ -0,0 +1,75 @@ +# number: 66 +# tmt: +# summary: Verify a random-key encrypted swap boots without an ordering cycle +# duration: 30m +# +# Regression test for https://github.com/bootc-dev/bootc/issues/2624. +# A crypttab entry keyed from /dev/urandom orders its systemd-cryptsetup@ +# unit after systemd-random-seed.service, and tmp.mount is +# After=swap.target, so local-fs.target ends up after random-seed. +# Anything bootc orders before random-seed must therefore not wait for +# local-fs.target itself: bootc 1.17 ordered systemd-tmpfiles-setup.service +# (After=local-fs.target) before it on native composefs boots, and systemd +# broke the resulting cycle by dropping a job, which could leave /tmp +# mounted over what tmpfiles had just created. +# +# Boot 0 sets up such a swap on a file in /var and reboots; boot 1 checks +# that there was no ordering cycle and that the swap and /tmp came up in +# order. +use std assert +use tap.nu +use bootc_testlib.nu + +const SWAP_NAME = "bootc-test-swap" +const SWAP_FILE = "/var/bootc-test-swap.img" + +def initial_setup [] { + tap begin "random-key encrypted swap ordering" + + ^fallocate -l 64M $SWAP_FILE + $"($SWAP_NAME) ($SWAP_FILE) /dev/urandom swap,cipher=aes-xts-plain64,size=256\n" | save --append /etc/crypttab + $"/dev/mapper/($SWAP_NAME) none swap defaults 0 0\n" | save --append /etc/fstab + # tmp.mount (After=swap.target) closes the loop; most images enable it + # already, but make sure. + systemctl enable tmp.mount + + bootc_testlib reboot +} + +# A monotonic timestamp property of a unit, in microseconds +def timestamp [unit: string, prop: string] { + ^systemctl show -P $prop $unit | str trim | into int +} + +def second_boot [] { + let cycles = (^journalctl -b 0 -o cat --no-pager + | lines + | where { |l| $l | str contains "ordering cycle" }) + assert equal ($cycles | length) 0 $"Ordering cycles at boot:\n($cycles | str join "\n")" + + let swap_unit = (^systemd-escape --path --suffix=swap $"/dev/mapper/($SWAP_NAME)" | str trim) + let cryptsetup_unit = (^systemd-escape --template=systemd-cryptsetup@.service $SWAP_NAME | str trim) + mut units = [$cryptsetup_unit $swap_unit "tmp.mount" "systemd-random-seed.service" "systemd-tmpfiles-setup.service"] + if (tap is_composefs) { + $units = ($units | append "bootc-early-tmpfiles.service") + } + for unit in $units { + let state = (^systemctl show -P ActiveState $unit | str trim) + assert equal $state "active" $"($unit) is ($state)" + } + + # The symptom from the issue: tmpfiles must only start once /tmp is mounted + let tmp_mounted = (timestamp tmp.mount ActiveEnterTimestampMonotonic) + let tmpfiles_started = (timestamp systemd-tmpfiles-setup.service InactiveExitTimestampMonotonic) + assert ($tmp_mounted <= $tmpfiles_started) $"systemd-tmpfiles-setup started at ($tmpfiles_started), before tmp.mount at ($tmp_mounted)" + + tap ok +} + +def main [] { + match $env.TMT_REBOOT_COUNT? { + null | "0" => initial_setup, + "1" => second_boot, + $o => { error make { msg: $"Invalid TMT_REBOOT_COUNT ($o)" } }, + } +} diff --git a/tmt/tests/tests.fmf b/tmt/tests/tests.fmf index 7b7c804992..38094cd514 100644 --- a/tmt/tests/tests.fmf +++ b/tmt/tests/tests.fmf @@ -276,3 +276,8 @@ check: summary: Install from a shared mount namespace without leaking the storage bind mount duration: 30m test: nu booted/test-install-shared-propagation.nu + +/test-66-encrypted-swap-ordering: + summary: Verify a random-key encrypted swap boots without an ordering cycle + duration: 30m + test: nu booted/test-encrypted-swap-ordering.nu