diff --git a/mac-app/Sources/CloudMachineCore/AgentRepair.swift b/mac-app/Sources/CloudMachineCore/AgentRepair.swift index 8f78053..e0b33f6 100644 --- a/mac-app/Sources/CloudMachineCore/AgentRepair.swift +++ b/mac-app/Sources/CloudMachineCore/AgentRepair.swift @@ -54,18 +54,64 @@ public enum AgentRepair { return result.stdout } - /// Agents that are loaded but cannot start. Not loaded = not listed: - /// installing agents is a setup step, not a repair. + /// Agents launchd will not run: loaded but unable to start, or - for the + /// agents that are safe to reload - installed (plist on disk) but not + /// loaded. The second case is what a failed reload leaves behind: on 12 Nov + /// 2026 `bootout` succeeded, every `bootstrap` failed, and `backup-health` + /// stayed unloaded for two days while this check said nothing. No plist = + /// never installed = a setup step, not a repair. `vm-start` is not reloaded + /// just for being unloaded: loading it starts the machine. public static func brokenAgents() async -> [String] { + await brokenAgents( + names: all, + plistExists: { FileManager.default.fileExists(atPath: plist($0).path) }, + lookup: { await lookup(label: $0) }) + } + + static func brokenAgents( + names: [String], + plistExists: (String) -> Bool, + lookup: (String) async -> JobLookup + ) async -> [String] { var broken: [String] = [] - for name in all { - if let output = await printOutput(label: name), cannotStart(printOutput: output) { - broken.append(name) + for name in names { + switch await lookup(name) { + case .loaded(let output): + if cannotStart(printOutput: output) { broken.append(name) } + case .notFound: + if safeToReload.contains(name) && plistExists(name) { broken.append(name) } + case .unknown: + break } } return broken } + /// What `launchctl print` said about one agent. Only launchd's own "not + /// found" means unloaded; a timeout or a session without a GUI domain (SSH, + /// sudo) is `unknown` and never a reason to reload. + enum JobLookup: Equatable { + case loaded(String) + case notFound + case unknown + } + + static func jobLookup(exitCode: Int32, stdout: String, stderr: String) -> JobLookup { + if exitCode == 0 { return .loaded(stdout) } + if exitCode == 113 && (stderr + stdout).contains("Could not find service") { + return .notFound + } + return .unknown + } + + static func lookup(label: String) async -> JobLookup { + guard + let result = try? await ProcessRunner.run( + "/bin/launchctl", ["print", "gui/\(getuid())/\(prefix)\(label)"], timeout: 15) + else { return .unknown } + return jobLookup(exitCode: result.exitCode, stdout: result.stdout, stderr: result.stderr) + } + // MARK: - Reloading static func plist(_ name: String) -> URL { @@ -101,7 +147,7 @@ public enum AgentRepair { public static func repairBroken() async -> [String] { var repaired: [String] = [] for name in await brokenAgents() { - CMLogger.log("launchd agent \(name) cannot start (spawn failed) - reloading") + CMLogger.log("launchd agent \(name) is not loaded or cannot start - reloading") if await reload(name) { repaired.append(name) } } return repaired diff --git a/mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift b/mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift index c8824a1..f696da7 100644 --- a/mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift @@ -63,6 +63,72 @@ final class AgentRepairTests: XCTestCase { XCTAssertTrue(Set(AgentRepair.all).isDisjoint(with: AgentRepair.retired)) } + /// 12 Nov 2026: after the 1.3.28 upgrade the reload did `bootout`, then + /// every `bootstrap` failed - `backup-health` stayed UNLOADED for two days, + /// while `brokenAgents` (which skipped agents that are not loaded) and + /// `vm-status` said "Agents: OK". A plist on disk means the agent was + /// installed; not being loaded is then a fault to repair. + func testInstalledButUnloadedAgentIsBroken() async { + let healthy = "\tstate = not running\n\tlast exit code = 0\n" + let broken = await AgentRepair.brokenAgents( + names: ["backup-health", "vm-guard"], + plistExists: { _ in true }, + lookup: { $0 == "vm-guard" ? .loaded(healthy) : .notFound }) + XCTAssertEqual(broken, ["backup-health"]) + } + + /// A `launchctl print` that timed out, or ran where there is no GUI domain + /// (SSH, sudo), says nothing about the agent. Reloading on that would + /// `bootout` a healthy agent - possibly the very `backup-health` running + /// the check, which launchd then kills before it can bootstrap itself. + func testUnknownLaunchdAnswerIsNotBroken() async { + let broken = await AgentRepair.brokenAgents( + names: ["backup-health", "vm-guard"], plistExists: { _ in true }, + lookup: { _ in .unknown }) + XCTAssertEqual(broken, []) + } + + /// Without a plist the agent was never installed - that is setup, not repair. + func testAgentThatWasNeverInstalledIsNotBroken() async { + let broken = await AgentRepair.brokenAgents( + names: ["backup-health"], plistExists: { _ in false }, lookup: { _ in .notFound }) + XCTAssertEqual(broken, []) + } + + /// `vm-start` is reloaded only when launchd cannot START it: loading it + /// starts the machine, which someone may have stopped on purpose. + func testUnloadedMachineStartIsNotReloaded() async { + let broken = await AgentRepair.brokenAgents( + names: ["vm-start"], plistExists: { _ in true }, lookup: { _ in .notFound }) + XCTAssertEqual(broken, []) + } + + func testSpawnFailedLoadedAgentIsStillBroken() async { + let failed = "\tstate = not running\n\tjob state = spawn failed\n" + let broken = await AgentRepair.brokenAgents( + names: ["vm-start"], plistExists: { _ in true }, lookup: { _ in .loaded(failed) }) + XCTAssertEqual(broken, ["vm-start"]) + } + + /// Copied from `launchctl print` on 14 Nov 2026 (macOS 27.2): an unknown + /// label exits 113 with "Could not find service" on stderr. + func testOnlyLaunchdsOwnNotFoundCountsAsUnloaded() { + XCTAssertEqual( + AgentRepair.jobLookup( + exitCode: 113, stdout: "", + stderr: + "Bad request.\nCould not find service \"com.renacode.cloudmachine.backup-health\" in domain for user gui: 501\n" + ), .notFound) + XCTAssertEqual( + AgentRepair.jobLookup(exitCode: 0, stdout: "\tstate = running\n", stderr: ""), + .loaded("\tstate = running\n")) + XCTAssertEqual( + AgentRepair.jobLookup( + exitCode: 125, stdout: "", stderr: "Domain does not support specified action"), + .unknown) + XCTAssertEqual(AgentRepair.jobLookup(exitCode: 1, stdout: "", stderr: ""), .unknown) + } + // MARK: - The retired layout /// Records what the migration did, without launchd. diff --git a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift index 99b37ee..1a05de6 100644 --- a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift @@ -222,6 +222,36 @@ final class VMLayoutTests: XCTestCase { } } + /// 12 and 13 Nov 2026: punchfs was (re)started and the bind ran before its + /// FUSE mount showed `bands` ("special device /srv/tm-punch/bands does not + /// exist"); the bind failed, and Samba, which Requires it, stayed down for + /// a day. The bind must wait for the mount instead of racing it. + func testBandsBindWaitsForThePunchfsMount() throws { + let unit = try String( + contentsOf: Self.repoRoot.appendingPathComponent("vm/systemd/tm-bands-bind.service"), + encoding: .utf8) + let pre = unit.split(separator: "\n").first { $0.hasPrefix("ExecStartPre=") } + XCTAssertNotNil(pre, "tm-bands-bind needs an ExecStartPre that waits for punchfs") + XCTAssertTrue(pre?.contains("mountpoint -q /srv/tm-punch") ?? false) + XCTAssertTrue(pre?.contains("/srv/tm-punch/bands") ?? false) + XCTAssertTrue(unit.contains("TimeoutStartSec=")) + } + + /// 13 Nov 2026 06:10: unattended-upgrades updated libc and needrestart + /// restarted punchfs and Samba in the middle of a backup. The backup stack + /// is restarted only by CloudMachine (vm-update) or at the next boot. + func testUpgradesDoNotRestartTheBackupStack() throws { + let vm = Self.repoRoot.appendingPathComponent("vm") + let yaml = try String(contentsOf: vm.appendingPathComponent("cmsmb.yaml"), encoding: .utf8) + XCTAssertTrue(yaml.contains("path: /etc/needrestart/conf.d/cloudmachine.conf")) + let conf = try String( + contentsOf: vm.appendingPathComponent("needrestart-cloudmachine.conf"), encoding: .utf8) + for unit in ["rclone-tm", "punchfs", "tm-bands-bind", "smbd", "nmbd", "smb-throttle"] { + XCTAssertTrue(conf.contains(unit), "needrestart may still restart \(unit)") + } + XCTAssertTrue(conf.contains("$nrconf{override_rc}")) + } + /// The rclone download is pinned to a version AND a checksum, checked /// before anything is installed. func testRcloneIsPinnedWithAChecksum() throws { diff --git a/vm/cmsmb.yaml b/vm/cmsmb.yaml index fe4c81c..2339d61 100644 --- a/vm/cmsmb.yaml +++ b/vm/cmsmb.yaml @@ -76,6 +76,9 @@ provision: - mode: data path: /etc/systemd/system/tm-meta-sync.timer file: systemd/tm-meta-sync.timer +- mode: data + path: /etc/needrestart/conf.d/cloudmachine.conf + file: needrestart-cloudmachine.conf - mode: system script: | #!/bin/bash diff --git a/vm/needrestart-cloudmachine.conf b/vm/needrestart-cloudmachine.conf new file mode 100644 index 0000000..44b94c5 --- /dev/null +++ b/vm/needrestart-cloudmachine.conf @@ -0,0 +1,8 @@ +# CloudMachine: unattended-upgrades must not restart the backup stack. +# +# 13 Nov 2026 06:10 a libc update made needrestart restart punchfs and Samba +# in the middle of a Time Machine backup; the bands bind then raced punchfs, +# failed, and Samba stayed down for a day. These units are restarted only by +# CloudMachine (`cloudmachine-agent vm-update`) or at the next boot of the +# machine, which picks up the updated libraries. +$nrconf{override_rc}->{qr(^(rclone-tm|punchfs|tm-bands-bind|smbd|nmbd|smb-throttle|tm-meta-sync)\.service$)} = 0; diff --git a/vm/systemd/tm-bands-bind.service b/vm/systemd/tm-bands-bind.service index 7383fa8..14eab4f 100644 --- a/vm/systemd/tm-bands-bind.service +++ b/vm/systemd/tm-bands-bind.service @@ -13,6 +13,12 @@ ConditionPathExists=/etc/cloudmachine/bundle.env Type=oneshot RemainAfterExit=yes EnvironmentFile=/etc/cloudmachine/bundle.env +# punchfs is Type=simple: systemd calls it started before its FUSE mount is up. +# Binding then fails ("special device /srv/tm-punch/bands does not exist") and +# Samba, which Requires this unit, stays down - it happened at boot on 12 Nov +# and after a restart on 13 Nov 2026, a day without backups. Wait for the mount. +TimeoutStartSec=150 +ExecStartPre=/usr/bin/timeout 120 /bin/sh -c 'until mountpoint -q /srv/tm-punch && [ -d /srv/tm-punch/bands ]; do sleep 1; done' ExecStart=/bin/mount --bind /srv/tm-punch/bands "/srv/tm-local/${CM_BUNDLE}/bands" ExecStop=/bin/umount -l "/srv/tm-local/${CM_BUNDLE}/bands"