Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 52 additions & 6 deletions mac-app/Sources/CloudMachineCore/AgentRepair.swift
Original file line number Diff line number Diff line change
Expand Up @@ -54,18 +54,64 @@ public enum AgentRepair {
return result.stdout
}

/// Agents that are loaded but cannot start. Not loaded = not listed:
/// installing agents is a setup step, not a repair.
/// Agents launchd will not run: loaded but unable to start, or - for the
/// agents that are safe to reload - installed (plist on disk) but not
/// loaded. The second case is what a failed reload leaves behind: on 12 Nov
/// 2026 `bootout` succeeded, every `bootstrap` failed, and `backup-health`
/// stayed unloaded for two days while this check said nothing. No plist =
/// never installed = a setup step, not a repair. `vm-start` is not reloaded
/// just for being unloaded: loading it starts the machine.
public static func brokenAgents() async -> [String] {
await brokenAgents(
names: all,
plistExists: { FileManager.default.fileExists(atPath: plist($0).path) },
lookup: { await lookup(label: $0) })
}

static func brokenAgents(
names: [String],
plistExists: (String) -> Bool,
lookup: (String) async -> JobLookup
) async -> [String] {
var broken: [String] = []
for name in all {
if let output = await printOutput(label: name), cannotStart(printOutput: output) {
broken.append(name)
for name in names {
switch await lookup(name) {
case .loaded(let output):
if cannotStart(printOutput: output) { broken.append(name) }
case .notFound:
if safeToReload.contains(name) && plistExists(name) { broken.append(name) }
case .unknown:
break
}
}
return broken
}

/// What `launchctl print` said about one agent. Only launchd's own "not
/// found" means unloaded; a timeout or a session without a GUI domain (SSH,
/// sudo) is `unknown` and never a reason to reload.
enum JobLookup: Equatable {
case loaded(String)
case notFound
case unknown
}

static func jobLookup(exitCode: Int32, stdout: String, stderr: String) -> JobLookup {
if exitCode == 0 { return .loaded(stdout) }
if exitCode == 113 && (stderr + stdout).contains("Could not find service") {
return .notFound
}
return .unknown
}

static func lookup(label: String) async -> JobLookup {
guard
let result = try? await ProcessRunner.run(
"/bin/launchctl", ["print", "gui/\(getuid())/\(prefix)\(label)"], timeout: 15)
else { return .unknown }
return jobLookup(exitCode: result.exitCode, stdout: result.stdout, stderr: result.stderr)
}

// MARK: - Reloading

static func plist(_ name: String) -> URL {
Expand Down Expand Up @@ -101,7 +147,7 @@ public enum AgentRepair {
public static func repairBroken() async -> [String] {
var repaired: [String] = []
for name in await brokenAgents() {
CMLogger.log("launchd agent \(name) cannot start (spawn failed) - reloading")
CMLogger.log("launchd agent \(name) is not loaded or cannot start - reloading")
if await reload(name) { repaired.append(name) }
}
return repaired
Expand Down
66 changes: 66 additions & 0 deletions mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,72 @@ final class AgentRepairTests: XCTestCase {
XCTAssertTrue(Set(AgentRepair.all).isDisjoint(with: AgentRepair.retired))
}

/// 12 Nov 2026: after the 1.3.28 upgrade the reload did `bootout`, then
/// every `bootstrap` failed - `backup-health` stayed UNLOADED for two days,
/// while `brokenAgents` (which skipped agents that are not loaded) and
/// `vm-status` said "Agents: OK". A plist on disk means the agent was
/// installed; not being loaded is then a fault to repair.
func testInstalledButUnloadedAgentIsBroken() async {
let healthy = "\tstate = not running\n\tlast exit code = 0\n"
let broken = await AgentRepair.brokenAgents(
names: ["backup-health", "vm-guard"],
plistExists: { _ in true },
lookup: { $0 == "vm-guard" ? .loaded(healthy) : .notFound })
XCTAssertEqual(broken, ["backup-health"])
}

/// A `launchctl print` that timed out, or ran where there is no GUI domain
/// (SSH, sudo), says nothing about the agent. Reloading on that would
/// `bootout` a healthy agent - possibly the very `backup-health` running
/// the check, which launchd then kills before it can bootstrap itself.
func testUnknownLaunchdAnswerIsNotBroken() async {
let broken = await AgentRepair.brokenAgents(
names: ["backup-health", "vm-guard"], plistExists: { _ in true },
lookup: { _ in .unknown })
XCTAssertEqual(broken, [])
}

/// Without a plist the agent was never installed - that is setup, not repair.
func testAgentThatWasNeverInstalledIsNotBroken() async {
let broken = await AgentRepair.brokenAgents(
names: ["backup-health"], plistExists: { _ in false }, lookup: { _ in .notFound })
XCTAssertEqual(broken, [])
}

/// `vm-start` is reloaded only when launchd cannot START it: loading it
/// starts the machine, which someone may have stopped on purpose.
func testUnloadedMachineStartIsNotReloaded() async {
let broken = await AgentRepair.brokenAgents(
names: ["vm-start"], plistExists: { _ in true }, lookup: { _ in .notFound })
XCTAssertEqual(broken, [])
}

func testSpawnFailedLoadedAgentIsStillBroken() async {
let failed = "\tstate = not running\n\tjob state = spawn failed\n"
let broken = await AgentRepair.brokenAgents(
names: ["vm-start"], plistExists: { _ in true }, lookup: { _ in .loaded(failed) })
XCTAssertEqual(broken, ["vm-start"])
}

/// Copied from `launchctl print` on 14 Nov 2026 (macOS 27.2): an unknown
/// label exits 113 with "Could not find service" on stderr.
func testOnlyLaunchdsOwnNotFoundCountsAsUnloaded() {
XCTAssertEqual(
AgentRepair.jobLookup(
exitCode: 113, stdout: "",
stderr:
"Bad request.\nCould not find service \"com.renacode.cloudmachine.backup-health\" in domain for user gui: 501\n"
), .notFound)
XCTAssertEqual(
AgentRepair.jobLookup(exitCode: 0, stdout: "\tstate = running\n", stderr: ""),
.loaded("\tstate = running\n"))
XCTAssertEqual(
AgentRepair.jobLookup(
exitCode: 125, stdout: "", stderr: "Domain does not support specified action"),
.unknown)
XCTAssertEqual(AgentRepair.jobLookup(exitCode: 1, stdout: "", stderr: ""), .unknown)
}

// MARK: - The retired layout

/// Records what the migration did, without launchd.
Expand Down
30 changes: 30 additions & 0 deletions mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,36 @@ final class VMLayoutTests: XCTestCase {
}
}

/// 12 and 13 Nov 2026: punchfs was (re)started and the bind ran before its
/// FUSE mount showed `bands` ("special device /srv/tm-punch/bands does not
/// exist"); the bind failed, and Samba, which Requires it, stayed down for
/// a day. The bind must wait for the mount instead of racing it.
func testBandsBindWaitsForThePunchfsMount() throws {
let unit = try String(
contentsOf: Self.repoRoot.appendingPathComponent("vm/systemd/tm-bands-bind.service"),
encoding: .utf8)
let pre = unit.split(separator: "\n").first { $0.hasPrefix("ExecStartPre=") }
XCTAssertNotNil(pre, "tm-bands-bind needs an ExecStartPre that waits for punchfs")
XCTAssertTrue(pre?.contains("mountpoint -q /srv/tm-punch") ?? false)
XCTAssertTrue(pre?.contains("/srv/tm-punch/bands") ?? false)
XCTAssertTrue(unit.contains("TimeoutStartSec="))
}

/// 13 Nov 2026 06:10: unattended-upgrades updated libc and needrestart
/// restarted punchfs and Samba in the middle of a backup. The backup stack
/// is restarted only by CloudMachine (vm-update) or at the next boot.
func testUpgradesDoNotRestartTheBackupStack() throws {
let vm = Self.repoRoot.appendingPathComponent("vm")
let yaml = try String(contentsOf: vm.appendingPathComponent("cmsmb.yaml"), encoding: .utf8)
XCTAssertTrue(yaml.contains("path: /etc/needrestart/conf.d/cloudmachine.conf"))
let conf = try String(
contentsOf: vm.appendingPathComponent("needrestart-cloudmachine.conf"), encoding: .utf8)
for unit in ["rclone-tm", "punchfs", "tm-bands-bind", "smbd", "nmbd", "smb-throttle"] {
XCTAssertTrue(conf.contains(unit), "needrestart may still restart \(unit)")
}
XCTAssertTrue(conf.contains("$nrconf{override_rc}"))
}

/// The rclone download is pinned to a version AND a checksum, checked
/// before anything is installed.
func testRcloneIsPinnedWithAChecksum() throws {
Expand Down
3 changes: 3 additions & 0 deletions vm/cmsmb.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ provision:
- mode: data
path: /etc/systemd/system/tm-meta-sync.timer
file: systemd/tm-meta-sync.timer
- mode: data
path: /etc/needrestart/conf.d/cloudmachine.conf
file: needrestart-cloudmachine.conf
- mode: system
script: |
#!/bin/bash
Expand Down
8 changes: 8 additions & 0 deletions vm/needrestart-cloudmachine.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# CloudMachine: unattended-upgrades must not restart the backup stack.
#
# 13 Nov 2026 06:10 a libc update made needrestart restart punchfs and Samba
# in the middle of a Time Machine backup; the bands bind then raced punchfs,
# failed, and Samba stayed down for a day. These units are restarted only by
# CloudMachine (`cloudmachine-agent vm-update`) or at the next boot of the
# machine, which picks up the updated libraries.
$nrconf{override_rc}->{qr(^(rclone-tm|punchfs|tm-bands-bind|smbd|nmbd|smb-throttle|tm-meta-sync)\.service$)} = 0;
6 changes: 6 additions & 0 deletions vm/systemd/tm-bands-bind.service
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@ ConditionPathExists=/etc/cloudmachine/bundle.env
Type=oneshot
RemainAfterExit=yes
EnvironmentFile=/etc/cloudmachine/bundle.env
# punchfs is Type=simple: systemd calls it started before its FUSE mount is up.
# Binding then fails ("special device /srv/tm-punch/bands does not exist") and
# Samba, which Requires this unit, stays down - it happened at boot on 12 Nov
# and after a restart on 13 Nov 2026, a day without backups. Wait for the mount.
TimeoutStartSec=150
ExecStartPre=/usr/bin/timeout 120 /bin/sh -c 'until mountpoint -q /srv/tm-punch && [ -d /srv/tm-punch/bands ]; do sleep 1; done'
ExecStart=/bin/mount --bind /srv/tm-punch/bands "/srv/tm-local/${CM_BUNDLE}/bands"
ExecStop=/bin/umount -l "/srv/tm-local/${CM_BUNDLE}/bands"

Expand Down
Loading