Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -28,24 +28,38 @@ final class CloudMachineController: ObservableObject {
/// good as one from five seconds ago.
private static let backupCycleInterval: TimeInterval = 300
private var backupCycleCheckedAt: Date?
/// `AgentRepair.afterLaunch`, started once per process.
private var agentMaintenance: Task<Void, Never>?
/// One failed read of the Time Machine preferences is not "no Full Disk Access".
private var preferencesReads = BackupHealth.ReadConfirmation()

// MARK: - Refresh cycle

func startAutoRefresh(interval: TimeInterval = 10) {
// Homebrew quits the app for an upgrade and reopens it afterwards, so
// this is the first code that runs after the bundle was replaced - see
// `AgentRepair` for what breaks if nobody reloads the agents. ONCE per
// process and detached: the dashboard and the menu bar both call
// `startAutoRefresh`, and each call cancels the previous refresh task -
// on 12 Nov 2026 that cancelled a reload between `bootout` and
// `bootstrap` and left the watchdogs unloaded.
if agentMaintenance == nil {
agentMaintenance = Task.detached { await AgentRepair.afterLaunch() }
}
let maintenance = agentMaintenance
refreshTask?.cancel()
refreshTask = Task { [weak self] in
// Homebrew quits the app for an upgrade and reopens it afterwards, so
// this is the first code that runs after the bundle was replaced - see
// `AgentRepair` for what breaks if nobody reloads the agents.
await AgentRepair.afterLaunch()
var cycles = 0
while !Task.isCancelled {
await self?.refreshAll()
// An agent can stop being startable later too, so check every few minutes.
// An agent can stop being startable later too, so check every few
// minutes - never while the launch reload is still running (a failing
// bootstrap can take tens of seconds; the panel does not wait for it).
cycles += 1
if cycles % 30 == 0 { await AgentRepair.repairBroken() }
if cycles % 30 == 0 {
await maintenance?.value
await AgentRepair.repairBroken()
}
try? await Task.sleep(nanoseconds: UInt64(interval) * 1_000_000_000)
}
}
Expand Down
104 changes: 79 additions & 25 deletions mac-app/Sources/CloudMachineCore/AgentRepair.swift
Original file line number Diff line number Diff line change
Expand Up @@ -122,24 +122,68 @@ public enum AgentRepair {
/// `bootout` + `bootstrap`. `bootout` finishes asynchronously and a
/// `bootstrap` right after it fails with error 5, so it is retried.
@discardableResult
static func reload(_ name: String) async -> Bool {
let plist = plist(name)
guard FileManager.default.fileExists(atPath: plist.path) else { return false }
let domain = "gui/\(getuid())"
_ = try? await ProcessRunner.run(
"/bin/launchctl", ["bootout", "\(domain)/\(prefix)\(name)"], timeout: 30)
for _ in 0..<10 {
try? await Task.sleep(nanoseconds: 1_000_000_000)
if let result = try? await ProcessRunner.run(
"/bin/launchctl", ["bootstrap", domain, plist.path], timeout: 30),
result.succeeded
{
CMLogger.log("Reloaded launchd agent \(name)")
return true
static func reload(_ name: String, steps: ReloadSteps = .live) async -> Bool {
guard steps.plistExists(name) else { return false }
// `bootout` and `bootstrap` must finish together, so the reload runs
// detached from the caller: cancelling the caller must not stop it
// between the two. 12 Nov 2026 14:53 the app's refresh task was cancelled
// (a second `startAutoRefresh`) right after `bootout`; `Task.sleep`
// returned at once, `ProcessRunner` killed every `launchctl bootstrap`,
// ten attempts failed within one second and backup-health stayed unloaded.
return await Task.detached {
await steps.bootout(name)
for _ in 0..<10 {
await steps.pause()
if await steps.bootstrap(name) {
steps.log("Reloaded launchd agent \(name)")
return true
}
}
}
CMLogger.log("Could not reload launchd agent \(name) - bootstrap kept failing")
return false
steps.log("Could not reload launchd agent \(name) - bootstrap kept failing")
return false
}.value
}

/// The launchd calls of a reload - replaceable, so the tests run it
/// without launchd.
struct ReloadSteps: Sendable {
var plistExists: @Sendable (String) -> Bool
var bootout: @Sendable (String) async -> Void
var bootstrap: @Sendable (String) async -> Bool
/// `bootout` finishes asynchronously; a `bootstrap` right after it fails
/// with error 5, hence a pause before every attempt.
var pause: @Sendable () async -> Void
var log: @Sendable (String) -> Void

static let live = ReloadSteps(
plistExists: { FileManager.default.fileExists(atPath: plist($0).path) },
bootout: { name in
_ = try? await ProcessRunner.run(
"/bin/launchctl", ["bootout", "gui/\(getuid())/\(prefix)\(name)"], timeout: 30)
},
bootstrap: { name in
guard
let result = try? await ProcessRunner.run(
"/bin/launchctl", ["bootstrap", "gui/\(getuid())", plist(name).path], timeout: 30)
else { return false }
if !result.succeeded {
let text = (result.stderr + result.stdout).trimmingCharacters(in: .whitespacesAndNewlines)
CMLogger.log("launchctl bootstrap \(name): code \(result.exitCode) \(text)")
}
return result.succeeded
},
pause: { try? await Task.sleep(nanoseconds: 1_000_000_000) },
log: { CMLogger.log($0) })
}

/// Reloads every agent in `names` (a failure does not skip the rest).
/// `true` only when all of them were reloaded.
static func reloadAfterUpgrade(
_ names: [String], reload: (String) async -> Bool
) async -> Bool {
var allReloaded = true
for name in names where !(await reload(name)) { allReloaded = false }
return allReloaded
}

/// Reloads every agent launchd cannot start. Returns the ones reloaded.
Expand Down Expand Up @@ -283,14 +327,24 @@ public enum AgentRepair {
await removeRetiredAgents()
guard let version = AppVersionReader.current() else { return }
let current = version.summary
let stamp = try? String(contentsOf: stampFile, encoding: .utf8)
if versionChanged(current: current, stamp: stamp) {
CMLogger.log("App version is now \(current) - reloading the launchd agents")
for name in safeToReload where FileManager.default.fileExists(atPath: plist(name).path) {
await reload(name)
}
try? current.write(to: stampFile, atomically: true, encoding: .utf8)
}
await reloadIfUpgraded(
current: current,
stamp: try? String(contentsOf: stampFile, encoding: .utf8),
installed: safeToReload.filter { FileManager.default.fileExists(atPath: plist($0).path) },
reload: { await reload($0) },
writeStamp: { try? $0.write(to: stampFile, atomically: true, encoding: .utf8) })
await repairBroken()
}

/// After an upgrade, reloads the installed safe agents. The stamp means
/// "the agents run this version": written after a failed reload, it would
/// stop every later launch from trying again.
static func reloadIfUpgraded(
current: String, stamp: String?, installed: [String],
reload: (String) async -> Bool, writeStamp: (String) -> Void
) async {
guard versionChanged(current: current, stamp: stamp) else { return }
CMLogger.log("App version is now \(current) - reloading the launchd agents")
if await reloadAfterUpgrade(installed, reload: reload) { writeStamp(current) }
}
}
106 changes: 106 additions & 0 deletions mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,112 @@ final class AgentRepairTests: XCTestCase {
XCTAssertEqual(AgentRepair.jobLookup(exitCode: 1, stdout: "", stderr: ""), .unknown)
}

// MARK: - Reloading

/// Records a reload without launchd. `bootstrap` behaves like
/// `ProcessRunner` does: in a cancelled task the process is terminated at
/// once, so the call fails.
private final class FakeLaunchctl: @unchecked Sendable {
let lock = NSLock()
var calls: [String] = []
var logs: [String] = []
var bootstrapSucceeds = true

private func record(_ call: String) {
lock.lock()
calls.append(call)
lock.unlock()
}

var steps: AgentRepair.ReloadSteps {
AgentRepair.ReloadSteps(
plistExists: { _ in true },
bootout: { self.record("bootout \($0)") },
bootstrap: {
self.record("bootstrap \($0)")
return self.bootstrapSucceeds && !Task.isCancelled
},
pause: {},
log: { message in
self.lock.lock()
self.logs.append(message)
self.lock.unlock()
})
}
}

/// 12 Nov 2026 14:53: the dashboard and the menu bar both called
/// `startAutoRefresh`, the second call cancelled the first task right after
/// its `bootout`, and all ten `bootstrap` attempts failed within ONE second
/// (`Task.sleep` returns at once in a cancelled task, `ProcessRunner` kills
/// the process). backup-health and vm-guard were left unloaded.
func testReloadFinishesEvenWhenTheCallerIsCancelled() async {
let fake = FakeLaunchctl()
let task = Task { await AgentRepair.reload("backup-health", steps: fake.steps) }
task.cancel()
let reloaded = await task.value
XCTAssertTrue(reloaded)
XCTAssertEqual(fake.calls, ["bootout backup-health", "bootstrap backup-health"])
}

func testReloadGivesUpAfterTenFailedBootstraps() async {
let fake = FakeLaunchctl()
fake.bootstrapSucceeds = false
let reloaded = await AgentRepair.reload("vm-guard", steps: fake.steps)
XCTAssertFalse(reloaded)
XCTAssertEqual(fake.calls.filter { $0.hasPrefix("bootstrap") }.count, 10)
XCTAssertTrue(fake.logs.contains { $0.contains("bootstrap kept failing") })
}

/// The stamp says "the agents run this version". Written after a failed
/// reload, it stopped every later launch from trying again.
func testUpgradeStampOnlyAfterEveryReloadSucceeded() async {
var tried: [String] = []
let allOK = await AgentRepair.reloadAfterUpgrade(["backup-health", "vm-guard"]) {
tried.append($0)
return $0 == "vm-guard"
}
XCTAssertFalse(allOK)
XCTAssertEqual(tried, ["backup-health", "vm-guard"], "one failure must not skip the rest")
let fine = await AgentRepair.reloadAfterUpgrade(["backup-health"]) { _ in true }
XCTAssertTrue(fine)
}

/// The whole upgrade step: no stamp after a failed reload (the next launch
/// tries again), a stamp after a good one, nothing at all when the version
/// did not change.
func testUpgradeStepWritesTheStampOnlyWhenTheReloadWorked() async {
var written: [String] = []
var reloaded: [String] = []
await AgentRepair.reloadIfUpgraded(
current: "1.3.29 (211) e2eb360", stamp: "1.3.28 (210) aca535e",
installed: ["backup-health", "vm-guard"],
reload: {
reloaded.append($0)
return false
},
writeStamp: { written.append($0) })
XCTAssertEqual(reloaded, ["backup-health", "vm-guard"])
XCTAssertEqual(written, [])

await AgentRepair.reloadIfUpgraded(
current: "1.3.29 (211) e2eb360", stamp: "1.3.28 (210) aca535e",
installed: ["backup-health"], reload: { _ in true }, writeStamp: { written.append($0) })
XCTAssertEqual(written, ["1.3.29 (211) e2eb360"])

reloaded = []
await AgentRepair.reloadIfUpgraded(
current: "1.3.29 (211) e2eb360", stamp: "1.3.29 (211) e2eb360\n",
installed: ["backup-health"],
reload: {
reloaded.append($0)
return true
},
writeStamp: { written.append($0) })
XCTAssertEqual(reloaded, [])
XCTAssertEqual(written.count, 1)
}

// MARK: - The retired layout

/// Records what the migration did, without launchd.
Expand Down
Loading