From 93edd127fa75b77e53a5c6c0bd1d2eaa2c7da710 Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Sat, 14 Nov 2026 07:37:09 +0100 Subject: [PATCH] =?UTF-8?q?Agenty:=20prze=C5=82adowanie=20odporne=20na=20a?= =?UTF-8?q?nulowanie;=20znacznik=20wersji=20tylko=20po=20sukcesie?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 12.11 14:53 okno i pasek menu wywołały startAutoRefresh dwa razy; drugie wywołanie anulowało zadanie pierwsze tuż po `launchctl bootout`. W anulowanym zadaniu Task.sleep wraca od razu, a ProcessRunner zabija launchctl, więc 10 prób bootstrap padło w 1 s, a znacznik wersji zapisał się mimo to. backup-health i vm-guard zostały wyładowane, strażnik backupu nie działał 2 dni. - reload: bootout+bootstrap w Task.detached, kroki wstrzykiwalne (testy) - znacznik wersji tylko gdy wszystkie przeładowania się udały - afterLaunch raz na proces, panel na niego nie czeka - błąd bootstrap trafia do dziennika (kod + komunikat) Mutacja (bez detached) sprawdzona: test anulowania pada. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AU1Mhtkx8DukvjdJr4eLsf --- .../Services/CloudMachineController.swift | 26 ++++- .../CloudMachineCore/AgentRepair.swift | 104 ++++++++++++----- .../AgentRepairTests.swift | 106 ++++++++++++++++++ 3 files changed, 205 insertions(+), 31 deletions(-) diff --git a/mac-app/Sources/CloudMachineApp/Services/CloudMachineController.swift b/mac-app/Sources/CloudMachineApp/Services/CloudMachineController.swift index 574dd59..25965c1 100644 --- a/mac-app/Sources/CloudMachineApp/Services/CloudMachineController.swift +++ b/mac-app/Sources/CloudMachineApp/Services/CloudMachineController.swift @@ -28,24 +28,38 @@ final class CloudMachineController: ObservableObject { /// good as one from five seconds ago. private static let backupCycleInterval: TimeInterval = 300 private var backupCycleCheckedAt: Date? + /// `AgentRepair.afterLaunch`, started once per process. + private var agentMaintenance: Task? /// One failed read of the Time Machine preferences is not "no Full Disk Access". private var preferencesReads = BackupHealth.ReadConfirmation() // MARK: - Refresh cycle func startAutoRefresh(interval: TimeInterval = 10) { + // Homebrew quits the app for an upgrade and reopens it afterwards, so + // this is the first code that runs after the bundle was replaced - see + // `AgentRepair` for what breaks if nobody reloads the agents. ONCE per + // process and detached: the dashboard and the menu bar both call + // `startAutoRefresh`, and each call cancels the previous refresh task - + // on 12 Nov 2026 that cancelled a reload between `bootout` and + // `bootstrap` and left the watchdogs unloaded. + if agentMaintenance == nil { + agentMaintenance = Task.detached { await AgentRepair.afterLaunch() } + } + let maintenance = agentMaintenance refreshTask?.cancel() refreshTask = Task { [weak self] in - // Homebrew quits the app for an upgrade and reopens it afterwards, so - // this is the first code that runs after the bundle was replaced - see - // `AgentRepair` for what breaks if nobody reloads the agents. - await AgentRepair.afterLaunch() var cycles = 0 while !Task.isCancelled { await self?.refreshAll() - // An agent can stop being startable later too, so check every few minutes. + // An agent can stop being startable later too, so check every few + // minutes - never while the launch reload is still running (a failing + // bootstrap can take tens of seconds; the panel does not wait for it). cycles += 1 - if cycles % 30 == 0 { await AgentRepair.repairBroken() } + if cycles % 30 == 0 { + await maintenance?.value + await AgentRepair.repairBroken() + } try? await Task.sleep(nanoseconds: UInt64(interval) * 1_000_000_000) } } diff --git a/mac-app/Sources/CloudMachineCore/AgentRepair.swift b/mac-app/Sources/CloudMachineCore/AgentRepair.swift index e0b33f6..0198bfa 100644 --- a/mac-app/Sources/CloudMachineCore/AgentRepair.swift +++ b/mac-app/Sources/CloudMachineCore/AgentRepair.swift @@ -122,24 +122,68 @@ public enum AgentRepair { /// `bootout` + `bootstrap`. `bootout` finishes asynchronously and a /// `bootstrap` right after it fails with error 5, so it is retried. @discardableResult - static func reload(_ name: String) async -> Bool { - let plist = plist(name) - guard FileManager.default.fileExists(atPath: plist.path) else { return false } - let domain = "gui/\(getuid())" - _ = try? await ProcessRunner.run( - "/bin/launchctl", ["bootout", "\(domain)/\(prefix)\(name)"], timeout: 30) - for _ in 0..<10 { - try? await Task.sleep(nanoseconds: 1_000_000_000) - if let result = try? await ProcessRunner.run( - "/bin/launchctl", ["bootstrap", domain, plist.path], timeout: 30), - result.succeeded - { - CMLogger.log("Reloaded launchd agent \(name)") - return true + static func reload(_ name: String, steps: ReloadSteps = .live) async -> Bool { + guard steps.plistExists(name) else { return false } + // `bootout` and `bootstrap` must finish together, so the reload runs + // detached from the caller: cancelling the caller must not stop it + // between the two. 12 Nov 2026 14:53 the app's refresh task was cancelled + // (a second `startAutoRefresh`) right after `bootout`; `Task.sleep` + // returned at once, `ProcessRunner` killed every `launchctl bootstrap`, + // ten attempts failed within one second and backup-health stayed unloaded. + return await Task.detached { + await steps.bootout(name) + for _ in 0..<10 { + await steps.pause() + if await steps.bootstrap(name) { + steps.log("Reloaded launchd agent \(name)") + return true + } } - } - CMLogger.log("Could not reload launchd agent \(name) - bootstrap kept failing") - return false + steps.log("Could not reload launchd agent \(name) - bootstrap kept failing") + return false + }.value + } + + /// The launchd calls of a reload - replaceable, so the tests run it + /// without launchd. + struct ReloadSteps: Sendable { + var plistExists: @Sendable (String) -> Bool + var bootout: @Sendable (String) async -> Void + var bootstrap: @Sendable (String) async -> Bool + /// `bootout` finishes asynchronously; a `bootstrap` right after it fails + /// with error 5, hence a pause before every attempt. + var pause: @Sendable () async -> Void + var log: @Sendable (String) -> Void + + static let live = ReloadSteps( + plistExists: { FileManager.default.fileExists(atPath: plist($0).path) }, + bootout: { name in + _ = try? await ProcessRunner.run( + "/bin/launchctl", ["bootout", "gui/\(getuid())/\(prefix)\(name)"], timeout: 30) + }, + bootstrap: { name in + guard + let result = try? await ProcessRunner.run( + "/bin/launchctl", ["bootstrap", "gui/\(getuid())", plist(name).path], timeout: 30) + else { return false } + if !result.succeeded { + let text = (result.stderr + result.stdout).trimmingCharacters(in: .whitespacesAndNewlines) + CMLogger.log("launchctl bootstrap \(name): code \(result.exitCode) \(text)") + } + return result.succeeded + }, + pause: { try? await Task.sleep(nanoseconds: 1_000_000_000) }, + log: { CMLogger.log($0) }) + } + + /// Reloads every agent in `names` (a failure does not skip the rest). + /// `true` only when all of them were reloaded. + static func reloadAfterUpgrade( + _ names: [String], reload: (String) async -> Bool + ) async -> Bool { + var allReloaded = true + for name in names where !(await reload(name)) { allReloaded = false } + return allReloaded } /// Reloads every agent launchd cannot start. Returns the ones reloaded. @@ -283,14 +327,24 @@ public enum AgentRepair { await removeRetiredAgents() guard let version = AppVersionReader.current() else { return } let current = version.summary - let stamp = try? String(contentsOf: stampFile, encoding: .utf8) - if versionChanged(current: current, stamp: stamp) { - CMLogger.log("App version is now \(current) - reloading the launchd agents") - for name in safeToReload where FileManager.default.fileExists(atPath: plist(name).path) { - await reload(name) - } - try? current.write(to: stampFile, atomically: true, encoding: .utf8) - } + await reloadIfUpgraded( + current: current, + stamp: try? String(contentsOf: stampFile, encoding: .utf8), + installed: safeToReload.filter { FileManager.default.fileExists(atPath: plist($0).path) }, + reload: { await reload($0) }, + writeStamp: { try? $0.write(to: stampFile, atomically: true, encoding: .utf8) }) await repairBroken() } + + /// After an upgrade, reloads the installed safe agents. The stamp means + /// "the agents run this version": written after a failed reload, it would + /// stop every later launch from trying again. + static func reloadIfUpgraded( + current: String, stamp: String?, installed: [String], + reload: (String) async -> Bool, writeStamp: (String) -> Void + ) async { + guard versionChanged(current: current, stamp: stamp) else { return } + CMLogger.log("App version is now \(current) - reloading the launchd agents") + if await reloadAfterUpgrade(installed, reload: reload) { writeStamp(current) } + } } diff --git a/mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift b/mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift index f696da7..9fc7cea 100644 --- a/mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift @@ -129,6 +129,112 @@ final class AgentRepairTests: XCTestCase { XCTAssertEqual(AgentRepair.jobLookup(exitCode: 1, stdout: "", stderr: ""), .unknown) } + // MARK: - Reloading + + /// Records a reload without launchd. `bootstrap` behaves like + /// `ProcessRunner` does: in a cancelled task the process is terminated at + /// once, so the call fails. + private final class FakeLaunchctl: @unchecked Sendable { + let lock = NSLock() + var calls: [String] = [] + var logs: [String] = [] + var bootstrapSucceeds = true + + private func record(_ call: String) { + lock.lock() + calls.append(call) + lock.unlock() + } + + var steps: AgentRepair.ReloadSteps { + AgentRepair.ReloadSteps( + plistExists: { _ in true }, + bootout: { self.record("bootout \($0)") }, + bootstrap: { + self.record("bootstrap \($0)") + return self.bootstrapSucceeds && !Task.isCancelled + }, + pause: {}, + log: { message in + self.lock.lock() + self.logs.append(message) + self.lock.unlock() + }) + } + } + + /// 12 Nov 2026 14:53: the dashboard and the menu bar both called + /// `startAutoRefresh`, the second call cancelled the first task right after + /// its `bootout`, and all ten `bootstrap` attempts failed within ONE second + /// (`Task.sleep` returns at once in a cancelled task, `ProcessRunner` kills + /// the process). backup-health and vm-guard were left unloaded. + func testReloadFinishesEvenWhenTheCallerIsCancelled() async { + let fake = FakeLaunchctl() + let task = Task { await AgentRepair.reload("backup-health", steps: fake.steps) } + task.cancel() + let reloaded = await task.value + XCTAssertTrue(reloaded) + XCTAssertEqual(fake.calls, ["bootout backup-health", "bootstrap backup-health"]) + } + + func testReloadGivesUpAfterTenFailedBootstraps() async { + let fake = FakeLaunchctl() + fake.bootstrapSucceeds = false + let reloaded = await AgentRepair.reload("vm-guard", steps: fake.steps) + XCTAssertFalse(reloaded) + XCTAssertEqual(fake.calls.filter { $0.hasPrefix("bootstrap") }.count, 10) + XCTAssertTrue(fake.logs.contains { $0.contains("bootstrap kept failing") }) + } + + /// The stamp says "the agents run this version". Written after a failed + /// reload, it stopped every later launch from trying again. + func testUpgradeStampOnlyAfterEveryReloadSucceeded() async { + var tried: [String] = [] + let allOK = await AgentRepair.reloadAfterUpgrade(["backup-health", "vm-guard"]) { + tried.append($0) + return $0 == "vm-guard" + } + XCTAssertFalse(allOK) + XCTAssertEqual(tried, ["backup-health", "vm-guard"], "one failure must not skip the rest") + let fine = await AgentRepair.reloadAfterUpgrade(["backup-health"]) { _ in true } + XCTAssertTrue(fine) + } + + /// The whole upgrade step: no stamp after a failed reload (the next launch + /// tries again), a stamp after a good one, nothing at all when the version + /// did not change. + func testUpgradeStepWritesTheStampOnlyWhenTheReloadWorked() async { + var written: [String] = [] + var reloaded: [String] = [] + await AgentRepair.reloadIfUpgraded( + current: "1.3.29 (211) e2eb360", stamp: "1.3.28 (210) aca535e", + installed: ["backup-health", "vm-guard"], + reload: { + reloaded.append($0) + return false + }, + writeStamp: { written.append($0) }) + XCTAssertEqual(reloaded, ["backup-health", "vm-guard"]) + XCTAssertEqual(written, []) + + await AgentRepair.reloadIfUpgraded( + current: "1.3.29 (211) e2eb360", stamp: "1.3.28 (210) aca535e", + installed: ["backup-health"], reload: { _ in true }, writeStamp: { written.append($0) }) + XCTAssertEqual(written, ["1.3.29 (211) e2eb360"]) + + reloaded = [] + await AgentRepair.reloadIfUpgraded( + current: "1.3.29 (211) e2eb360", stamp: "1.3.29 (211) e2eb360\n", + installed: ["backup-health"], + reload: { + reloaded.append($0) + return true + }, + writeStamp: { written.append($0) }) + XCTAssertEqual(reloaded, []) + XCTAssertEqual(written.count, 1) + } + // MARK: - The retired layout /// Records what the migration did, without launchd.