Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 46 additions & 18 deletions Sources/DilatedCore/ClosedLidWatchdog.swift
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,39 @@ public struct ClosedLidWatchdog: Sendable {
private let sleep: @Sendable (Duration) async throws -> Void
private let readHeartbeat: @Sendable () -> String?
private let heartbeatTimeout: Duration?
private let monotonic: @Sendable () -> Duration
private let ticker: @Sendable (Duration, @escaping @Sendable () -> Void) -> @Sendable () -> Void

/// How long the heartbeat may sit unchanged before the app counts as hung: the governor's
/// lease plus a margin for a slow tick or a briefly busy machine.
public static let heartbeatTimeout = Duration.seconds(ClosedLidGovernor.leaseSeconds + 30)

/// Time since an arbitrary origin on a clock that stops while the Mac sleeps.
public static let suspendingNow: @Sendable () -> Duration = {
suspendingOrigin.duration(to: SuspendingClock.now)
}
private static let suspendingOrigin = SuspendingClock.now

/// Calls `handler` every `interval` on a serial queue until the returned closure is called.
public static let timerTicker:
@Sendable (Duration, @escaping @Sendable () -> Void) ->
@Sendable () -> Void = { pollInterval, handler in
let timer = DispatchSource.makeTimerSource(
queue: DispatchQueue(label: "dilated.closed-lid-watchdog.poll"))
let interval = pollInterval.components
let nanos = max(
Int(interval.seconds) * 1_000_000_000 + Int(interval.attoseconds / 1_000_000_000),
1_000_000)
timer.setEventHandler(handler: handler)
timer.schedule(deadline: .now() + .nanoseconds(nanos), repeating: .nanoseconds(nanos))
timer.resume()
return { timer.cancel() }
}

/// `onExit` registers a handler called once when the app exits (possibly at once, possibly
/// from any thread) and returns a cancel closure. `acquireLock` returns a release closure, or
/// nil when the lock couldn't be taken in time.
/// nil when the lock couldn't be taken in time. `monotonic` and `ticker` default to the
/// suspending clock and a real timer; tests inject a manual clock and drive each poll by hand.
public init(
generation: UUID, pollInterval: Duration = .seconds(10),
readMarker: @escaping @Sendable () -> ClosedLidSessionMarker?,
Expand All @@ -53,8 +78,13 @@ public struct ClosedLidWatchdog: Sendable {
acquireLock: @escaping @Sendable () async -> (@Sendable () -> Void)? = { {} },
sleep: @escaping @Sendable (Duration) async throws -> Void = { try await Task.sleep(for: $0) },
readHeartbeat: @escaping @Sendable () -> String? = { nil },
heartbeatTimeout: Duration? = nil
heartbeatTimeout: Duration? = nil,
monotonic: @escaping @Sendable () -> Duration = ClosedLidWatchdog.suspendingNow,
ticker: @escaping @Sendable (Duration, @escaping @Sendable () -> Void) -> @Sendable () -> Void =
ClosedLidWatchdog.timerTicker
) {
self.monotonic = monotonic
self.ticker = ticker
self.readHeartbeat = readHeartbeat
self.heartbeatTimeout = heartbeatTimeout
self.generation = generation
Expand Down Expand Up @@ -82,22 +112,15 @@ public struct ClosedLidWatchdog: Sendable {
// swift_task_dealloc when a spawned task awaits here (CI crash reports, 2026-11-02).
let (stream, events) = AsyncStream.makeStream(of: Event.self)
let cancelExit = onExit { events.yield(.exited) }
let timer = DispatchSource.makeTimerSource(
queue: DispatchQueue(label: "dilated.closed-lid-watchdog.poll"))
let stall = HeartbeatStallTracker(timeout: heartbeatTimeout)
timer.setEventHandler {
let stall = HeartbeatStallTracker(timeout: heartbeatTimeout, now: monotonic)
_ = stall.isStalled(readHeartbeat())
let cancelTimer = ticker(pollInterval) {
if !markerIsOurs {
events.yield(.markerGone)
} else if stall.isStalled(readHeartbeat()) {
events.yield(.heartbeatStalled)
}
}
let interval = pollInterval.components
let nanos = max(
Int(interval.seconds) * 1_000_000_000 + Int(interval.attoseconds / 1_000_000_000), 1_000_000)
timer.schedule(deadline: .now() + .nanoseconds(nanos), repeating: .nanoseconds(nanos))
_ = stall.isStalled(readHeartbeat())
timer.resume()
let event: Event? = await withTaskCancellationHandler {
var first: Event?
for await next in stream {
Expand All @@ -108,7 +131,7 @@ public struct ClosedLidWatchdog: Sendable {
} onCancel: {
events.finish()
}
timer.cancel()
cancelTimer()
cancelExit()
events.finish()
switch event {
Expand Down Expand Up @@ -143,19 +166,24 @@ public struct ClosedLidWatchdog: Sendable {
private final class HeartbeatStallTracker: @unchecked Sendable {
private let timeout: Duration?
private var lastValue: String?
private var changedAt = SuspendingClock.now
private let now: @Sendable () -> Duration
private var changedAt: Duration

init(timeout: Duration?) { self.timeout = timeout }
init(timeout: Duration?, now: @escaping @Sendable () -> Duration) {
self.timeout = timeout
self.now = now
changedAt = now()
}

func isStalled(_ value: String?) -> Bool {
guard let timeout else { return false }
let now = SuspendingClock.now
let current = now()
if value != lastValue {
lastValue = value
changedAt = now
changedAt = current
return false
}
return changedAt.duration(to: now) >= timeout
return current - changedAt >= timeout
}
}

Expand Down
4 changes: 3 additions & 1 deletion Sources/DilatedUI/PackageQuit.swift
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@ import AppKit

func run() async {
try? await Task.sleep(for: delay)
terminate()
// Armed first: with `.terminateLater` AppKit may run a modal loop inside terminate() and
// not return, so the fallback can't wait until after it.
let forceExit = forceExit
DispatchQueue.global().asyncAfter(deadline: .now() + grace) { forceExit() }
terminate()
}
}
77 changes: 66 additions & 11 deletions Tests/DilatedCoreTests/ClosedLidHeartbeatTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -51,18 +51,48 @@ private final class MutableClock: @unchecked Sendable {
func advance(_ seconds: TimeInterval) { lock.withLock { current += seconds } }
}

/// A monotonic clock the test advances by hand.
private final class ManualMonotonic: @unchecked Sendable {
private let lock = NSLock()
private var current = Duration.zero
var now: Duration { lock.withLock { current } }
func advance(_ duration: Duration) { lock.withLock { current += duration } }
}

/// Stands in for the poll timer: the test fires each poll itself, in lockstep with the clock
/// and the heartbeat, so no real elapsed time decides anything.
private final class ManualTicks: @unchecked Sendable {
private let lock = NSLock()
private var handler: (@Sendable () -> Void)?
var ticker: @Sendable (Duration, @escaping @Sendable () -> Void) -> @Sendable () -> Void {
{ [self] _, handler in
lock.withLock { self.handler = handler }
return { [self] in lock.withLock { self.handler = nil } }
}
}
/// Waits (by yielding, not by time) until `run()` has registered its poll.
func waitUntilArmed() async {
while lock.withLock({ handler == nil }) { await Task.yield() }
}
func fire() { lock.withLock { handler }?() }
}

private func watchdog(
_ generation: UUID, store: ClosedLidMarkerStore, power: FakePower, pid: Int32, start: Int64,
timeout: Duration = .milliseconds(300)
timeout: Duration = .milliseconds(300), pollInterval: Duration = .milliseconds(20),
monotonic: @escaping @Sendable () -> Duration = ClosedLidWatchdog.suspendingNow,
ticker: @escaping @Sendable (Duration, @escaping @Sendable () -> Void) -> @Sendable () -> Void =
ClosedLidWatchdog.timerTicker
) -> ClosedLidWatchdog {
ClosedLidWatchdog(
generation: generation, pollInterval: .milliseconds(20),
generation: generation, pollInterval: pollInterval,
readMarker: { store.read() },
onExit: { handler in
ProcessExit.register(pid: pid, startMicros: start, handler: handler)
},
power: power, clearMarker: { store.clear(generation: generation) },
readHeartbeat: { store.readHeartbeat() }, heartbeatTimeout: timeout)
readHeartbeat: { store.readHeartbeat() }, heartbeatTimeout: timeout,
monotonic: monotonic, ticker: ticker)
}

struct ClosedLidHeartbeatTests {
Expand All @@ -74,8 +104,18 @@ struct ClosedLidHeartbeatTests {
try store.write(marker(generation, pid: getpid(), start: start))
store.writeHeartbeat(generation: generation, count: 1, at: Date())
let power = FakePower()
let dog = watchdog(generation, store: store, power: power, pid: getpid(), start: start)
#expect(await dog.run() == .reverted)
let clock = ManualMonotonic()
let ticks = ManualTicks()
let dog = watchdog(
generation, store: store, power: power, pid: getpid(), start: start,
monotonic: { clock.now }, ticker: ticks.ticker)
let outcome = Task { await dog.run() }
await ticks.waitUntilArmed()
clock.advance(.milliseconds(299))
ticks.fire()
clock.advance(.milliseconds(1))
ticks.fire()
#expect(await outcome.value == .reverted)
#expect(await power.calls == [false])
#expect(store.read() == nil)
}
Expand All @@ -87,16 +127,23 @@ struct ClosedLidHeartbeatTests {
try store.write(marker(generation, pid: getpid(), start: start))
store.writeHeartbeat(generation: generation, count: 0, at: Date())
let power = FakePower()
let dog = watchdog(generation, store: store, power: power, pid: getpid(), start: start)
let clock = ManualMonotonic()
let ticks = ManualTicks()
let dog = watchdog(
generation, store: store, power: power, pid: getpid(), start: start,
monotonic: { clock.now }, ticker: ticks.ticker)
let outcome = Task { await dog.run() }
await ticks.waitUntilArmed()
// Three times the timeout, wall-clock field frozen: only the count moves.
let frozen = Date(timeIntervalSince1970: 1_800_000_000)
for count in 1...45 {
store.writeHeartbeat(generation: generation, count: UInt64(count), at: frozen)
try await Task.sleep(for: .milliseconds(20))
clock.advance(.milliseconds(20))
ticks.fire()
}
#expect(await power.calls.isEmpty)
store.clear(generation: generation)
ticks.fire()
#expect(await outcome.value == .markerGone)
#expect(await power.calls.isEmpty)
}
Expand All @@ -107,16 +154,23 @@ struct ClosedLidHeartbeatTests {
let start = try #require(ProcessStartTime.micros(pid: getpid()))
try store.write(marker(generation, pid: getpid(), start: start))
let power = FakePower()
let dog = watchdog(generation, store: store, power: power, pid: getpid(), start: start)
let clock = ManualMonotonic()
let ticks = ManualTicks()
let dog = watchdog(
generation, store: store, power: power, pid: getpid(), start: start,
monotonic: { clock.now }, ticker: ticks.ticker)
let outcome = Task { await dog.run() }
await ticks.waitUntilArmed()
for count in 1...30 {
store.writeHeartbeat(
generation: generation, count: UInt64(count),
at: Date(timeIntervalSinceNow: 86_400 * 365))
try await Task.sleep(for: .milliseconds(20))
clock.advance(.milliseconds(20))
ticks.fire()
}
#expect(await power.calls.isEmpty)
store.clear(generation: generation)
ticks.fire()
#expect(await outcome.value == .markerGone)
}

Expand Down Expand Up @@ -221,9 +275,10 @@ struct ClosedLidHeartbeatTests {
let power = FakePower()
let dog = watchdog(
generation, store: store, power: power, pid: pid, start: start,
timeout: .milliseconds(600))
timeout: .seconds(5), pollInterval: .milliseconds(100))
let outcome = Task { await dog.run() }
try await Task.sleep(for: .milliseconds(1000))
// Well under the timeout, so a slow runner can't make the live loop look stalled.
try await Task.sleep(for: .seconds(2))
#expect(await power.calls.isEmpty)
kill(pid, SIGSTOP)
#expect(await outcome.value == .reverted)
Expand Down
16 changes: 16 additions & 0 deletions Tests/DilatedUITests/PackageSheetDismissalTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,22 @@ import Testing
#expect(forced.value == 1)
}

@Test func forceExitIsArmedBeforeATerminateThatNeverReturns() async {
let forced = Counter()
let gate = DispatchSemaphore(value: 0)
var quit = PackageQuit()
quit.delay = .milliseconds(1)
quit.grace = 0.05
// Stands in for `.terminateLater`: blocks until the fallback fires (or 10s, failing the test).
quit.terminate = { _ = gate.wait(timeout: .now() + 10) }
quit.forceExit = {
forced.bump()
gate.signal()
}
await quit.run()
#expect(forced.value == 1)
}

private func pump() { RunLoop.main.run(until: Date().addingTimeInterval(0.5)) }
}

Expand Down
Loading