Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 47 additions & 6 deletions src/main/kotlin/com/gitvantage/app/AppState.kt
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,20 @@ class AppState(private val scope: CoroutineScope) {
private set

private val scanLimiter = Semaphore(8) // cap concurrent git subprocesses

/**
* Where scans nobody is waiting on run, at most two at a time. See [rescanRepos].
*
* Every git launch is a `fork()` of this process, and while the kernel copies it the UI thread
* stalls too: a field report measured 11–15ms per launch after 13 days of uptime. Eight scans in
* parallel launch git back to back, which leaves the UI no gap to draw in. Two scans still launch
* git many times a second, but now there is time between launches, so the UI can draw between
* them and a sweep is spread out instead of one long stall.
*
* Handed to [RepoScanner.scan] rather than wrapped around it: the scan switches to its own
* dispatcher, which would escape the limit.
*/
private val backgroundScans = Dispatchers.IO.limitedParallelism(2)
private var toastJob: Job? = null

// Open GitHub issues/PRs, keyed by repo id. Deliberately NOT folded into [repos]: those are
Expand Down Expand Up @@ -508,6 +522,12 @@ class AppState(private val scope: CoroutineScope) {
/** One repo's in-progress burst: the ceiling it must scan by, and when it is currently due. */
private class WatchBurst(val deadline: Long, @Volatile var dueAt: Long)

/**
* When the last overflow sweep finished, so [WatchPolicy.fullRescanDueAt] can space them out.
* Written by [watchDrainLoop] and read on the watcher's thread.
*/
@Volatile private var lastFullSweepEnd: Long? = null

/**
* Each repo's nested worktrees, as [actionFor] needs them — kept beside [repos] rather than read
* out of it.
Expand Down Expand Up @@ -780,7 +800,8 @@ class AppState(private val scope: CoroutineScope) {
private fun noteWatchEvent(id: String) {
val now = System.currentTimeMillis()
val burst = watchBursts.computeIfAbsent(id) { WatchBurst(WatchPolicy.deadlineFor(now), now) }
burst.dueAt = WatchPolicy.dueAt(now, burst.deadline)
val due = WatchPolicy.dueAt(now, burst.deadline)
burst.dueAt = if (id == ALL_REPOS) WatchPolicy.fullRescanDueAt(due, lastFullSweepEnd) else due
watchWake.trySend(Unit)
}

Expand Down Expand Up @@ -814,8 +835,9 @@ class AppState(private val scope: CoroutineScope) {
if (ids.isNotEmpty()) {
Perf.count("watch.rescans", ids.size.toLong())
ids.forEach(WatchStats::rescan)
Perf.timed("watch.rescan.batch") { rescanRepos(ids, fetch = false) }
Perf.timed("watch.rescan.batch") { rescanRepos(ids, fetch = false, background = true) }
}
if (ALL_REPOS in due) lastFullSweepEnd = System.currentTimeMillis()
}
}
when (val next = watchBursts.values.minOfOrNull { it.dueAt }) {
Expand All @@ -834,6 +856,9 @@ class AppState(private val scope: CoroutineScope) {
* up, and answering each one with an immediate sweep of every repo is how a dropped event turns
* into a scan storm that guarantees the next one. The ceiling still applies, so a tree churning
* hard enough to overflow repeatedly cannot postpone the sweep it is the whole reason for.
* Sweeps are also spaced [WatchPolicy.FULL_RESCAN_MIN_INTERVAL_MS] apart, because a debounce
* only merges overflows that arrive close together. A tree that overflows every few seconds
* would otherwise get a sweep each time.
*
* Not a fetch: the watcher is reporting local filesystem churn, and a sweep that opened a
* network connection per repo would make an overload worse in a way the user pays for twice.
Expand Down Expand Up @@ -2835,12 +2860,28 @@ class AppState(private val scope: CoroutineScope) {
}
}

/** Re-scan the given repos in place (git state may have changed after push/fetch). */
private suspend fun rescanRepos(ids: Collection<String>, fetch: Boolean, userInitiated: Boolean = true) {
/**
* Re-scan the given repos in place (git state may have changed after push/fetch).
*
* [background] is for scans nobody is waiting on: the file watcher's and the periodic
* auto-refresh. They run on [backgroundScans] instead of taking [scanLimiter] permits, so a sweep
* of every repo can't hold the UI up and doesn't queue ahead of something the user just clicked.
*/
private suspend fun rescanRepos(
ids: Collection<String>,
fetch: Boolean,
userInitiated: Boolean = true,
background: Boolean = false,
) {
val es = ids.mapNotNull { entries[it] }
val scanned = withContext(Dispatchers.Default) {
es.map { e ->
async { scanLimiter.withPermit { runCatching { RepoScanner.scan(e, fetch, userInitiated) }.getOrNull() } }
async {
runCatching {
if (background) RepoScanner.scan(e, fetch, userInitiated, backgroundScans)
else scanLimiter.withPermit { RepoScanner.scan(e, fetch, userInitiated) }
}.getOrNull()
}
}.awaitAll()
}.filterNotNull()
// Before the prevBehind baseline moves below — the advance stamp needs the previous value.
Expand Down Expand Up @@ -2959,7 +3000,7 @@ class AppState(private val scope: CoroutineScope) {
try {
val now = System.currentTimeMillis()
val due = order.filter { id -> id == selectedId || isFetchDue(id, now) }
if (due.isNotEmpty()) rescanRepos(due, fetch = true, userInitiated = false)
if (due.isNotEmpty()) rescanRepos(due, fetch = true, userInitiated = false, background = true)
} finally {
running = false
}
Expand Down
16 changes: 14 additions & 2 deletions src/main/kotlin/com/gitvantage/git/RepoScanner.kt
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import com.gitvantage.model.RegistryEntry
import com.gitvantage.model.Repo
import com.gitvantage.model.Stash
import java.io.File
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext

Expand All @@ -39,9 +40,20 @@ object RepoScanner {
* Refresh button, and is pure noise when it came from the 5-minute timer — which fires per
* repo, forever, and would otherwise evict every real entry from a 500-deep console within
* a couple of idle hours.
*
* [dispatcher] is where the scan's blocking git calls run. A caller that wants to limit how
* many scans run at once passes a `limitedParallelism` view. Wrapping the call in its own
* `withContext` won't do it, because this function would switch away to [Dispatchers.IO] and
* the limit would no longer apply. The whole scan blocks without suspending, so it holds its
* slot from start to finish.
*/
suspend fun scan(entry: RegistryEntry, fetch: Boolean, userInitiated: Boolean = true): Repo =
withContext(Dispatchers.IO) { Activity.scan { Perf.timed("scan.repo") { scanNow(entry, fetch, userInitiated) } } }
suspend fun scan(
entry: RegistryEntry,
fetch: Boolean,
userInitiated: Boolean = true,
dispatcher: CoroutineDispatcher = Dispatchers.IO,
): Repo =
withContext(dispatcher) { Activity.scan { Perf.timed("scan.repo") { scanNow(entry, fetch, userInitiated) } } }

private fun scanNow(entry: RegistryEntry, fetch: Boolean, userInitiated: Boolean): Repo {
val dir = File(entry.path)
Expand Down
21 changes: 21 additions & 0 deletions src/main/kotlin/com/gitvantage/model/WatchPolicy.kt
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,27 @@ object WatchPolicy {
*/
fun dueAt(lastEventAt: Long, deadline: Long): Long = lastEventAt + delayMs(lastEventAt, deadline)

/**
* The shortest gap between the end of one overflow sweep and the start of the next.
*
* An overflow can't be traced to a repo, so it rescans all of them, and one sweep is several
* hundred git launches. A field report (42 repos, 1.6.1) had 4 overflows a minute while a coding
* session worked in a repo with 22 nested worktrees. Each one swept every repo, and each git
* launch blocked the UI thread for about 13ms, so the app stalled for seconds at a time.
*
* A sweep finds every change made before it starts, however many overflows asked for it. Holding
* the next one off doesn't lose events; it only delays when a repo whose own event was dropped
* catches up. A repo that changes again in the meantime still gets its own scan right away.
*/
const val FULL_RESCAN_MIN_INTERVAL_MS = 120_000L

/**
* When an overflow sweep normally due at [dueAt] may run, given that the previous one ended at
* [lastSweepEnd] (null if there hasn't been one yet).
*/
fun fullRescanDueAt(dueAt: Long, lastSweepEnd: Long?): Long =
if (lastSweepEnd == null) dueAt else maxOf(dueAt, lastSweepEnd + FULL_RESCAN_MIN_INTERVAL_MS)

/**
* Whether a change to [paths] is a change to the repo itself, rather than to one of the
* checkouts [nested] inside its folder.
Expand Down
33 changes: 33 additions & 0 deletions src/test/kotlin/com/gitvantage/model/WatchPolicyTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -276,3 +276,36 @@ val WatchIgnoredDirs by testSuite {
assert(!WatchPolicy.ignoresStale(root, checked, listOf(p("README.md"))))
}
}

/**
* Spacing between overflow sweeps. Each sweep rescans every repo, so overflows that keep coming
* must not turn into sweeps that keep coming.
*/
val WatchFullRescanSpacing by testSuite {
val now = 1_800_000_000_000L

test("the first sweep runs on the ordinary debounce") {
val due = WatchPolicy.dueAt(now, WatchPolicy.deadlineFor(now))
assert(WatchPolicy.fullRescanDueAt(due, lastSweepEnd = null) == due)
}

test("a sweep soon after the last one waits until the interval is up") {
val lastEnd = now - 10_000
val due = WatchPolicy.dueAt(now, WatchPolicy.deadlineFor(now))
assert(WatchPolicy.fullRescanDueAt(due, lastEnd) == lastEnd + WatchPolicy.FULL_RESCAN_MIN_INTERVAL_MS)
}

test("one long after the last runs on the ordinary debounce") {
val lastEnd = now - WatchPolicy.FULL_RESCAN_MIN_INTERVAL_MS - 1
val due = WatchPolicy.dueAt(now, WatchPolicy.deadlineFor(now))
assert(WatchPolicy.fullRescanDueAt(due, lastEnd) == due)
}

test("further overflows while one is held off don't push it any later") {
val lastEnd = now - 10_000
val first = WatchPolicy.fullRescanDueAt(WatchPolicy.dueAt(now, WatchPolicy.deadlineFor(now)), lastEnd)
val later = now + 30_000
val again = WatchPolicy.fullRescanDueAt(WatchPolicy.dueAt(later, WatchPolicy.deadlineFor(now)), lastEnd)
assert(again == first)
}
}
Loading