diff --git a/docs/architecture.md b/docs/architecture.md index 5f66f53..4e9d90e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -45,7 +45,7 @@ type Scanner interface { } ``` -Scanners report progress via context-attached callbacks for real-time UI updates; the walk batch reports under a single "projects" label, stat scanners under their own names. Sizing collects two figures per artifact via an in-process walk (`scanner.Measure`): **disk** (allocated blocks, `st_blocks×512` — sparse-accurate and matching `du`) and **apparent** (sum of logical file sizes). Directories contribute their own blocks to disk (real on ext4, ~0 on APFS); symlinks are never followed. Hard-linked inodes (`Nlink>1`) are counted once per artifact and recorded (keyed by `(dev, ino)`) so shared blocks can be netted out across artifacts. The walk engine does not size inline — it collects every matched artifact during the single pass, then sizes them across a bounded worker pool (`min(NumCPU, 8)`) so the traversals and their I/O overlap. Disk is the primary figure (sorting, `--min-size`, totals); apparent surfaces only when a file is materially sparse. +Scanners report progress via context-attached callbacks for real-time UI updates; the walk batch reports under a single "projects" label, stat scanners under their own names. Sizing collects two figures per artifact via an in-process walk (`scanner.Measure`): **disk** (allocated blocks, `st_blocks×512` — sparse-accurate and matching `du`) and **apparent** (sum of logical file sizes). Directories contribute their own blocks to disk (real on ext4, ~0 on APFS); symlinks are never followed. Hard-linked inodes (`Nlink>1`) are counted once per artifact and recorded (keyed by `(dev, ino)`) so shared blocks can be netted out across artifacts. The walk engine does not size inline — it collects every matched artifact during the single pass, then sizes them across a bounded worker pool (`min(NumCPU, 8)`) so the traversals and their I/O overlap. The stat scanners (global/xcode/llm) size the same way: they discover artifacts via `stat` first, then defer to that same `sizePending` pool instead of measuring each inline. Disk is the primary figure (sorting, `--min-size`, totals); apparent surfaces only when a file is materially sparse. ### Walk engine diff --git a/internal/scanner/global.go b/internal/scanner/global.go index 0d08fab..9fc54ba 100644 --- a/internal/scanner/global.go +++ b/internal/scanner/global.go @@ -176,7 +176,7 @@ func (s *GlobalScanner) Scan(ctx context.Context, root string) ([]model.ScanResu continue } - results = append(results, sized(model.ScanResult{ + results = append(results, model.ScanResult{ Path: full, Ecosystem: model.EcoGlobal, Category: c.category, @@ -185,7 +185,7 @@ func (s *GlobalScanner) Scan(ctx context.Context, root string) ([]model.ScanResu ProjectRoot: filepath.Dir(full), Label: c.description, Recommendation: c.rec, - })) + }) ReportProgress(ctx, len(results)) } @@ -197,6 +197,12 @@ func (s *GlobalScanner) Scan(ctx context.Context, root string) ([]model.ScanResu if runtime.GOOS == "darwin" && isUnderRoot(home, absRoot) { results = append(results, s.scanCodeSignClones(ctx, len(results))...) } + + // Size all caches (and any code-sign clones) in one bounded worker pool + // rather than serially as each is discovered. + if err := sizePending(ctx, results); err != nil { + return results, err + } return results, nil } @@ -299,7 +305,7 @@ func (s *GlobalScanner) scanCodeSignClones(ctx context.Context, found int) []mod } } - out = append(out, sized(model.ScanResult{ + out = append(out, model.ScanResult{ Path: m, Ecosystem: model.EcoGlobal, Category: model.CatCache, @@ -308,7 +314,7 @@ func (s *GlobalScanner) scanCodeSignClones(ctx context.Context, found int) []mod ProjectRoot: filepath.Dir(m), Label: codeSignCloneLabel(m, name), Recommendation: rec, - })) + }) ReportProgress(ctx, found+len(out)) } return out diff --git a/internal/scanner/llm.go b/internal/scanner/llm.go index ce285e7..dcafce9 100644 --- a/internal/scanner/llm.go +++ b/internal/scanner/llm.go @@ -53,7 +53,7 @@ func (s *LLMScanner) Scan(ctx context.Context, root string) ([]model.ScanResult, return false default: } - results = append(results, sized(model.ScanResult{ + results = append(results, model.ScanResult{ Path: path, Ecosystem: model.EcoLLM, Category: model.CatCache, @@ -63,7 +63,7 @@ func (s *LLMScanner) Scan(ctx context.Context, root string) ([]model.ScanResult, Label: label, Recommendation: rec, LastUsedAt: ModTime(path), - })) + }) ReportProgress(ctx, len(results)) return true } @@ -120,6 +120,11 @@ func (s *LLMScanner) Scan(ctx context.Context, root string) ([]model.ScanResult, } } + // Size every discovered store in one bounded worker pool rather than + // serially as each is found (one tree walk per store overlaps the rest). + if err := sizePending(ctx, results); err != nil { + return results, err + } return results, nil } diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go index 37b1fbb..2bfdc9c 100644 --- a/internal/scanner/scanner.go +++ b/internal/scanner/scanner.go @@ -227,9 +227,10 @@ func DirSize(path string) int64 { return Measure(path).Disk } -// sized fills Size, ApparentSize and Links on r by measuring r.Path. Convenience -// for the stat-based scanners (xcode/global/llm) that build results one at a -// time, mirroring what the walk engine's sizePending does in bulk. +// sized fills Size, ApparentSize and Links on r by measuring r.Path — the +// single-result form of what the walk engine's sizePending does in bulk (the +// stat-based scanners now defer to sizePending; this remains for callers that +// size one result at a time). func sized(r model.ScanResult) model.ScanResult { st := Measure(r.Path) r.Size = st.Disk diff --git a/internal/scanner/xcode.go b/internal/scanner/xcode.go index bd23ea5..343ad06 100644 --- a/internal/scanner/xcode.go +++ b/internal/scanner/xcode.go @@ -120,16 +120,22 @@ func (s *XcodeScanner) Scan(ctx context.Context, root string) ([]model.ScanResul continue } - results = append(results, sized(model.ScanResult{ + results = append(results, model.ScanResult{ Path: full, Ecosystem: model.EcoXcode, Category: a.category, LastMod: ModTime(full), Safety: a.safety, ProjectRoot: filepath.Dir(full), - })) + }) ReportProgress(ctx, len(results)) } + + // Size every artifact (including expanded children) in one bounded worker + // pool rather than serially as each is discovered. + if err := sizePending(ctx, results); err != nil { + return results, err + } return results, nil } @@ -151,14 +157,14 @@ func expandChildren(ctx context.Context, parent string, a xcodeArtifact) []model default: } child := filepath.Join(parent, e.Name()) - out = append(out, sized(model.ScanResult{ + out = append(out, model.ScanResult{ Path: child, Ecosystem: model.EcoXcode, Category: a.category, LastMod: ModTime(child), Safety: a.safety, ProjectRoot: parent, - })) + }) ReportProgress(ctx, len(out)) } return out