From 66a0d23757093f3c764dd40c82773311b31352d9 Mon Sep 17 00:00:00 2001 From: jjscripts Date: Fri, 15 May 2026 22:20:43 +0800 Subject: [PATCH 1/2] feat: add --sort/--reverse flags and persistent hash cache for dupes --- Cargo.lock | 143 ++++++++++++++++++++++++-- Cargo.toml | 1 + benches/walker_bench.rs | 8 +- src/analyzer.rs | 107 ++++++++++++++++++- src/cache.rs | 222 ++++++++++++++++++++++++++++++++++++++++ src/dupes.rs | 34 +++++- src/lib.rs | 1 + src/main.rs | 81 +++++++++++++-- 8 files changed, 569 insertions(+), 28 deletions(-) create mode 100644 src/cache.rs diff --git a/Cargo.lock b/Cargo.lock index 12d7163..16a432d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,6 +107,7 @@ dependencies = [ "crossterm", "ctrlc", "dialoguer", + "dirs", "ignore", "indicatif", "insta", @@ -516,6 +517,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "dispatch2" version = "0.3.1" @@ -619,6 +641,17 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.4.2" @@ -853,6 +886,15 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libredox" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +dependencies = [ + "libc", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -977,6 +1019,12 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "owo-colors" version = "4.3.0" @@ -1150,6 +1198,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + [[package]] name = "regex" version = "1.12.3" @@ -1413,7 +1472,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -1666,13 +1725,22 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -1684,34 +1752,67 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -1724,24 +1825,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index 4610ddd..bb38a52 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ ctrlc = "3" ratatui = "0.29" crossterm = "0.28" num-format = "0.4" +dirs = "5" [target.'cfg(unix)'.dependencies] pager = "0.16" diff --git a/benches/walker_bench.rs b/benches/walker_bench.rs index 3248ccb..a0392c7 100644 --- a/benches/walker_bench.rs +++ b/benches/walker_bench.rs @@ -103,9 +103,7 @@ fn bench_walkers(c: &mut Criterion) { group.bench_function("deep_narrow_ignore_off", |b| { b.iter(|| count_ignore(deep.path(), false)) }); - group.bench_function("deep_narrow_jwalk", |b| { - b.iter(|| count_jwalk(deep.path())) - }); + group.bench_function("deep_narrow_jwalk", |b| b.iter(|| count_jwalk(deep.path()))); group.bench_function("realistic_ignore_on", |b| { b.iter(|| count_ignore(real.path(), true)) @@ -113,9 +111,7 @@ fn bench_walkers(c: &mut Criterion) { group.bench_function("realistic_ignore_off", |b| { b.iter(|| count_ignore(real.path(), false)) }); - group.bench_function("realistic_jwalk", |b| { - b.iter(|| count_jwalk(real.path())) - }); + group.bench_function("realistic_jwalk", |b| b.iter(|| count_jwalk(real.path()))); group.finish(); } diff --git a/src/analyzer.rs b/src/analyzer.rs index 84b0a02..c109087 100644 --- a/src/analyzer.rs +++ b/src/analyzer.rs @@ -13,6 +13,15 @@ pub struct CategorySummary { pub stale_count: usize, } +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum SortKey { + Size, + Count, + StaleSize, + StaleCount, + Name, +} + pub fn analyze(files: &[FileEntry], stale_years: u64) -> Vec { let mut map: HashMap<&'static str, CategorySummary> = HashMap::new(); let now = SystemTime::now(); @@ -40,6 +49,102 @@ pub fn analyze(files: &[FileEntry], stale_years: u64) -> Vec { } let mut summaries: Vec = map.into_values().collect(); - summaries.sort_by_key(|s| std::cmp::Reverse(s.total_size)); + sort_summaries(&mut summaries, SortKey::Size, false); summaries } + +pub fn sort_summaries(summaries: &mut [CategorySummary], key: SortKey, reverse: bool) { + match key { + SortKey::Size => summaries.sort_by_key(|s| std::cmp::Reverse(s.total_size)), + SortKey::Count => summaries.sort_by_key(|s| std::cmp::Reverse(s.file_count)), + SortKey::StaleSize => summaries.sort_by_key(|s| std::cmp::Reverse(s.stale_size)), + SortKey::StaleCount => summaries.sort_by_key(|s| std::cmp::Reverse(s.stale_count)), + SortKey::Name => summaries.sort_by(|a, b| a.category.cmp(&b.category)), + } + if reverse { + summaries.reverse(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn s( + category: &str, + total: u64, + count: usize, + stale_size: u64, + stale_count: usize, + ) -> CategorySummary { + CategorySummary { + category: category.to_string(), + total_size: total, + file_count: count, + stale_size, + stale_count, + } + } + + fn names(v: &[CategorySummary]) -> Vec<&str> { + v.iter().map(|s| s.category.as_str()).collect() + } + + #[test] + fn sort_by_size_desc_by_default() { + let mut v = vec![ + s("a", 100, 1, 0, 0), + s("b", 300, 1, 0, 0), + s("c", 200, 1, 0, 0), + ]; + sort_summaries(&mut v, SortKey::Size, false); + assert_eq!(names(&v), vec!["b", "c", "a"]); + } + + #[test] + fn sort_by_count() { + let mut v = vec![s("a", 0, 5, 0, 0), s("b", 0, 20, 0, 0), s("c", 0, 10, 0, 0)]; + sort_summaries(&mut v, SortKey::Count, false); + assert_eq!(names(&v), vec!["b", "c", "a"]); + } + + #[test] + fn sort_by_stale_size() { + let mut v = vec![ + s("a", 0, 0, 50, 0), + s("b", 0, 0, 200, 0), + s("c", 0, 0, 100, 0), + ]; + sort_summaries(&mut v, SortKey::StaleSize, false); + assert_eq!(names(&v), vec!["b", "c", "a"]); + } + + #[test] + fn sort_by_stale_count() { + let mut v = vec![s("a", 0, 0, 0, 2), s("b", 0, 0, 0, 7), s("c", 0, 0, 0, 4)]; + sort_summaries(&mut v, SortKey::StaleCount, false); + assert_eq!(names(&v), vec!["b", "c", "a"]); + } + + #[test] + fn sort_by_name_is_alphabetical() { + let mut v = vec![ + s("video", 0, 0, 0, 0), + s("audio", 0, 0, 0, 0), + s("images", 0, 0, 0, 0), + ]; + sort_summaries(&mut v, SortKey::Name, false); + assert_eq!(names(&v), vec!["audio", "images", "video"]); + } + + #[test] + fn reverse_flips_order() { + let mut v = vec![ + s("a", 100, 1, 0, 0), + s("b", 300, 1, 0, 0), + s("c", 200, 1, 0, 0), + ]; + sort_summaries(&mut v, SortKey::Size, true); + assert_eq!(names(&v), vec!["a", "c", "b"]); + } +} diff --git a/src/cache.rs b/src/cache.rs new file mode 100644 index 0000000..d32ffa8 --- /dev/null +++ b/src/cache.rs @@ -0,0 +1,222 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::SystemTime; + +const HASH_ALGORITHM: &str = "blake3"; +const CACHE_FILENAME: &str = "hashes.json"; + +#[derive(Serialize, Deserialize, Clone)] +struct CacheEntry { + mtime_ns: u128, + size: u64, + hash_hex: String, +} + +#[derive(Serialize, Deserialize, Default)] +struct CacheFile { + algorithm: String, + entries: HashMap, +} + +pub struct HashCache { + inner: Mutex>, + dirty: Mutex, +} + +impl HashCache { + pub fn load() -> Self { + let path = match cache_path() { + Some(p) => p, + None => return Self::empty(), + }; + let bytes = match fs::read(&path) { + Ok(b) => b, + Err(_) => return Self::empty(), + }; + let parsed: CacheFile = match serde_json::from_slice(&bytes) { + Ok(p) => p, + Err(_) => return Self::empty(), + }; + if parsed.algorithm != HASH_ALGORITHM { + return Self::empty(); + } + HashCache { + inner: Mutex::new(parsed.entries), + dirty: Mutex::new(false), + } + } + + pub fn empty() -> Self { + HashCache { + inner: Mutex::new(HashMap::new()), + dirty: Mutex::new(false), + } + } + + pub fn get(&self, path: &Path, mtime: SystemTime, size: u64) -> Option<[u8; 32]> { + let key = path.to_string_lossy().into_owned(); + let mtime_ns = systime_to_ns(mtime)?; + let guard = self.inner.lock().ok()?; + let entry = guard.get(&key)?; + if entry.mtime_ns != mtime_ns || entry.size != size { + return None; + } + decode_hash(&entry.hash_hex) + } + + pub fn insert(&self, path: &Path, mtime: SystemTime, size: u64, hash: [u8; 32]) { + let Some(mtime_ns) = systime_to_ns(mtime) else { + return; + }; + let key = path.to_string_lossy().into_owned(); + let entry = CacheEntry { + mtime_ns, + size, + hash_hex: encode_hash(&hash), + }; + if let Ok(mut guard) = self.inner.lock() { + guard.insert(key, entry); + } + if let Ok(mut d) = self.dirty.lock() { + *d = true; + } + } + + pub fn save(&self) -> std::io::Result<()> { + let dirty = self.dirty.lock().map(|g| *g).unwrap_or(false); + if !dirty { + return Ok(()); + } + let Some(path) = cache_path() else { + return Ok(()); + }; + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let entries = self.inner.lock().map(|g| g.clone()).unwrap_or_default(); + let pruned: HashMap = entries + .into_iter() + .filter(|(k, _)| Path::new(k).exists()) + .collect(); + let file = CacheFile { + algorithm: HASH_ALGORITHM.to_string(), + entries: pruned, + }; + let bytes = serde_json::to_vec(&file).map_err(std::io::Error::other)?; + fs::write(&path, bytes)?; + Ok(()) + } +} + +pub fn cache_path() -> Option { + let base = dirs::cache_dir()?; + Some(base.join("bigfiles").join(CACHE_FILENAME)) +} + +pub fn clear() -> std::io::Result<()> { + let Some(path) = cache_path() else { + return Ok(()); + }; + match fs::remove_file(&path) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +fn systime_to_ns(t: SystemTime) -> Option { + t.duration_since(SystemTime::UNIX_EPOCH) + .ok() + .map(|d| d.as_nanos()) +} + +fn encode_hash(h: &[u8; 32]) -> String { + let mut s = String::with_capacity(64); + for b in h { + use std::fmt::Write as _; + let _ = write!(s, "{:02x}", b); + } + s +} + +fn decode_hash(s: &str) -> Option<[u8; 32]> { + if s.len() != 64 { + return None; + } + let mut out = [0u8; 32]; + for (i, byte) in out.iter_mut().enumerate() { + let hi = u8::from_str_radix(&s[i * 2..i * 2 + 1], 16).ok()?; + let lo = u8::from_str_radix(&s[i * 2 + 1..i * 2 + 2], 16).ok()?; + *byte = (hi << 4) | lo; + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::File; + use std::io::Write; + use tempfile::tempdir; + + #[test] + fn hash_roundtrip() { + let h = [0xABu8; 32]; + let encoded = encode_hash(&h); + assert_eq!(encoded.len(), 64); + let decoded = decode_hash(&encoded).unwrap(); + assert_eq!(decoded, h); + } + + #[test] + fn miss_on_empty_cache() { + let dir = tempdir().unwrap(); + let p = dir.path().join("x"); + File::create(&p).unwrap().write_all(b"hi").unwrap(); + let meta = fs::metadata(&p).unwrap(); + let c = HashCache::empty(); + assert!(c.get(&p, meta.modified().unwrap(), meta.len()).is_none()); + } + + #[test] + fn hit_after_insert() { + let dir = tempdir().unwrap(); + let p = dir.path().join("x"); + File::create(&p).unwrap().write_all(b"hi").unwrap(); + let meta = fs::metadata(&p).unwrap(); + let mtime = meta.modified().unwrap(); + let h = [0x42u8; 32]; + let c = HashCache::empty(); + c.insert(&p, mtime, meta.len(), h); + assert_eq!(c.get(&p, mtime, meta.len()), Some(h)); + } + + #[test] + fn miss_when_size_changes() { + let dir = tempdir().unwrap(); + let p = dir.path().join("x"); + File::create(&p).unwrap().write_all(b"hi").unwrap(); + let meta = fs::metadata(&p).unwrap(); + let mtime = meta.modified().unwrap(); + let h = [0x42u8; 32]; + let c = HashCache::empty(); + c.insert(&p, mtime, meta.len(), h); + assert!(c.get(&p, mtime, meta.len() + 1).is_none()); + } + + #[test] + fn miss_when_mtime_changes() { + let dir = tempdir().unwrap(); + let p = dir.path().join("x"); + File::create(&p).unwrap().write_all(b"hi").unwrap(); + let meta = fs::metadata(&p).unwrap(); + let h = [0x42u8; 32]; + let c = HashCache::empty(); + c.insert(&p, meta.modified().unwrap(), meta.len(), h); + let later = meta.modified().unwrap() + std::time::Duration::from_secs(1); + assert!(c.get(&p, later, meta.len()).is_none()); + } +} diff --git a/src/dupes.rs b/src/dupes.rs index fdf9459..e0dce14 100644 --- a/src/dupes.rs +++ b/src/dupes.rs @@ -1,3 +1,4 @@ +use crate::cache::HashCache; use crate::format::bytes as format_bytes; use crate::walker::{FileEntry, InodeKey}; use dialoguer::{theme::ColorfulTheme, Confirm, Select}; @@ -32,10 +33,17 @@ impl DupeGroup { } pub fn find(files: &[FileEntry], min_size: u64) -> Vec { + let cache = HashCache::empty(); + find_with_cache(files, min_size, &cache) +} + +pub fn find_with_cache(files: &[FileEntry], min_size: u64, cache: &HashCache) -> Vec { let by_size = group_by_size(files, min_size); let mut groups: Vec = by_size .into_par_iter() - .flat_map_iter(|(size, candidates)| process_size_bucket(size, candidates).into_iter()) + .flat_map_iter(|(size, candidates)| { + process_size_bucket(size, candidates, cache).into_iter() + }) .collect(); groups.sort_by_key(|g| std::cmp::Reverse(g.reclaimable())); @@ -54,7 +62,11 @@ fn group_by_size(files: &[FileEntry], min_size: u64) -> HashMap) -> Vec { +fn process_size_bucket( + size: u64, + candidates: Vec<&FileEntry>, + cache: &HashCache, +) -> Vec { let by_inode = collapse_hardlinks(&candidates); if by_inode.len() < 2 { return Vec::new(); @@ -62,7 +74,7 @@ fn process_size_bucket(size: u64, candidates: Vec<&FileEntry>) -> Vec let by_partial = group_by_partial_hash(&by_inode); let mut out = Vec::new(); for partial_group in by_partial { - for full_group in group_by_full_hash(&partial_group) { + for full_group in group_by_full_hash(&partial_group, cache) { let mut entries: Vec = full_group; entries.sort_by(|a, b| a.primary_path().cmp(b.primary_path())); out.push(DupeGroup { size, entries }); @@ -108,10 +120,10 @@ fn group_by_partial_hash(entries: &[DupeEntry]) -> Vec> { by_hash.into_values().filter(|v| v.len() >= 2).collect() } -fn group_by_full_hash(entries: &[DupeEntry]) -> Vec> { +fn group_by_full_hash(entries: &[DupeEntry], cache: &HashCache) -> Vec> { let hashed: Vec<([u8; 32], &DupeEntry)> = entries .par_iter() - .filter_map(|e| full_hash(e.primary_path()).map(|h| (h, e))) + .filter_map(|e| cached_full_hash(e.primary_path(), cache).map(|h| (h, e))) .collect(); let mut by_hash: HashMap<[u8; 32], Vec> = HashMap::new(); @@ -123,6 +135,18 @@ fn group_by_full_hash(entries: &[DupeEntry]) -> Vec> { by_hash.into_values().filter(|v| v.len() >= 2).collect() } +fn cached_full_hash(path: &Path, cache: &HashCache) -> Option<[u8; 32]> { + let meta = fs::metadata(path).ok()?; + let mtime = meta.modified().ok()?; + let size = meta.len(); + if let Some(h) = cache.get(path, mtime, size) { + return Some(h); + } + let h = full_hash(path)?; + cache.insert(path, mtime, size, h); + Some(h) +} + fn partial_hash(path: &Path) -> Option<[u8; 32]> { let mut file = File::open(path).ok()?; let mut buf = vec![0u8; PARTIAL_HASH_BYTES as usize]; diff --git a/src/lib.rs b/src/lib.rs index 708807a..6c4fbbe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ pub mod analyzer; +pub mod cache; pub mod classifier; pub mod dupes; pub mod format; diff --git a/src/main.rs b/src/main.rs index 805d7c2..3cd3c37 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,11 @@ mod delete; +use bigfiles::analyzer::SortKey; use bigfiles::format::Units; use bigfiles::walker::{ScanResult, WalkOptions}; use bigfiles::{analyzer, dupes, format, renderer, walker, INTERRUPTED}; use clap::builder::styling::{AnsiColor, Effects, Styles}; -use clap::{Parser, Subcommand}; +use clap::{Parser, Subcommand, ValueEnum}; use indicatif::{ProgressBar, ProgressStyle}; use std::io::IsTerminal; use std::path::PathBuf; @@ -82,6 +83,36 @@ struct Cli { /// Output raw JSON (default scan only) #[arg(short, long)] json: bool, + + /// Sort categories by: size, count, stale-size, stale-count, name (default scan only) + #[arg(long, value_enum, default_value_t = SortKeyArg::Size)] + sort: SortKeyArg, + + /// Reverse the sort order (default scan only) + #[arg(long)] + reverse: bool, +} + +#[derive(Copy, Clone, Debug, ValueEnum)] +#[clap(rename_all = "kebab-case")] +enum SortKeyArg { + Size, + Count, + StaleSize, + StaleCount, + Name, +} + +impl From for SortKey { + fn from(v: SortKeyArg) -> Self { + match v { + SortKeyArg::Size => SortKey::Size, + SortKeyArg::Count => SortKey::Count, + SortKeyArg::StaleSize => SortKey::StaleSize, + SortKeyArg::StaleCount => SortKey::StaleCount, + SortKeyArg::Name => SortKey::Name, + } + } } #[derive(Subcommand)] @@ -95,6 +126,14 @@ enum Command { /// Interactively delete duplicate copies (keep one per group) #[arg(long)] delete: bool, + + /// Skip the persistent hash cache (read and write disabled for this run) + #[arg(long)] + no_cache: bool, + + /// Delete the persistent hash cache before running + #[arg(long)] + clear_cache: bool, }, /// Interactively delete stale files Delete, @@ -135,15 +174,21 @@ fn main() -> ExitCode { return ExitCode::from(EXIT_USAGE_ERROR); } - if cli.command.is_some() && (cli.top.is_some() || cli.json) { + let non_default_sort = !matches!(cli.sort, SortKeyArg::Size) || cli.reverse; + if cli.command.is_some() && (cli.top.is_some() || cli.json || non_default_sort) { eprintln!( - "bigfiles: --top and --json only apply to the default scan; ignoring for this subcommand" + "bigfiles: --top, --json, --sort, --reverse only apply to the default scan; ignoring for this subcommand" ); } match &cli.command { None => run_scan(&cli), - Some(Command::Dupes { min_size, delete }) => run_dupes(&cli, *min_size, *delete), + Some(Command::Dupes { + min_size, + delete, + no_cache, + clear_cache, + }) => run_dupes(&cli, *min_size, *delete, *no_cache, *clear_cache), Some(Command::Delete) => run_delete(&cli), Some(Command::Tui) => run_tui(&cli), } @@ -218,7 +263,8 @@ fn setup_pager(_cli: &Cli) {} fn run_scan(cli: &Cli) -> ExitCode { let scan = scan_with_progress(cli, !cli.json); let total: u64 = scan.files.iter().map(|f| f.size).sum(); - let summaries = analyzer::analyze(&scan.files, cli.stale_years); + let mut summaries = analyzer::analyze(&scan.files, cli.stale_years); + analyzer::sort_summaries(&mut summaries, cli.sort.into(), cli.reverse); if cli.json { let envelope = serde_json::json!({ @@ -245,9 +291,30 @@ fn run_scan(cli: &Cli) -> ExitCode { ExitCode::from(EXIT_SUCCESS) } -fn run_dupes(cli: &Cli, min_size: u64, delete: bool) -> ExitCode { +fn run_dupes( + cli: &Cli, + min_size: u64, + delete: bool, + no_cache: bool, + clear_cache: bool, +) -> ExitCode { + if clear_cache { + if let Err(e) = bigfiles::cache::clear() { + eprintln!("bigfiles: failed to clear hash cache: {}", e); + } + } let scan = scan_with_progress(cli, true); - let groups = dupes::find(&scan.files, min_size); + let cache = if no_cache { + bigfiles::cache::HashCache::empty() + } else { + bigfiles::cache::HashCache::load() + }; + let groups = dupes::find_with_cache(&scan.files, min_size, &cache); + if !no_cache { + if let Err(e) = cache.save() { + eprintln!("bigfiles: failed to save hash cache: {}", e); + } + } if delete { match dupes::delete_interactive(&groups, &cli.path) { Ok(()) => ExitCode::from(EXIT_SUCCESS), From 02e3e7fd9799991a55e3cdd60fe8110b1f9ea967 Mon Sep 17 00:00:00 2001 From: jjscripts Date: Fri, 15 May 2026 22:29:46 +0800 Subject: [PATCH 2/2] feat: --sort flag, persistent dupes cache, and `audit` subcommand --- README.md | 1 + deny.toml | 1 + src/audit.rs | 372 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/main.rs | 13 ++ 5 files changed, 388 insertions(+) create mode 100644 src/audit.rs diff --git a/README.md b/README.md index 7b10e4d..eaf96a3 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![CI](https://github.com/Par-python/bigfiles/actions/workflows/ci.yml/badge.svg)](https://github.com/Par-python/bigfiles/actions/workflows/ci.yml) [![crates.io](https://img.shields.io/crates/v/bigfiles.svg)](https://crates.io/crates/bigfiles) [![Downloads](https://img.shields.io/crates/d/bigfiles.svg)](https://crates.io/crates/bigfiles) +[![Stars](https://img.shields.io/github/stars/Par-python/bigfiles.svg?style=flat)](https://github.com/Par-python/bigfiles/stargazers) [![License: AGPL v3](https://img.shields.io/badge/license-AGPL--3.0--or--later-blue.svg)](LICENSE) A small Rust CLI that walks a directory in parallel, groups files by type, flags stale ones, finds duplicates (hardlink-aware), and renders a color-coded summary in the terminal. Cross-platform: Linux, macOS, Windows. diff --git a/deny.toml b/deny.toml index 0438061..91df8e8 100644 --- a/deny.toml +++ b/deny.toml @@ -19,6 +19,7 @@ allow = [ "Unlicense", "Zlib", "ISC", + "MPL-2.0", "AGPL-3.0-or-later", ] confidence-threshold = 0.8 diff --git a/src/audit.rs b/src/audit.rs new file mode 100644 index 0000000..31388e5 --- /dev/null +++ b/src/audit.rs @@ -0,0 +1,372 @@ +use crate::analyzer::CategorySummary; +use crate::format::bytes as format_bytes; +use crate::walker::FileEntry; +use owo_colors::OwoColorize; +use std::collections::HashMap; +use std::fmt::Write as _; +use std::path::Path; +use std::time::{Duration, SystemTime}; + +const INSTALLER_EXTS: &[&str] = &["dmg", "pkg", "iso", "exe", "msi", "deb", "rpm"]; +const TOP_N_CONCENTRATION: usize = 10; +const TOP_EXTENSIONS: usize = 5; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Severity { + Info, + Notable, + Heavy, +} + +#[derive(Debug, Clone)] +pub struct Insight { + pub headline: String, + pub severity: Severity, +} + +pub fn analyze( + files: &[FileEntry], + summaries: &[CategorySummary], + total_size: u64, + stale_years: u64, +) -> Vec { + let mut out = Vec::new(); + if total_size == 0 { + return out; + } + + if let Some(i) = heaviest_category(summaries, total_size) { + out.push(i); + } + out.extend(top_extensions(files, total_size)); + if let Some(i) = installer_junk(files, total_size) { + out.push(i); + } + if let Some(i) = top_n_concentration(files, total_size) { + out.push(i); + } + if let Some(i) = stale_concentration(files, total_size, stale_years) { + out.push(i); + } + out +} + +fn heaviest_category(summaries: &[CategorySummary], total: u64) -> Option { + let top = summaries.iter().max_by_key(|s| s.total_size)?; + if top.total_size == 0 { + return None; + } + let pct = percent(top.total_size, total); + let severity = if pct >= 40.0 { + Severity::Heavy + } else if pct >= 20.0 { + Severity::Notable + } else { + Severity::Info + }; + Some(Insight { + headline: format!( + "{} is your heaviest category: {} across {} files ({:.0}% of total)", + top.category, + format_bytes(top.total_size), + count_str(top.file_count), + pct + ), + severity, + }) +} + +fn top_extensions(files: &[FileEntry], total: u64) -> Vec { + let mut by_ext: HashMap<&str, (u64, usize)> = HashMap::new(); + for f in files { + let entry = by_ext.entry(f.extension.as_str()).or_insert((0, 0)); + entry.0 += f.size; + entry.1 += 1; + } + let mut ranked: Vec<(&str, u64, usize)> = + by_ext.into_iter().map(|(e, (s, c))| (e, s, c)).collect(); + ranked.sort_by_key(|(_, s, _)| std::cmp::Reverse(*s)); + + let mut out = Vec::new(); + for (ext, size, count) in ranked.into_iter().take(TOP_EXTENSIONS) { + if size == 0 { + continue; + } + let pct = percent(size, total); + if pct < 1.0 { + continue; + } + let label = if ext == "none" { + "(no extension)".to_string() + } else { + format!(".{}", ext) + }; + let severity = if pct >= 25.0 { + Severity::Heavy + } else if pct >= 10.0 { + Severity::Notable + } else { + Severity::Info + }; + out.push(Insight { + headline: format!( + "{} files account for {} ({:.0}% of total, {} files)", + label, + format_bytes(size), + pct, + count_str(count) + ), + severity, + }); + } + out +} + +fn installer_junk(files: &[FileEntry], total: u64) -> Option { + let mut size = 0u64; + let mut count = 0usize; + for f in files { + if INSTALLER_EXTS.contains(&f.extension.as_str()) { + size += f.size; + count += 1; + } + } + if count == 0 || size == 0 { + return None; + } + let pct = percent(size, total); + let severity = if pct >= 10.0 { + Severity::Notable + } else { + Severity::Info + }; + Some(Insight { + headline: format!( + "Installer files (.dmg/.pkg/.iso/.exe/.msi/.deb/.rpm): {} across {} files — usually safe to delete after use", + format_bytes(size), + count_str(count) + ), + severity, + }) +} + +fn top_n_concentration(files: &[FileEntry], total: u64) -> Option { + if files.len() <= TOP_N_CONCENTRATION { + return None; + } + let mut sizes: Vec = files.iter().map(|f| f.size).collect(); + sizes.sort_unstable_by_key(|s| std::cmp::Reverse(*s)); + let top_sum: u64 = sizes.iter().take(TOP_N_CONCENTRATION).sum(); + if top_sum == 0 { + return None; + } + let pct = percent(top_sum, total); + if pct < 5.0 { + return None; + } + let severity = if pct >= 50.0 { + Severity::Heavy + } else if pct >= 25.0 { + Severity::Notable + } else { + Severity::Info + }; + Some(Insight { + headline: format!( + "Your top {} files alone are {} ({:.0}% of total) — concentrated weight", + TOP_N_CONCENTRATION, + format_bytes(top_sum), + pct + ), + severity, + }) +} + +fn stale_concentration(files: &[FileEntry], total: u64, stale_years: u64) -> Option { + let now = SystemTime::now(); + let threshold = Duration::from_secs(stale_years * 365 * 24 * 60 * 60); + let mut stale_size = 0u64; + let mut stale_count = 0usize; + for f in files { + if let Ok(age) = now.duration_since(f.modified) { + if age > threshold { + stale_size += f.size; + stale_count += 1; + } + } + } + if stale_count == 0 || stale_size == 0 { + return None; + } + let pct = percent(stale_size, total); + let severity = if pct >= 40.0 { + Severity::Heavy + } else if pct >= 20.0 { + Severity::Notable + } else { + Severity::Info + }; + Some(Insight { + headline: format!( + "{} ({:.0}% of total) is stale — older than {} year{} across {} files", + format_bytes(stale_size), + pct, + stale_years, + if stale_years == 1 { "" } else { "s" }, + count_str(stale_count) + ), + severity, + }) +} + +pub fn render(insights: &[Insight], total: u64, root: &Path) { + print!("{}", render_to_string(insights, total, root)); +} + +pub fn render_to_string(insights: &[Insight], total: u64, root: &Path) -> String { + let mut out = String::new(); + let _ = writeln!(out); + let _ = writeln!( + out, + " {} {} {}", + "bigfiles audit".bold(), + format_bytes(total).bold().cyan(), + root.display().dimmed() + ); + let _ = writeln!(out); + + if insights.is_empty() { + let _ = writeln!(out, " {}", "Nothing notable to report.".dimmed()); + let _ = writeln!(out); + return out; + } + + for i in insights { + let bullet = match i.severity { + Severity::Heavy => "!".red().bold().to_string(), + Severity::Notable => "•".yellow().bold().to_string(), + Severity::Info => "·".dimmed().to_string(), + }; + let _ = writeln!(out, " {} {}", bullet, i.headline); + } + let _ = writeln!(out); + let _ = writeln!( + out, + " {}", + "Run `bigfiles` for the full category breakdown, or `bigfiles dupes` to find duplicates." + .dimmed() + ); + let _ = writeln!(out); + out +} + +fn percent(part: u64, whole: u64) -> f64 { + if whole == 0 { + return 0.0; + } + (part as f64 / whole as f64) * 100.0 +} + +fn count_str(n: usize) -> String { + n.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + use std::time::{Duration, SystemTime}; + + fn fe(path: &str, size: u64, ext: &str, age_secs: u64) -> FileEntry { + FileEntry { + path: PathBuf::from(path), + size, + extension: ext.to_string(), + modified: SystemTime::now() - Duration::from_secs(age_secs), + inode: None, + } + } + + fn cs(category: &str, total: u64, count: usize) -> CategorySummary { + CategorySummary { + category: category.to_string(), + total_size: total, + file_count: count, + stale_size: 0, + stale_count: 0, + } + } + + #[test] + fn empty_total_yields_no_insights() { + let v = analyze(&[], &[], 0, 2); + assert!(v.is_empty()); + } + + #[test] + fn heaviest_category_is_reported() { + let summaries = vec![cs("video", 800, 10), cs("audio", 200, 50)]; + let files = vec![fe("a.mp4", 800, "mp4", 0), fe("b.mp3", 200, "mp3", 0)]; + let v = analyze(&files, &summaries, 1000, 2); + let h = v.iter().find(|i| i.headline.contains("heaviest")).unwrap(); + assert!(h.headline.contains("video")); + assert!(h.headline.contains("80%")); + assert_eq!(h.severity, Severity::Heavy); + } + + #[test] + fn installer_junk_detected() { + let files = vec![ + fe("a.dmg", 500, "dmg", 0), + fe("b.pkg", 300, "pkg", 0), + fe("c.txt", 100, "txt", 0), + ]; + let summaries = vec![cs("archives", 800, 2), cs("other", 100, 1)]; + let v = analyze(&files, &summaries, 900, 2); + assert!(v.iter().any(|i| i.headline.contains("Installer files"))); + } + + #[test] + fn stale_concentration_uses_threshold() { + let three_years = 3 * 365 * 24 * 60 * 60; + let files = vec![ + fe("old.mp4", 600, "mp4", three_years), + fe("new.mp4", 400, "mp4", 0), + ]; + let summaries = vec![cs("video", 1000, 2)]; + let v = analyze(&files, &summaries, 1000, 2); + let stale = v.iter().find(|i| i.headline.contains("stale")).unwrap(); + assert!(stale.headline.contains("60%")); + } + + #[test] + fn top_n_concentration_skipped_for_few_files() { + let files = vec![fe("a", 100, "txt", 0), fe("b", 100, "txt", 0)]; + let summaries = vec![cs("other", 200, 2)]; + let v = analyze(&files, &summaries, 200, 2); + assert!(!v.iter().any(|i| i.headline.contains("top 10"))); + } + + #[test] + fn top_extensions_reported() { + let mut files = Vec::new(); + for _ in 0..20 { + files.push(fe("v.mp4", 100, "mp4", 0)); + } + for _ in 0..5 { + files.push(fe("a.mp3", 10, "mp3", 0)); + } + let summaries = vec![cs("video", 2000, 20), cs("audio", 50, 5)]; + let v = analyze(&files, &summaries, 2050, 2); + assert!(v.iter().any(|i| i.headline.contains(".mp4"))); + } + + #[test] + fn renders_without_panicking() { + let summaries = vec![cs("video", 800, 10)]; + let files = vec![fe("a.mp4", 800, "mp4", 0)]; + let insights = analyze(&files, &summaries, 1000, 2); + let s = render_to_string(&insights, 1000, Path::new("/test")); + assert!(s.contains("bigfiles audit")); + } +} diff --git a/src/lib.rs b/src/lib.rs index 6c4fbbe..640f4a9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ pub mod analyzer; +pub mod audit; pub mod cache; pub mod classifier; pub mod dupes; diff --git a/src/main.rs b/src/main.rs index 3cd3c37..01f7b78 100644 --- a/src/main.rs +++ b/src/main.rs @@ -139,6 +139,8 @@ enum Command { Delete, /// Interactive directory browser (ncdu-style) Tui, + /// Quick insights about what's eating your disk + Audit, } fn main() -> ExitCode { @@ -191,6 +193,7 @@ fn main() -> ExitCode { }) => run_dupes(&cli, *min_size, *delete, *no_cache, *clear_cache), Some(Command::Delete) => run_delete(&cli), Some(Command::Tui) => run_tui(&cli), + Some(Command::Audit) => run_audit(&cli), } } @@ -341,6 +344,16 @@ fn run_tui(cli: &Cli) -> ExitCode { } } +fn run_audit(cli: &Cli) -> ExitCode { + let scan = scan_with_progress(cli, true); + let total: u64 = scan.files.iter().map(|f| f.size).sum(); + let summaries = analyzer::analyze(&scan.files, cli.stale_years); + let insights = bigfiles::audit::analyze(&scan.files, &summaries, total, cli.stale_years); + setup_pager(cli); + bigfiles::audit::render(&insights, total, &cli.path); + ExitCode::from(EXIT_SUCCESS) +} + fn run_delete(cli: &Cli) -> ExitCode { let scan = scan_with_progress(cli, true); match delete::run(&scan.files, cli.stale_years) {