From 8670afa29da075a09e509a23f5b79039025f2255 Mon Sep 17 00:00:00 2001 From: kingchenc Date: Thu, 22 Oct 2026 22:01:56 +0200 Subject: [PATCH] NTFS MFT engine and a per-file parallel duplicate pipeline with hash cache (0.10.0) Scanning: NTFS MFT engine (Windows, administrator) - New mft module: reads the volume's Master File Table straight from the volume (boot sector -> $MFT data runs -> 8 MB chunks read in parallel over four handles), applies update-sequence fixups and parses $STANDARD_INFORMATION, $FILE_NAME (8.3 aliases skipped, one entry per hard link), the unnamed $DATA stream (logical / allocated / compressed size) and reparse points. Metafiles, symbolic links and junctions are left out. All reads are bounds-checked. - Without exclusions the tree is built straight from the records (no paths); with exclusions, and for the duplicate search, it emits the usual batches. - Cloud placeholders and OneDrive-synced folders (from the OneDrive environment variables) are sized / listed through Windows, because only the cloud filter knows what is stored locally. - Options::mft, CLI --mft; ScanTree::engine() and mft_fallback() say which engine ran and why the MFT was not used (no rights, not NTFS, ...). - Measured (administrator, NVMe system drive): C:\Users\me 196 s -> 37 s with the walk vs the MFT, all of C: (4.8 M files, 4.9 M folders) in ~30 s; file counts and folder sizes match the walk. On a SATA SSD with millions of files the table is larger than the folders and the walk stays faster, which the settings text says. Duplicates - New dupes module: candidates of a repeated size are opened once, in parallel per file (not per size group), for their identity (hard links collapsed) and a BLAKE3 of the first and last 16 KB (files up to 32 KB are thereby hashed whole); only what still matches is hashed in full, big files on several threads via mmap. same-file dependency dropped. - HashCache: path + size + modification time -> identity and hashes, zstd-compressed; find_duplicates_cached reuses it. CLI --cache FILE; the GUI keeps one in its data folder. A repeated search of a 461-group Steam library: 1.2 s -> 0.1 s, identical groups. - Progress reports phases (list, compare ends, hash) with totals. - Exclude globs are case-insensitive on Windows. GUI - Engine "Ultra (NTFS MFT)" with a restart-as-administrator button (and the note that Windows blocks drag & drop into an elevated window); "Admin" next to the version; the scan card shows "NTFS MFT" and a status line explains a fallback. - Duplicate search skips program and game files unless included (the card offers a search that includes them); the progress card shows the three phases with counts, bytes and a real percentage. Tests: 34 core (cache round trip and reuse, same ends / different middle, in-order and out-of-order tree batches), 8 CLI, 23 UI helper tests. --- Cargo.lock | 6 +- Cargo.toml | 2 +- README.md | 31 +- crates/diskghost-cli/src/main.rs | 64 ++- crates/diskghost-core/Cargo.toml | 3 +- crates/diskghost-core/src/dupes.rs | 502 +++++++++++++++++ crates/diskghost-core/src/lib.rs | 204 +++---- crates/diskghost-core/src/mft.rs | 770 ++++++++++++++++++++++++++ crates/diskghost-core/src/snapshot.rs | 4 +- crates/diskghost-core/src/tree.rs | 163 +++++- crates/diskghost-core/src/walk.rs | 59 +- gui/src-tauri/Cargo.lock | 6 +- gui/src-tauri/Cargo.toml | 2 +- gui/src-tauri/src/main.rs | 145 ++++- gui/src-tauri/tauri.conf.json | 2 +- gui/ui/app.js | 111 +++- gui/ui/lib.js | 13 +- gui/ui/lib.test.js | 12 + 18 files changed, 1894 insertions(+), 205 deletions(-) create mode 100644 crates/diskghost-core/src/dupes.rs create mode 100644 crates/diskghost-core/src/mft.rs diff --git a/Cargo.lock b/Cargo.lock index 65d74b6..6025877 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,6 +113,7 @@ dependencies = [ "constant_time_eq", "cpufeatures", "memmap2", + "rayon-core", ] [[package]] @@ -329,7 +330,7 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "diskghost-cli" -version = "0.9.2" +version = "0.10.0" dependencies = [ "clap", "diskghost-core", @@ -339,14 +340,13 @@ dependencies = [ [[package]] name = "diskghost-core" -version = "0.9.2" +version = "0.10.0" dependencies = [ "blake3", "criterion", "fs4", "globset", "rayon", - "same-file", "serde", "serde_json", "trash", diff --git a/Cargo.toml b/Cargo.toml index eff3a03..eadae2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ exclude = ["gui"] resolver = "2" [workspace.package] -version = "0.9.2" +version = "0.10.0" edition = "2021" license = "MIT" authors = ["kingchenc"] diff --git a/README.md b/README.md index aa29917..97b1261 100644 --- a/README.md +++ b/README.md @@ -13,16 +13,29 @@ Find where your space went, hunt down byte-identical duplicates, and reclaim the come straight from the directory listing (no file is opened), every scan has its own thread pool, and the tree is built while the walk is still running: 4.9 M files / 1.2 TB on an NVMe SSD in about 6 s. -- **Correct duplicate detection** — group by size → **collapse hard links** - (same physical file, so `wasted` is real) → **first-block pre-hash** → full - **BLAKE3** hash, all in parallel. +- **Correct, fast duplicate detection** — group by size → open every + candidate **once** (collapsing **hard links**, so `wasted` is real, and + hashing its first and last 16 KB) → full **BLAKE3** only for what still + matches; every step parallel per file, big files hashed on several threads. + An optional **hash cache** (`--cache`, automatic in the GUI) remembers hashes + by path, size and modification time: searching the same place again only + reads files that changed (a 461-group Steam library: 1.2 s → 0.1 s). - **Reclaim duplicates** — delete, send to the **OS trash/recycle bin**, or replace a copy with a **hard link**. Dry-run by default. - **Delete a file or folder** — `diskghost rm `, permanently or to the OS trash. Parallel unlink for speed, never follows symlinks (can't escape the target), refuses a drive root, dry-run by default, reports how long it took. -- **Scan options** — exclude globs (`node_modules`, `*.tmp`), max depth, - follow-symlinks, `--threads` (2-4 suit a spinning disk). +- **Scan options** — exclude globs (`node_modules`, `*.tmp`; case-insensitive + on Windows), max depth, follow-symlinks, `--threads` (2-4 suit a spinning + disk), `--mft`. +- **NTFS MFT engine** (`--mft`, Windows, administrator) — reads the volume's + Master File Table in large parallel blocks and builds the tree straight from + its records instead of opening every folder. Measured on a system drive: + `C:\Users\me` 196 s → 37 s, all of C: (4.8 M files, 4.9 M folders) in about + 30 s. On a SATA SSD with millions of files the table is bigger than the + folders and the normal walk can be quicker. Cloud placeholders and OneDrive + folders are sized and listed through Windows; without rights or NTFS it falls + back to the walk and says why. - **Live progress + cancel** — long scans report files/bytes and can be stopped; a cancelled scan is clearly labelled as a partial result. - **One walk, whole tree** — the scan builds an in-memory tree, so the GUI @@ -129,7 +142,8 @@ A modern Tauri UI lives in `gui/`: temp files, Windows/macOS system files …), turn single patterns off, or add your own; patterns are validated before they are used. - **Settings** — scan engine (**Fast** for SSD/NVMe: many directories at once; - **Gentle** for spinning disks, network drives or scanning in the background), + **Gentle** for spinning disks, network drives or scanning in the background; + **Ultra** = the NTFS MFT engine, with a *restart as administrator* button), list length, max depth, symlinks, parallel scans, snapshots kept per folder, duplicate minimum size, Recycle Bin vs. permanent delete, window size at start (1440p if it fits the screen, otherwise maximized; or 1080p / 4K / maximized), @@ -138,7 +152,9 @@ A modern Tauri UI lives in `gui/`: whole file), whatever the names. Copies inside installed programs and games (Steam, Epic, Battle.net, Program Files, …; DLLs, game archives) are there on purpose, so they are hidden by default, shown with a lock when asked for, and - never removed while hidden; the program's copy is the one kept. **Folder + never removed while hidden (by default not even read: the search skips + them); the program's copy is the one kept. The search shows its three phases + with progress, and hashes are cached between runs. **Folder pairs** sums up which folders share identical files; filter by category or name, then reclaim with one click (dry-run first). - **What changed** — every complete scan is kept as a snapshot (in the @@ -197,6 +213,7 @@ Diskghost/ - [x] GUI dashboard: parallel scans + queue, drives, nested treemap, file-type and size-range charts, exclude templates, settings - [x] Own parallel walker (no per-file open), file-level snapshots (zstd, streaming diff), non-blocking GUI - [x] Automatic "since last scan" comparison; duplicate finder that protects program and game files, folder pairs +- [x] NTFS MFT engine (administrator), per-file parallel duplicate pipeline with a persistent hash cache ## Development diff --git a/crates/diskghost-cli/src/main.rs b/crates/diskghost-cli/src/main.rs index 51b7282..c7d099a 100644 --- a/crates/diskghost-cli/src/main.rs +++ b/crates/diskghost-cli/src/main.rs @@ -11,8 +11,8 @@ use std::sync::Arc; use clap::{Args, Parser, Subcommand, ValueEnum}; use diskghost_core::{ - diff, find_duplicates_with_progress, format_timestamp, human_delta, human_size, reclaim, - remove_path, validate_globs, DiffReport, DupGroup, Options, Progress, ReclaimAction, + diff, find_duplicates_cached, format_timestamp, human_delta, human_size, phase, reclaim, + remove_path, validate_globs, DiffReport, DupGroup, HashCache, Options, Progress, ReclaimAction, RemoveMode, RemoveReport, ScanReport, ScanTree, Snapshot, }; use serde::Serialize; @@ -28,11 +28,25 @@ fn with_progress(job: impl FnOnce(&Progress) -> T) -> T { let (p, d) = (progress.clone(), done.clone()); let handle = std::thread::spawn(move || { while !d.load(Ordering::Relaxed) { - eprint!( - "\r working… {} files, {} ", - p.files(), - human_size(p.bytes()) - ); + match p.phase() { + phase::SAMPLE => eprint!( + "\r comparing file ends… {} / {} files ", + p.files(), + p.total_files() + ), + phase::HASH => eprint!( + "\r hashing… {} / {} files, {} / {} ", + p.files(), + p.total_files(), + human_size(p.bytes()), + human_size(p.total_bytes()) + ), + _ => eprint!( + "\r working… {} files, {} ", + p.files(), + human_size(p.bytes()) + ), + } let _ = std::io::stderr().flush(); std::thread::sleep(std::time::Duration::from_millis(120)); } @@ -119,6 +133,10 @@ enum Command { /// Actually perform the reclaim (default: dry-run). #[arg(long)] apply: bool, + /// Keep file hashes in this file between runs: a repeated search only + /// reads files whose size or modification time changed. + #[arg(long, value_name = "FILE")] + cache: Option, }, /// Delete a file or folder — permanently, or to the OS trash. Dry-run by default. Rm { @@ -150,6 +168,11 @@ struct WalkArgs { /// spinning disk). #[arg(long)] threads: Option, + /// Read the NTFS Master File Table instead of listing directories (much + /// faster on Windows; needs an administrator console, falls back to the + /// directory walk otherwise). + #[arg(long)] + mft: bool, } impl WalkArgs { @@ -159,6 +182,7 @@ impl WalkArgs { follow_symlinks: self.follow_symlinks, exclude: self.exclude.clone(), threads: self.threads, + mft: self.mft, } } } @@ -231,6 +255,8 @@ fn main() -> ExitCode { let out = ScanOutput { report: &report, diff: changes.as_ref(), + engine: tree.engine(), + mft_fallback: tree.mft_fallback(), }; println!("{}", serde_json::to_string_pretty(&out).unwrap()); } else { @@ -239,7 +265,15 @@ fn main() -> ExitCode { println!(); print_diff(d); } - println!("\nScanned in {}", fmt_duration(elapsed)); + let engine = if tree.engine() == "mft" { + " (NTFS MFT)" + } else { + "" + }; + println!("\nScanned in {}{engine}", fmt_duration(elapsed)); + if let Some(why) = tree.mft_fallback() { + eprintln!("note: --mft not used: {why}"); + } } if let (Some(file), Some(snap)) = (&save, &snapshot) { if let Err(e) = snap.save(file) { @@ -279,6 +313,7 @@ fn main() -> ExitCode { walk, reclaim: reclaim_arg, apply, + cache, } => { if !path.is_dir() { eprintln!("error: not a directory: {}", path.display()); @@ -286,9 +321,15 @@ fn main() -> ExitCode { } warn_invalid_globs(&walk.exclude); let opts = walk.to_options(); + let hashes = cache.as_deref().map(HashCache::load).unwrap_or_default(); let groups = with_progress(|p| { - find_duplicates_with_progress(&path, min_mb * 1024 * 1024, &opts, p) + find_duplicates_cached(&path, min_mb * 1024 * 1024, &opts, p, &hashes) }); + if let Some(file) = &cache { + if let Err(e) = hashes.save(file) { + eprintln!("warning: cannot save hash cache: {e}"); + } + } if cli.json { println!("{}", serde_json::to_string_pretty(&groups).unwrap()); } else { @@ -345,6 +386,11 @@ struct ScanOutput<'a> { report: &'a ScanReport, #[serde(skip_serializing_if = "Option::is_none")] diff: Option<&'a DiffReport>, + /// "walk" or "mft". + engine: &'a str, + /// Why --mft was asked for but not used. + #[serde(skip_serializing_if = "Option::is_none")] + mft_fallback: Option<&'a str>, } fn print_diff(d: &DiffReport) { diff --git a/crates/diskghost-core/Cargo.toml b/crates/diskghost-core/Cargo.toml index 140d43c..23a80fb 100644 --- a/crates/diskghost-core/Cargo.toml +++ b/crates/diskghost-core/Cargo.toml @@ -10,8 +10,7 @@ description = "Fast parallel disk scanning and duplicate detection — the engin [dependencies] rayon = "1" -blake3 = { version = "1", features = ["mmap"] } -same-file = "1" +blake3 = { version = "1", features = ["mmap", "rayon"] } globset = "0.4" trash = "5" fs4 = "1" diff --git a/crates/diskghost-core/src/dupes.rs b/crates/diskghost-core/src/dupes.rs new file mode 100644 index 0000000..b19c159 --- /dev/null +++ b/crates/diskghost-core/src/dupes.rs @@ -0,0 +1,502 @@ +//! Duplicate detection: files with byte-identical content. +//! +//! 1. The walk lists every file at least `min_size` bytes long; only sizes +//! that occur more than once can hold duplicates. +//! 2. Every same-size candidate is opened once — in parallel, file by file +//! — to learn its identity (so hard links to one physical file are never +//! called duplicates) and to hash its first and last 16 KB. Files up to +//! 32 KB are thereby hashed completely. +//! 3. Only files whose size and both ends match another one are hashed in +//! full with BLAKE3, again in parallel per file (big files additionally +//! use several threads each, through a memory map). +//! +//! A [`HashCache`] remembers identity and hashes by path, size and +//! modification time, so searching the same place again reads only files +//! that changed. + +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::Ordering; +use std::sync::Mutex; + +use rayon::prelude::*; + +use crate::snapshot::{get_varint, put_varint}; +use crate::walk::walk; +use crate::{phase, DupGroup, Options, Progress}; + +/// Bytes read from each end of a candidate for the quick comparison. +const SAMPLE: u64 = 16 * 1024; + +/// Files at least this big are hashed with several threads each. +const PARALLEL_HASH_MIN: u64 = 64 << 20; + +const CACHE_MAGIC: &[u8; 8] = b"DGHASH01"; + +/// What is known about one file's content. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Known { + size: u64, + /// Modification time (ns since the epoch); a cached entry only counts + /// while size and time are unchanged. + modified: u64, + /// (volume, file index) on Windows, (device, inode) elsewhere. + id: (u64, u64), + /// BLAKE3 of the first and last [`SAMPLE`] bytes. + sample: [u8; 32], + /// BLAKE3 of the whole file, once computed. + full: Option<[u8; 32]>, +} + +/// Hashes of files seen by earlier duplicate searches, keyed by path. Share +/// one between searches (it is internally synchronised) and persist it with +/// [`HashCache::save`] / [`HashCache::load`]. +#[derive(Default)] +pub struct HashCache { + entries: Mutex>, +} + +impl HashCache { + /// An empty cache. + pub fn new() -> HashCache { + HashCache::default() + } + + /// Number of files the cache knows. + pub fn len(&self) -> usize { + self.entries.lock().map_or(0, |e| e.len()) + } + + /// Whether the cache knows no file. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + fn get(&self, path: &Path, size: u64, modified: u64) -> Option { + if modified == 0 { + return None; // no reliable time: always look at the file + } + let e = self.entries.lock().ok()?; + e.get(path) + .filter(|k| k.size == size && k.modified == modified) + .copied() + } + + /// Replace everything known below `root` with `fresh` — files that are + /// gone or changed drop out, entries elsewhere stay. + fn replace_under(&self, root: &Path, fresh: Vec<(PathBuf, Known)>) { + if let Ok(mut e) = self.entries.lock() { + e.retain(|p, _| !p.starts_with(root)); + e.extend(fresh); + } + } + + /// Read a cache written by [`HashCache::save`]. A missing, foreign or + /// damaged file yields an empty cache: it only ever costs a re-read. + pub fn load(path: &Path) -> HashCache { + let cache = HashCache::new(); + if let Ok(map) = Self::read(path) { + if let Ok(mut e) = cache.entries.lock() { + *e = map; + } + } + cache + } + + fn read(path: &Path) -> std::io::Result> { + let bad = || std::io::Error::new(std::io::ErrorKind::InvalidData, "bad hash cache"); + let bytes = std::fs::read(path)?; + let body = bytes.strip_prefix(CACHE_MAGIC).ok_or_else(bad)?; + let mut r = BufReader::new(zstd::stream::read::Decoder::with_buffer(body)?); + let n = get_varint(&mut r).map_err(|_| bad())?; + let mut map = HashMap::with_capacity(n.min(1 << 24) as usize); + let mut name = Vec::new(); + for _ in 0..n { + let mut num = || get_varint(&mut r).map_err(|_| bad()); + let len = num()? as usize; + let (size, modified, id0, id1, has_full) = (num()?, num()?, num()?, num()?, num()?); + if len > 1 << 16 { + return Err(bad()); + } + name.resize(len, 0); + r.read_exact(&mut name)?; + let mut sample = [0u8; 32]; + r.read_exact(&mut sample)?; + let full = if has_full == 1 { + let mut f = [0u8; 32]; + r.read_exact(&mut f)?; + Some(f) + } else { + None + }; + let path = PathBuf::from(String::from_utf8(name.clone()).map_err(|_| bad())?); + map.insert( + path, + Known { + size, + modified, + id: (id0, id1), + sample, + full, + }, + ); + } + let _ = r.fill_buf(); + Ok(map) + } + + /// Write the cache (zstd-compressed), replacing `path` atomically. + pub fn save(&self, path: &Path) -> std::io::Result<()> { + let entries = self + .entries + .lock() + .map_err(|_| std::io::Error::other("hash cache poisoned"))?; + let mut raw = Vec::with_capacity(entries.len() * 96); + let named: Vec<(&str, &Known)> = entries + .iter() + .filter_map(|(p, k)| Some((p.to_str()?, k))) + .collect(); + put_varint(&mut raw, named.len() as u64); + for (p, k) in named { + for v in [ + p.len() as u64, + k.size, + k.modified, + k.id.0, + k.id.1, + u64::from(k.full.is_some()), + ] { + put_varint(&mut raw, v); + } + raw.extend_from_slice(p.as_bytes()); + raw.extend_from_slice(&k.sample); + if let Some(f) = k.full { + raw.extend_from_slice(&f); + } + } + drop(entries); + let mut out = CACHE_MAGIC.to_vec(); + out.extend(zstd::bulk::compress(&raw, 3)?); + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir)?; + } + let tmp = path.with_extension("tmp"); + std::fs::File::create(&tmp)?.write_all(&out)?; + std::fs::rename(tmp, path) + } +} + +/// One same-size candidate. +struct Cand { + path: PathBuf, + size: u64, + modified: u64, + known: Option, +} + +/// Identity of an open file: (volume, file index) / (device, inode). +#[cfg(windows)] +fn file_id(file: &std::fs::File) -> Option<(u64, u64)> { + use std::os::windows::io::AsRawHandle; + #[repr(C)] + #[derive(Default)] + struct ByHandleFileInformation { + attributes: u32, + creation: [u32; 2], + access: [u32; 2], + write: [u32; 2], + volume_serial: u32, + size_high: u32, + size_low: u32, + links: u32, + index_high: u32, + index_low: u32, + } + #[link(name = "kernel32")] + extern "system" { + fn GetFileInformationByHandle( + file: *mut std::ffi::c_void, + info: *mut ByHandleFileInformation, + ) -> i32; + } + let mut info = ByHandleFileInformation::default(); + let ok = unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } != 0; + ok.then(|| { + ( + u64::from(info.volume_serial), + (u64::from(info.index_high) << 32) | u64::from(info.index_low), + ) + }) +} + +#[cfg(unix)] +fn file_id(file: &std::fs::File) -> Option<(u64, u64)> { + use std::os::unix::fs::MetadataExt; + let m = file.metadata().ok()?; + Some((m.dev(), m.ino())) +} + +#[cfg(not(any(windows, unix)))] +fn file_id(_file: &std::fs::File) -> Option<(u64, u64)> { + None +} + +/// Open the file once: identity plus the hash of both ends (the whole file +/// when it is small enough). +fn sample(c: &Cand) -> Option { + let mut f = std::fs::File::open(&c.path).ok()?; + let id = file_id(&f).unwrap_or((u64::MAX, 0)); + let mut h = blake3::Hasher::new(); + let mut buf = vec![0u8; SAMPLE as usize]; + let whole = c.size <= 2 * SAMPLE; + if whole { + let n = std::io::copy(&mut f, &mut h).ok()?; + if n != c.size { + return None; // changed under us + } + } else { + f.read_exact(&mut buf).ok()?; + h.update(&buf); + f.seek(SeekFrom::Start(c.size - SAMPLE)).ok()?; + f.read_exact(&mut buf).ok()?; + h.update(&buf); + } + let sample = *h.finalize().as_bytes(); + Some(Known { + size: c.size, + modified: c.modified, + id, + sample, + full: whole.then_some(sample), + }) +} + +/// BLAKE3 of the whole file: memory-mapped (several threads for big files), +/// streamed where mapping is not possible. +fn full_hash(path: &Path, size: u64) -> Option<[u8; 32]> { + let mut h = blake3::Hasher::new(); + let mapped = if size >= PARALLEL_HASH_MIN { + h.update_mmap_rayon(path) + } else { + h.update_mmap(path) + }; + if mapped.is_err() { + h = blake3::Hasher::new(); + let mut f = std::fs::File::open(path).ok()?; + std::io::copy(&mut f, &mut h).ok()?; + } + Some(*h.finalize().as_bytes()) +} + +/// See the module docs. `cache` is read and updated. +pub(crate) fn find( + root: &Path, + min_size: u64, + opts: &Options, + progress: &Progress, + cache: &HashCache, +) -> Vec { + // 1. Every file big enough; duplicates group by *logical* (content) size. + let mut cands: Vec = Vec::new(); + let _ = walk(root, opts, progress, |b| { + for (f, modified) in b.files.into_iter().zip(b.modified) { + if f.logical_size >= min_size && f.logical_size > 0 { + cands.push(Cand { + path: f.path, + size: f.logical_size, + modified, + known: None, + }); + } + } + }); + if progress.cancelled() { + return Vec::new(); + } + let mut per_size: HashMap = HashMap::new(); + for c in &cands { + *per_size.entry(c.size).or_default() += 1; + } + cands.retain(|c| per_size[&c.size] > 1); + drop(per_size); + + // 2. Identity and both ends, file by file in parallel (cache first). + let sample_bytes = cands.iter().map(|c| c.size.min(2 * SAMPLE)).sum(); + progress.enter(phase::SAMPLE, cands.len() as u64, sample_bytes); + cands.par_iter_mut().for_each(|c| { + if progress.cancelled() { + return; + } + c.known = cache.get(&c.path, c.size, c.modified).or_else(|| sample(c)); + progress.files.fetch_add(1, Ordering::Relaxed); + progress + .bytes + .fetch_add(c.size.min(2 * SAMPLE), Ordering::Relaxed); + }); + if progress.cancelled() { + return Vec::new(); + } + + // Collapse hard links (one physical file reached by several paths), then + // keep only files whose size and both ends match another file. + let mut seen_ids: HashMap<(u64, (u64, u64)), ()> = HashMap::new(); + let mut by_sample: HashMap<(u64, [u8; 32]), Vec> = HashMap::new(); + for (i, c) in cands.iter().enumerate() { + let Some(k) = c.known else { continue }; + if k.id.0 != u64::MAX && seen_ids.insert((c.size, k.id), ()).is_some() { + continue; + } + by_sample.entry((c.size, k.sample)).or_default().push(i); + } + let mut alike: Vec = by_sample + .into_values() + .filter(|v| v.len() > 1) + .flatten() + .collect(); + + // 3. Whole-file hashes for whatever still looks alike, in parallel. + let need: Vec = alike + .iter() + .copied() + .filter(|&i| cands[i].known.is_some_and(|k| k.full.is_none())) + .collect(); + let need_bytes = need.iter().map(|&i| cands[i].size).sum(); + progress.enter(phase::HASH, need.len() as u64, need_bytes); + let fulls: Vec<(usize, Option<[u8; 32]>)> = need + .par_iter() + .map(|&i| { + if progress.cancelled() { + return (i, None); + } + let c = &cands[i]; + let h = full_hash(&c.path, c.size); + progress.files.fetch_add(1, Ordering::Relaxed); + progress.bytes.fetch_add(c.size, Ordering::Relaxed); + (i, h) + }) + .collect(); + for (i, h) in fulls { + if let Some(k) = cands[i].known.as_mut() { + k.full = h; + } + } + + // Remember what was learned (also for files that turned out unique). + cache.replace_under( + root, + cands + .iter() + .filter(|c| !progress.cancelled() || c.known.is_some_and(|k| k.full.is_some())) + .filter_map(|c| Some((c.path.clone(), c.known?))) + .collect(), + ); + if progress.cancelled() { + return Vec::new(); + } + + // Group by size and full hash. + alike.sort_unstable(); + let mut by_full: HashMap<(u64, [u8; 32]), Vec> = HashMap::new(); + for i in alike { + let c = &cands[i]; + if let Some(full) = c.known.and_then(|k| k.full) { + by_full + .entry((c.size, full)) + .or_default() + .push(c.path.clone()); + } + } + let mut groups: Vec = by_full + .into_iter() + .filter(|(_, files)| files.len() > 1) + .map(|((size, full), mut files)| { + files.sort(); // stable order so "keep the first" is deterministic + DupGroup { + hash: blake3::Hash::from(full).to_hex().to_string(), + size, + wasted: size.saturating_mul(files.len() as u64 - 1), + files, + } + }) + .collect(); + groups.sort_by(|a, b| b.wasted.cmp(&a.wasted).then_with(|| a.files.cmp(&b.files))); + groups +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmpdir(label: &str) -> PathBuf { + let mut d = std::env::temp_dir(); + d.push(format!("diskghost-dupes-{}-{label}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d + } + + fn write(path: &Path, content: &[u8]) { + if let Some(p) = path.parent() { + std::fs::create_dir_all(p).unwrap(); + } + std::fs::write(path, content).unwrap(); + } + + #[test] + fn same_ends_different_middle_is_not_a_duplicate() { + // Big files that agree in their first and last 16 KB but differ in + // the middle must survive the quick check and fail the full hash. + let d = tmpdir("middle"); + let mut a = vec![7u8; 200_000]; + let b = a.clone(); + write(&d.join("a.bin"), &a); + write(&d.join("b.bin"), &b); // true duplicate of a + a[100_000] = 8; + write(&d.join("c.bin"), &a); // same ends, other middle + let cache = HashCache::new(); + let g = find(&d, 1, &Options::default(), &Progress::default(), &cache); + assert_eq!(g.len(), 1); + assert_eq!(g[0].files, vec![d.join("a.bin"), d.join("b.bin")]); + assert_eq!(g[0].wasted, 200_000); + assert_eq!(cache.len(), 3); + std::fs::remove_dir_all(&d).ok(); + } + + #[test] + fn cache_round_trip_and_reuse() { + let d = tmpdir("cache"); + write(&d.join("x/1.bin"), &[1u8; 50_000]); + write(&d.join("y/1.bin"), &[1u8; 50_000]); + write(&d.join("small1"), b"hello"); + write(&d.join("small2"), b"hello"); + let cache = HashCache::new(); + let first = find(&d, 1, &Options::default(), &Progress::default(), &cache); + assert_eq!(first.len(), 2); + + let file = d.join("cache/hashes.bin"); + cache.save(&file).unwrap(); + let back = HashCache::load(&file); + assert_eq!(back.len(), 4); + + // With everything cached the second search reads no file content: + // the hash phase has nothing to do and the result is the same. + let p = Progress::default(); + let again = find(&d, 1, &Options::default(), &p, &back); + assert_eq!(again, first); + assert_eq!(p.phase(), phase::HASH); + assert_eq!(p.total_files(), 0); + + // A changed file is looked at again. + std::thread::sleep(std::time::Duration::from_millis(20)); + write(&d.join("y/1.bin"), &[2u8; 50_000]); + let changed = find(&d, 1, &Options::default(), &Progress::default(), &back); + assert_eq!(changed.len(), 1); + + // Damaged or foreign cache files just mean an empty cache. + std::fs::write(&file, b"garbage").unwrap(); + assert!(HashCache::load(&file).is_empty()); + assert!(HashCache::load(&d.join("missing")).is_empty()); + std::fs::remove_dir_all(&d).ok(); + } +} diff --git a/crates/diskghost-core/src/lib.rs b/crates/diskghost-core/src/lib.rs index bb63aad..35fe166 100644 --- a/crates/diskghost-core/src/lib.rs +++ b/crates/diskghost-core/src/lib.rs @@ -16,17 +16,21 @@ //! [`Options`] controls the walk: exclude globs, max depth, follow-symlinks. //! Everything is `serde`-serialisable so a CLI, a GUI or an agent can consume it. +mod dupes; +#[cfg(windows)] +mod mft; mod snapshot; mod tree; mod walk; +pub use dupes::HashCache; + pub use snapshot::{ diff, format_timestamp, DiffReport, DirDelta, FileDelta, Snapshot, SnapshotError, DIFF_FILE_LIMIT, SNAPSHOT_FORMAT, }; pub use tree::ScanTree; -use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; @@ -48,6 +52,11 @@ pub struct Options { /// Threads listing directories (`None` = twice the CPU cores, which keeps /// SSDs busy; a spinning disk is better served by 2-4). pub threads: Option, + /// Read the NTFS Master File Table instead of listing directories: a + /// whole drive in seconds. Needs administrator rights and a local NTFS + /// volume, and does not follow links; otherwise the directory walk is + /// used. See [`ScanTree::engine`] for which one ran. + pub mft: bool, } /// Live progress + cancellation shared with a running scan / duplicate search. @@ -58,17 +67,50 @@ pub struct Progress { files: Arc, bytes: Arc, cancel: Arc, + phase: Arc, + total_files: Arc, + total_bytes: Arc, +} + +/// What a duplicate search is doing (see [`Progress::phase`]). +pub mod phase { + /// Listing files (also the only phase of a scan). + pub const WALK: u64 = 0; + /// Reading the start and end of every same-size candidate. + pub const SAMPLE: u64 = 1; + /// Hashing whole files that still look alike. + pub const HASH: u64 = 2; } impl Progress { - /// Files seen so far. + /// Files seen (or, past the walk, processed) so far. pub fn files(&self) -> u64 { self.files.load(Ordering::Relaxed) } - /// Bytes seen so far. + /// Bytes seen (or, past the walk, read) so far. pub fn bytes(&self) -> u64 { self.bytes.load(Ordering::Relaxed) } + /// The current [`phase`]; a scan stays in [`phase::WALK`]. + pub fn phase(&self) -> u64 { + self.phase.load(Ordering::Relaxed) + } + /// Files the current phase will process (0 while walking: unknown). + pub fn total_files(&self) -> u64 { + self.total_files.load(Ordering::Relaxed) + } + /// Bytes the current phase will read (0 while walking: unknown). + pub fn total_bytes(&self) -> u64 { + self.total_bytes.load(Ordering::Relaxed) + } + /// Start a new phase: counters restart at 0 against the given totals. + fn enter(&self, phase: u64, total_files: u64, total_bytes: u64) { + self.files.store(0, Ordering::Relaxed); + self.bytes.store(0, Ordering::Relaxed); + self.total_files.store(total_files, Ordering::Relaxed); + self.total_bytes.store(total_bytes, Ordering::Relaxed); + self.phase.store(phase, Ordering::Relaxed); + } /// Ask the running operation to stop as soon as possible. pub fn cancel(&self) { self.cancel.store(true, Ordering::Relaxed); @@ -168,7 +210,7 @@ pub struct ScanReport { /// A group of byte-identical files (hard links to the same physical file are /// collapsed, so `wasted` reflects space that is actually reclaimable). -#[derive(Debug, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct DupGroup { pub hash: String, pub size: u64, @@ -220,13 +262,15 @@ pub struct RemoveReport { pub dry_run: bool, } -/// First N bytes hashed as a cheap pre-filter before a full-file hash. -const PREFIX_LEN: usize = 4096; - +/// Exclude patterns as one matcher. On Windows names compare without regard +/// to case, like the filesystem does (`*.DLL` excludes `x.dll`). pub(crate) fn build_globset(patterns: &[String]) -> globset::GlobSet { let mut builder = globset::GlobSetBuilder::new(); for p in patterns { - if let Ok(g) = globset::Glob::new(p) { + if let Ok(g) = globset::GlobBuilder::new(p) + .case_insensitive(cfg!(windows)) + .build() + { builder.add(g); } } @@ -295,7 +339,7 @@ pub(crate) fn on_disk_size(meta: &std::fs::Metadata, path: &Path) -> u64 { /// Actual allocated size via `GetCompressedFileSizeW`. Returns `None` on error /// so the caller can fall back to the logical length. #[cfg(windows)] -fn compressed_size(path: &Path) -> Option { +pub(crate) fn compressed_size(path: &Path) -> Option { use std::os::windows::ffi::OsStrExt; #[link(name = "kernel32")] extern "system" { @@ -350,7 +394,7 @@ pub fn walk_files(root: &Path) -> Vec { /// Every file below `root` (honouring `opts`), in no particular order. fn collect_files(root: &Path, opts: &Options, progress: &Progress) -> Vec { let mut files = Vec::new(); - walk::walk(root, opts, progress, |b| files.extend(b.files)); + let _ = walk::walk(root, opts, progress, |b| files.extend(b.files)); files } @@ -387,82 +431,33 @@ pub fn find_duplicates(root: &Path, min_size: u64) -> Vec { } /// Find groups of byte-identical files under `root`, ignoring files smaller than -/// `min_size` bytes and zero-byte files. Reports live progress and can be cancelled. +/// `min_size` bytes and zero-byte files. Reports live progress (see +/// [`Progress::phase`]) and can be cancelled (no groups are returned then). /// -/// Cheap and correct: group by size, collapse hard links, pre-hash the first -/// [`PREFIX_LEN`] bytes, then full-hash the survivors (BLAKE3). Parallel across -/// size groups. +/// Cheap and correct: group by size, open each candidate once (collapsing hard +/// links and hashing both ends), then BLAKE3 the whole of what still matches — +/// every step in parallel per file. See [`find_duplicates_cached`] to reuse +/// hashes between searches. pub fn find_duplicates_with_progress( root: &Path, min_size: u64, opts: &Options, progress: &Progress, ) -> Vec { - // Duplicate detection groups by *logical* (content) size — two byte-identical - // files are duplicates regardless of how they happen to be allocated on disk. - let files: Vec = collect_files(root, opts, progress) - .into_iter() - .filter(|f| f.logical_size >= min_size && f.logical_size > 0) - .collect(); - - let mut by_size: HashMap> = HashMap::new(); - for f in files { - by_size.entry(f.logical_size).or_default().push(f.path); - } - - let candidates: Vec<(u64, Vec)> = - by_size.into_iter().filter(|(_, v)| v.len() > 1).collect(); - - let mut groups: Vec = candidates - .par_iter() - .flat_map(|(size, paths)| { - if progress.cancelled() { - return Vec::new(); // stop hashing new groups once cancelled - } - let reps = dedup_hard_links(paths); - if reps.len() < 2 { - return Vec::new(); - } - - let mut by_prefix: HashMap<[u8; 32], Vec> = HashMap::new(); - for p in reps { - if let Ok(h) = hash_prefix(&p, PREFIX_LEN) { - by_prefix.entry(h).or_default().push(p); - } - } - - let mut out = Vec::new(); - for prefix_group in by_prefix.into_values() { - if prefix_group.len() < 2 { - continue; - } - let mut by_hash: HashMap> = HashMap::new(); - for p in prefix_group { - if let Ok(h) = hash_file(&p) { - by_hash.entry(h).or_default().push(p); - } - } - for (hash, group) in by_hash { - if group.len() < 2 { - continue; - } - let mut group = group; - group.sort(); // stable order so "keep the first" is deterministic - let wasted = size.saturating_mul(group.len() as u64 - 1); - out.push(DupGroup { - hash, - size: *size, - files: group, - wasted, - }); - } - } - out - }) - .collect(); + dupes::find(root, min_size, opts, progress, &HashCache::new()) +} - groups.sort_by_key(|g| std::cmp::Reverse(g.wasted)); - groups +/// [`find_duplicates_with_progress`] that takes hashes from `cache` for files +/// whose size and modification time are unchanged and records what it learns, +/// so a repeated search only reads files that changed. +pub fn find_duplicates_cached( + root: &Path, + min_size: u64, + opts: &Options, + progress: &Progress, + cache: &HashCache, +) -> Vec { + dupes::find(root, min_size, opts, progress, cache) } /// Reclaim the space of redundant duplicate copies: keep `keep`, act on every @@ -556,7 +551,7 @@ pub fn remove_path( ..Progress::default() }; dirs.push(path.to_path_buf()); - walk::walk(path, &Options::default(), &listing, |b| { + let _ = walk::walk(path, &Options::default(), &listing, |b| { for f in b.files { bytes = bytes.saturating_add(f.size); files.push(f.path); @@ -652,57 +647,6 @@ pub fn remove_path( } } -/// Keep one path per distinct physical file. Hard links share a file identity, -/// so they must not be counted as duplicates. [`same_file::Handle`] provides that -/// identity cross-platform. Paths whose identity can't be read are kept. -fn dedup_hard_links(paths: &[PathBuf]) -> Vec { - let mut seen: HashSet = HashSet::new(); - let mut reps = Vec::new(); - for p in paths { - match same_file::Handle::from_path(p) { - Ok(h) => { - if seen.insert(h) { - reps.push(p.clone()); - } - } - Err(_) => reps.push(p.clone()), - } - } - reps -} - -fn hash_prefix(path: &Path, len: usize) -> std::io::Result<[u8; 32]> { - use std::io::Read; - let mut file = std::fs::File::open(path)?; - let mut buf = vec![0u8; len]; - let mut read = 0usize; - while read < buf.len() { - let n = file.read(&mut buf[read..])?; - if n == 0 { - break; - } - read += n; - } - let mut hasher = blake3::Hasher::new(); - hasher.update(&buf[..read]); - Ok(*hasher.finalize().as_bytes()) -} - -fn hash_file(path: &Path) -> std::io::Result { - // Memory-map the file so BLAKE3 reads straight from the page cache with no - // buffered-read syscalls or intermediate copies — the fast path for the - // large files that dominate full-hash cost. - let mut hasher = blake3::Hasher::new(); - if hasher.update_mmap(path).is_ok() { - return Ok(hasher.finalize().to_hex().to_string()); - } - // Fallback: stream the file (mmap unsupported, empty, or a special file). - let mut hasher = blake3::Hasher::new(); - let mut file = std::fs::File::open(path)?; - std::io::copy(&mut file, &mut hasher)?; - Ok(hasher.finalize().to_hex().to_string()) -} - /// Format a byte count as a human-readable string, e.g. `1.5 GB`. pub fn human_size(bytes: u64) -> String { const UNITS: [&str; 6] = ["B", "KB", "MB", "GB", "TB", "PB"]; diff --git a/crates/diskghost-core/src/mft.rs b/crates/diskghost-core/src/mft.rs new file mode 100644 index 0000000..c1dff17 --- /dev/null +++ b/crates/diskghost-core/src/mft.rs @@ -0,0 +1,770 @@ +//! Scanning an NTFS volume by reading its Master File Table directly. +//! +//! Listing a directory costs Windows an open, a query and a close; on a system +//! drive with millions of folders that — not the disk — is the limit. Every +//! file and folder of an NTFS volume is one 1 KB record in the MFT, so reading +//! that table sequentially in big blocks (and parsing the records in parallel) +//! yields the whole tree in a few seconds. It needs administrator rights (the +//! volume itself is opened) and works on local NTFS volumes only; callers fall +//! back to the directory walk otherwise. +//! +//! What is counted matches the walk: the unnamed data stream of each file +//! (logical size, or the allocated size for sparse / compressed / cloud files), +//! one entry per hard link, symbolic links and junctions skipped, NTFS +//! metafiles (`$MFT`, `$Extend`, …) left out. + +use std::ffi::OsString; +use std::os::windows::ffi::OsStringExt; +use std::os::windows::fs::{FileExt, OpenOptionsExt}; +use std::path::{Component, Path, PathBuf, Prefix}; +use std::sync::atomic::Ordering; + +use rayon::prelude::*; + +use crate::walk::Batch; +use crate::{build_globset, is_excluded, FileEntry, Options, Progress}; + +/// Record number of the volume's root directory. +const ROOT: u64 = 5; +/// Records below this are NTFS metafiles (except the root). +const FIRST_USER_RECORD: u64 = 24; +/// Bytes read from the volume per request. +const CHUNK: u64 = 8 << 20; + +const ATTR_STANDARD_INFORMATION: u32 = 0x10; +const ATTR_FILE_NAME: u32 = 0x30; +const ATTR_DATA: u32 = 0x80; +const ATTR_REPARSE_POINT: u32 = 0xC0; +const ATTR_END: u32 = 0xFFFF_FFFF; + +/// File attributes that make the on-disk size differ from the logical size +/// (the same set the walk asks `GetCompressedFileSizeW` about). +const WEIRD: u32 = 0x200 | 0x400 | 0x800 | 0x1000 | 0x4_0000 | 0x40_0000; + +/// Why the MFT could not be used. +#[derive(Debug)] +pub(crate) enum MftError { + /// Not a local drive letter path. + NotADrive, + /// The volume could not be opened (usually: not running as administrator). + Open(std::io::Error), + /// Not NTFS, or the table could not be understood. + NotNtfs(&'static str), + Io(std::io::Error), +} + +impl std::fmt::Display for MftError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + MftError::NotADrive => write!(f, "not a local drive"), + MftError::Open(e) => write!( + f, + "cannot open the volume, administrator rights needed ({e})" + ), + MftError::NotNtfs(why) => write!(f, "{why}"), + MftError::Io(e) => write!(f, "read error: {e}"), + } + } +} + +impl From for MftError { + fn from(e: std::io::Error) -> Self { + MftError::Io(e) + } +} + +fn u16_at(b: &[u8], o: usize) -> Option { + Some(u16::from_le_bytes(b.get(o..o + 2)?.try_into().ok()?)) +} +fn u32_at(b: &[u8], o: usize) -> Option { + Some(u32::from_le_bytes(b.get(o..o + 4)?.try_into().ok()?)) +} +fn u64_at(b: &[u8], o: usize) -> Option { + Some(u64::from_le_bytes(b.get(o..o + 8)?.try_into().ok()?)) +} + +/// The drive letter of `root` (`C:\Users` -> `C`). +fn drive_letter(root: &Path) -> Option { + match root.components().next()? { + Component::Prefix(p) => match p.kind() { + Prefix::Disk(d) | Prefix::VerbatimDisk(d) => Some(d as char), + _ => None, + }, + _ => None, + } +} + +/// Geometry from the NTFS boot sector. +struct Geometry { + cluster: u64, + record: u64, + mft_lcn: u64, +} + +fn geometry(boot: &[u8]) -> Result { + if boot.get(3..11) != Some(b"NTFS ") { + return Err(MftError::NotNtfs("not an NTFS volume")); + } + let sector = u64::from(u16_at(boot, 0x0B).unwrap_or(0)); + let per_cluster = u64::from(boot[0x0D]); + let cluster = sector * per_cluster; + let mft_lcn = u64_at(boot, 0x30).unwrap_or(0); + let raw = boot[0x40] as i8; + let record = if raw < 0 { + 1u64 << (-raw as u32) + } else { + raw as u64 * cluster + }; + if cluster == 0 || !(512..=65536).contains(&record) || mft_lcn == 0 { + return Err(MftError::NotNtfs("unexpected boot sector")); + } + Ok(Geometry { + cluster, + record, + mft_lcn, + }) +} + +/// Undo the update-sequence protection of one record in place. `false` if +/// the record is torn or not a FILE record. +fn fixup(rec: &mut [u8]) -> bool { + if rec.get(0..4) != Some(b"FILE") { + return false; + } + let (Some(off), Some(count)) = (u16_at(rec, 4), u16_at(rec, 6)) else { + return false; + }; + let (off, count) = (off as usize, count as usize); + let Some(usn) = u16_at(rec, off) else { + return false; + }; + for i in 1..count { + let end = i * 512; + if end > rec.len() || u16_at(rec, end - 2) != Some(usn) { + return false; + } + let Some(fix) = u16_at(rec, off + 2 * i) else { + return false; + }; + rec[end - 2..end].copy_from_slice(&fix.to_le_bytes()); + } + true +} + +/// One attribute of a record. +struct Attr<'a> { + kind: u32, + named: bool, + flags: u16, + /// Resident value, or `None` for a non-resident attribute. + value: Option<&'a [u8]>, + /// Whole attribute (for non-resident fields). + raw: &'a [u8], +} + +fn attrs(rec: &[u8]) -> impl Iterator> { + let mut off = u16_at(rec, 0x14).map_or(usize::MAX, usize::from); + let used = u32_at(rec, 0x18).map_or(0, |u| u as usize).min(rec.len()); + std::iter::from_fn(move || { + let kind = u32_at(rec, off)?; + if kind == ATTR_END || off >= used { + return None; + } + let len = u32_at(rec, off + 4)? as usize; + if len < 0x18 || off + len > used { + return None; + } + let raw = &rec[off..off + len]; + off += len; + let non_resident = raw[8] != 0; + let value = if non_resident { + None + } else { + let vlen = u32_at(raw, 0x10)? as usize; + let voff = u16_at(raw, 0x14)? as usize; + Some(raw.get(voff..voff + vlen)?) + }; + Some(Attr { + kind, + named: raw[9] != 0, + flags: u16_at(raw, 0x0C)?, + value, + raw, + }) + }) +} + +/// Data runs of a non-resident attribute as (first cluster, cluster count). +fn runs(raw: &[u8]) -> Vec<(u64, u64)> { + let mut out = Vec::new(); + let Some(mut p) = u16_at(raw, 0x20).map(usize::from) else { + return out; + }; + let mut lcn: i64 = 0; + while let Some(&head) = raw.get(p) { + if head == 0 { + break; + } + let (len_size, off_size) = ((head & 0x0F) as usize, (head >> 4) as usize); + let Some(bytes) = raw.get(p + 1..p + 1 + len_size + off_size) else { + break; + }; + let mut length = 0u64; + for (i, b) in bytes[..len_size].iter().enumerate() { + length |= u64::from(*b) << (8 * i); + } + if off_size > 0 { + let mut delta = 0i64; + for (i, b) in bytes[len_size..].iter().enumerate() { + delta |= i64::from(*b) << (8 * i); + } + // Sign-extend the delta. + let shift = 64 - 8 * off_size as u32; + delta = (delta << shift) >> shift; + lcn += delta; + out.push((lcn as u64, length)); + } // a run without offset is sparse: not part of the table + p += 1 + len_size + off_size; + } + out +} + +/// What one record (or an extension record of it) contributes. +#[derive(Default)] +struct Rec { + /// Base record this belongs to (itself for a base record). + base: u64, + is_dir: bool, + /// (parent record, name) per non-DOS file name — several for hard links. + names: Vec<(u64, OsString)>, + attributes: u32, + modified: u64, + /// A symbolic link, junction or other name surrogate: never followed. + surrogate: bool, + /// Any reparse point (cloud placeholders included): its on-disk size is + /// the allocated one, like the walk's `GetCompressedFileSizeW`. + reparse: bool, + /// From the unnamed data stream's first extent: (logical, allocated). + data: Option<(u64, u64)>, +} + +/// Parse one fixed-up record. +fn parse(index: u64, rec: &[u8]) -> Option { + let flags = u16_at(rec, 0x16)?; + if flags & 1 == 0 { + return None; // not in use + } + let base = u64_at(rec, 0x20)? & 0xFFFF_FFFF_FFFF; + let mut r = Rec { + base: if base == 0 { index } else { base }, + is_dir: flags & 2 != 0, + ..Rec::default() + }; + let mut dos_names: Vec<(u64, OsString)> = Vec::new(); + let mut units: Vec = Vec::new(); + for a in attrs(rec) { + match a.kind { + ATTR_STANDARD_INFORMATION => { + if let Some(v) = a.value { + // FILETIME (100 ns since 1601) -> ns since 1970. + let ft = u64_at(v, 0x08).unwrap_or(0); + r.modified = ft + .saturating_sub(116_444_736_000_000_000) + .saturating_mul(100); + r.attributes = u32_at(v, 0x20).unwrap_or(0); + } + } + ATTR_FILE_NAME => { + let Some(v) = a.value else { continue }; + let parent = u64_at(v, 0)? & 0xFFFF_FFFF_FFFF; + let len = *v.get(0x40)? as usize; + let space = *v.get(0x41)?; + units.clear(); + units.extend( + v.get(0x42..0x42 + 2 * len)? + .chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])), + ); + let name = OsString::from_wide(&units); + if space == 2 { + dos_names.push((parent, name)); // 8.3 alias of a long name + } else { + r.names.push((parent, name)); + } + } + ATTR_DATA if !a.named => { + let size = match a.value { + Some(v) => Some((v.len() as u64, v.len() as u64)), + // Sizes are only valid in the extent starting at VCN 0. + None if u64_at(a.raw, 0x10) == Some(0) => { + let logical = u64_at(a.raw, 0x30)?; + let allocated = u64_at(a.raw, 0x28)?; + let compressed_or_sparse = a.flags & 0x0001 != 0 || a.flags & 0x8000 != 0; + let on_disk = if compressed_or_sparse { + u64_at(a.raw, 0x40).unwrap_or(allocated) + } else { + allocated + }; + Some((logical, on_disk)) + } + None => None, + }; + if size.is_some() { + r.data = size; + } + } + ATTR_REPARSE_POINT => { + let tag = a.value.and_then(|v| u32_at(v, 0)).unwrap_or(0); + r.reparse = true; + r.surrogate = tag & 0x2000_0000 != 0; // symlink, junction, … + } + _ => {} + } + } + if r.names.is_empty() { + r.names = dos_names; // a name that only exists in 8.3 form + } + Some(r) +} + +/// A merged base record. +struct Node { + is_dir: bool, + names: Vec<(u64, OsString)>, + modified: u64, + surrogate: bool, + /// A reparse point that is not a link — a cloud placeholder (OneDrive …): + /// only its filter driver knows what is stored locally. + cloud: bool, + /// (on-disk, logical) size of the unnamed data stream. + size: (u64, u64), +} + +/// Prints phase timings to stderr when `DISKGHOST_TRACE` is set. +struct Trace(Option); + +impl Trace { + fn new() -> Trace { + Trace(std::env::var_os("DISKGHOST_TRACE").map(|_| std::time::Instant::now())) + } + fn mark(&mut self, what: &str) { + if let Some(t) = self.0 { + eprintln!( + "mft: {what:<28} {:>8.0} ms", + t.elapsed().as_secs_f64() * 1000.0 + ); + self.0 = Some(std::time::Instant::now()); + } + } +} + +/// A zeroed, page-aligned buffer (volume reads must be sector-aligned). +struct Aligned { + buf: Vec, + start: usize, + len: usize, +} + +impl Aligned { + fn new(len: usize) -> Aligned { + let buf = vec![0u8; len + 4096]; + let start = buf.as_ptr().align_offset(4096); + Aligned { buf, start, len } + } + fn as_mut(&mut self) -> &mut [u8] { + &mut self.buf[self.start..self.start + self.len] + } +} + +/// Volume handles read from in parallel: synchronous reads on one handle are +/// served one after another, so each reader thread gets its own. +const HANDLES: usize = 4; + +/// Read and parse the whole MFT of the volume holding `root`; the result is +/// indexed by record number. +fn read_table( + root: &Path, + progress: &Progress, + trace: &mut Trace, +) -> Result>, MftError> { + let letter = drive_letter(root).ok_or(MftError::NotADrive)?; + // Buffered on purpose: reading without the cache was measured to be no + // faster the first time, and through it a repeated scan comes from RAM. + let open = || { + std::fs::OpenOptions::new() + .read(true) + .share_mode(7) // read | write | delete: the volume stays usable + .open(format!(r"\\.\{letter}:")) + .map_err(MftError::Open) + }; + let volume = open()?; + trace.mark("open volume"); + + let mut boot = Aligned::new(4096); + volume.seek_read(boot.as_mut(), 0)?; + let g = geometry(boot.as_mut())?; + + // Record 0 describes the MFT itself: where its pieces are on the volume. + let mut first_buf = Aligned::new(g.record.max(g.cluster).max(4096) as usize); + volume.seek_read(first_buf.as_mut(), g.mft_lcn * g.cluster)?; + let first = &mut first_buf.as_mut()[..g.record as usize]; + if !fixup(first) { + return Err(MftError::NotNtfs("unreadable $MFT record")); + } + let mut table_size = 0; + let mut extents = Vec::new(); + for a in attrs(first) { + if a.kind == ATTR_DATA && !a.named && a.value.is_none() { + if u64_at(a.raw, 0x10) == Some(0) { + table_size = u64_at(a.raw, 0x30).unwrap_or(0); + } + extents.extend(runs(a.raw)); + } + } + if extents.is_empty() || table_size == 0 { + return Err(MftError::NotNtfs("no $MFT data runs")); + } + + // Byte ranges of the table on the volume, cut into chunks; each chunk + // knows the record number it starts with. + let mut chunks: Vec<(u64, u64, u64)> = Vec::new(); // (volume offset, bytes, first record) + let mut record_no = 0u64; + let mut left = table_size; + for (lcn, clusters) in extents { + let mut off = lcn * g.cluster; + let mut bytes = (clusters * g.cluster).min(left); + left -= bytes; + while bytes > 0 { + let n = bytes.min(CHUNK); + chunks.push((off, n, record_no)); + record_no += n / g.record; + off += n; + bytes -= n; + } + if left == 0 { + break; + } + } + let mut handles = vec![volume]; + for _ in 1..HANDLES { + handles.push(open()?); + } + trace.mark("boot sector + other handles"); + + let record = g.record as usize; + let parsed: Vec> = chunks + .par_iter() + .enumerate() + .map(|(c, &(off, n, first_no))| { + if progress.cancelled() { + return Ok(Vec::new()); + } + let volume = &handles[c % HANDLES]; + // Whole clusters (the table may end mid-cluster). + let len = n.div_ceil(g.cluster) * g.cluster; + let mut buf = Aligned::new(len as usize); + let buf = buf.as_mut(); + let mut got = 0; + while got < buf.len() { + let r = volume.seek_read(&mut buf[got..], off + got as u64)?; + if r == 0 { + break; + } + got += r; + } + let got = got.min(n as usize); + let mut out = Vec::with_capacity(got / record); + for (i, rec) in buf[..got].chunks_exact_mut(record).enumerate() { + let index = first_no + i as u64; + if fixup(rec) { + if let Some(r) = parse(index, rec) { + out.push((index, r)); + } + } + } + progress + .files + .fetch_add(out.len() as u64, Ordering::Relaxed); + Ok(out) + }) + .collect::>()?; + trace.mark(&format!("read + parse {} MB", table_size >> 20)); + + // Merge extension records into their base records. + let mut nodes: Vec> = Vec::new(); + nodes.resize_with(record_no as usize, || None); + let mut extensions = Vec::new(); + for (index, r) in parsed.into_iter().flatten() { + if r.base != index { + extensions.push(r); + continue; + } + let Some(slot) = nodes.get_mut(index as usize) else { + continue; + }; + *slot = Some(Node { + is_dir: r.is_dir, + names: r.names, + modified: r.modified, + surrogate: r.surrogate, + cloud: r.reparse && !r.surrogate, + size: size_of(r.data, r.attributes, r.reparse), + }); + } + for r in extensions { + if let Some(Some(n)) = nodes.get_mut(r.base as usize) { + n.names.extend(r.names); + if r.data.is_some() && n.size == (0, 0) { + n.size = size_of(r.data, r.attributes, r.reparse); + } + n.surrogate |= r.surrogate; + n.cloud |= r.reparse && !r.surrogate; + } + } + trace.mark("merge records"); + Ok(nodes) +} + +/// (on-disk, logical) size like the walk reports it: the logical length, +/// except where it differs from what the disk holds (sparse, compressed, +/// cloud placeholders and other reparse points) — then the allocated size. +fn size_of(data: Option<(u64, u64)>, attributes: u32, reparse: bool) -> (u64, u64) { + let (logical, allocated) = data.unwrap_or((0, 0)); + if attributes & WEIRD != 0 || reparse { + (allocated, logical) + } else { + (logical, logical) + } +} + +/// One directory entry: (parent record, child record, which of the child's +/// names). +pub(crate) type Link = (u32, u32, u32); + +/// The parsed table of a volume plus a parent -> children index. +pub(crate) struct Table { + nodes: Vec>, + /// Folders synced by OneDrive: its filter hides and resizes files there, + /// so they are listed through Windows like other cloud folders. + synced: Vec, + /// Every entry, sorted by parent: a directory's entries are one slice. + links: Vec, + trace: Trace, +} + +/// What a tree needs to know about one entry. +pub(crate) struct Entry<'a> { + pub(crate) record: u64, + pub(crate) name: &'a OsString, + pub(crate) is_dir: bool, + /// A cloud placeholder (file or folder): sizes and contents must be + /// asked from Windows. + pub(crate) cloud: bool, + pub(crate) size: u64, + pub(crate) logical: u64, + pub(crate) modified: u64, +} + +impl Table { + /// Read the MFT of the volume holding `root`. + pub(crate) fn read(root: &Path, progress: &Progress) -> Result { + let mut trace = Trace::new(); + let nodes = read_table(root, progress, &mut trace)?; + let mut links: Vec = Vec::new(); + for (index, n) in nodes.iter().enumerate() { + let Some(n) = n else { continue }; + let index = index as u64; + if (index < FIRST_USER_RECORD && index != ROOT) || n.surrogate { + continue; + } + for (k, (parent, _)) in n.names.iter().enumerate() { + if *parent != index && *parent <= u64::from(u32::MAX) { + links.push((*parent as u32, index as u32, k as u32)); + } + } + } + links.par_sort_unstable_by_key(|l| l.0); + trace.mark("index children"); + let mut table = Table { + nodes, + synced: Vec::new(), + links, + trace, + }; + let letter = drive_letter(root); + for var in ["OneDrive", "OneDriveConsumer", "OneDriveCommercial"] { + let Some(dir) = std::env::var_os(var).map(PathBuf::from) else { + continue; + }; + if drive_letter(&dir).map(|d| d.to_ascii_uppercase()) + == letter.map(|d| d.to_ascii_uppercase()) + { + if let Ok(record) = table.find(&dir) { + table.synced.push(record); + } + } + } + Ok(table) + } + + /// The entries of directory `dir`. + pub(crate) fn children(&self, dir: u64) -> impl Iterator> { + let lo = self.links.partition_point(|l| u64::from(l.0) < dir); + let hi = self.links.partition_point(|l| u64::from(l.0) <= dir); + self.links[lo..hi].iter().filter_map(|l| { + let n = self.nodes[l.1 as usize].as_ref()?; + Some(Entry { + record: u64::from(l.1), + name: &n.names[l.2 as usize].1, + is_dir: n.is_dir, + cloud: n.cloud || self.synced.contains(&u64::from(l.1)), + size: n.size.0, + logical: n.size.1, + modified: n.modified, + }) + }) + } + + /// The record of the folder `root` names (matched without regard to + /// case, like Windows does) and its path as spelled on disk. + pub(crate) fn find(&self, root: &Path) -> Result { + let mut at = ROOT; + for c in root.components() { + match c { + Component::Prefix(_) | Component::RootDir => {} + Component::Normal(want) => { + let want = want.to_string_lossy().to_lowercase(); + let next = self + .children(at) + .find(|e| e.is_dir && e.name.to_string_lossy().to_lowercase() == want); + at = next + .ok_or(MftError::NotNtfs("folder not found in the MFT"))? + .record; + } + _ => return Err(MftError::NotADrive), + } + } + Ok(at) + } + + /// Whether `record` is a cloud placeholder or a OneDrive-synced folder. + pub(crate) fn is_cloud(&self, record: u64) -> bool { + self.synced.contains(&record) + || self + .nodes + .get(record as usize) + .and_then(Option::as_ref) + .is_some_and(|n| n.cloud) + } + + pub(crate) fn mark(&mut self, what: &str) { + self.trace.mark(what); + } +} + +/// Walk `root` from the MFT, handing one [`Batch`] per directory to `sink` — +/// the same contract as [`crate::walk::walk`], for callers that need paths +/// (exclusions, duplicate search). Honours exclusions and max depth (links +/// are never followed); cloud folders are listed the ordinary way. +pub(crate) fn walk( + root: &Path, + opts: &Options, + progress: &Progress, + sink: &mut dyn FnMut(Batch), +) -> Result<(), MftError> { + let mut table = Table::read(root, progress)?; + if progress.cancelled() { + return Ok(()); + } + let start = table.find(root)?; + if table.is_cloud(start) { + // The folder itself is synced: all of it through Windows. + walk_cloud(&[(root.to_path_buf(), 0)], opts, progress, sink); + return Ok(()); + } + + let glob = build_globset(&opts.exclude); + let mut cloud_dirs: Vec<(PathBuf, usize)> = Vec::new(); + let mut queue = vec![(start, root.to_path_buf(), 0usize)]; + let mut bytes = 0u64; + while let Some((dir, path, depth)) = queue.pop() { + if progress.cancelled() { + return Ok(()); + } + let mut b = Batch { + dir: path.clone(), + files: Vec::new(), + modified: Vec::new(), + subdirs: Vec::new(), + others: Vec::new(), + skipped: 0, + }; + let child_depth = depth + 1; + if opts.max_depth.is_some_and(|m| child_depth > m) { + sink(b); + continue; + } + let descend = opts.max_depth.map_or(true, |m| child_depth < m); + for e in table.children(dir) { + let p = path.join(e.name); + if !glob.is_empty() && is_excluded(&glob, root, &p) { + continue; + } + if e.is_dir { + if descend { + if e.cloud { + // A cloud folder: list it the ordinary way, which + // asks the cloud filter what really is there. + cloud_dirs.push((p.clone(), child_depth)); + } else { + queue.push((e.record, p.clone(), child_depth)); + } + } + b.subdirs.push(p); + } else { + let size = if e.cloud { + crate::compressed_size(&p).unwrap_or(e.size) + } else { + e.size + }; + bytes += size; + b.files.push(FileEntry { + path: p, + size, + logical_size: e.logical, + }); + b.modified.push(e.modified); + } + } + progress.bytes.store(bytes, Ordering::Relaxed); + sink(b); + } + table.mark("emit directories"); + walk_cloud(&cloud_dirs, opts, progress, sink); + table.mark("walk cloud folders"); + Ok(()) +} + +/// List cloud folders found in the MFT the ordinary way, carrying the depth +/// limit on. +pub(crate) fn walk_cloud( + dirs: &[(PathBuf, usize)], + opts: &Options, + progress: &Progress, + sink: &mut dyn FnMut(Batch), +) { + for (dir, depth) in dirs { + if progress.cancelled() { + break; + } + let local = Options { + mft: false, + max_depth: opts.max_depth.map(|m| m.saturating_sub(*depth)), + ..opts.clone() + }; + if local.max_depth == Some(0) { + continue; + } + let _ = crate::walk::walk_dyn(dir, &local, progress, sink); + } +} diff --git a/crates/diskghost-core/src/snapshot.rs b/crates/diskghost-core/src/snapshot.rs index 283dc73..80d4068 100644 --- a/crates/diskghost-core/src/snapshot.rs +++ b/crates/diskghost-core/src/snapshot.rs @@ -167,7 +167,7 @@ fn path_of_key(root: &Path, key: &[u8]) -> PathBuf { p } -fn put_varint(out: &mut Vec, mut v: u64) { +pub(crate) fn put_varint(out: &mut Vec, mut v: u64) { while v >= 0x80 { out.push((v as u8) | 0x80); v >>= 7; @@ -184,7 +184,7 @@ fn get_u8(r: &mut impl BufRead) -> Result { Ok(b) } -fn get_varint(r: &mut impl BufRead) -> Result { +pub(crate) fn get_varint(r: &mut impl BufRead) -> Result { let mut v = 0u64; for shift in (0..64).step_by(7) { let b = get_u8(r)?; diff --git a/crates/diskghost-core/src/tree.rs b/crates/diskghost-core/src/tree.rs index 7826e09..294f0cb 100644 --- a/crates/diskghost-core/src/tree.rs +++ b/crates/diskghost-core/src/tree.rs @@ -15,7 +15,7 @@ use std::collections::{BinaryHeap, HashMap}; use std::ffi::OsStr; use std::path::{Component, Path, PathBuf}; -use crate::walk::{walk, Batch}; +use crate::walk::{walk, Batch, Engine}; use crate::{ disk_space, DirSize, ExtStat, FileEntry, FileStats, Options, Progress, ScanReport, SizeBucket, SIZE_BUCKET_BOUNDS, @@ -62,6 +62,9 @@ pub struct ScanTree { exts: Vec>, skipped: u64, cancelled: bool, + engine: Engine, + /// Why the MFT was asked for but the directory walk ran. + mft_fallback: Option, disk_total: u64, disk_free: u64, } @@ -199,6 +202,131 @@ impl Builder { } } +#[cfg(windows)] +impl Builder { + /// Build the tree straight from the NTFS Master File Table — no path is + /// made except for cloud placeholders, which are sized (files) or listed + /// (folders) the ordinary way, through the cloud filter. + fn add_mft( + &mut self, + table: &crate::mft::Table, + opts: &Options, + progress: &Progress, + ) -> Result<(), crate::mft::MftError> { + use std::sync::atomic::Ordering; + let start = table.find(&self.tree.root)?; + let mut queue = vec![(start, 0usize, 0usize)]; // (record, node, depth) + let mut cloud = Vec::new(); + if table.is_cloud(start) { + // The folder itself is synced: all of it through Windows. + queue.clear(); + cloud.push((0usize, 0usize)); + } + while let Some((dir, id, depth)) = queue.pop() { + if progress.cancelled() { + break; + } + let child_depth = depth + 1; + if opts.max_depth.is_some_and(|m| child_depth > m) { + continue; + } + let descend = opts.max_depth.map_or(true, |m| child_depth < m); + let (mut size, mut count) = (0u64, 0u64); + for e in table.children(dir) { + if e.is_dir { + let child = self.push(id, e.name); + if descend { + if e.cloud { + cloud.push((child, child_depth)); + } else { + queue.push((e.record, child, child_depth)); + } + } + } else { + let mut bytes = e.size; + if e.cloud { + let path = self.tree.path_of(id).join(e.name); + bytes = crate::compressed_size(&path).unwrap_or(bytes); + } + let ext = self.ext_id(e.name); + self.tree.nodes[id].files.push(FileRec { + name: e.name.as_os_str().into(), + size: bytes, + logical_size: e.logical, + ext, + }); + size += bytes; + count += 1; + } + } + let node = &mut self.tree.nodes[id]; + node.size += size; + node.file_count += count; + progress.files.fetch_add(count, Ordering::Relaxed); + progress.bytes.fetch_add(size, Ordering::Relaxed); + } + // Cloud folders, listed the ordinary way into this same tree. + for (id, depth) in cloud { + if progress.cancelled() { + break; + } + let local = Options { + mft: false, + max_depth: opts.max_depth.map(|m| m.saturating_sub(depth)), + ..opts.clone() + }; + if local.max_depth == Some(0) { + continue; + } + let path = self.tree.path_of(id); + self.pending.insert(path.clone(), id); + let _ = walk(&path, &local, progress, |b| self.add(b)); + } + Ok(()) + } +} + +/// Fill `b` from the MFT if asked and possible (no exclusions: those need +/// paths), else from the directory walk. Returns the engine used and, if the +/// MFT was asked for but not used, why. +#[cfg(windows)] +fn fill( + b: &mut Builder, + root: &Path, + opts: &Options, + progress: &Progress, +) -> (Engine, Option) { + if opts.mft && opts.exclude.is_empty() && !opts.follow_symlinks { + let direct = crate::mft::Table::read(root, progress).and_then(|mut table| { + b.add_mft(&table, opts, progress)?; + table.mark("build tree"); + Ok(()) + }); + return match direct { + Ok(()) => (Engine::Mft, None), + Err(e) => { + let walk_opts = Options { + mft: false, + ..opts.clone() + }; + let (engine, _) = walk(root, &walk_opts, progress, |batch| b.add(batch)); + (engine, Some(e.to_string())) + } + }; + } + walk(root, opts, progress, |batch| b.add(batch)) +} + +#[cfg(not(windows))] +fn fill( + b: &mut Builder, + root: &Path, + opts: &Options, + progress: &Progress, +) -> (Engine, Option) { + walk(root, opts, progress, |batch| b.add(batch)) +} + impl Node { fn new(name: &OsStr, parent: Option) -> Node { Node { @@ -224,14 +352,26 @@ impl ScanTree { exts: Vec::new(), skipped: 0, cancelled: false, + engine: Engine::Walk, + mft_fallback: None, disk_total: 0, disk_free: 0, }, pending: HashMap::new(), ext_ids: HashMap::new(), }; - walk(root, opts, progress, |batch| b.add(batch)); - b.finish(progress.cancelled()) + let (engine, fallback) = fill(&mut b, root, opts, progress); + b.tree.engine = engine; + b.tree.mft_fallback = fallback; + let started = std::time::Instant::now(); + let tree = b.finish(progress.cancelled()); + if std::env::var_os("DISKGHOST_TRACE").is_some() { + eprintln!( + "tree: finish {:.0} ms", + started.elapsed().as_secs_f64() * 1000.0 + ); + } + tree } /// The directory this tree was built from. @@ -281,6 +421,20 @@ impl ScanTree { self.cancelled } + /// How the tree was read: `"mft"` (NTFS Master File Table) or `"walk"`. + pub fn engine(&self) -> &'static str { + match self.engine { + Engine::Mft => "mft", + Engine::Walk => "walk", + } + } + + /// Why the MFT engine was requested but not used (e.g. no administrator + /// rights), if so. + pub fn mft_fallback(&self) -> Option<&str> { + self.mft_fallback.as_deref() + } + /// Re-query the filesystem's total/free bytes (e.g. after deleting files). pub fn refresh_disk_space(&mut self) { let (total, free) = disk_space(&self.root); @@ -719,6 +873,7 @@ mod tests { logical_size: size, }) .collect(), + modified: vec![0; files.len()], subdirs: subdirs.iter().map(PathBuf::from).collect(), others: Vec::new(), skipped: 0, @@ -730,6 +885,8 @@ mod tests { exts: Vec::new(), skipped: 0, cancelled: false, + engine: Engine::Walk, + mft_fallback: None, disk_total: 0, disk_free: 0, }, diff --git a/crates/diskghost-core/src/walk.rs b/crates/diskghost-core/src/walk.rs index 1517377..9dd566b 100644 --- a/crates/diskghost-core/src/walk.rs +++ b/crates/diskghost-core/src/walk.rs @@ -19,6 +19,9 @@ use crate::{build_globset, is_excluded, on_disk_size, FileEntry, Options, Progre pub(crate) struct Batch { pub(crate) dir: PathBuf, pub(crate) files: Vec, + /// Last-modified time of each of `files` (nanoseconds since the Unix + /// epoch, 0 if unknown) — from the same listing, so it costs nothing. + pub(crate) modified: Vec, /// Subdirectories (also those not descended into because of `max_depth`). pub(crate) subdirs: Vec, /// Entries neither counted nor followed: symlinks and junctions when links @@ -58,7 +61,40 @@ impl Ctx<'_> { /// Walk `root`, calling `sink` on the caller's thread with each directory's /// [`Batch`] as soon as it is listed (in no particular order). Honours /// exclusions, max depth, link following and cancellation. -pub(crate) fn walk(root: &Path, opts: &Options, progress: &Progress, mut sink: impl FnMut(Batch)) { +pub(crate) fn walk( + root: &Path, + opts: &Options, + progress: &Progress, + mut sink: impl FnMut(Batch), +) -> (Engine, Option) { + walk_dyn(root, opts, progress, &mut sink) +} + +/// [`walk`] with a type-erased sink, so the MFT engine can hand folders it +/// cannot read itself back to the directory walk. +pub(crate) fn walk_dyn( + root: &Path, + opts: &Options, + progress: &Progress, + sink: &mut dyn FnMut(Batch), +) -> (Engine, Option) { + // Why the MFT was asked for but not used. + let mut fallback = None; + if opts.mft { + #[cfg(windows)] + if opts.follow_symlinks { + fallback = Some("links are followed only by the directory walk".to_string()); + } else { + match crate::mft::walk(root, opts, progress, sink) { + Ok(()) => return (Engine::Mft, None), + Err(e) => fallback = Some(e.to_string()), + } + } + #[cfg(not(windows))] + { + fallback = Some("the MFT engine exists on Windows only".to_string()); + } + } let threads = opts.threads.unwrap_or_else(default_threads).max(1); let (tx, rx) = mpsc::channel(); let ctx = Ctx { @@ -90,6 +126,17 @@ pub(crate) fn walk(root: &Path, opts: &Options, progress: &Progress, mut sink: i sink(batch); } }); + (Engine::Walk, fallback) +} + +/// How a tree was read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Engine { + /// Directory by directory. + Walk, + /// From the NTFS Master File Table (Windows only). + #[cfg_attr(not(windows), allow(dead_code))] + Mft, } fn visit<'s, 'c: 's>(s: &rayon::Scope<'s>, ctx: &'c Ctx<'c>, dir: PathBuf, depth: usize) { @@ -99,6 +146,7 @@ fn visit<'s, 'c: 's>(s: &rayon::Scope<'s>, ctx: &'c Ctx<'c>, dir: PathBuf, depth let mut b = Batch { dir, files: Vec::new(), + modified: Vec::new(), subdirs: Vec::new(), others: Vec::new(), skipped: 0, @@ -170,6 +218,7 @@ fn visit<'s, 'c: 's>(s: &rayon::Scope<'s>, ctx: &'c Ctx<'c>, dir: PathBuf, depth }; let size = on_disk_size(&meta, &path); bytes += size; + b.modified.push(modified_ns(&meta)); b.files.push(FileEntry { path, size, @@ -185,3 +234,11 @@ fn visit<'s, 'c: 's>(s: &rayon::Scope<'s>, ctx: &'c Ctx<'c>, dir: PathBuf, depth ctx.progress.bytes.fetch_add(bytes, Ordering::Relaxed); let _ = ctx.tx.send(b); } + +/// Modification time in nanoseconds since the Unix epoch (0 if unknown). +pub(crate) fn modified_ns(meta: &std::fs::Metadata) -> u64 { + meta.modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map_or(0, |d| u64::try_from(d.as_nanos()).unwrap_or(u64::MAX)) +} diff --git a/gui/src-tauri/Cargo.lock b/gui/src-tauri/Cargo.lock index c1c9ed3..898d854 100644 --- a/gui/src-tauri/Cargo.lock +++ b/gui/src-tauri/Cargo.lock @@ -149,6 +149,7 @@ dependencies = [ "constant_time_eq", "cpufeatures 0.3.0", "memmap2", + "rayon-core", ] [[package]] @@ -644,13 +645,12 @@ dependencies = [ [[package]] name = "diskghost-core" -version = "0.9.2" +version = "0.10.0" dependencies = [ "blake3", "fs4", "globset", "rayon", - "same-file", "serde", "serde_json", "trash", @@ -659,7 +659,7 @@ dependencies = [ [[package]] name = "diskghost-gui" -version = "0.9.2" +version = "0.10.0" dependencies = [ "diskghost-core", "serde", diff --git a/gui/src-tauri/Cargo.toml b/gui/src-tauri/Cargo.toml index 0ed3619..2b7abb2 100644 --- a/gui/src-tauri/Cargo.toml +++ b/gui/src-tauri/Cargo.toml @@ -4,7 +4,7 @@ [package] name = "diskghost-gui" -version = "0.9.2" +version = "0.10.0" edition = "2021" license = "MIT" description = "Diskghost desktop GUI (Tauri) — modern disk usage & duplicate finder." diff --git a/gui/src-tauri/src/main.rs b/gui/src-tauri/src/main.rs index 1a258ab..b122c0b 100644 --- a/gui/src-tauri/src/main.rs +++ b/gui/src-tauri/src/main.rs @@ -7,8 +7,8 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, RwLock}; use diskghost_core::{ - diff, disk_space, find_duplicates_with_progress, reclaim, remove_path, validate_globs, - DiffReport, DirSize, DupGroup, FileStats, Options, Progress, ReclaimAction, ReclaimReport, + diff, disk_space, find_duplicates_cached, reclaim, remove_path, validate_globs, DiffReport, + DirSize, DupGroup, FileStats, HashCache, Options, Progress, ReclaimAction, ReclaimReport, RemoveMode, RemoveReport, ScanReport, ScanTree, Snapshot, }; use serde::{Deserialize, Serialize}; @@ -51,6 +51,9 @@ type Shared = Arc>; struct AppState { jobs: Mutex>, trees: Mutex>, + /// File hashes of earlier duplicate searches (loaded on first use, saved + /// after each search), so searching again only reads changed files. + hashes: Mutex>>, } /// Payload pushed to the frontend as a scan/search runs. @@ -59,6 +62,23 @@ struct ProgressPayload { job: u64, files: u64, bytes: u64, + /// Duplicate search phase (0 listing, 1 comparing ends, 2 hashing). + phase: u64, + total_files: u64, + total_bytes: u64, +} + +impl ProgressPayload { + fn of(job: u64, p: &Progress) -> ProgressPayload { + ProgressPayload { + job, + files: p.files(), + bytes: p.bytes(), + phase: p.phase(), + total_files: p.total_files(), + total_bytes: p.total_bytes(), + } + } } /// Walk options coming from the frontend. @@ -70,6 +90,8 @@ struct WalkOpts { follow_symlinks: bool, /// `None` = automatic. threads: Option, + /// Read the NTFS Master File Table (needs administrator rights). + mft: bool, } impl WalkOpts { @@ -79,6 +101,7 @@ impl WalkOpts { follow_symlinks: self.follow_symlinks, exclude: self.exclude.clone(), threads: self.threads, + mft: self.mft, } } } @@ -93,6 +116,10 @@ struct View { nested: Vec>, /// Bytes per extension and per size range below the viewed folder. stats: FileStats, + /// How the tree was read: "mft" or "walk". + engine: &'static str, + /// Why the MFT was asked for but not used. + mft_fallback: Option, } fn view_of(tree: &ScanTree, path: &Path, top: usize) -> Option { @@ -110,6 +137,8 @@ fn view_of(tree: &ScanTree, path: &Path, top: usize) -> Option { report, nested, stats, + engine: tree.engine(), + mft_fallback: tree.mft_fallback().map(str::to_string), }) } @@ -145,14 +174,7 @@ fn all_trees(app: &tauri::AppHandle) -> Vec { fn spawn_emitter(app: tauri::AppHandle, job: u64, progress: Progress, done: Arc) { std::thread::spawn(move || { while !done.load(Ordering::Relaxed) { - let _ = app.emit( - "progress", - ProgressPayload { - job, - files: progress.files(), - bytes: progress.bytes(), - }, - ); + let _ = app.emit("progress", ProgressPayload::of(job, &progress)); std::thread::sleep(std::time::Duration::from_millis(120)); } }); @@ -191,14 +213,9 @@ impl Drop for Running { jobs.remove(&self.job); } // A last event with the final counts. - let _ = self.app.emit( - "progress", - ProgressPayload { - job: self.job, - files: self.progress.files(), - bytes: self.progress.bytes(), - }, - ); + let _ = self + .app + .emit("progress", ProgressPayload::of(self.job, &self.progress)); } } @@ -456,16 +473,36 @@ async fn find_dupes( if !p.is_dir() { return Err(format!("not a directory: {path}")); } - Ok(find_duplicates_with_progress( + let file = hash_cache_file(&app)?; + let cache = { + let state = app.state::(); + let mut slot = state.hashes.lock().map_err(|e| e.to_string())?; + slot.get_or_insert_with(|| Arc::new(HashCache::load(&file))) + .clone() + }; + let groups = find_duplicates_cached( &p, mb * 1024 * 1024, &opts.to_options(), &running.progress, - )) + &cache, + ); + drop(running); + // Persist in the background; the result is shown right away. + std::thread::spawn(move || { + let _ = cache.save(&file); + }); + Ok(groups) }) .await } +/// Where duplicate-search hashes are kept between runs. +fn hash_cache_file(app: &tauri::AppHandle) -> Result { + let base = app.path().app_data_dir().map_err(|e| e.to_string())?; + Ok(base.join("hashes.bin")) +} + /// Request cancellation of job `job`, if it is still running. #[tauri::command] fn cancel(state: tauri::State<'_, AppState>, job: u64) { @@ -601,6 +638,69 @@ async fn check_globs(patterns: Vec) -> Vec { validate_globs(&patterns) } +/// Whether the app runs with administrator rights (needed for the MFT engine). +#[tauri::command] +fn is_elevated() -> bool { + #[cfg(windows)] + { + #[link(name = "shell32")] + extern "system" { + fn IsUserAnAdmin() -> i32; + } + unsafe { IsUserAnAdmin() != 0 } + } + #[cfg(not(windows))] + { + false + } +} + +/// Start the app again with administrator rights (Windows asks for consent) +/// and quit this instance. An error if the user declined. +#[tauri::command] +fn restart_elevated(app: tauri::AppHandle) -> Result<(), String> { + #[cfg(windows)] + { + use std::os::windows::ffi::OsStrExt; + #[link(name = "shell32")] + extern "system" { + fn ShellExecuteW( + hwnd: *mut std::ffi::c_void, + op: *const u16, + file: *const u16, + params: *const u16, + dir: *const u16, + show: i32, + ) -> isize; + } + let wide = |s: &std::ffi::OsStr| -> Vec { + s.encode_wide().chain(std::iter::once(0)).collect() + }; + let exe = std::env::current_exe().map_err(|e| e.to_string())?; + let (op, file) = (wide("runas".as_ref()), wide(exe.as_os_str())); + let r = unsafe { + ShellExecuteW( + std::ptr::null_mut(), + op.as_ptr(), + file.as_ptr(), + std::ptr::null(), + std::ptr::null(), + 1, // SW_SHOWNORMAL + ) + }; + if r <= 32 { + return Err("administrator rights were not granted".into()); + } + app.exit(0); + Ok(()) + } + #[cfg(not(windows))] + { + let _ = app; + Err("only available on Windows".into()) + } +} + /// The app's version, for the title bar. #[tauri::command] fn app_version() -> &'static str { @@ -826,6 +926,7 @@ fn main() { app.manage(AppState { jobs: Mutex::new(HashMap::new()), trees: Mutex::new(HashMap::new()), + hashes: Mutex::new(None), }); place_window(app.handle()); Ok(()) @@ -846,7 +947,9 @@ fn main() { check_globs, load_settings, save_settings, - app_version + app_version, + is_elevated, + restart_elevated ]) .run(tauri::generate_context!()) .expect("error while running Diskghost"); diff --git a/gui/src-tauri/tauri.conf.json b/gui/src-tauri/tauri.conf.json index 57d105e..aa43a66 100644 --- a/gui/src-tauri/tauri.conf.json +++ b/gui/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Diskghost", - "version": "0.9.2", + "version": "0.10.0", "identifier": "org.diskghost.app", "build": { "frontendDist": "../ui" diff --git a/gui/ui/app.js b/gui/ui/app.js index 67c7557..ec42792 100644 --- a/gui/ui/app.js +++ b/gui/ui/app.js @@ -9,6 +9,7 @@ const { invoke } = window.__TAURI__.core; const $ = (id) => document.getElementById(id); let settings = mergeSettings(null); +let elevated = false; // running as administrator (the MFT engine needs it) let drives = []; // Every scan / duplicate search the user started, oldest first. A job is @@ -104,6 +105,15 @@ const I18N = { setParallel: "Parallel scans", setParallelD: "Scans running at the same time; more wait in the queue. 1 is best for a single HDD.", setDupes: "Duplicates", setMinMb: "Ignore files smaller than (MB)", setDelete: "Deleting", setTrash: "Send deleted items to the Recycle Bin", setTrashD: "Off deletes permanently.", + engineMft: "Ultra — NTFS MFT (administrator)", + engineMftD: "Reads the drive's file table (MFT) in one go instead of opening every folder, also for the duplicate search. Pays off most on the system drive C:, where Windows slows folder listing down (measured: 5× faster). On a SATA SSD with millions of files the table is larger than the folders and Fast can be quicker. Needs administrator rights and a local NTFS drive, otherwise Fast is used automatically. OneDrive folders are always read through Windows; links are not followed.", + mftNeedsAdmin: "Diskghost is not running as administrator.", restartAdmin: "Restart as administrator", + adminDropNote: "Windows does not allow dragging folders from Explorer onto a program running as administrator — use Browse or the drive list there.", + adminBadge: "Admin", mftFallback: "MFT not used", + phaseWalk: "Listing files", phaseWalkD: "Finding files of the same size…", + phaseSample: "Comparing start and end", phaseSampleD: "each file is opened once", + phaseHash: "Checking whole contents (BLAKE3)", + protSkipped: "Program & game files", protSkippedSub: "not searched — click to include them (searches again)", setEngine: "Scan engine", engineFast: "Fast — SSD / NVMe (default)", engineGentle: "Gentle — HDD / in the background", engineFastD: "Lists many folders at once (2 threads per CPU core). Fastest on SSDs; on a spinning disk the head jumps around and it can get slower.", engineGentleD: "3 threads. Best for spinning disks and network drives, and keeps the PC responsive while it scans — slower on SSDs.", @@ -198,6 +208,15 @@ const I18N = { setParallel: "Parallele Scans", setParallelD: "Gleichzeitig laufende Scans; weitere warten. 1 ist am besten für eine einzelne HDD.", setDupes: "Duplikate", setMinMb: "Dateien kleiner als (MB) ignorieren", setDelete: "Löschen", setTrash: "Gelöschtes in den Papierkorb", setTrashD: "Aus löscht endgültig.", + engineMft: "Ultra — NTFS MFT (Administrator)", + engineMftD: "Liest die Dateitabelle (MFT) des Laufwerks am Stück, statt jeden Ordner zu öffnen, auch für die Duplikatsuche. Bringt am meisten auf dem Systemlaufwerk C:, wo Windows das Auflisten bremst (gemessen: 5× schneller). Auf einer SATA-SSD mit Millionen Dateien ist die Tabelle größer als die Ordner, dort kann „Schnell“ schneller sein. Braucht Administratorrechte und ein lokales NTFS-Laufwerk, sonst wird automatisch „Schnell“ benutzt. OneDrive-Ordner werden immer über Windows gelesen; Links werden nicht verfolgt.", + mftNeedsAdmin: "Diskghost läuft nicht als Administrator.", restartAdmin: "Als Administrator neu starten", + adminDropNote: "Windows erlaubt kein Ziehen von Ordnern aus dem Explorer auf ein Programm mit Administratorrechten — dort Durchsuchen oder die Laufwerksliste nutzen.", + adminBadge: "Admin", mftFallback: "MFT nicht benutzt", + phaseWalk: "Dateien auflisten", phaseWalkD: "Suche Dateien gleicher Größe…", + phaseSample: "Anfang und Ende vergleichen", phaseSampleD: "jede Datei wird einmal geöffnet", + phaseHash: "Ganze Inhalte prüfen (BLAKE3)", + protSkipped: "Programm- & Spieldateien", protSkippedSub: "nicht durchsucht — klicken zum Einbeziehen (sucht neu)", setEngine: "Scan-Engine", engineFast: "Schnell — SSD / NVMe (Standard)", engineGentle: "Schonend — HDD / im Hintergrund", engineFastD: "Liest viele Ordner gleichzeitig (2 Threads pro CPU-Kern). Am schnellsten auf SSDs; auf einer HDD springt der Lesekopf hin und her und es kann langsamer werden.", engineGentleD: "3 Threads. Am besten für HDDs und Netzlaufwerke, und der PC bleibt beim Scannen flüssig — auf SSDs langsamer.", @@ -310,6 +329,7 @@ function walkOpts() { // "gentle" keeps a spinning disk from seeking itself to death and leaves // the machine responsive; "fast" lets the walker pick (2 x CPU threads). threads: settings.engine === "gentle" ? 3 : null, + mft: settings.engine === "mft", }; } @@ -425,8 +445,15 @@ function renderSettings() { field(t("setParallel"), t("setParallelD"), numberInput(settings.parallel, 1, 8, (v) => { set({ parallel: v ?? 2 }); pump(); })), checkField(t("setFollow"), t("setFollowD"), settings.follow_symlinks, (v) => set({ follow_symlinks: v }))), el("div", { class: "set-grid" }, - field(t("setEngine"), settings.engine === "gentle" ? t("engineGentleD") : t("engineFastD"), - selectInput(settings.engine, [["fast", t("engineFast")], ["gentle", t("engineGentle")]], (v) => set({ engine: v }, true))), + field(t("setEngine"), t({ fast: "engineFastD", gentle: "engineGentleD", mft: "engineMftD" }[settings.engine]), + selectInput(settings.engine, [["fast", t("engineFast")], ["gentle", t("engineGentle")], ["mft", t("engineMft")]], + (v) => set({ engine: v }, true))), + settings.engine === "mft" && !elevated + ? el("div", { class: "field" }, + el("span", { class: "fl", text: t("mftNeedsAdmin") }), + el("button", { type: "button", class: "primary", onclick: () => invoke("restart_elevated").catch((e) => status(errText(e), true)) }, `🛡 ${t("restartAdmin")}`), + el("span", { class: "fd", text: t("adminDropNote") })) + : null, field(t("setKeep"), t("setKeepD"), numberInput(settings.keep_snapshots, 1, 100, (v) => set({ keep_snapshots: v ?? 10 }))))), el("section", { class: "set-sec" }, el("h2", { text: `${t("setDupes")} & ${t("setDelete")}` }), @@ -488,10 +515,18 @@ function renderDrives() { const active = () => jobs.find((j) => j.id === activeId) || null; const byId = (id) => jobs.find((j) => j.id === id); -function enqueue(kind, path, { select = true } = {}) { +// Walk options of a duplicate search: unless program and game files are +// included on purpose, they are not even read. +function dupeOpts(includeProt) { + const o = walkOpts(); + if (!includeProt) o.exclude = [...new Set([...o.exclude, ...protectedExcludes()])]; + return o; +} + +function enqueue(kind, path, { select = true, includeProt = false } = {}) { path = String(path || "").trim(); if (!path) return status(t("enterPath"), true); - const opts = walkOpts(); + const opts = kind === "dupes" ? dupeOpts(includeProt) : walkOpts(); // The same scan again: just show it (Rescan walks it afresh). const same = jobs.find((j) => j.kind === kind && j.root === path && j.state !== "error" && JSON.stringify(j.opts) === JSON.stringify(opts)); @@ -503,6 +538,7 @@ function enqueue(kind, path, { select = true } = {}) { id: nextId++, kind, root: path, path, opts, top: settings.top, mb: settings.min_mb, state: "queued", files: 0, bytes: 0, startedAt: 0, elapsedMs: null, view: null, groups: null, error: null, tab: "folders", + includeProt, cache: new Map(), // folder -> view, so going back is instant back: [], // folders visited before, for Back }; @@ -553,6 +589,7 @@ async function runJob(j) { j.returnTo = null; if (back && back !== j.root) navigate(j, back, { keepTab: true, record: false }); if (!view.cancelled) autoCompare(j); + if (view.mft_fallback) status(`${t("mftFallback")}: ${view.mft_fallback}`, true); } else { j.groups = await invoke("find_dupes", { job: j.id, path: j.root, mb: j.mb, opts: j.opts }); j.elapsedMs = performance.now() - j.startedAt; @@ -603,7 +640,7 @@ function selectJob(id) { } function rescan(j) { - j.opts = walkOpts(); + j.opts = j.kind === "dupes" ? dupeOpts(j.includeProt) : walkOpts(); j.top = settings.top; j.mb = settings.min_mb; j.returnTo = j.kind === "scan" ? j.path : null; @@ -813,16 +850,41 @@ function queuedView(j) { el("div", {}, el("button", { type: "button", onclick: () => closeJob(j) }, t("removeFromQueue"))))); } +// Title, 0..1 fraction (or null) and detail line of a running job: a scan +// estimates against the drive's used space, a duplicate search reports its +// three phases against their known totals. +function progressInfo(j) { + if (j.kind !== "dupes") { + const est = estimate(j); + return { + title: `${t("stRunning")}${j.opts?.mft ? " · NTFS MFT" : ""}`, + frac: est, + detail: est == null ? t("noEstimate") : `≈ ${pct(est, 1)} ${t("estimated")}`, + }; + } + const ph = j.phase || 0; + const title = `${ph + 1}/3 · ${t(["phaseWalk", "phaseSample", "phaseHash"][ph])}`; + if (ph === 0 || !j.totalFiles) return { title, frac: null, detail: t("phaseWalkD") }; + if (ph === 1) { + return { title, frac: j.files / j.totalFiles, detail: `${num(j.files)} / ${num(j.totalFiles)} ${t("files")} · ${t("phaseSampleD")}` }; + } + return { + title, + frac: j.totalBytes ? j.bytes / j.totalBytes : j.files / j.totalFiles, + detail: `${human(j.bytes)} / ${human(j.totalBytes)} · ${num(j.files)} / ${num(j.totalFiles)} ${t("files")}`, + }; +} + function progressView(j) { const secs = (performance.now() - j.startedAt) / 1000; - const est = estimate(j); + const p = progressInfo(j); return el("div", { class: "center" }, el("div", { class: "card progress-card" }, - el("h2", { text: j.kind === "dupes" ? t("stSearching") : t("stRunning") }), + el("h2", { id: "pv-title", text: p.title }), el("div", { class: "path", text: j.root }), el("div", { class: "big", id: "pv-bytes", text: human(j.bytes) }), - el("div", { class: "meter big" + (est == null ? " indeterminate" : ""), id: "pv-meter" }, - el("span", { class: "used", style: est == null ? {} : { width: `${est * 100}%` } })), - el("div", { class: "muted", id: "pv-est", text: est == null ? t("noEstimate") : `≈ ${pct(est, 1)} ${t("estimated")}` }), + el("div", { class: "meter big" + (p.frac == null ? " indeterminate" : ""), id: "pv-meter" }, + el("span", { class: "used", style: p.frac == null ? {} : { width: `${p.frac * 100}%` } })), + el("div", { class: "muted", id: "pv-est", text: p.detail }), el("div", { class: "progress-grid" }, kpi(t("kFiles"), num(j.files), null, { id: "pv-files" }), kpi(t("elapsed"), fmtMs(secs * 1000), null, { id: "pv-time" }), @@ -835,15 +897,17 @@ function progressView(j) { function updateProgressView(j) { const secs = (performance.now() - j.startedAt) / 1000; const set = (id, v) => { const n = $(id); if (n) n.querySelector?.(".v") ? (n.querySelector(".v").textContent = v) : (n.textContent = v); }; + const p = progressInfo(j); + set("pv-title", p.title); set("pv-bytes", human(j.bytes)); set("pv-files", num(j.files)); set("pv-time", fmtMs(secs * 1000)); set("pv-rate", secs > 0.5 ? num(Math.round(j.files / secs)) : "–"); - const est = estimate(j); - if (est != null) { - const bar = $("pv-meter")?.firstChild; - if (bar) bar.style.width = `${est * 100}%`; - set("pv-est", `≈ ${pct(est, 1)} ${t("estimated")}`); + set("pv-est", p.detail); + const meter = $("pv-meter"); + if (meter) { + meter.classList.toggle("indeterminate", p.frac == null); + meter.firstChild.style.width = p.frac == null ? "" : `${Math.min(1, p.frac) * 100}%`; } } @@ -939,7 +1003,7 @@ function kpiRow(j, v) { el("span", { class: "meter" }, el("span", { class: used / v.disk_total > 0.9 ? "hot" : "used", style: { width: `${(100 * used) / v.disk_total}%` } })))); } const secs = (j.elapsedMs || 0) / 1000; - row.append(kpi(t("kScan"), fmtMs(j.elapsedMs || 0), + row.append(kpi(v.engine === "mft" ? `${t("kScan")} · NTFS MFT` : t("kScan"), fmtMs(j.elapsedMs || 0), `${secs > 0 ? `${num(Math.round(v.total_files / secs))} ${t("rate")} · ` : ""}${v.skipped ? `${num(v.skipped)} ${t("kSkipped")}` : t("kNoSkip")}`, {}, v.skipped ? "" : "")); if (j.diff) { @@ -1502,8 +1566,11 @@ function dupesView(j) { const sum = (list) => list.reduce((a, g) => a + g.wasted, 0); wrap.append(el("div", { class: "kpis" }, kpi(t("ownReclaim"), human(sum(own)), `${num(own.length)} ${t("groups")} · ${num(own.reduce((a, g) => a + g.files.length, 0))} ${t("files")}`), - kpi(t("protHidden"), human(sum(prot)), `${num(prot.length)} ${t("groups")} — ${t("protHiddenSub")}`, - { role: "button", tabindex: "0", style: { cursor: "pointer" }, onclick: () => { j.dupShowProt = !j.dupShowProt; renderMain(); } }), + j.includeProt + ? kpi(t("protHidden"), human(sum(prot)), `${num(prot.length)} ${t("groups")} — ${t("protHiddenSub")}`, + { role: "button", tabindex: "0", style: { cursor: "pointer" }, onclick: () => { j.dupShowProt = !j.dupShowProt; renderMain(); } }) + : kpi(t("protSkipped"), "—", t("protSkippedSub"), + { role: "button", tabindex: "0", style: { cursor: "pointer" }, onclick: () => { j.includeProt = true; j.dupShowProt = true; rescan(j); } }), kpi(t("kScan"), fmtMs(j.elapsedMs || 0), null))); wrap.append(el("div", { class: "muted", text: t("dupHint") })); @@ -1672,6 +1739,9 @@ try { const j = e.payload && byId(e.payload.job); if (!j || j.state !== "running") return; j.files = e.payload.files; + j.phase = e.payload.phase; + j.totalFiles = e.payload.total_files; + j.totalBytes = e.payload.total_bytes; j.bytes = e.payload.bytes; if (paintQueued) return; paintQueued = true; @@ -1723,7 +1793,10 @@ try { } catch (_) { /* events unavailable */ } (async function start() { - invoke("app_version").then((v) => { $("version").textContent = `v${v}`; }).catch(() => {}); + Promise.all([invoke("app_version").catch(() => ""), invoke("is_elevated").catch(() => false)]).then(([v, admin]) => { + elevated = Boolean(admin); + $("version").textContent = `${v ? `v${v}` : ""}${elevated ? ` · ${t("adminBadge")}` : ""}`; + }); try { settings = mergeSettings(await invoke("load_settings")); } catch (_) { /* defaults */ } applyI18n(); updateExcludeChip(); diff --git a/gui/ui/lib.js b/gui/ui/lib.js index 3c09bf5..f1eddb7 100644 --- a/gui/ui/lib.js +++ b/gui/ui/lib.js @@ -197,7 +197,7 @@ // Before version 2 at most 2 scans ran at once; that default is gone. if (saved.version >= 2) s.parallel = int(saved.parallel, 1, 8, s.parallel); s.keep_snapshots = int(saved.keep_snapshots, 1, 100, s.keep_snapshots); - if (["fast", "gentle"].includes(saved.engine)) s.engine = saved.engine; + if (["fast", "gentle", "mft"].includes(saved.engine)) s.engine = saved.engine; if (typeof saved.follow_symlinks === "boolean") s.follow_symlinks = saved.follow_symlinks; if (typeof saved.trash === "boolean") s.trash = saved.trash; const ids = new Set(EXCLUDE_PRESETS.map((p) => p.id)); @@ -295,6 +295,15 @@ return parts.slice(0, -1).some((p) => PROTECTED_DIRS.has(p.toLowerCase())); } + // The same protection as exclude patterns, so a duplicate search does not + // even read program and game files: their folders as path components, + // their file types as `*.ext`. + function protectedExcludes() { + const exts = new Set(PROTECTED_EXTS); + for (const [ext, cat] of EXT_CATEGORY) if (cat === "program" || cat === "gamedata") exts.add(ext); + return [...PROTECTED_DIRS, ...[...exts].sort().map((e) => `*.${e}`)]; + } + // Duplicate groups by the set of folders their copies sit in, so "these // two folders share 300 identical files" is one line instead of 300. function folderSets(groups) { @@ -377,6 +386,6 @@ human, humanDelta, fmtMs, baseName, fmtDate, crumbs, squarify, deltaTint, changesByPath, pct, EXCLUDE_PRESETS, DEFAULT_SETTINGS, mergeSettings, activeExcludes, categoryOf, categorize, bucketLabel, jobsToStart, driveKey, mapItems, parentPath, isUnder, SETTINGS_VERSION, - isProtectedPath, folderSets, + isProtectedPath, folderSets, protectedExcludes, }; }); diff --git a/gui/ui/lib.test.js b/gui/ui/lib.test.js index 640dcc7..877d0c8 100644 --- a/gui/ui/lib.test.js +++ b/gui/ui/lib.test.js @@ -264,3 +264,15 @@ test("folderSets groups duplicates by the folders their copies are in", () => { assert.equal(sets[1].wasted, 150); assert.deepEqual(lib.folderSets([]), []); }); + +test("protectedExcludes covers the protected folders and program file types", () => { + const ex = lib.protectedExcludes(); + assert.ok(ex.includes("steamapps")); + assert.ok(ex.includes("program files")); + assert.ok(ex.includes("*.dll")); + assert.ok(ex.includes("*.pak")); + assert.ok(ex.includes("*.pck")); + assert.ok(!ex.includes("*.jpg")); + assert.equal(new Set(ex).size, ex.length); + assert.equal(lib.mergeSettings({ engine: "mft" }).engine, "mft"); +});