From 88bf05537e29914757fc4fead8bd426dd8e87736 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Fri, 6 Nov 2026 07:13:08 +0200 Subject: [PATCH 1/3] fsfork -l: Check all of a fork's files in one git apply When a fork's whole patch did not undo in the directory, each changed file was checked on its own: a `git diff` and a `git apply --check` for each. A fork with an upstream pull in it changes a thousand files, and listing the forks of turso took over ten minutes. Git checks every file's changes in one `git apply --reverse --check --verbose`, saying which file it is checking before any error about it, so the files that do not undo are counted from one run, without reading their paths back from what git says. --- cmd/fsfork/forks.rs | 12 +------ fsfork/git.rs | 76 +++++++++++++++++++++++++++++++-------------- 2 files changed, 53 insertions(+), 35 deletions(-) diff --git a/cmd/fsfork/forks.rs b/cmd/fsfork/forks.rs index f1d3020..e68bfc0 100644 --- a/cmd/fsfork/forks.rs +++ b/cmd/fsfork/forks.rs @@ -244,17 +244,7 @@ fn state(repo: &git::Repo, dir: &Path, fork: &Found) -> io::Result<(State, Syste .iter() .filter_map(|path| modified(&fork.fork, path)) .fold(made, SystemTime::max); - if repo.applied(&before, &after, None, dir)? { - return Ok((State::Merged, touched)); - } - let mut files = 0; - for path in &changed { - if !repo.applied(&before, &after, Some(path), dir)? { - files += 1; - } - } - // Each file's changes may be there without the whole patch undoing. - let state = match files { + let state = match repo.unapplied(&before, &after, dir)? { 0 => State::Merged, files => State::OnlyHere { files }, }; diff --git a/fsfork/git.rs b/fsfork/git.rs index 985dda7..f2c6a0a 100644 --- a/fsfork/git.rs +++ b/fsfork/git.rs @@ -122,40 +122,58 @@ impl Repo { .collect()) } - /// Whether `dir` has the changes between the trees `before` and `after`, - /// or with `path`, those to it only: whether they could be undone in - /// `dir`, as they could if they were applied to it, even if it changed - /// after. - pub fn applied( - &self, - before: &str, - after: &str, - path: Option<&Path>, - dir: &Path, - ) -> io::Result { - let mut diff = self.git(); - diff.args(["--literal-pathspecs", "diff", "--binary", "--no-renames"]) - .args([before, after]); - if let Some(path) = path { - diff.arg("--").arg(path); - } - let patch = diff.stderr(process::Stdio::inherit()).output()?; + /// How many of the files changed between the trees `before` and `after` + /// have changes that `dir` does not: that could not be undone in `dir`, + /// as they could if they were applied to it, even if it changed after. + /// + /// Git checks every file's changes in one run, saying which file it is + /// checking before any error about it, so that the files are told apart + /// without reading their paths back from what it says. + pub fn unapplied(&self, before: &str, after: &str, dir: &Path) -> io::Result { + let patch = self + .git() + .args(["diff", "--binary", "--no-renames", before, after]) + .stderr(process::Stdio::inherit()) + .output()?; if !patch.status.success() { return Err(io::Error::other("git could not compare the trees")); } let file = self.dir.join("patch"); fs::write(&file, &patch.stdout)?; - let undone = status( + let checked = output( self.git() .arg("--work-tree") .arg(dir) - .args(["apply", "--reverse", "--check"]) + .args(["apply", "--reverse", "--check", "--verbose"]) .arg(&file) .current_dir(dir) - .stdout(process::Stdio::null()) - .stderr(process::Stdio::null()), + .env("LC_ALL", "C") + .env_remove("LANGUAGE") + .stdin(process::Stdio::null()) + .stdout(process::Stdio::null()), )?; - Ok(undone.success()) + if checked.status.success() { + return Ok(0); + } + let mut files = 0; + let mut checking = false; + let mut counted = false; + for line in checked.stderr.split(|&b| b == b'\n') { + if line.starts_with(b"Checking patch ") { + checking = true; + counted = false; + } else if line.starts_with(b"error: ") && checking && !counted { + files += 1; + counted = true; + } + } + match (checked.status.code(), files) { + (Some(1), 1..) => Ok(files), + _ => Err(io::Error::other(format!( + "{}: git could not check the changes in it", + shown(dir).display() + ))), + } } /// Records the files in `tree` that git does not ignore as a tree, @@ -221,6 +239,16 @@ fn bare() -> Command { /// Runs `command`, with the terminal's signals, which the caller ignores, as /// they are by default, and waits for it to exit. fn status(command: &mut Command) -> io::Result { + with_signals(command).status() +} + +/// Runs `command` as `status` does, and returns what it printed too. +fn output(command: &mut Command) -> io::Result { + with_signals(command).output() +} + +/// `command`, with the terminal's signals restored to their defaults in it. +fn with_signals(command: &mut Command) -> &mut Command { unsafe { command.pre_exec(|| { for signal in SIGNALS { @@ -229,5 +257,5 @@ fn status(command: &mut Command) -> io::Result { Ok(()) }); } - command.status() + command } From 0f62a4df4b90cd31fc7b545a525cfec6255f8475 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Fri, 6 Nov 2026 07:13:46 +0200 Subject: [PATCH 2/3] fsfork -l, fsdiff: Hash again only the files whose size or time changed Every fork's files are clones of its base's, with new inodes and change times, so git hashed every file of every fork and base again: a few seconds each, for 4,500 files in turso. Git now compares only a file's size and modification time, which clones keep. It compares only the whole seconds of the time then, so a file whose time differs from the one the index was made from only within the second is hashed again first, comparing all its stat. It stays in the index, so that git does not ignore it. --- fsfork/git.rs | 101 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 95 insertions(+), 6 deletions(-) diff --git a/fsfork/git.rs b/fsfork/git.rs index f2c6a0a..a64a4ac 100644 --- a/fsfork/git.rs +++ b/fsfork/git.rs @@ -8,10 +8,11 @@ //! configuration but its defaults. use std::{ + cell::RefCell, env, ffi::OsStr, fs, io, - os::unix::{ffi::OsStrExt, process::CommandExt}, + os::unix::{ffi::OsStrExt, fs::MetadataExt, process::CommandExt}, path::{Path, PathBuf}, process::{self, Command, ExitStatus}, }; @@ -35,6 +36,9 @@ const REPOSITORY: &[&str] = &[ /// A repository of the command's own, removed when dropped. pub struct Repo { dir: PathBuf, + /// The directory last recorded as `old`, whose files the index of `old` + /// says how they were when hashed. + indexed: RefCell>, } impl Repo { @@ -50,7 +54,10 @@ impl Repo { Err(e) => return Err(e), } }; - let repo = Repo { dir }; + let repo = Repo { + dir, + indexed: RefCell::new(None), + }; let made = status(bare().args(["init", "--quiet", "--bare"]).arg(&repo.dir))?; if !made.success() { return Err(io::Error::other("git could not make a repository")); @@ -66,9 +73,11 @@ impl Repo { pub fn record(&self, old: &Path, new: &Path) -> io::Result<(String, String)> { let old_index = self.dir.join("old.index"); let new_index = self.dir.join("new.index"); - let before = self.snapshot(old, &old_index)?; + let last = self.indexed.borrow_mut().take(); + let before = self.snapshot(old, &old_index, last.as_deref())?; + *self.indexed.borrow_mut() = Some(old.to_path_buf()); fs::copy(&old_index, &new_index)?; - let after = self.snapshot(new, &new_index)?; + let after = self.snapshot(new, &new_index, Some(old))?; Ok((before, after)) } @@ -177,10 +186,29 @@ impl Repo { } /// Records the files in `tree` that git does not ignore as a tree, - /// keeping the index in `index`, and returns the tree's ID. - fn snapshot(&self, tree: &Path, index: &Path) -> io::Result { + /// keeping the index in `index`, which says how the files in `from` were + /// when hashed, if it is not new, and returns the tree's ID. + /// + /// A file is hashed again only if its size or modification time differs + /// from the index's. Its inode and change time are not compared: a fork's + /// files are clones of its base's, with new ones but the same times, and + /// comparing them would hash every file in every fork and base again. + /// Git compares only the whole seconds of the time then, so a file whose + /// time differs from `from`'s only within the second is hashed again + /// first, comparing all the file's stat, staying in the index so that + /// git does not ignore it. + fn snapshot(&self, tree: &Path, index: &Path, from: Option<&Path>) -> io::Result { + if let Some(from) = from { + self.rehash_changed_within_second(tree, index, from)?; + } let added = status( self.git() + .args([ + "-c", + "core.checkStat=minimal", + "-c", + "core.trustCtime=false", + ]) .arg("--work-tree") .arg(tree) .args(["add", "--all", "--", "."]) @@ -207,6 +235,67 @@ impl Repo { Ok(String::from_utf8_lossy(&written.stdout).trim().to_string()) } + /// Hashes again into `index` the files in `tree` whose size and whole + /// seconds of modification time are those of the same file in `from`, but + /// whose time is not. + fn rehash_changed_within_second( + &self, + tree: &Path, + index: &Path, + from: &Path, + ) -> io::Result<()> { + let listed = self + .git() + .args(["ls-files", "-z"]) + .env("GIT_INDEX_FILE", index) + .stderr(process::Stdio::inherit()) + .output()?; + if !listed.status.success() { + return Err(io::Error::other("git could not list its index")); + } + let mut changed = Vec::new(); + for path in listed.stdout.split(|&b| b == 0).filter(|p| !p.is_empty()) { + let path = Path::new(OsStr::from_bytes(path)); + let (Ok(was), Ok(is)) = ( + fs::symlink_metadata(from.join(path)), + fs::symlink_metadata(tree.join(path)), + ) else { + continue; + }; + if was.file_type() == is.file_type() + && !is.is_dir() + && was.size() == is.size() + && was.mtime() == is.mtime() + && was.mtime_nsec() != is.mtime_nsec() + { + changed.extend_from_slice(path.as_os_str().as_bytes()); + changed.push(0); + } + } + if changed.is_empty() { + return Ok(()); + } + let file = self.dir.join("rehash"); + fs::write(&file, &changed)?; + let rehashed = status( + self.git() + .arg("--work-tree") + .arg(tree) + .args(["update-index", "-z", "--stdin"]) + .current_dir(tree) + .env("GIT_INDEX_FILE", index) + .stdin(fs::File::open(&file)?) + .stdout(process::Stdio::null()), + )?; + if !rehashed.success() { + return Err(io::Error::other(format!( + "{}: git could not read it", + shown(tree).display() + ))); + } + Ok(()) + } + /// A command that runs git on the repository, as `bare` does. fn git(&self) -> Command { let mut git = bare(); From ac21b715c953f7509ab80cd0a4d07e495092bca9 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Fri, 6 Nov 2026 07:14:04 +0200 Subject: [PATCH 3/3] fsfork -l: Print each fork as soon as it is looked into The list was printed only once every fork had been looked into, after seconds with nothing to show. The header is printed at once, and each fork as soon as it is looked into, in the order they were made, with the columns as wide as the forks' paths, known before any is looked into, and the longest age. --- cmd/fsfork/forks.rs | 110 +++++++++++++++++++++++++++++--------------- cmd/fsfork/fsfork.1 | 2 +- 2 files changed, 73 insertions(+), 39 deletions(-) diff --git a/cmd/fsfork/forks.rs b/cmd/fsfork/forks.rs index e68bfc0..d707901 100644 --- a/cmd/fsfork/forks.rs +++ b/cmd/fsfork/forks.rs @@ -57,17 +57,34 @@ impl Looked { } } -/// Lists the forks of `dir`, under a header, those left alone longest first, -/// and returns whether all could be looked into. +/// Lists the forks of `dir`, under a header, each as soon as it is looked +/// into, in the order they were made, and returns whether all could be +/// looked into. pub fn list(dir: &Path) -> io::Result { - let (forks, all) = look(dir)?; let now = SystemTime::now(); - let header = ["FORK", "UPDATED", "STATUS"].map(String::from); - let rows: Vec<_> = std::iter::once(header) - .chain(forks.iter().map(|fork| row(fork, now, &fork.state))) - .collect(); - print(&rows)?; - Ok(all) + let widths = std::cell::Cell::new([0; 3]); + each( + dir, + |found| { + let header = ["FORK", "UPDATED", "STATUS"]; + let paths = found.iter().map(|found| { + let path = if found.left_over { + &found.base + } else { + &found.fork + }; + message::shown(path).display().to_string().chars().count() + }); + let age = if found.is_empty() { 0 } else { AGE_WIDTH }; + widths.set([ + paths.fold(header[0].len(), usize::max), + age.max(header[1].len()), + 0, + ]); + print(&header.map(String::from), widths.get()) + }, + |fork| print(&row(&fork, now, &fork.state), widths.get()), + ) } /// How long a fork must be left alone for `prune` to remove it: one just @@ -189,9 +206,33 @@ fn remove(fork: &Looked) -> io::Result<()> { /// Looks into the forks of `dir`, and returns them, those left alone longest /// first, and whether all could be looked into, having said why not. fn look(dir: &Path) -> io::Result<(Vec, bool)> { - let found = fsfork::forks(dir)?; - if found.is_empty() { - return Ok((Vec::new(), true)); + let mut forks = Vec::new(); + let all = each( + dir, + |_| Ok(()), + |fork| { + forks.push(fork); + Ok(()) + }, + )?; + // Those left alone longest, the likeliest to be done with, first. + forks.sort_by_key(|fork| fork.touched); + Ok((forks, all)) +} + +/// Finds the forks of `dir`, gives them to `found`, even if there are none, and +/// then looks into each, in the order they were made, giving it to `looked` +/// as soon as it is; and returns whether all could be looked into, having +/// said why not. +fn each( + dir: &Path, + found: impl FnOnce(&[Found]) -> io::Result<()>, + mut looked: impl FnMut(Looked) -> io::Result<()>, +) -> io::Result { + let forks = fsfork::forks(dir)?; + found(&forks)?; + if forks.is_empty() { + return Ok(true); } let dir = dir.canonicalize()?; // The terminal's signals reach git too, which exits; `fsfork` outlives @@ -201,9 +242,8 @@ fn look(dir: &Path) -> io::Result<(Vec, bool)> { } let repo = git::Repo::new()?; let mut all = true; - let mut forks: Vec = found - .into_iter() - .map(|found| match state(&repo, &dir, &found) { + for found in forks { + let fork = match state(&repo, &dir, &found) { Ok((state, touched)) => Looked { found, state, @@ -219,11 +259,10 @@ fn look(dir: &Path) -> io::Result<(Vec, bool)> { touched, } } - }) - .collect(); - // Those left alone longest, the likeliest to be done with, first. - forks.sort_by_key(|fork| fork.touched); - Ok((forks, all)) + }; + looked(fork)?; + } + Ok(all) } /// What is in `fork`, of `dir`, and when it was last changed, or made, if it @@ -269,27 +308,22 @@ fn row(fork: &Looked, now: SystemTime, what: &dyn fmt::Display) -> [String; 3] { ] } -/// Prints `rows` on stdout, in columns as wide as the widest of each, and -/// three spaces apart. -fn print(rows: &[[String; N]]) -> io::Result<()> { - let mut widths = [0; N]; - for row in rows { - for (width, column) in widths.iter_mut().zip(row) { - *width = (*width).max(column.chars().count()); +/// The widest `age` says, as in "59 minutes ago", for its column. +const AGE_WIDTH: usize = 14; + +/// Prints `row` on stdout, its columns as wide as `widths` says, but the +/// last, and three spaces apart, and flushes it, to be seen at once. +fn print(row: &[String; N], widths: [usize; N]) -> io::Result<()> { + let mut line = String::new(); + for (i, (column, width)) in row.iter().zip(widths).enumerate() { + if i + 1 == N { + line.push_str(column); + } else { + line.push_str(&format!("{column:width$} ")); } } let mut out = io::stdout().lock(); - for row in rows { - let mut line = String::new(); - for (i, (column, width)) in row.iter().zip(widths).enumerate() { - if i + 1 == N { - line.push_str(column); - } else { - line.push_str(&format!("{column:width$} ")); - } - } - writeln!(out, "{line}")?; - } + writeln!(out, "{line}")?; out.flush() } diff --git a/cmd/fsfork/fsfork.1 b/cmd/fsfork/fsfork.1 index 61027e8..015886c 100644 --- a/cmd/fsfork/fsfork.1 +++ b/cmd/fsfork/fsfork.1 @@ -142,7 +142,7 @@ It could not be compared with its base, as .B fsfork says on standard error. .PP -Those left alone longest come first. +Each is listed as soon as it is looked into, in the order they were made. Forks made elsewhere, with .I fork given, are not listed.