From 14d848e34254a6927a4bc18ae23796063c262107 Mon Sep 17 00:00:00 2001 From: Wayn_Liu Date: Thu, 3 Sep 2026 16:46:17 +0800 Subject: [PATCH] feat(diff): content-level diff between local files and the remote project `push --dry-run` answers which files would change; there was no way to see what changed inside them. `olcli diff` prints a unified diff of the local tree against the project's current contents. The remote side is fetched fresh on every run. `.olcli.json` records remote paths, never remote contents, so there is no stored snapshot to diff against - comparing "against the last pull" would have meant inventing a content cache rather than reusing one. Fetching fresh is also what makes the diff describe what a subsequent push will overwrite, which is the question the command exists to answer. It costs one request: downloadProject returns the whole project as a single archive, the same call pull and sync already make. `a/` is the remote and `b/` is local, so `+` is content push would upload and `-` is content it would overwrite. Binary files are reported as differing without a patch. Both sides pass through the same ignore layers and dotfile rule, so artifacts sitting on Overleaf are not reported as local deletions. `diff --name-only` and `push --dry-run` deliberately answer different questions: push selects by modification time, diff by content. What they do share - the local walk-and-filter loop, which push and sync each had their own drifting copy of - moves to src/scan.ts. `push --dry-run` now says what it measures and points at `olcli diff`. Comparison, rendering and remote-tree filtering are pure functions in src/diff.ts, unit-tested without an Overleaf account. New dependency `diff` has no dependencies of its own. latexdiff integration follows separately. Refs #45 --- CHANGELOG.md | 23 +++++ README.md | 35 +++++++ SKILL.md | 14 +++ package-lock.json | 18 ++++ package.json | 2 + src/cli.ts | 257 ++++++++++++++++++++++++++++++++-------------- src/diff.ts | 219 +++++++++++++++++++++++++++++++++++++++ src/scan.ts | 83 +++++++++++++++ test/diff.test.ts | 219 +++++++++++++++++++++++++++++++++++++++ test/e2e.sh | 45 ++++++++ test/scan.test.ts | 70 +++++++++++++ 11 files changed, 908 insertions(+), 77 deletions(-) create mode 100644 src/diff.ts create mode 100644 src/scan.ts create mode 100644 test/diff.test.ts create mode 100644 test/scan.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index febfbb4..f3ae6d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ All notable changes to this project will be documented in this file. +## [Unreleased] + +### Added +- **`olcli diff [project] [dir]`** ([#45](https://github.com/aloth/olcli/issues/45)) - content-level preview of what a push would change + - `push --dry-run` answers *which files*; there was no way to see *what changed inside them* short of pulling into a scratch directory and running `diff(1)` by hand + - Unified diff to stdout, colourized when stdout is a TTY. `--name-only` for paths only, `--file ` for a single file, `-U ` for context width + - **The remote side is fetched fresh on every run**, and the command says so in `--help` and in its output footer. `.olcli.json` records remote *paths*, never remote *contents*, so there is no stored snapshot to compare against - "diff against the last pull" would have meant inventing a content cache, not reusing one. Fetching fresh is also what makes the diff describe what a subsequent `push` will overwrite, which is the question the command exists to answer + - Cost of fetching fresh is one request: `downloadProject` returns the whole project as a single archive, the same call `pull` and `sync` already make. Per-file fetching would have been one request per file and still could not have identified which files differ without downloading them + - `a/` is the remote and `b/` is local, so a `+` line is content `push` would upload and a `-` line is content it would overwrite + - Binary files (PDFs, images) are reported as `Binary files ... differ`, detected by a NUL byte in the first 8000 bytes. No attempt is made to be cleverer + - Both sides pass through the same ignore layers and the same dotfile rule. Filtering only the local side would have listed `output.pdf` and every stray `.aux` on Overleaf as a local deletion on every run + - Remote-only files are reported but flagged as untouched by a plain `push`, since only `push --delete` removes them + - Archive entries whose names escape the target directory are dropped, consistent with what `pull` refuses to extract + +### Changed +- Local file scanning extracted into `src/scan.ts`. `push` and `sync` each carried their own copy of the same walk-and-filter loop and the two had already drifted (`sync` guarded against a missing directory, `push` did not); `diff` would have made a third. Same reasoning as `src/rename-plan.ts` in 0.9.0 +- `push --dry-run` now notes that its list is selected by modification time and points at `olcli diff` for content changes. The two commands answer different questions and will disagree - a file touched but not edited appears in `push --dry-run` and not in `diff` - so the overlap is resolved by making each one say what it measures rather than by merging them + +### Notes +- New runtime dependency: [`diff`](https://www.npmjs.com/package/diff) `^9.0.0`, which has no dependencies of its own +- Comparison, rendering and remote-tree filtering live in `src/diff.ts` as pure functions, so they are unit-tested without an Overleaf account (`npm test`). Like `rename-plan.ts`, they are not re-exported from the package root +- `latexdiff` integration (`--latexdiff`, `--pdf`) is deliberately left out of this change and will follow separately + ## [0.9.1] - 2026-09-01 ### Fixed diff --git a/README.md b/README.md index 16c3896..9d646f6 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Work with Overleaf projects directly from your command line. Edit locally with y - ⬇️ **Pull** project files to local directory for offline editing - ⬆️ **Push** local changes back to Overleaf - 🔄 **Sync** bidirectionally with smart conflict detection +- 🔍 **Diff** local files against the live remote before pushing - 🔀 **Git remote** — use Overleaf as a native git remote ([docs](docs/GIT-REMOTE.md)) - ✌️ **Two-way deletions** — files removed locally are deleted on Overleaf on next sync - 🗑️ **Delete** and ✏️ **rename** remote files by path @@ -132,6 +133,7 @@ All commands auto-detect the project when run from a synced directory (contains | `olcli pull [project] [dir]` | Download project files to local directory | | `olcli push [dir]` | Upload local changes to Overleaf (`--delete` also removes files deleted locally) | | `olcli sync [dir]` | Bidirectional sync (pull + push) | +| `olcli diff [project] [dir]` | Show content-level changes between local files and the remote | | `olcli upload [project]` | Upload a single file (`--to ` sets the remote destination) | | `olcli download [project]` | Download a single file | | `olcli delete [project]` | Delete a remote file or folder (alias: `rm`) | @@ -192,6 +194,39 @@ Useful in multi-doc projects: each `-r` run compiles the file as if it were the - **Propagates local deletions** — use `--no-delete` to opt out - Use `--dry-run` to preview without applying +### Diff + +`olcli diff` compares the bytes of your local files against the project's +current contents and prints a unified diff. + +```bash +olcli diff # every changed file, as patches +olcli diff --name-only # just the changed paths +olcli diff --file main.tex # one file +olcli diff -U 8 # wider context +``` + +**The remote side is fetched fresh on every run.** The diff describes the +project as it is at that moment — which is what a subsequent `push` would +overwrite — not a comparison against your last `pull`. `.olcli.json` records +remote *paths*, never remote *contents*, so there is no stored snapshot to +compare against; and the whole project arrives in a single request, the same +one `pull` makes, so fetching fresh costs one round trip rather than one per +file. A collaborator editing between `diff` and `push` can still change the +outcome, which is why the fetch time is printed. + +In the output, `a/` is the remote and `b/` is local: a `+` line is content +`push` would upload, a `-` line is content it would overwrite. Files that +differ only in bytes that are not text (PDFs, images) are reported as +`Binary files ... differ`. Both sides pass through the same ignore layers, so +build artifacts sitting on Overleaf are not reported as locally deleted. + +`diff --name-only` and `push --dry-run` answer different questions and will +disagree. `push --dry-run` lists files whose **modification time** is newer +than the last pull, because that is what `push` uploads; `diff` lists files +whose **contents** actually differ. A file you touched without editing appears +in the first and not the second. + #### How deletion propagation works `olcli` records a manifest of remote files in `.olcli.json`. On next sync: diff --git a/SKILL.md b/SKILL.md index 999533c..7fa90eb 100644 --- a/SKILL.md +++ b/SKILL.md @@ -133,6 +133,18 @@ olcli sync # Bidirectional sync (pull + push, propagates local dele olcli sync --no-delete # Sync without propagating local deletions to remote ``` +### Review changes before pushing + +```bash +olcli diff # unified diff of every changed file +olcli diff --name-only # changed paths only +olcli diff --file main.tex # a single file +``` + +The remote side is fetched fresh each run, so this shows what a subsequent +`push` would overwrite — not a comparison against the last `pull`. `a/` is the +remote, `b/` is local. Binary files are reported as differing without a patch. + ### Delete or rename remote files ```bash @@ -243,6 +255,7 @@ zip arxiv.zip *.tex main.bbl figures/*.pdf | `olcli pull [project] [dir]` | Download project files | | `olcli push [dir]` | Upload local changes | | `olcli sync [dir]` | Bidirectional sync | +| `olcli diff [project] [dir]` | Content-level diff of local files vs. the live remote | | `olcli upload [project]` | Upload a single file (`--to ` sets the remote destination) | | `olcli download [project]` | Download a single file | | `olcli delete [project]` | Delete a remote file or folder (alias: `rm`) | @@ -266,6 +279,7 @@ zip arxiv.zip *.tex main.bbl figures/*.pdf - **Auto-detect project**: Run commands from a synced directory (contains `.olcli.json`) to skip the project argument - **Dry run**: Use `olcli push --dry-run` or `olcli sync --dry-run` to preview before applying +- **Preview content**: `push --dry-run` lists files by modification time; `olcli diff` compares actual contents, so the two lists can differ - **Force overwrite**: Use `olcli pull --force` to overwrite local changes - **Two-way deletes**: `olcli sync` propagates *local* deletions to the remote; use `--no-delete` to opt out per run - **Build artifacts**: `.aux`, `.bbl`, `.log`, `.synctex.gz` etc. are filtered by default. Add custom patterns to a `.olignore` file (gitignore-style) diff --git a/package-lock.json b/package-lock.json index 3af01e3..2e9b8cb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "cheerio": "^1.0.0", "commander": "^12.1.0", "conf": "^13.0.0", + "diff": "^9.0.0", "ignore": "^7.0.5", "ora": "^8.0.1", "tough-cookie": "^4.1.4", @@ -27,6 +28,7 @@ }, "devDependencies": { "@types/adm-zip": "^0.5.7", + "@types/diff": "^7.0.2", "@types/node": "^22.0.0", "@types/tough-cookie": "^4.0.5", "tsx": "^4.7.0", @@ -540,6 +542,13 @@ "@types/node": "*" } }, + "node_modules/@types/diff": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@types/diff/-/diff-7.0.2.tgz", + "integrity": "sha512-JSWRMozjFKsGlEjiiKajUjIJVKuKdE3oVy2DNtK+fUo8q82nhFZ2CPQwicAIkXrofahDXrWJ7mjelvZphMS98Q==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/node": { "version": "22.19.7", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.7.tgz", @@ -971,6 +980,15 @@ "node": ">= 0.8" } }, + "node_modules/diff": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-9.0.0.tgz", + "integrity": "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/dom-serializer": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", diff --git a/package.json b/package.json index a652d1d..59c4582 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "cheerio": "^1.0.0", "commander": "^12.1.0", "conf": "^13.0.0", + "diff": "^9.0.0", "ignore": "^7.0.5", "ora": "^8.0.1", "tough-cookie": "^4.1.4", @@ -70,6 +71,7 @@ }, "devDependencies": { "@types/adm-zip": "^0.5.7", + "@types/diff": "^7.0.2", "@types/node": "^22.0.0", "@types/tough-cookie": "^4.0.5", "tsx": "^4.7.0", diff --git a/src/cli.ts b/src/cli.ts index 08f1c50..99deedc 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -13,12 +13,12 @@ import { writeFileSync, readFileSync, existsSync, mkdirSync } from 'node:fs'; import { join, dirname, basename } from 'node:path'; import { fileURLToPath } from 'node:url'; import { OverleafClient } from './client.js'; -import { resolveRemotePath, resolveWithin } from './paths.js'; +import { resolveRemotePath, resolveWithin, normalizeRemotePath } from './paths.js'; import { planProjectRenames } from './rename-plan.js'; +import { scanLocalFiles } from './scan.js'; +import { compareTrees, filterRemoteTree, renderFileDiff, statusLetter } from './diff.js'; import { loadIgnore, - shouldIgnore, - buildTexSiblingSet, DEFAULT_IGNORE_PATTERNS, type IgnoreContext, } from './ignore.js'; @@ -1218,56 +1218,22 @@ program }); // Get list of files to upload - const { readdirSync, statSync } = await import('node:fs'); + const { files: localFileList, ignored: filesIgnored } = scanLocalFiles(targetDir, ignoreCtx); const filesToUpload: { path: string; relativePath: string }[] = []; - const filesIgnored: string[] = []; // Every local file that survives ignore filtering, regardless of mtime. // filesToUpload is mtime-filtered and therefore useless as a deletion // baseline: an unchanged file would look "absent" and get deleted. const allLocalPaths = new Set(); - function scanDir(currentDir: string, relativeBase: string = '') { - const entries = readdirSync(currentDir, { withFileTypes: true }); - // Pre-compute sibling .tex set for the PDF special rule. - const texSiblings = buildTexSiblingSet( - entries.filter((e) => !e.isDirectory()).map((e) => e.name), - ); - for (const entry of entries) { - // Skip hidden files and .olcli.json (always — predates ignore subsystem) - if (entry.name.startsWith('.')) continue; - - const fullPath = join(currentDir, entry.name); - const relativePath = relativeBase ? `${relativeBase}/${entry.name}` : entry.name; - - if (entry.isDirectory()) { - // Test directory ignore (gitignore semantics: trailing slash matches dir) - if (shouldIgnore(`${relativePath}/`, ignoreCtx)) { - filesIgnored.push(`${relativePath}/`); - continue; - } - scanDir(fullPath, relativePath); - } else { - if (shouldIgnore(relativePath, ignoreCtx, texSiblings)) { - filesIgnored.push(relativePath); - continue; - } - allLocalPaths.add(relativePath); - // Check if file is newer than last pull (unless --all) - if (options.all || !lastPull) { - filesToUpload.push({ path: fullPath, relativePath }); - } else { - const stats = statSync(fullPath); - if (stats.mtime > lastPull) { - filesToUpload.push({ path: fullPath, relativePath }); - } - } - } + for (const file of localFileList) { + allLocalPaths.add(file.relativePath); + // Check if file is newer than last pull (unless --all) + if (options.all || !lastPull || file.mtime > lastPull) { + filesToUpload.push({ path: file.path, relativePath: file.relativePath }); } } - scanDir(targetDir); - if (options.showIgnored && filesIgnored.length > 0) { spinner.stop(); console.log(chalk.bold(chalk.dim(`Ignored ${filesIgnored.length} file(s)/dir(s):`))); @@ -1318,6 +1284,12 @@ program if (noBaseline) { console.log(chalk.dim(' --delete skipped: no manifest yet (first push from this directory)')); } + // This list is mtime-based: a file touched but not edited is in it, and + // a file whose bytes already match the remote is too. `olcli diff` + // compares content instead. + if (filesToUpload.length > 0) { + console.log(chalk.dim(' selected by modification time — run `olcli diff` to see content changes')); + } return; } @@ -1500,42 +1472,18 @@ program // Track local modifications const localFiles = new Map(); - const filesIgnored: string[] = []; - const { readdirSync, statSync } = await import('node:fs'); - - function scanLocalFiles(currentDir: string, relativeBase: string = '') { - if (!existsSync(currentDir)) return; - const entries = readdirSync(currentDir, { withFileTypes: true }); - const texSiblings = buildTexSiblingSet( - entries.filter((e) => !e.isDirectory()).map((e) => e.name), - ); - for (const entry of entries) { - if (entry.name.startsWith('.')) continue; - const fullPath = join(currentDir, entry.name); - const relativePath = relativeBase ? `${relativeBase}/${entry.name}` : entry.name; - if (entry.isDirectory()) { - if (shouldIgnore(`${relativePath}/`, ignoreCtx)) { - filesIgnored.push(`${relativePath}/`); - continue; - } - scanLocalFiles(fullPath, relativePath); - } else { - if (shouldIgnore(relativePath, ignoreCtx, texSiblings)) { - filesIgnored.push(relativePath); - continue; - } - const stats = statSync(fullPath); - localFiles.set(relativePath, { - mtime: stats.mtime, - content: readFileSync(fullPath) - }); - } - } - } + let filesIgnored: string[] = []; // Read local files before overwriting - if (existsSync(metaPath)) { - scanLocalFiles(targetDir); + if (existsSync(targetDir) && existsSync(metaPath)) { + const scan = scanLocalFiles(targetDir, ignoreCtx); + filesIgnored = scan.ignored; + for (const file of scan.files) { + localFiles.set(file.relativePath, { + mtime: file.mtime, + content: readFileSync(file.path) + }); + } } if (options.showIgnored && filesIgnored.length > 0) { @@ -1730,6 +1678,161 @@ program } }); +program + .command('diff [project] [dir]') + .description('Show content-level differences between local files and the remote project') + .option('--name-only', 'List changed paths instead of printing patches') + .option('--file ', 'Diff a single file') + .option('-U, --unified ', 'Lines of context around each hunk (default: 3)', parseInt) + .option('--no-default-ignore', 'Disable built-in LaTeX artifact ignore list (only .olignore applies)') + .option('--no-ignore', 'Disable all ignore filtering') + .option('--cookie ', 'Session cookie override') + .addHelpText('after', ` +The remote side is fetched fresh on every run, so the diff describes the +project as it is right now - which is what a subsequent push would overwrite. +It is not a comparison against the last pull. A collaborator editing between +diff and push can still change the outcome; the fetch time is printed for that +reason.`) + .action(async (project, dir, options) => { + const targetDir = dir || '.'; + + if (!existsSync(targetDir)) { + console.error(chalk.red(`Directory not found: ${targetDir}`)); + process.exit(1); + } + + const spinner = ora('Connecting...').start(); + try { + const client = await getClient(options.cookie); + + let resolved; + try { + resolved = await resolveProject(client, project, targetDir); + } catch (error: any) { + spinner.fail(error.message); + console.error('Either run from a directory with .olcli.json or pass a project name/ID'); + process.exit(1); + } + const { id: projectId, name: projectName } = resolved; + + // The whole project arrives as one zip in a single request - the same + // call pull and sync already make. Fetching per-file would mean one + // request per file and could not tell us which files differ without + // downloading them anyway. + spinner.text = 'Fetching remote project...'; + const zipBuffer = await client.downloadProject(projectId); + const fetchedAt = new Date(); + + const AdmZip = (await import('adm-zip')).default; + const zip = new AdmZip(zipBuffer); + + const ignoreCtx = loadIgnore(targetDir, { + noDefaults: options.defaultIgnore === false, + disableAll: options.ignore === false, + }); + + // Both sides go through the same filters; see filterRemoteTree. + const remoteFiles = filterRemoteTree( + zip.getEntries() + .filter((e) => !e.isDirectory) + .map((e) => ({ path: e.entryName, data: e.getData() })), + ignoreCtx, + (path) => resolveWithin(targetDir, path) !== null, + ); + + const scan = scanLocalFiles(targetDir, ignoreCtx); + const localFiles = new Map(); + for (const file of scan.files) { + localFiles.set(file.relativePath, readFileSync(file.path)); + } + + let entries = compareTrees(localFiles, remoteFiles).filter((e) => e.status !== 'unchanged'); + + if (options.file) { + const wanted = normalizeRemotePath(options.file); + entries = entries.filter((e) => e.path === wanted); + if (entries.length === 0) { + spinner.info(`No differences in ${wanted}`); + if (!localFiles.has(wanted) && !remoteFiles.has(wanted)) { + console.log(chalk.dim(' (file is on neither side, or is filtered by an ignore rule)')); + } + return; + } + } + + spinner.stop(); + + if (entries.length === 0) { + console.log(chalk.green(`No differences — local files match "${projectName}"`)); + console.log(chalk.dim(` remote fetched ${fetchedAt.toISOString()}`)); + return; + } + + if (options.nameOnly) { + for (const e of entries) { + const colour = e.status === 'added' ? chalk.green + : e.status === 'deleted' ? chalk.red + : chalk.yellow; + console.log(`${colour(statusLetter(e.status))} ${e.path}`); + } + } else { + for (const e of entries) { + const patch = renderFileDiff( + e, + localFiles.get(e.path), + remoteFiles.get(e.path), + { context: options.unified }, + ); + patch.replace(/\n$/, '').split('\n').forEach((line, index) => { + console.log(colourizeDiffLine(line, index, e.binary)); + }); + } + } + + console.log(); + // With --file the summary would count only the one file asked for, which + // reads as "this is all that differs". Report totals only for a full run. + if (!options.file) { + const counts = { + added: entries.filter((e) => e.status === 'added').length, + modified: entries.filter((e) => e.status === 'modified').length, + deleted: entries.filter((e) => e.status === 'deleted').length, + }; + console.log(chalk.bold( + `${entries.length} file(s) differ from "${projectName}": ` + + `${counts.added} added, ${counts.modified} modified, ${counts.deleted} remote-only` + )); + if (counts.deleted > 0) { + console.log(chalk.dim(' remote-only files are left alone by push; use push --delete to remove them')); + } + } + console.log(chalk.dim(` a/ = remote as of ${fetchedAt.toISOString()}, b/ = local`)); + + setLastProject(projectId); + } catch (error: any) { + spinner.fail(`Failed: ${error.message}`); + process.exit(1); + } + }); + +/** + * Colourize one line of a rendered patch. chalk already no-ops when stdout is + * not a TTY, so this needs no flag of its own. + * + * The file headers are identified by position, not by prefix: a removed line + * whose own content starts with `--` renders as `--- something` and would + * otherwise be mistaken for the `---` header and shown as unchanged. + */ +function colourizeDiffLine(line: string, index: number, binary: boolean): string { + if (index === 0) return chalk.bold(line); // our `diff --olcli` header + if (binary) return chalk.magenta(line); // the single summary line + if (index <= 2) return chalk.bold(line); // `---` / `+++` + if (line.startsWith('@@')) return chalk.cyan(line); + if (line.startsWith('+')) return chalk.green(line); + if (line.startsWith('-')) return chalk.red(line); + return line; +} + // ───────────────────────────────────────────────────────────────────────────── // HELP // ───────────────────────────────────────────────────────────────────────────── diff --git a/src/diff.ts b/src/diff.ts new file mode 100644 index 0000000..1e0c38a --- /dev/null +++ b/src/diff.ts @@ -0,0 +1,219 @@ +/** + * Content-level comparison between a local directory and a remote Overleaf + * project. + * + * Pure functions only: no network, no filesystem, no colour. The command in + * `cli.ts` fetches both sides and renders; everything decided here is decided + * from two maps of `path -> bytes`, which is what makes it unit-testable + * without an Overleaf account. Same reasoning as `rename-plan.ts`. + * + * Orientation is fixed and matters: **`a/` is the remote, `b/` is local**, so + * a `+` line is content that a subsequent `push` would put on Overleaf and a + * `-` line is content it would overwrite. Reading the diff the other way round + * would invert the meaning of every hunk. + */ + +import { createTwoFilesPatch } from 'diff'; +import { shouldIgnore, buildTexSiblingSet, type IgnoreContext } from './ignore.js'; + +export type FileStatus = 'added' | 'deleted' | 'modified' | 'unchanged'; + +export interface FileDiff { + /** Project-relative path, forward slashes. */ + path: string; + /** + * From the point of view of a push: + * added - local only; `push` uploads it + * deleted - remote only; plain `push` leaves it, `push --delete` removes it + * modified - both sides, different bytes; `push` overwrites the remote + * unchanged - both sides, identical bytes + */ + status: FileStatus; + /** True when either side looks binary; such files are never rendered as text. */ + binary: boolean; +} + +/** + * How many leading bytes to sniff when deciding whether a file is binary. + * Same window git uses. + */ +const BINARY_SNIFF_BYTES = 8000; + +/** + * A NUL byte near the start means "do not try to render this as text". + * + * Crude on purpose. The alternative - content-type sniffing per extension - + * gets PDFs and images right and then quietly mangles the next format nobody + * anticipated. Reporting "binary files differ" is always safe. + */ +export function isBinary(buf: Buffer): boolean { + const end = Math.min(buf.length, BINARY_SNIFF_BYTES); + for (let i = 0; i < end; i++) { + if (buf[i] === 0) return true; + } + return false; +} + +export interface RemoteEntry { + /** Entry name as it appears in the project archive. */ + path: string; + data: Buffer; +} + +/** + * Reduce a project archive listing to the files the local scan would also have + * reported, so the two sides of a diff are filtered identically. + * + * Applying the ignore layers to only the local side would list every artifact + * Overleaf keeps in the project - `output.pdf` above all - as a file missing + * locally, which is noise on every single run. The dotfile rule is mirrored + * for the same reason: the local scan never reports them, so a remote + * `.latexmkrc` would otherwise always look locally deleted. + * + * @param isSafePath Rejects archive entries whose names escape the target + * directory. `diff` never writes these files out, but an + * entry `pull` refuses to extract is not part of the project + * as far as this directory is concerned, and showing it + * would suggest a difference that no push could resolve. + */ +export function filterRemoteTree( + entries: RemoteEntry[], + ctx: IgnoreContext, + isSafePath: (path: string) => boolean = () => true, +): Map { + const normalized = entries.map((e) => ({ ...e, path: e.path.replace(/\\/g, '/') })); + const texSiblings = buildRemoteTexSiblings(normalized.map((e) => e.path)); + + const out = new Map(); + for (const entry of normalized) { + if (!isSafePath(entry.path)) continue; + if (entry.path.split('/').some((seg) => seg.startsWith('.'))) continue; + if (shouldIgnore(entry.path, ctx, texSiblings.get(folderOf(entry.path)))) continue; + out.set(entry.path, entry.data); + } + return out; +} + +function folderOf(path: string): string { + const idx = path.lastIndexOf('/'); + return idx === -1 ? '' : path.slice(0, idx); +} + +/** + * Per-folder sets of basenames that have a `.tex`/`.ltx` companion, which is + * what the ignore subsystem's PDF sibling rule needs. The local scanner gets + * this for free from `readdir`; a flat archive listing has to be regrouped. + */ +function buildRemoteTexSiblings(paths: string[]): Map> { + const byFolder = new Map(); + for (const path of paths) { + const folder = folderOf(path); + const name = folder ? path.slice(folder.length + 1) : path; + const list = byFolder.get(folder); + if (list) list.push(name); + else byFolder.set(folder, [name]); + } + + const out = new Map>(); + for (const [folder, names] of byFolder) { + out.set(folder, buildTexSiblingSet(names)); + } + return out; +} + +/** + * Compare two file trees. + * + * Both maps are keyed by project-relative, forward-slash paths. Callers are + * responsible for having applied the same ignore rules to both sides; + * comparing a filtered local tree against an unfiltered remote one would + * report every build artifact on Overleaf as "deleted". + * + * Results are sorted by path so output is stable across runs. + */ +export function compareTrees( + local: Map, + remote: Map, +): FileDiff[] { + const paths = new Set([...local.keys(), ...remote.keys()]); + const out: FileDiff[] = []; + + for (const path of [...paths].sort()) { + const localBuf = local.get(path); + const remoteBuf = remote.get(path); + + if (localBuf && !remoteBuf) { + out.push({ path, status: 'added', binary: isBinary(localBuf) }); + } else if (!localBuf && remoteBuf) { + out.push({ path, status: 'deleted', binary: isBinary(remoteBuf) }); + } else if (localBuf && remoteBuf) { + const identical = localBuf.equals(remoteBuf); + out.push({ + path, + status: identical ? 'unchanged' : 'modified', + binary: isBinary(localBuf) || isBinary(remoteBuf), + }); + } + } + + return out; +} + +export interface RenderOptions { + /** Lines of context around each hunk. Defaults to 3, like git. */ + context?: number; +} + +/** + * Render one file's change as a unified diff. + * + * Returns an empty string for unchanged files. Binary files get a single + * summary line instead of a patch. + */ +export function renderFileDiff( + entry: FileDiff, + localBuf: Buffer | undefined, + remoteBuf: Buffer | undefined, + options: RenderOptions = {}, +): string { + if (entry.status === 'unchanged') return ''; + + const oldName = entry.status === 'added' ? '/dev/null' : `a/${entry.path}`; + const newName = entry.status === 'deleted' ? '/dev/null' : `b/${entry.path}`; + const header = `diff --olcli a/${entry.path} b/${entry.path}`; + + if (entry.binary) { + return `${header}\nBinary files ${oldName} and ${newName} differ\n`; + } + + const oldText = remoteBuf ? remoteBuf.toString('utf-8') : ''; + const newText = localBuf ? localBuf.toString('utf-8') : ''; + + // A non-numeric --unified reaches us as NaN, which jsdiff turns into an + // empty patch rather than an error. Fall back instead. + const context = Number.isInteger(options.context) && options.context! >= 0 + ? options.context! + : 3; + + const patch = createTwoFilesPatch(oldName, newName, oldText, newText, undefined, undefined, { + context, + }); + + // jsdiff prefixes every patch with a '====' separator line that only makes + // sense when concatenating multiple patches into one file. Drop it and use + // a git-shaped header instead, so the output pastes into tools that already + // understand unified diffs. + const body = patch.replace(/^=+\n/, ''); + + return `${header}\n${body}`; +} + +/** One-letter status prefix for `--name-only` output, mirroring `git status`. */ +export function statusLetter(status: FileStatus): string { + switch (status) { + case 'added': return 'A'; + case 'deleted': return 'D'; + case 'modified': return 'M'; + case 'unchanged': return ' '; + } +} diff --git a/src/scan.ts b/src/scan.ts new file mode 100644 index 0000000..c16f5e4 --- /dev/null +++ b/src/scan.ts @@ -0,0 +1,83 @@ +/** + * Shared local-file scanning. + * + * `push`, `sync` and `diff` all need the same answer to "which files in this + * directory are in play": walk the tree, skip dotfiles, apply the ignore + * layers, and report what was skipped. That walk previously existed twice, + * once inside `push` and once inside `sync`, with the two copies already + * drifting apart. A third copy in `diff` would have made it three. + * + * Deliberately does NOT read file contents. `push` only ever reads the files + * it is about to upload, and eagerly loading every file here would change its + * memory profile on large projects. Callers that need contents read them from + * `path` themselves. + */ + +import { readdirSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import { shouldIgnore, buildTexSiblingSet, type IgnoreContext } from './ignore.js'; + +export interface LocalFile { + /** + * Project-relative path with forward slashes. This is the key that lines up + * with remote entry names, `.olcli.json` manifests, and ignore patterns. + */ + relativePath: string; + /** Path as passed to `fs`, relative to the scan root's own base. */ + path: string; + mtime: Date; +} + +export interface LocalScan { + files: LocalFile[]; + /** + * Paths skipped by the ignore layers, for `--show-ignored`. Directories + * carry a trailing slash and their contents are not descended into. + */ + ignored: string[]; +} + +/** + * Walk `root`, returning every file that survives ignore filtering. + * + * Hidden entries (anything starting with `.`) are skipped unconditionally, + * which is also what keeps `.olcli.json`, `.olauth` and `.git/` out of every + * caller. That rule predates the ignore subsystem and is not configurable. + */ +export function scanLocalFiles(root: string, ctx: IgnoreContext): LocalScan { + const files: LocalFile[] = []; + const ignored: string[] = []; + + function walk(currentDir: string, relativeBase: string): void { + const entries = readdirSync(currentDir, { withFileTypes: true }); + // Pre-compute the sibling .tex set for the PDF special rule. + const texSiblings = buildTexSiblingSet( + entries.filter((e) => !e.isDirectory()).map((e) => e.name), + ); + + for (const entry of entries) { + if (entry.name.startsWith('.')) continue; + + const fullPath = join(currentDir, entry.name); + const relativePath = relativeBase ? `${relativeBase}/${entry.name}` : entry.name; + + if (entry.isDirectory()) { + // gitignore semantics: a trailing slash matches a directory. + if (shouldIgnore(`${relativePath}/`, ctx)) { + ignored.push(`${relativePath}/`); + continue; + } + walk(fullPath, relativePath); + } else { + if (shouldIgnore(relativePath, ctx, texSiblings)) { + ignored.push(relativePath); + continue; + } + files.push({ relativePath, path: fullPath, mtime: statSync(fullPath).mtime }); + } + } + } + + walk(root, ''); + return { files, ignored }; +} diff --git a/test/diff.test.ts b/test/diff.test.ts new file mode 100644 index 0000000..99c07c4 --- /dev/null +++ b/test/diff.test.ts @@ -0,0 +1,219 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + compareTrees, + filterRemoteTree, + renderFileDiff, + isBinary, + statusLetter, +} from '../src/diff.js'; +import { loadIgnore } from '../src/ignore.js'; + +const buf = (s: string) => Buffer.from(s, 'utf-8'); +const remoteEntry = (path: string, content = 'x\n') => ({ path, data: buf(content) }); + +test('compareTrees: classifies added, deleted, modified and unchanged', () => { + const local = new Map([ + ['main.tex', buf('a\n')], + ['new.tex', buf('new\n')], + ['same.tex', buf('same\n')], + ]); + const remote = new Map([ + ['main.tex', buf('b\n')], + ['gone.tex', buf('gone\n')], + ['same.tex', buf('same\n')], + ]); + + const result = compareTrees(local, remote); + const byPath = Object.fromEntries(result.map((e) => [e.path, e.status])); + + assert.equal(byPath['main.tex'], 'modified'); + assert.equal(byPath['new.tex'], 'added'); + assert.equal(byPath['gone.tex'], 'deleted'); + assert.equal(byPath['same.tex'], 'unchanged'); +}); + +test('compareTrees: output is sorted by path for stable diffs', () => { + const local = new Map([['z.tex', buf('z')], ['a.tex', buf('a')], ['m.tex', buf('m')]]); + const result = compareTrees(local, new Map()); + assert.deepEqual(result.map((e) => e.path), ['a.tex', 'm.tex', 'z.tex']); +}); + +test('compareTrees: byte-identical files are unchanged, not modified', () => { + // Same characters, and the comparison must not go through a lossy decode. + const bytes = Buffer.from([0xc3, 0xa9, 0x0a]); + const result = compareTrees(new Map([['a.tex', bytes]]), new Map([['a.tex', Buffer.from(bytes)]])); + assert.equal(result[0].status, 'unchanged'); +}); + +test('compareTrees: trailing-newline-only difference is a real modification', () => { + const result = compareTrees( + new Map([['a.tex', buf('x\n')]]), + new Map([['a.tex', buf('x')]]), + ); + assert.equal(result[0].status, 'modified'); +}); + +test('compareTrees: CRLF vs LF is a real modification', () => { + const result = compareTrees( + new Map([['a.tex', buf('x\r\ny\r\n')]]), + new Map([['a.tex', buf('x\ny\n')]]), + ); + assert.equal(result[0].status, 'modified'); +}); + +test('isBinary: NUL bytes mark a file binary, plain text does not', () => { + assert.equal(isBinary(buf('\\documentclass{article}\n')), false); + assert.equal(isBinary(Buffer.from([0x25, 0x50, 0x44, 0x46, 0x00, 0x01])), true); + assert.equal(isBinary(Buffer.alloc(0)), false); +}); + +test('isBinary: only the first 8000 bytes are sniffed', () => { + const late = Buffer.concat([Buffer.alloc(9000, 0x41), Buffer.from([0x00])]); + assert.equal(isBinary(late), false); +}); + +test('compareTrees: a file that is binary on either side is flagged binary', () => { + const result = compareTrees( + new Map([['fig.pdf', buf('text now')]]), + new Map([['fig.pdf', Buffer.from([0x00, 0x01])]]), + ); + assert.equal(result[0].binary, true); +}); + +test('renderFileDiff: a/ is the remote and b/ is local, so + is what push would write', () => { + const entry = { path: 'main.tex', status: 'modified' as const, binary: false }; + const patch = renderFileDiff(entry, buf('hello\nthere\n'), buf('hello\nworld\n')); + + assert.match(patch, /^diff --olcli a\/main\.tex b\/main\.tex\n/); + assert.match(patch, /--- a\/main\.tex/); + assert.match(patch, /\+\+\+ b\/main\.tex/); + assert.match(patch, /^-world$/m, 'remote content appears as a removal'); + assert.match(patch, /^\+there$/m, 'local content appears as an addition'); + assert.doesNotMatch(patch, /^=+$/m, 'jsdiff separator line is stripped'); +}); + +test('renderFileDiff: added files diff against /dev/null', () => { + const entry = { path: 'new.tex', status: 'added' as const, binary: false }; + const patch = renderFileDiff(entry, buf('fresh\n'), undefined); + assert.match(patch, /--- \/dev\/null/); + assert.match(patch, /\+\+\+ b\/new\.tex/); + assert.match(patch, /^\+fresh$/m); +}); + +test('renderFileDiff: remote-only files diff towards /dev/null', () => { + const entry = { path: 'old.tex', status: 'deleted' as const, binary: false }; + const patch = renderFileDiff(entry, undefined, buf('stale\n')); + assert.match(patch, /--- a\/old\.tex/); + assert.match(patch, /\+\+\+ \/dev\/null/); + assert.match(patch, /^-stale$/m); +}); + +test('renderFileDiff: binary files get a summary line, never a patch', () => { + const entry = { path: 'figures/plot.pdf', status: 'modified' as const, binary: true }; + const patch = renderFileDiff(entry, Buffer.from([0x00, 0x01]), Buffer.from([0x00, 0x02])); + assert.equal( + patch, + 'diff --olcli a/figures/plot.pdf b/figures/plot.pdf\n' + + 'Binary files a/figures/plot.pdf and b/figures/plot.pdf differ\n', + ); + assert.doesNotMatch(patch, /@@/); +}); + +test('renderFileDiff: unchanged files render nothing', () => { + const entry = { path: 'a.tex', status: 'unchanged' as const, binary: false }; + assert.equal(renderFileDiff(entry, buf('x'), buf('x')), ''); +}); + +test('renderFileDiff: context width is configurable', () => { + const lines = Array.from({ length: 20 }, (_, i) => `line ${i}`).join('\n') + '\n'; + const changed = lines.replace('line 10', 'line TEN'); + const entry = { path: 'a.tex', status: 'modified' as const, binary: false }; + + const wide = renderFileDiff(entry, buf(changed), buf(lines), { context: 5 }); + const narrow = renderFileDiff(entry, buf(changed), buf(lines), { context: 1 }); + + assert.ok(wide.split('\n').length > narrow.split('\n').length); + assert.match(narrow, /@@ -10,3 \+10,3 @@/); +}); + +test('statusLetter: mirrors git status shorthand', () => { + assert.equal(statusLetter('added'), 'A'); + assert.equal(statusLetter('deleted'), 'D'); + assert.equal(statusLetter('modified'), 'M'); +}); + +test('filterRemoteTree: drops archive entries the ignore layers would skip locally', () => { + const ctx = loadIgnore('/nonexistent-project-root'); + const tree = filterRemoteTree([ + remoteEntry('main.tex'), + remoteEntry('output.pdf'), + remoteEntry('main.aux'), + remoteEntry('figures/diagram.png'), + ], ctx); + + assert.deepEqual([...tree.keys()].sort(), ['figures/diagram.png', 'main.tex']); +}); + +test('filterRemoteTree: applies the PDF sibling rule per folder, not globally', () => { + const ctx = loadIgnore('/nonexistent-project-root'); + const tree = filterRemoteTree([ + remoteEntry('main.tex'), + remoteEntry('main.pdf'), // sibling of main.tex -> ignored + remoteEntry('figures/main.pdf'), // no main.tex in figures/ -> kept + ], ctx); + + assert.ok(!tree.has('main.pdf')); + assert.ok(tree.has('figures/main.pdf')); +}); + +test('filterRemoteTree: mirrors the local scan by skipping dot entries', () => { + const ctx = loadIgnore('/nonexistent-project-root'); + const tree = filterRemoteTree([ + remoteEntry('main.tex'), + remoteEntry('.latexmkrc'), + remoteEntry('.github/workflows/build.yml'), + ], ctx); + + assert.deepEqual([...tree.keys()], ['main.tex']); +}); + +test('filterRemoteTree: rejects unsafe archive entries via the caller predicate', () => { + const ctx = loadIgnore('/nonexistent-project-root'); + const tree = filterRemoteTree( + [remoteEntry('main.tex'), remoteEntry('../escape.tex')], + ctx, + (path) => !path.startsWith('..'), + ); + + assert.deepEqual([...tree.keys()], ['main.tex']); +}); + +test('filterRemoteTree: normalizes backslash separators to forward slashes', () => { + const ctx = loadIgnore('/nonexistent-project-root'); + const tree = filterRemoteTree([remoteEntry('chapters\\intro.tex')], ctx); + assert.deepEqual([...tree.keys()], ['chapters/intro.tex']); +}); + +test('filterRemoteTree: --no-ignore keeps artifacts but still skips dot entries', () => { + const ctx = loadIgnore('/nonexistent-project-root', { disableAll: true }); + const tree = filterRemoteTree([ + remoteEntry('main.aux'), + remoteEntry('output.pdf'), + remoteEntry('.latexmkrc'), + ], ctx); + + assert.deepEqual([...tree.keys()].sort(), ['main.aux', 'output.pdf']); +}); + +test('renderFileDiff: a non-numeric context falls back to the default', () => { + const lines = Array.from({ length: 20 }, (_, i) => `line ${i}`).join('\n') + '\n'; + const changed = lines.replace('line 10', 'line TEN'); + const target = { path: 'a.tex', status: 'modified' as const, binary: false }; + + const bad = renderFileDiff(target, buf(changed), buf(lines), { context: NaN }); + const good = renderFileDiff(target, buf(changed), buf(lines), { context: 3 }); + + assert.equal(bad, good); + assert.match(bad, /^\+line TEN$/m); +}); diff --git a/test/e2e.sh b/test/e2e.sh index b137f18..09cfd86 100755 --- a/test/e2e.sh +++ b/test/e2e.sh @@ -562,6 +562,51 @@ fi sleep 1 # Rate limit +####################################### +# Test: Diff +####################################### + +log_section "Diff Tests" + +# The pulled directory is byte-identical to the remote at this point, so a +# diff must report nothing. Anything else means the two sides are being +# filtered differently. +run_test "diff reports no changes on a freshly pulled directory" \ + "cd '$PULL_DIR' && olcli diff | grep -q 'No differences'" + +DIFF_TEST_FILE="$PULL_DIR/${TEST_ID}.txt" +DIFF_ORIGINAL_CONTENT=$(cat "$DIFF_TEST_FILE") +echo "diff test modification - $TIMESTAMP" >> "$DIFF_TEST_FILE" + +run_test "diff --name-only marks a modified file with M" \ + "cd '$PULL_DIR' && olcli diff --name-only | grep -qE '^M +${TEST_ID}\\.txt$'" + +run_test "diff shows the added line as a local addition" \ + "cd '$PULL_DIR' && olcli diff --file '${TEST_ID}.txt' | grep -q '^+diff test modification'" + +run_test "diff --file limits output to the requested file" \ + "cd '$PULL_DIR' && test \$(olcli diff --file '${TEST_ID}.txt' | grep -c '^diff --olcli') -eq 1" + +# Restore, so the push tests below see the tree they expect. +printf '%s\n' "$DIFF_ORIGINAL_CONTENT" > "$DIFF_TEST_FILE" + +DIFF_NEW_FILE="$PULL_DIR/${TEST_ID}_diffonly.txt" +echo "local only - $TIMESTAMP" > "$DIFF_NEW_FILE" + +run_test "diff --name-only marks a local-only file with A" \ + "cd '$PULL_DIR' && olcli diff --name-only | grep -qE '^A +${TEST_ID}_diffonly\\.txt$'" + +run_test "diff ignores build artifacts on both sides" \ + "cd '$PULL_DIR' && touch '$PULL_DIR/scratch.aux' && ! olcli diff --name-only | grep -q 'scratch\\.aux'" + +rm -f "$DIFF_NEW_FILE" "$PULL_DIR/scratch.aux" + +# Back to a clean tree; verify the restore actually worked before pushing. +run_test "diff is clean again after restoring the tree" \ + "cd '$PULL_DIR' && olcli diff | grep -q 'No differences'" + +sleep 1 # Rate limit + ####################################### # Test: Push ####################################### diff --git a/test/scan.test.ts b/test/scan.test.ts new file mode 100644 index 0000000..18566a6 --- /dev/null +++ b/test/scan.test.ts @@ -0,0 +1,70 @@ +import { test, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { scanLocalFiles } from '../src/scan.js'; +import { loadIgnore } from '../src/ignore.js'; + +let root: string; + +before(() => { + root = mkdtempSync(join(tmpdir(), 'olcli-scan-')); + mkdirSync(join(root, 'figures'), { recursive: true }); + mkdirSync(join(root, 'build'), { recursive: true }); + mkdirSync(join(root, '.git'), { recursive: true }); + + writeFileSync(join(root, 'main.tex'), '\\documentclass{article}\n'); + writeFileSync(join(root, 'main.aux'), 'aux artifact\n'); + writeFileSync(join(root, 'main.pdf'), 'compiled output\n'); + writeFileSync(join(root, 'figures', 'diagram.pdf'), 'hand-made figure\n'); + writeFileSync(join(root, 'build', 'ignored.tex'), 'in a build dir\n'); + writeFileSync(join(root, '.olcli.json'), '{}\n'); + writeFileSync(join(root, '.git', 'HEAD'), 'ref: refs/heads/main\n'); +}); + +after(() => { + rmSync(root, { recursive: true, force: true }); +}); + +const paths = (r: string) => + scanLocalFiles(r, loadIgnore(r)).files.map((f) => f.relativePath).sort(); + +test('scanLocalFiles: returns project-relative forward-slash paths', () => { + assert.deepEqual(paths(root), ['figures/diagram.pdf', 'main.tex']); +}); + +test('scanLocalFiles: skips dotfiles and dot-directories without descending', () => { + const found = paths(root); + assert.ok(!found.includes('.olcli.json'), '.olcli.json must never be uploaded'); + assert.ok(!found.some((p) => p.startsWith('.git/')), '.git must not be walked'); +}); + +test('scanLocalFiles: applies the default ignore list and the PDF sibling rule', () => { + const found = paths(root); + assert.ok(!found.includes('main.aux'), 'build artifacts are ignored'); + assert.ok(!found.includes('main.pdf'), 'main.pdf is ignored next to main.tex'); + assert.ok(found.includes('figures/diagram.pdf'), 'a PDF with no .tex sibling is kept'); +}); + +test('scanLocalFiles: reports ignored directories with a trailing slash and does not descend', () => { + const { files, ignored } = scanLocalFiles(root, loadIgnore(root)); + assert.ok(ignored.includes('build/')); + assert.ok(!files.some((f) => f.relativePath.startsWith('build/'))); +}); + +test('scanLocalFiles: --no-ignore disables filtering but not the dotfile rule', () => { + const { files } = scanLocalFiles(root, loadIgnore(root, { disableAll: true })); + const found = files.map((f) => f.relativePath).sort(); + assert.ok(found.includes('main.aux')); + assert.ok(found.includes('build/ignored.tex')); + assert.ok(!found.includes('.olcli.json')); +}); + +test('scanLocalFiles: every file carries an fs-usable path and an mtime', () => { + const { files } = scanLocalFiles(root, loadIgnore(root)); + const main = files.find((f) => f.relativePath === 'main.tex'); + assert.ok(main); + assert.equal(main.path, join(root, 'main.tex')); + assert.ok(main.mtime instanceof Date); +});