From d1b6e76a252d3c6647513da6ef8858158e0b467e Mon Sep 17 00:00:00 2001 From: Speck Pratt Date: Sun, 15 Nov 2026 11:16:20 -0500 Subject: [PATCH 1/2] Add Image Matcher plugin A UI plugin that finds look-alike images by perceptual hash and lets the user merge them into one image (every file kept), delete the lesser copies, or handle a whole folder pair in one decision. Co-Authored-By: Claude Fable 5.2 --- plugins/image-matcher/README.md | 71 +++ plugins/image-matcher/image-matcher.css | 75 +++ plugins/image-matcher/image-matcher.yml | 11 + plugins/image-matcher/imcore.js | 451 +++++++++++++++++ plugins/image-matcher/imops.js | 408 ++++++++++++++++ plugins/image-matcher/imui.js | 614 ++++++++++++++++++++++++ 6 files changed, 1630 insertions(+) create mode 100644 plugins/image-matcher/README.md create mode 100644 plugins/image-matcher/image-matcher.css create mode 100644 plugins/image-matcher/image-matcher.yml create mode 100644 plugins/image-matcher/imcore.js create mode 100644 plugins/image-matcher/imops.js create mode 100644 plugins/image-matcher/imui.js diff --git a/plugins/image-matcher/README.md b/plugins/image-matcher/README.md new file mode 100644 index 00000000..09dd6d4b --- /dev/null +++ b/plugins/image-matcher/README.md @@ -0,0 +1,71 @@ +# Image Matcher + +A Stash UI plugin that finds images that look the same and lets you act on them: + +- **Merge**: the look-alikes become one image in Stash. Every file stays where it is on disk, the better copy is the one shown, and the image appears in each folder's gallery. +- **Delete**: keep one copy, delete the others from disk. Their tags, ratings and details move to the kept copy first. +- **Folder sets**: when two folders share several look-alikes ("the same set in two places"), merge or dedupe the whole pair in one decision, and choose which folder wins. + Each set card says how close its pairs are ("25 same hash · 9 within 2 bits") and *Show all* lists every pair with its distance, furthest first. The hash describes how a picture looks; files with the same hash can still differ in size or quality. +- **Leave out**: any pair on a set card, or any image in a group, can be left out of a merge or delete. A group with two pictures in each folder is shown as two pairs, never merged into one image. +- **One detail different**: the page compares the two thumbnails of every pair. A resized or recompressed copy differs a little everywhere; two drawings that differ in one detail differ a lot in one place. Those are marked "differs in one area" with a red box on the spot and are left out until you put them back. A very small or faint change can still pass. +- **All on this page**: under the list on the Images tab, one button merges (or deletes the other copies in) every group on the page, as the cards are set. +- **Colour**: a black-and-white copy matches its colour original (the hash only sees brightness). The colour copy is the suggested keeper unless the black-and-white one has more than twice the pixels; folder rules still come first. +- **Zoom**: a slider in the toolbar enlarges every thumbnail up to 4×; the size is remembered in the browser. +- **Folders left out of sets**: list folders (one per line, under *Folder rules*) that should never be compared as a set, such as a year folder of loose images; or use *Leave this folder out of sets* on a card. Subfolders are still compared unless the line ends in `/*`. +- **Archives**: images inside a cbz or zip are left out and counted on the page. A file in an archive cannot be deleted or re-attached on its own, so neither merge nor delete can work on it. +- **Guide**: *How this works* on the page explains the tabs, the buttons, the labels and what is left out. It is open on a first visit. A set card whose folders mostly do not match carries a caution. +- **Folder rules**: an ordered list of folders that always win. Stored in Stash's plugin settings. + +It opens from the stacked-squares button in the top bar, or at `/image-matcher`. + +## Requirements + +- Stash 0.31 or later (image perceptual hashes arrived in 0.31.0). +- Image phashes generated: Settings → Tasks → Generate → *Image perceptual hashes*. Images without one are skipped and counted on the page. + +## Install + +Install from the CommunityScripts source in Settings → Plugins → Available Plugins, or copy this folder into Stash's `plugins/` directory and press *Reload plugins*. + +## How merge works + +Stash has no "merge images" call, and this plugin does not write to the database or assume anything about the filesystem. It uses Stash's own scanner: + +1. Both files get the same custom fingerprint (type `imagematch`) through `fileSetFingerprints`. +2. The lesser image's tags, performers, rating, details, URLs and hand-made gallery memberships are copied to the kept image. +3. The lesser image record is removed with its file kept (`imagesDestroy`, `delete_file: false`). +4. That file is rescanned. Stash's scanner attaches a file with no image to any image sharing a fingerprint, so it joins the kept image. +5. The plugin checks that the file landed on exactly the kept image. + +The scanner matches on *any* shared fingerprint, including phash, so a detached file would also attach to every other look-alike with an equal phash. The plugin therefore hides the file's phash during the rescan and restores it afterwards. + +Real content hashes (md5, oshash) are never touched. The `imagematch` stamp stays as the record that a person declared the files the same. + +## Limits + +- One merge or delete runs at a time; the other cards are locked until it finishes. +- A delete never removes an image that holds more than one file (one that was merged); it is left alone and counted, and merging is the right action for it. +- **No undo.** Stash has no "split image" call. A merge can only be reversed by hand; a delete cannot be reversed at all. +- Matching uses the phash Stash already stores. It finds resized and recompressed copies well; crops, rotations and mirrors mostly not. +- Groups are chains: if A looks like B and B looks like C, all three are shown together even when A and C are further apart. +- Folder and zip galleries cannot be edited in Stash, so "merging galleries" means their details (tags, performers, rating, and so on) are synced; their contents converge because the merged images belong to both folders. +- In a folder-set delete, an image that also holds files in other folders is left alone; merge it instead. +- Very loose accuracy on very large libraries is slow. At *High* a 200,000-image library compares in well under a second; at *Low* a 34,000-image library takes about a third of a second on a desktop. +- `custom_fields` are not merged. + +## How it was tested + +- Unit tests of the matching and decision logic (17), end-to-end tests of merge, delete and folder sets against a throwaway Stash in Docker (12), and a scripted browser pass. All three pass on Stash v0.31.1 (release) and v0.32.0-12 (development image). +- Used by its author on a real library. +- The test files are kept with the development copy, not shipped here, so they are not installed with the plugin. They can be added to this folder if that is preferred. + +## Prior art and credit + +- The interface follows the closed Stash PR #6675 (image duplicate checker) and the scene duplicate checker in spirit; no code was copied. +- The merge-on-resolve rules (highest rating, union of tags, combined details, merge only when exactly one copy is kept) follow Immich's duplicates utility. +- "Folders that win" is the reference-folder idea from dupeGuru. +- The task-computes / page-displays split in RyanAtNight's DuplicateScorer informed the layout, though here everything runs in the page. + +## License + +AGPL-3.0-or-later, compatible with this repository's AGPL-3.0 licence. diff --git a/plugins/image-matcher/image-matcher.css b/plugins/image-matcher/image-matcher.css new file mode 100644 index 00000000..88d464e4 --- /dev/null +++ b/plugins/image-matcher/image-matcher.css @@ -0,0 +1,75 @@ +.imt-page { --imt-zoom: 1; max-width: 1100px; margin: 0 auto; padding: 1rem; } +.imt-zoom { margin-left: auto !important; } +.imt-zoom input { width: 9rem; } +.imt-muted { opacity: 0.7; font-size: 0.875rem; } +.imt-toolbar { display: flex; flex-wrap: wrap; align-items: center; gap: 0.75rem; margin: 1rem 0; } +.imt-toolbar label { display: flex; align-items: center; gap: 0.5rem; margin: 0; } +.imt-toolbar select { width: auto; } +.imt-summary { margin: 0.5rem 0; } +.imt-tabs { display: flex; gap: 0.5rem; margin: 0.75rem 0 1rem; } +.imt-card { padding: 1rem; margin-bottom: 1rem; } +.imt-message { padding: 0.5rem 0.75rem; margin: 0.5rem 0; border-left: 3px solid #4caf50; background: rgba(76, 175, 80, 0.12); } +.imt-error { padding: 0.5rem 0.75rem; margin: 0.5rem 0; border-left: 3px solid #db3737; background: rgba(219, 55, 55, 0.12); word-break: break-word; } +.imt-busy { display: flex; align-items: center; gap: 0.5rem; padding: 0.5rem 0; } +.imt-spinner { width: 1rem; height: 1rem; border: 2px solid currentColor; border-right-color: transparent; border-radius: 50%; animation: imt-spin 0.8s linear infinite; } +@keyframes imt-spin { to { transform: rotate(360deg); } } + +.imt-tiles { display: grid; grid-template-columns: repeat(auto-fill, minmax(min(100%, calc(170px * var(--imt-zoom))), 1fr)); gap: 0.75rem; margin-bottom: 0.75rem; } +.imt-tile { display: flex; flex-direction: column; gap: 0.4rem; padding: 0.5rem; border: 2px solid transparent; border-radius: 4px; background: rgba(0, 0, 0, 0.15); } +.imt-tile.imt-keep, .imt-side.imt-keep { border-color: #4caf50; } +.imt-thumb { display: flex; align-items: center; justify-content: center; container-type: size; height: calc(150px * var(--imt-zoom)); background: rgba(0, 0, 0, 0.3); border-radius: 3px; overflow: hidden; } +.imt-fit { position: relative; display: block; width: min(100cqw, calc(100cqh * var(--imt-ar))); height: min(100cqh, calc(100cqw / var(--imt-ar))); } +.imt-fit img { display: block; width: 100%; height: 100%; object-fit: contain; } +.imt-spot { position: absolute; box-sizing: border-box; border: 2px solid #ff5252; border-radius: 3px; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.7); pointer-events: none; } +.imt-diff { font-size: 0.8rem; color: #ff8a80; } +.imt-pair-dist.imt-diff { opacity: 1; font-size: 0.75rem; } +.imt-out .imt-pair-dist.imt-diff, .imt-out .imt-diff { text-decoration: none; opacity: 1; } +.imt-thumb-sm { height: calc(72px * var(--imt-zoom)); width: calc(96px * var(--imt-zoom)); max-width: 100%; } +.imt-pair { max-width: 100%; } +.imt-facts { font-size: 0.875rem; line-height: 1.35; } +.imt-path { font-size: 0.8rem; word-break: break-all; } +.imt-badge { align-self: flex-start; padding: 0.1rem 0.5rem; border-radius: 3px; font-size: 0.8rem; background: #4caf50; color: #fff; } + +.imt-sides { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); gap: 0.75rem; margin-bottom: 0.75rem; } +.imt-side { display: flex; align-items: flex-start; gap: 0.6rem; margin: 0; padding: 0.6rem; border: 2px solid transparent; border-radius: 4px; background: rgba(0, 0, 0, 0.15); cursor: pointer; } +.imt-side input { margin-top: 0.25rem; } +.imt-side .imt-badge { display: inline-block; margin-top: 0.3rem; } +.imt-pairs { display: flex; flex-wrap: wrap; align-items: center; gap: 0.5rem; margin-bottom: 0.75rem; } +.imt-pair { display: flex; flex-direction: column; gap: 2px; } +.imt-pair-row { display: flex; gap: 2px; } +.imt-more { padding: 0 0.5rem; } +.imt-pair-dist { font-size: 0.75rem; opacity: 0.7; text-align: center; } +.imt-out .imt-thumb img { opacity: 0.6; } +.imt-out .imt-pair-dist, .imt-out .imt-facts { text-decoration: line-through; opacity: 0.5; } +.imt-pair-out { padding: 0; font-size: 0.75rem; } +.imt-tile-btns { display: flex; flex-wrap: wrap; align-items: center; gap: 0.25rem; } +.imt-pair-dist.imt-pair-off { opacity: 1; color: #ffb74d; } +.imt-mix { font-size: 0.875rem; margin-bottom: 0.75rem; } + +.imt-skip { display: block; padding: 0; font-size: 0.75rem; } +.imt-rules h5 { margin-top: 0.75rem; } +.imt-found { max-height: 16rem; overflow-y: auto; margin-bottom: 0.6rem; padding: 0.25rem 0.5rem; background: rgba(0, 0, 0, 0.15); border-radius: 4px; } +.imt-found-row { display: flex; align-items: baseline; gap: 0.5rem; margin: 0; padding: 0.15rem 0; cursor: pointer; } +.imt-found-row .imt-muted { margin-left: auto; white-space: nowrap; } +.imt-help h5 { margin: 0.75rem 0 0.25rem; font-size: 1rem; } +.imt-help h5:first-child { margin-top: 0; } +.imt-help ul { margin: 0 0 0.25rem; padding-left: 1.2rem; } +.imt-help li { margin-bottom: 0.2rem; font-size: 0.9rem; line-height: 1.4; } +.imt-help-link { padding: 0; vertical-align: baseline; font-size: inherit; } +.imt-caution { padding: 0.5rem 0.75rem; margin-bottom: 0.75rem; border-left: 3px solid #ffb74d; background: rgba(255, 183, 77, 0.12); font-size: 0.875rem; } +.imt-bulk { border: 1px solid rgba(255, 255, 255, 0.25); } +.imt-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 0.5rem; } +.imt-remember { margin: 0 0 0 auto; font-size: 0.875rem; cursor: pointer; } +.imt-confirm { padding: 0.75rem; border: 1px solid #db3737; border-radius: 4px; } +.imt-confirm p { margin: 0 0 0.6rem; } +.imt-confirm .btn { margin-right: 0.5rem; } +.imt-rules textarea { font-family: monospace; margin-bottom: 0.6rem; } +.imt-pager { display: flex; align-items: center; justify-content: center; gap: 1rem; margin: 1rem 0; } + +@media (max-width: 576px) { + .imt-page { padding: 0.5rem; } + .imt-tiles { grid-template-columns: repeat(auto-fill, minmax(min(100%, calc(150px * var(--imt-zoom))), 1fr)); } + .imt-thumb { height: calc(120px * var(--imt-zoom)); } + .imt-zoom { margin-left: 0 !important; } + .imt-remember { margin-left: 0; } +} diff --git a/plugins/image-matcher/image-matcher.yml b/plugins/image-matcher/image-matcher.yml new file mode 100644 index 00000000..0f0a6414 --- /dev/null +++ b/plugins/image-matcher/image-matcher.yml @@ -0,0 +1,11 @@ +name: Image Matcher +description: Finds images that look the same (perceptual hash). Merge them into one image with every file kept, or delete the lesser copies, one group or one folder pair at a time. +version: 0.9.0 +url: https://github.com/stashapp/CommunityScripts/tree/main/plugins/image-matcher +ui: + javascript: + - imcore.js + - imops.js + - imui.js + css: + - image-matcher.css diff --git a/plugins/image-matcher/imcore.js b/plugins/image-matcher/imcore.js new file mode 100644 index 00000000..283d696f --- /dev/null +++ b/plugins/image-matcher/imcore.js @@ -0,0 +1,451 @@ +// Image Matcher - pure matching and decision logic (no network, no DOM). +// Runs in the browser as part of the Stash UI plugin and under node for tests. +// License: AGPL-3.0-or-later. +(function (root) { + "use strict"; + + var STAMP = "imagematch"; // custom fingerprint type that marks files a person merged + + function dirname(p) { + var i = Math.max(p.lastIndexOf("/"), p.lastIndexOf("\\")); + if (i < 0) return ""; + return i === 0 ? p.slice(0, 1) : p.slice(0, i); + } + + function popcount32(v) { + v = v - ((v >>> 1) & 0x55555555); + v = (v & 0x33333333) + ((v >>> 2) & 0x33333333); + return (((v + (v >>> 4)) & 0x0f0f0f0f) * 0x01010101) >>> 24; + } + + // "c0b7a1ee10ea7c3a" -> [hi, lo] as unsigned 32-bit numbers + function parseHash(hex) { + if (typeof hex !== "string" || !/^[0-9a-fA-F]{1,16}$/.test(hex)) return null; + var s = hex.padStart(16, "0"); + return [parseInt(s.slice(0, 8), 16) >>> 0, parseInt(s.slice(8), 16) >>> 0]; + } + + function hashDistance(a, b) { + return popcount32((a[0] ^ b[0]) >>> 0) + popcount32((a[1] ^ b[1]) >>> 0); + } + + function fingerprint(file, type) { + var fps = (file && file.fingerprints) || []; + for (var i = 0; i < fps.length; i++) if (fps[i].type === type) return fps[i].value; + return null; + } + + // GraphQL image (id + visual_files) -> the small record the matcher works on. + // The first visual file is the primary one. + function toItem(image) { + var files = (image.visual_files || []).filter(function (f) { return f && f.id; }).map(function (f) { + return { + id: String(f.id), path: f.path || "", size: Number(f.size) || 0, + width: f.width || 0, height: f.height || 0, phash: fingerprint(f, "phash"), + archived: !!(f.zip_file && f.zip_file.id), + }; + }); + var primary = files[0] || { path: "", size: 0, width: 0, height: 0 }; + var hash = null; + for (var i = 0; i < files.length && !hash; i++) hash = parseHash(files[i].phash); + var folders = []; + files.forEach(function (f) { var d = dirname(f.path); if (folders.indexOf(d) < 0) folders.push(d); }); + return { + id: String(image.id), files: files, folder: dirname(primary.path), folders: folders, + area: primary.width * primary.height, width: primary.width, height: primary.height, + size: primary.size, phash: hash, + archived: files.some(function (f) { return f.archived; }), // any file inside a cbz or zip + }; + } + + // Groups items whose hashes are within maxDist bits of each other (connected components). + // Pigeonhole banding: split the 64 bits into maxDist+1 bands; two hashes within maxDist + // must agree exactly on at least one band, so only items sharing a band value are compared. + function findGroups(items, maxDist) { + var pos = []; + for (var i = 0; i < items.length; i++) if (items[i].phash) pos.push(i); + var n = pos.length, hi = new Uint32Array(n), lo = new Uint32Array(n), k; + for (k = 0; k < n; k++) { hi[k] = items[pos[k]].phash[0]; lo[k] = items[pos[k]].phash[1]; } + + var parent = new Int32Array(n); + for (k = 0; k < n; k++) parent[k] = k; + function find(x) { while (parent[x] !== x) { parent[x] = parent[parent[x]]; x = parent[x]; } return x; } + function union(a, b) { a = find(a); b = find(b); if (a !== b) parent[b < a ? a : b] = b < a ? b : a; } + + // collapse exact-equal hashes first so large runs of identical hashes cost nothing later + var exact = new Map(), uniq = []; + for (k = 0; k < n; k++) { + var key = hi[k] + ":" + lo[k], rep = exact.get(key); + if (rep === undefined) { exact.set(key, k); uniq.push(k); } else union(rep, k); + } + + var compared = 0; + if (maxDist > 0) { + var bands = maxDist + 1; + for (var b = 0; b < bands; b++) { + var start = Math.floor((b * 64) / bands), end = Math.floor(((b + 1) * 64) / bands); + var buckets = new Map(); + for (var u = 0; u < uniq.length; u++) { + var p = uniq[u], bk = bandKey(hi[p], lo[p], start, end), arr = buckets.get(bk); + if (arr) arr.push(p); else buckets.set(bk, [p]); + } + buckets.forEach(function (arr) { + for (var x = 0; x < arr.length; x++) { + var ax = arr[x], hx = hi[ax], lx = lo[ax]; + for (var y = x + 1; y < arr.length; y++) { + var ay = arr[y]; + compared++; + if (popcount32((hx ^ hi[ay]) >>> 0) + popcount32((lx ^ lo[ay]) >>> 0) <= maxDist) union(ax, ay); + } + } + }); + } + } + + var byRoot = new Map(); + for (k = 0; k < n; k++) { + var r = find(k), g = byRoot.get(r); + if (g) g.push(pos[k]); else byRoot.set(r, [pos[k]]); + } + var groups = []; + byRoot.forEach(function (g) { if (g.length > 1) groups.push(g); }); + return { groups: groups, hashed: n, skipped: items.length - n, compared: compared }; + } + + // value of bits [start, end) of the 64-bit hash, bit 0 being the top bit of hi (width <= 32) + function bandKey(hi, lo, start, end) { + var width = end - start; + if (end <= 32) return width === 32 ? hi : (hi >>> (32 - end)) & ((1 << width) - 1); + if (start >= 32) return width === 32 ? lo : (lo >>> (64 - end)) & ((1 << width) - 1); + var hiBits = 32 - start, loBits = end - 32; + var top = hiBits === 32 ? hi : hi & ((1 << hiBits) - 1); + return top * Math.pow(2, loBits) + (lo >>> (32 - loBits)); + } + + // Folder rules: an ordered list of folder paths, first wins. A rule covers its subfolders. + function normalizeRules(rules) { + var out = []; + (rules || []).forEach(function (r) { + r = String(r || "").trim().replace(/[\/\\]+$/, ""); + if (r && out.indexOf(r) < 0) out.push(r); + }); + return out; + } + + function folderRank(folder, rules) { + for (var i = 0; i < rules.length; i++) { + var r = rules[i]; + if (folder === r || folder.indexOf(r + "/") === 0 || folder.indexOf(r + "\\") === 0) return i; + } + return Infinity; + } + + var REASONS = ["lowest id", "larger file", "larger resolution", "in colour", "folder rule"]; + + // Do two look-alikes differ in one place? a and b are brightness grids (n x n bytes) of the two + // thumbnails. A resized or recompressed copy differs a little everywhere; two drawings that differ + // in one detail differ a lot in a few neighbouring cells and hardly at all elsewhere. + // When the pictures differ nearly everywhere (the typical window is far off, or a fifth of the + // windows are hot) they are different pictures, not copies: that is `different`. + // Returns { spot, different, flag, hot, peak, base, box }; flag is spot or different, and box is + // the differing area of a spot as fractions of the picture. + var DIFF_N = 128, DIFF_CELL = 4, HOT_MIN = 22, HOT_RATIO = 8, HOT_MAX_SHARE = 0.2, DIFFERENT_BASE = 8; + function localDiff(a, b, n, cell) { + n = n || DIFF_N; cell = cell || DIFF_CELL; + var per = n / cell, fine = new Float64Array(per * per), len = n * n, sum = 0, i, x, y; + // an overall change of brightness or contrast is not a local difference: fit b = gain * a + lift + // (least squares) and look at what is left over + var ma = 0, mb = 0, cov = 0, va = 0; + for (i = 0; i < len; i++) { ma += a[i]; mb += b[i]; } + ma /= len; mb /= len; + for (i = 0; i < len; i++) { cov += (a[i] - ma) * (b[i] - mb); va += (a[i] - ma) * (a[i] - ma); } + var gain = va > 0 ? cov / va : 1, lift = mb - gain * ma; + for (y = 0; y < n; y++) for (x = 0; x < n; x++) { + i = y * n + x; + fine[Math.floor(y / cell) * per + Math.floor(x / cell)] += Math.abs(b[i] - gain * a[i] - lift); + } + // overlapping windows of 2 x 2 cells, so a small detail that straddles a cell edge still counts whole + var wper = per - 1, wins = new Float64Array(wper * wper), sorted = [], peak = 0, area = cell * cell * 4; + for (y = 0; y < wper; y++) for (x = 0; x < wper; x++) { + var v = (fine[y * per + x] + fine[y * per + x + 1] + fine[(y + 1) * per + x] + fine[(y + 1) * per + x + 1]) / area; + wins[y * wper + x] = v; sorted.push(v); if (v > peak) peak = v; + } + sorted.sort(function (p, q) { return p - q; }); + var base = sorted[Math.floor(sorted.length / 2)], limit = Math.max(HOT_MIN, base * HOT_RATIO); + var hot = 0, x0 = wper, y0 = wper, x1 = -1, y1 = -1; + for (i = 0; i < wins.length; i++) { + if (wins[i] <= limit) continue; + hot++; + var cx = i % wper, cy = Math.floor(i / wper); + if (cx < x0) x0 = cx; if (cx > x1) x1 = cx; if (cy < y0) y0 = cy; if (cy > y1) y1 = cy; + } + // judged against a fixed limit too, because a high median raises the relative one out of reach + var hotFixed = 0; + for (i = 0; i < wins.length; i++) if (wins[i] > HOT_MIN) hotFixed++; + var different = base > DIFFERENT_BASE || hotFixed > wins.length * HOT_MAX_SHARE; + var spot = !different && hot >= 1 && hot <= wins.length * HOT_MAX_SHARE; + return { spot: spot, different: different, flag: spot || different, hot: hot, peak: peak, base: base, + box: spot ? { x: x0 / per, y: y0 / per, w: (x1 - x0 + 2) / per, h: (y1 - y0 + 2) / per } : null }; + } + // canvas rgba -> brightness bytes + function luminance(rgba) { + var out = new Uint8Array(rgba.length / 4); + for (var i = 0, j = 0; i + 3 < rgba.length; i += 4, j++) out[j] = Math.round(0.299 * rgba[i] + 0.587 * rgba[i + 1] + 0.114 * rgba[i + 2]); + return out; + } + + // Is a picture in colour? rgba is canvas pixel data (4 bytes per pixel). The perceptual hash + // only sees brightness, so a black-and-white copy matches its colour original; this tells them apart. + // A toned print (sepia) counts as colour. + function isColor(rgba) { + var total = 0, n = 0; + for (var i = 0; i + 3 < rgba.length; i += 4) { + var r = rgba[i], g = rgba[i + 1], b = rgba[i + 2]; + total += Math.max(r, g, b) - Math.min(r, g, b); n++; + } + return n > 0 && total / n > 6; + } + + // negative result means a is the better copy + function compareItems(a, b, rules) { + var ra = folderRank(a.folder, rules), rb = folderRank(b.folder, rules); + if (ra !== rb) return { result: ra < rb ? -1 : 1, reason: "folder rule" }; + // colour beats black and white, unless the black-and-white copy has more than twice the pixels. + // color is true/false once the page has looked at the thumbnail, and absent before that. + if (typeof a.color === "boolean" && typeof b.color === "boolean" && a.color !== b.color) { + var col = a.color ? a : b, grey = a.color ? b : a; + if (grey.area <= col.area * 2) return { result: a.color ? -1 : 1, reason: "in colour" }; + } + if (a.area !== b.area) return { result: a.area > b.area ? -1 : 1, reason: "larger resolution" }; + if (a.size !== b.size) return { result: a.size > b.size ? -1 : 1, reason: "larger file" }; + return { result: Number(a.id) < Number(b.id) ? -1 : 1, reason: "lowest id" }; + } + + // Which copy to keep: folder rule first, then colour over black and white, then resolution, then file size, then lowest id. + function chooseKeeper(members, rules) { + rules = rules || []; + var best = members[0]; + for (var i = 1; i < members.length; i++) if (compareItems(members[i], best, rules).result < 0) best = members[i]; + var reason = members.length > 1 ? "lowest id" : "only copy"; + members.forEach(function (m) { + if (m === best) return; + var r = compareItems(best, m, rules).reason; + if (REASONS.indexOf(r) > REASONS.indexOf(reason)) reason = r; + }); + return { keeper: best, reason: reason }; + } + + // "Always prefer this folder": put the kept folder where it actually outranks the other one. + // Rules are first-match and cover subfolders, so it goes ahead of the first rule that already + // covers either folder; with no such rule it goes last. + function rememberRule(rules, keepFolder, otherFolder) { + var out = normalizeRules(rules).filter(function (r) { return r !== keepFolder; }); + var at = Math.min(folderRank(keepFolder, out), folderRank(otherFolder, out)); + out.splice(at === Infinity ? out.length : at, 0, keepFolder); + return out; + } + + function setKey(a, b) { return "set:" + (a < b ? a + "|" + b : b + "|" + a); } + function groupKey(ids) { return ids.map(Number).sort(function (a, b) { return a - b; }).join("-"); } + + function defaultKeepSide(set, items, rules) { + var ra = folderRank(set.folderA, rules), rb = folderRank(set.folderB, rules); + if (ra !== rb) return { side: ra < rb ? "A" : "B", reason: "folder rule" }; + var a = 0, b = 0; + set.matches.forEach(function (m) { + var ka = chooseKeeper(m.a.map(function (i) { return items[i]; }), []).keeper; + var kb = chooseKeeper(m.b.map(function (i) { return items[i]; }), []).keeper; + var c = compareItems(ka, kb, []); + if (c.reason === "lowest id") return; + if (c.result < 0) a++; else b++; + }); + if (a === b) return { side: "A", reason: "no difference found" }; + return { side: a > b ? "A" : "B", reason: "better copies in " + Math.max(a, b) + " of " + set.matches.length }; + } + + // Folder pairs that share look-alike images: "the same set in two places". + // How far apart a folder-set match is: the worst phash distance between any image on one side + // and any image on the other, so a label never understates. null when a hash is missing. + function matchDistance(items, match) { + var worst = null; + match.a.forEach(function (i) { + match.b.forEach(function (j) { + if (!items[i].phash || !items[j].phash) return; + var d = hashDistance(items[i].phash, items[j].phash); + if (worst === null || d > worst) worst = d; + }); + }); + return worst; + } + // The mix of a set: pairs with the same hash, pairs that differ, and the largest difference. + function distanceMix(distances) { + var mix = { same: 0, near: 0, max: 0, unknown: 0 }; + distances.forEach(function (d) { + if (d === null) mix.unknown++; else if (d === 0) mix.same++; else { mix.near++; if (d > mix.max) mix.max = d; } + }); + return mix; + } + + // A look-alike group can hold several images in each of two folders (two different pictures + // that happen to be close, each present in both folders). Treating that as one match would + // merge them all into one image, so pair each image with its nearest counterpart instead. + // One image against several on the other side stays one match (copies of the same picture). + function splitMatch(items, m) { + if (m.a.length < 2 || m.b.length < 2) return [m]; + var cross = []; + m.a.forEach(function (i) { + m.b.forEach(function (j) { + var d = items[i].phash && items[j].phash ? hashDistance(items[i].phash, items[j].phash) : 65; + cross.push({ i: i, j: j, d: d }); + }); + }); + cross.sort(function (p, q) { return p.d - q.d || p.i - q.i || p.j - q.j; }); + var subOfA = new Map(), subOfB = new Map(), subs = []; + cross.forEach(function (c) { + if (subOfA.has(c.i) || subOfB.has(c.j)) return; + var sub = { group: m.group, a: [c.i], b: [c.j] }; + subs.push(sub); subOfA.set(c.i, sub); subOfB.set(c.j, sub); + }); + cross.forEach(function (c) { // sides of unequal size: a leftover joins its nearest counterpart's pair + if (!subOfA.has(c.i) && subOfB.has(c.j)) { subOfB.get(c.j).a.push(c.i); subOfA.set(c.i, subOfB.get(c.j)); } + else if (!subOfB.has(c.j) && subOfA.has(c.i)) { subOfA.get(c.i).b.push(c.j); subOfB.set(c.j, subOfA.get(c.i)); } + }); + return subs; + } + // stable name for one match of a set, used to leave it out of an action + function pairKey(items, m) { return groupKey(m.a.concat(m.b).map(function (i) { return items[i].id; })); } + + // Is this folder left out of folder sets? A line names one folder exactly (its subfolders still + // count); a line ending in /* covers the folder and everything under it. + function folderSkipped(folder, skip) { + for (var i = 0; i < skip.length; i++) { + var s = skip[i]; + if (s.slice(-2) === "/*") { var root = s.slice(0, -2); if (folder === root || folder.indexOf(root + "/") === 0 || folder.indexOf(root + "\\") === 0) return true; } + else if (folder === s) return true; + } + return false; + } + function normalizeSkips(list) { + var out = []; + (list || []).forEach(function (r) { + r = String(r || "").trim(); + var deep = /[\/\\]\*$/.test(r); + r = r.replace(/[\/\\]\*$/, "").replace(/[\/\\]+$/, ""); + if (r && deep) r += "/*"; + if (r && out.indexOf(r) < 0) out.push(r); + }); + return out; + } + + function buildSets(items, groups, rules, minMatches, skip) { + rules = rules || []; skip = skip || []; + minMatches = minMatches || 2; + var folderCounts = new Map(); + // an image already merged across folders counts in each folder that holds one of its files + function foldersOf(it) { return it.folders && it.folders.length ? it.folders : [it.folder]; } + items.forEach(function (it) { foldersOf(it).forEach(function (f) { folderCounts.set(f, (folderCounts.get(f) || 0) + 1); }); }); + + var sets = new Map(); + groups.forEach(function (group, gi) { + var byFolder = new Map(); + group.forEach(function (idx) { + foldersOf(items[idx]).forEach(function (f) { + if (skip.length && folderSkipped(f, skip)) return; + var arr = byFolder.get(f); + if (arr) arr.push(idx); else byFolder.set(f, [idx]); + }); + }); + var folders = Array.from(byFolder.keys()).sort(); + for (var x = 0; x < folders.length; x++) { + for (var y = x + 1; y < folders.length; y++) { + // an image holding files in both folders is already merged across them; a match + // needs a different image on the other side + var a = byFolder.get(folders[x]), b = byFolder.get(folders[y]); + var bOnly = b.filter(function (i) { return a.indexOf(i) < 0; }), aOnly = a.filter(function (i) { return b.indexOf(i) < 0; }); + var match = bOnly.length ? { group: gi, a: a, b: bOnly } : aOnly.length ? { group: gi, a: aOnly, b: b } : null; + if (!match) continue; + var key = setKey(folders[x], folders[y]), s = sets.get(key); + if (!s) { s = { key: key, folderA: folders[x], folderB: folders[y], matches: [] }; sets.set(key, s); } + splitMatch(items, match).forEach(function (sub) { s.matches.push(sub); }); + } + } + }); + + var out = []; + sets.forEach(function (s) { + if (s.matches.length < minMatches) return; + s.countA = folderCounts.get(s.folderA) || 0; + s.countB = folderCounts.get(s.folderB) || 0; + s.matchedA = s.matches.reduce(function (t, m) { return t + m.a.length; }, 0); + s.matchedB = s.matches.reduce(function (t, m) { return t + m.b.length; }, 0); + var pick = defaultKeepSide(s, items, rules); + s.keep = pick.side; s.reason = pick.reason; + out.push(s); + }); + out.sort(function (p, q) { return q.matches.length - p.matches.length || (p.key < q.key ? -1 : 1); }); + return out; + } + + // For one folder pair: which image is kept and which are folded in, per match. + // skip: optional { pairKey: true } of matches to leave out + function setPlans(set, items, keepSide, skip) { + return set.matches.filter(function (m) { return !(skip && skip[pairKey(items, m)]); }).map(function (m) { + var keepIdx = keepSide === "A" ? m.a : m.b, loseIdx = keepSide === "A" ? m.b : m.a; + var keeper = chooseKeeper(keepIdx.map(function (i) { return items[i]; }), []).keeper; + return { keeperId: keeper.id, loserIds: loseIdx.map(function (i) { return items[i].id; }) }; + }); + } + + function unionList(lists) { + var out = []; + lists.forEach(function (l) { l.forEach(function (v) { if (out.indexOf(v) < 0) out.push(v); }); }); + return out; + } + + // What the kept record should absorb from the others. Returns only the fields that change. + // Works for images and galleries (same field names in Stash). + function mergeMeta(keeper, losers) { + var patch = {}, all = [keeper].concat(losers); + + var ratings = all.map(function (m) { return m.rating100; }).filter(function (r) { return r != null; }); + if (ratings.length) { + var max = Math.max.apply(null, ratings); + if (max !== keeper.rating100) patch.rating100 = max; + } + if (!keeper.organized && losers.some(function (m) { return m.organized; })) patch.organized = true; + + ["title", "code", "date", "photographer"].forEach(function (f) { + if (keeper[f]) return; + for (var i = 0; i < losers.length; i++) if (losers[i][f]) { patch[f] = losers[i][f]; return; } + }); + + var details = []; + all.forEach(function (m) { var d = (m.details || "").trim(); if (d && details.indexOf(d) < 0) details.push(d); }); + var joined = details.join("\n\n"); + if (joined !== (keeper.details || "").trim()) patch.details = joined; + + var urls = unionList(all.map(function (m) { return m.urls || []; })); + if (urls.length !== (keeper.urls || []).length) patch.urls = urls; + + if (!keeper.studio) { + for (var s = 0; s < losers.length; s++) if (losers[s].studio) { patch.studio_id = losers[s].studio.id; break; } + } + + [["tags", "tag_ids"], ["performers", "performer_ids"], ["scenes", "scene_ids"]].forEach(function (pair) { + if (!all.some(function (m) { return m[pair[0]]; })) return; + var ids = unionList(all.map(function (m) { return (m[pair[0]] || []).map(function (t) { return t.id; }); })); + if (ids.length !== (keeper[pair[0]] || []).length) patch[pair[1]] = ids; + }); + return patch; + } + + var api = { + STAMP: STAMP, dirname: dirname, popcount32: popcount32, parseHash: parseHash, hashDistance: hashDistance, + fingerprint: fingerprint, toItem: toItem, findGroups: findGroups, bandKey: bandKey, + normalizeRules: normalizeRules, folderRank: folderRank, chooseKeeper: chooseKeeper, compareItems: compareItems, + buildSets: buildSets, matchDistance: matchDistance, isColor: isColor, rememberRule: rememberRule, folderRank: folderRank, folderSkipped: folderSkipped, normalizeSkips: normalizeSkips, localDiff: localDiff, luminance: luminance, DIFF_N: DIFF_N, splitMatch: splitMatch, pairKey: pairKey, distanceMix: distanceMix, setKey: setKey, groupKey: groupKey, setPlans: setPlans, mergeMeta: mergeMeta, + }; + if (typeof module !== "undefined" && module.exports) module.exports = api; + root.ImageMatcher = root.ImageMatcher || {}; + root.ImageMatcher.core = api; +})(typeof globalThis !== "undefined" ? globalThis : this); diff --git a/plugins/image-matcher/imops.js b/plugins/image-matcher/imops.js new file mode 100644 index 00000000..23cb10b9 --- /dev/null +++ b/plugins/image-matcher/imops.js @@ -0,0 +1,408 @@ +// Image Matcher - the operations, as sequences of ordinary Stash GraphQL calls. +// No database writes and no filesystem assumptions: merging works by stamping the files +// with a shared custom fingerprint, detaching the lesser image record and letting Stash's +// own scanner attach its file to the kept image. +// License: AGPL-3.0-or-later. +(function (root) { + "use strict"; + + var PLUGIN_ID = "image-matcher"; + var FILE_FIELDS = "id path size width height fingerprints { type value }"; + var FILES = "visual_files { __typename ... on ImageFile { " + FILE_FIELDS + " zip_file { id } } ... on VideoFile { " + FILE_FIELDS + " zip_file { id } } }"; + var LIGHT = "id " + FILES; + var META = "id rating100 organized o_counter title code date details photographer urls studio { id } tags { id } performers { id } " + + "galleries { id folder { path } files { id } } " + FILES; + var GALLERY_META = "id rating100 organized title code date details photographer urls studio { id } tags { id } performers { id } scenes { id }"; + + function createOps(gql, core, options) { + options = options || {}; + var sleep = options.sleep || function (ms) { return new Promise(function (r) { setTimeout(r, ms); }); }; + var q = JSON.stringify; // GraphQL string literals + + // ---- configuration (stored in Stash's plugin settings, not in the browser) ---- + function parseList(v) { + if (Array.isArray(v)) return v; + try { var p = JSON.parse(v || "[]"); return Array.isArray(p) ? p : []; } catch (e) { return []; } + } + function loadConfig() { + return gql("{ configuration { plugins } }").then(function (d) { + var raw = ((d.configuration || {}).plugins || {})[PLUGIN_ID] || {}; + return { + distance: typeof raw.distance === "number" ? raw.distance : 4, + folderRules: core.normalizeRules(parseList(raw.folderRules)), + dismissed: parseList(raw.dismissed), + setSkip: core.normalizeSkips(parseList(raw.setSkip)), + }; + }); + } + function saveConfig(cfg) { + var input = { + distance: cfg.distance, + folderRules: JSON.stringify(core.normalizeRules(cfg.folderRules)), + dismissed: JSON.stringify(cfg.dismissed || []), + setSkip: JSON.stringify(core.normalizeSkips(cfg.setSkip)), + }; + return gql("mutation($id: ID!, $input: Map!) { configurePlugin(plugin_id: $id, input: $input) }", { id: PLUGIN_ID, input: input }); + } + + // ---- reading ---- + // Images inside an archive (cbz, zip) are left out: a file in an archive cannot be deleted, + // detached or rescanned on its own, so neither merge nor delete can work on it. + // The returned list carries .archived, the number left out. + function loadImages(onProgress) { + var per = options.pageSize || 1000, items = [], seen = 0; + items.archived = 0; + function page(p) { + return gql("query($p: Int!, $n: Int!) { findImages(filter: { page: $p, per_page: $n, sort: \"id\", direction: ASC }) { count images { " + LIGHT + " } } }", { p: p, n: per }) + .then(function (d) { + var imgs = d.findImages.images; + imgs.forEach(function (im) { var it = core.toItem(im); if (it.archived) items.archived++; else items.push(it); }); + seen += imgs.length; + if (onProgress) onProgress(seen, d.findImages.count); + if (imgs.length < per || seen >= d.findImages.count) return items; + return page(p + 1); + }); + } + return page(1); + } + + function aliased(prefix, ids, body) { + return ids.map(function (id) { return prefix + id + ": " + body(id); }).join("\n"); + } + function chunked(ids, size, fn) { + var out = {}, chain = Promise.resolve(); + for (var i = 0; i < ids.length; i += size) (function (part) { + chain = chain.then(function () { return fn(part); }).then(function (d) { Object.assign(out, d); }); + })(ids.slice(i, i + size)); + return chain.then(function () { return out; }); + } + + function imageMeta(ids) { + return chunked(ids, 40, function (part) { + return gql("{ " + aliased("i", part, function (id) { return "findImage(id: " + q(id) + ") { " + META + " }"; }) + " }"); + }).then(function (d) { + var out = {}; + ids.forEach(function (id) { + if (!d["i" + id]) throw new Error("image " + id + " no longer exists; run Find matches again"); + out[id] = d["i" + id]; + }); + return out; + }); + } + + function fileInfo(fileIds) { + return chunked(fileIds, 60, function (part) { + return gql("{ " + aliased("f", part, function (id) { + return "findFile(id: " + q(id) + ") { __typename ... on ImageFile { id fingerprints { type value } } ... on VideoFile { id fingerprints { type value } } }"; + }) + " }"); + }).then(function (d) { + var out = {}; + fileIds.forEach(function (id) { out[id] = d["f" + id]; }); + return out; + }); + } + + // Which images hold each file. Newer Stash answers directly (ImageFile.images); older + // releases are asked which images have a file at that path. + // A VideoFile (an image clip) has no such field on any version, so it always goes by path. + var hasFileImages = null; + function fileOwners(files) { // [{ id, path, type? }] -> { fileId: [imageId, ...] } + var ready = hasFileImages !== null ? Promise.resolve() : gql('{ __type(name: "ImageFile") { fields { name } } }').then(function (d) { + hasFileImages = (((d.__type || {}).fields) || []).some(function (f) { return f.name === "images"; }); + }); + return ready.then(function () { + var idx = files.map(function (f, i) { return i; }); + return chunked(idx, 40, function (part) { + return gql("{ " + part.map(function (i) { + return hasFileImages && files[i].type !== "VideoFile" + ? "o" + i + ": findFile(id: " + q(files[i].id) + ") { ... on ImageFile { images { id } } }" + : "o" + i + ": findImages(image_filter: { path: { value: " + q(files[i].path) + ", modifier: EQUALS } }, filter: { per_page: -1 }) { images { id } }"; + }).join("\n") + " }"); + }).then(function (d) { + var out = {}; + files.forEach(function (f, i) { out[f.id] = (((d["o" + i] || {}).images) || []).map(function (im) { return im.id; }); }); + return out; + }); + }); + } + + function setFingerprints(entries) { // [{ fileId, fps: [{type, value|null}] }] + if (!entries.length) return Promise.resolve(); + return chunked(entries.map(function (e, i) { return i; }), 60, function (part) { + return gql("mutation { " + part.map(function (i) { + var e = entries[i]; + var fps = e.fps.map(function (f) { return "{ type: " + q(f.type) + ", value: " + (f.value == null ? "null" : q(f.value)) + " }"; }).join(", "); + return "s" + i + ": fileSetFingerprints(input: { id: " + q(e.fileId) + ", fingerprints: [" + fps + "] })"; + }).join("\n") + " }"); + }); + } + + function waitForJob(jobId) { + var started = Date.now(), delay = 150; + function poll() { + return gql("query($id: ID!) { findJob(input: { id: $id }) { status error } }", { id: jobId }).then(function (d) { + var job = d.findJob; + if (!job || job.status === "FINISHED") return; + if (job.status === "FAILED" || job.status === "CANCELLED") throw new Error("Stash scan " + job.status.toLowerCase() + (job.error ? ": " + job.error : "")); + if (Date.now() - started > (options.jobTimeout || 30 * 60 * 1000)) throw new Error("timed out waiting for the Stash scan"); + delay = Math.min(delay * 1.5, 1500); + return sleep(delay).then(poll); + }); + } + return poll(); + } + + // Scans exactly these files, 50 per job. Never their folders: a folder scan is recursive and + // would walk (and import from) everything under a large folder. + function scanPaths(paths) { + var chain = Promise.resolve(); + for (var i = 0; i < paths.length; i += 50) (function (part) { + chain = chain.then(function () { + return gql("mutation($paths: [String!]) { metadataScan(input: { paths: $paths }) }", { paths: part }) + .then(function (d) { return waitForJob(d.metadataScan); }); + }); + })(paths.slice(i, i + 50)); + return chain; + } + + // ---- shared: copy what the lesser records know onto the kept one ---- + function transferMeta(plan, meta) { + var keeper = meta[plan.keeperId], losers = plan.loserIds.map(function (id) { return meta[id]; }); + var patch = core.mergeMeta(keeper, losers), chain = Promise.resolve(); + if (Object.keys(patch).length) { + patch.id = keeper.id; + chain = chain.then(function () { return gql("mutation($input: ImageUpdateInput!) { imageUpdate(input: $input) { id } }", { input: patch }); }); + } + // hand-made galleries only: folder and zip galleries follow the files and cannot be edited + var have = keeper.galleries.map(function (g) { return g.id; }), manual = []; + losers.forEach(function (l) { + l.galleries.forEach(function (g) { + if (!g.folder && !(g.files || []).length && have.indexOf(g.id) < 0 && manual.indexOf(g.id) < 0) manual.push(g.id); + }); + }); + manual.forEach(function (gid) { + chain = chain.then(function () { + return gql("mutation($g: ID!, $i: [ID!]!) { addGalleryImages(input: { gallery_id: $g, image_ids: $i }) }", { g: gid, i: [keeper.id] }); + }); + }); + var extraO = Math.min(100, losers.reduce(function (t, l) { return t + (l.o_counter || 0); }, 0)); + for (var n = 0; n < extraO; n++) chain = chain.then(function () { return gql("mutation($id: ID!) { imageIncrementO(id: $id) }", { id: keeper.id }); }); + return chain; + } + + function checkPlans(plans) { + var seen = {}; + plans.forEach(function (p) { + [p.keeperId].concat(p.loserIds).forEach(function (id) { + if (seen[id]) throw new Error("image " + id + " appears in two merges at once"); + seen[id] = true; + }); + if (!p.loserIds.length) throw new Error("nothing to fold into image " + p.keeperId); + }); + } + function allIds(plans) { return plans.reduce(function (a, p) { return a.concat([p.keeperId], p.loserIds); }, []); } + + // ---- merge: one image record, every file kept where it is, keeper's file stays primary ---- + function mergeImages(plans, onStep) { + var step = onStep || function () {}; + var meta, work = [], restore = [], detached = false; + return Promise.resolve().then(function () { + checkPlans(plans); + return imageMeta(allIds(plans)); + }).then(function (m) { + meta = m; + var entries = []; + plans.forEach(function (plan) { + // A fresh stamp for every merge: one left behind by an earlier, failed merge must never match. + var keeper = meta[plan.keeperId]; + var stamp = "im-" + keeper.id + "-" + Date.now().toString(36) + Math.random().toString(36).slice(2, 8); + keeper.visual_files.forEach(function (f) { entries.push({ fileId: f.id, fps: [{ type: core.STAMP, value: stamp }] }); }); + var w = { keeperId: keeper.id, primaryFileId: keeper.visual_files[0].id, stamp: stamp, loserFiles: [] }; + plan.loserIds.forEach(function (id) { + meta[id].visual_files.forEach(function (f) { + var fps = [{ type: core.STAMP, value: stamp }], ph = core.fingerprint(f, "phash"); + // The scanner attaches a detached file to every image sharing any fingerprint. + // Hide the phash for the moment so only the stamp can match, then put it back. + if (ph) { fps.push({ type: "phash", value: null }); restore.push({ fileId: f.id, fps: [{ type: "phash", value: ph }] }); } + entries.push({ fileId: f.id, fps: fps }); + w.loserFiles.push({ id: f.id, path: f.path, type: f.__typename }); + }); + }); + work.push(w); + }); + step("Marking files"); + return setFingerprints(entries); + }).then(function () { + var ids = work.reduce(function (a, w) { return a.concat(w.loserFiles.map(function (f) { return f.id; })); }, []); + return fileInfo(ids).then(function (info) { + work.forEach(function (w) { + w.loserFiles.forEach(function (f) { + var fi = info[f.id]; + if (!fi || core.fingerprint(fi, core.STAMP) !== w.stamp || core.fingerprint(fi, "phash")) + throw new Error("could not mark file " + f.id + "; nothing was merged"); + }); + }); + }); + }).then(function () { + step("Copying tags and ratings"); + return plans.reduce(function (c, plan) { return c.then(function () { return transferMeta(plan, meta); }); }, Promise.resolve()); + }).then(function () { + step("Detaching the lesser records"); + var ids = plans.reduce(function (a, p) { return a.concat(p.loserIds); }, []); + // From here on assume the records are gone: a destroy whose answer was lost may still + // have happened, and putting the phashes back on loose files is the dangerous mistake. + detached = true; + return gql("mutation($ids: [ID!]!) { imagesDestroy(input: { ids: $ids, delete_file: false, delete_generated: true }) }", { ids: ids }); + }).then(function () { + step("Rescanning so Stash attaches the files"); + var paths = []; + work.forEach(function (w) { w.loserFiles.forEach(function (f) { if (paths.indexOf(f.path) < 0) paths.push(f.path); }); }); + return scanPaths(paths); + }).then(function () { + step("Checking the result"); + var files = work.reduce(function (a, w) { return a.concat(w.loserFiles); }, []); + return fileOwners(files).then(function (info) { + var bad = [], badIds = {}, loose = 0; + work.forEach(function (w) { + w.loserFiles.forEach(function (f) { + var owners = info[f.id] || []; + if (owners.length === 1 && owners[0] === w.keeperId) return; + badIds[f.id] = true; + if (!owners.length) loose++; + bad.push("file " + f.id + (owners.length ? " is on image(s) [" + owners.join(", ") + "]" : " is on no image") + ", expected " + w.keeperId); + }); + }); + // the hash goes back on every file that landed; only the ones that did not stay hidden + var ok = restore.filter(function (r) { return !badIds[r.fileId]; }); + restore = restore.filter(function (r) { return badIds[r.fileId]; }); + return setFingerprints(ok).then(function () { + if (!bad.length) { detached = false; return; } + var e = new Error("merge did not land cleanly: " + bad.join("; ") + "."); + e.loose = loose; e.misplaced = bad.length - loose; + throw e; + }); + }); + }).then(function () { + // the kept image's own file must still be the one shown + return imageMeta(work.map(function (w) { return w.keeperId; })).then(function (now) { + return work.reduce(function (c, w) { + if (now[w.keeperId].visual_files[0].id === w.primaryFileId) return c; + return c.then(function () { + return gql("mutation($input: ImageUpdateInput!) { imageUpdate(input: $input) { id } }", { input: { id: w.keeperId, primary_file_id: w.primaryFileId } }); + }); + }, Promise.resolve()); + }); + }).then(function () { + return { merged: work.reduce(function (t, w) { return t + w.loserFiles.length; }, 0), images: plans.length }; + }).catch(function (err) { + // Before the detach nothing has changed, so the hidden phashes go straight back. + // After it they must stay hidden: a detached file with its phash back would attach to + // every equal-phash image on the next scan. Stamp-only, it can only join the kept image. + if (!detached) return setFingerprints(restore).catch(function () {}).then(function () { throw err; }); + if (!/[.!?]$/.test(err.message)) err.message += "."; + if (err.misplaced) err.message += " A file that is on another image needs sorting out by hand in Stash."; + if (err.loose || err.misplaced === undefined) err.message += " Files still on no image stay marked, so a scan of their folder attaches them to the kept image." + + " Afterwards run Generate with image perceptual hashes on, to rebuild the hashes that were set aside."; + throw err; + }); + } + + // ---- dedupe: keep one copy, delete the others from disk ---- + // Deleting an image deletes all of its files. A lesser image holding more than one file (one + // that was merged, here or elsewhere) is left alone and counted in `skipped`; merging is the + // right action for it. Works from what Stash says now, not from what the page loaded earlier. + function dedupeImages(plans, onStep) { + var step = onStep || function () {}, meta, skipped = 0; + return Promise.resolve().then(function () { + checkPlans(plans); + return imageMeta(allIds(plans)); + }).then(function (m) { + meta = m; + plans = plans.map(function (p) { + return { keeperId: p.keeperId, loserIds: p.loserIds.filter(function (id) { + if (meta[id].visual_files.length > 1) { skipped++; return false; } + return true; + }) }; + }).filter(function (p) { return p.loserIds.length; }); + if (!plans.length) return { deleted: 0, images: 0, skipped: skipped, plans: plans }; + return dedupeNow(plans, meta, step).then(function (res) { res.skipped = skipped; res.plans = plans; return res; }); + }); + } + function dedupeNow(plans, meta, step) { + return Promise.resolve().then(function () { + step("Copying tags and ratings"); + return plans.reduce(function (c, plan) { return c.then(function () { return transferMeta(plan, meta); }); }, Promise.resolve()); + }).then(function () { + step("Deleting the lesser copies"); + var ids = plans.reduce(function (a, p) { return a.concat(p.loserIds); }, []); + return gql("mutation($ids: [ID!]!) { imagesDestroy(input: { ids: $ids, delete_file: true, delete_generated: true }) }", { ids: ids }) + .then(function () { return { deleted: ids.length, images: plans.length }; }); + }); + } + + // One operation at a time. A merge restamps the kept image's files, so a second merge that + // shares a keeper would orphan the files the first one has already detached. + var running = false; + function exclusive(fn) { + return function () { + if (running) return Promise.reject(new Error("Another merge or delete is still running. Wait for it to finish.")); + running = true; + var done = function () { running = false; }; + var p; + try { p = Promise.resolve(fn.apply(null, arguments)); } catch (err) { p = Promise.reject(err); } + return p.then(function (r) { done(); return r; }, function (err) { done(); throw err; }); + }; + } + + // ---- folder pairs ---- + function folderGalleryIds(meta, folder) { + var ids = []; + Object.keys(meta).forEach(function (id) { + meta[id].galleries.forEach(function (g) { if (g.folder && g.folder.path === folder && ids.indexOf(g.id) < 0) ids.push(g.id); }); + }); + return ids; + } + + // Gallery "merge" = both folder galleries end up knowing the same things. + // Their contents converge on their own once the images are merged. + function syncGalleries(keepIds, otherIds, bothWays) { + var ids = keepIds.concat(otherIds); + if (!keepIds.length || !otherIds.length) return Promise.resolve(0); + return gql("{ " + aliased("g", ids, function (id) { return "findGallery(id: " + q(id) + ") { " + GALLERY_META + " }"; }) + " }").then(function (d) { + var keep = keepIds.map(function (id) { return d["g" + id]; }).filter(Boolean); + var other = otherIds.map(function (id) { return d["g" + id]; }).filter(Boolean); + var updates = []; + keep.forEach(function (g) { var p = core.mergeMeta(g, other); if (Object.keys(p).length) { p.id = g.id; updates.push(p); } }); + if (bothWays) other.forEach(function (g) { var p = core.mergeMeta(g, keep); if (Object.keys(p).length) { p.id = g.id; updates.push(p); } }); + return updates.reduce(function (c, input) { + return c.then(function () { return gql("mutation($input: GalleryUpdateInput!) { galleryUpdate(input: $input) { id } }", { input: input }); }); + }, Promise.resolve()).then(function () { return updates.length; }); + }); + } + + function runSet(plans, keepFolder, otherFolder, mode, onStep) { + var keepG, otherG; + return imageMeta(allIds(plans)).then(function (meta) { + keepG = folderGalleryIds(meta, keepFolder); + otherG = folderGalleryIds(meta, otherFolder); + return mode === "merge" ? mergeImages(plans, onStep) : dedupeImages(plans, onStep); + }).then(function (res) { + res.skipped = res.skipped || 0; res.plans = res.plans || plans; + if (onStep) onStep("Syncing gallery details"); + return syncGalleries(keepG, otherG, mode === "merge").then(function (n) { res.galleriesUpdated = n; return res; }); + }); + } + function mergeSet(plans, keepFolder, otherFolder, onStep) { return runSet(plans, keepFolder, otherFolder, "merge", onStep); } + function dedupeSet(plans, keepFolder, otherFolder, onStep) { return runSet(plans, keepFolder, otherFolder, "dedupe", onStep); } + + return { + loadConfig: loadConfig, saveConfig: saveConfig, loadImages: loadImages, imageMeta: imageMeta, fileInfo: fileInfo, fileOwners: fileOwners, + mergeImages: exclusive(mergeImages), dedupeImages: exclusive(dedupeImages), mergeSet: exclusive(mergeSet), dedupeSet: exclusive(dedupeSet), scanPaths: scanPaths, + }; + } + + var api = { createOps: createOps, PLUGIN_ID: PLUGIN_ID }; + if (typeof module !== "undefined" && module.exports) module.exports = api; + root.ImageMatcher = root.ImageMatcher || {}; + root.ImageMatcher.ops = api; +})(typeof globalThis !== "undefined" ? globalThis : this); diff --git a/plugins/image-matcher/imui.js b/plugins/image-matcher/imui.js new file mode 100644 index 00000000..52595a0a --- /dev/null +++ b/plugins/image-matcher/imui.js @@ -0,0 +1,614 @@ +// Image Matcher - the page. Every look-alike group and every folder pair ends in an action: +// merge (one image, all files kept), delete the lesser copies, or "not a match". +// License: AGPL-3.0-or-later. +(function () { + "use strict"; + var PluginApi = window.PluginApi; + if (!PluginApi || !window.ImageMatcher || !window.ImageMatcher.core || !window.ImageMatcher.ops) return; + + var React = PluginApi.React, h = React.createElement; + var core = window.ImageMatcher.core; + // not under /plugin/: Stash serves plugin assets there, so a refresh of the page would 404 + var ROUTE = "/image-matcher"; + var LEVELS = [[0, "Exact"], [4, "High"], [8, "Medium"], [10, "Low"]]; + var SETS_PER_PAGE = 8, GROUPS_PER_PAGE = 20; + + function url(path) { return new URL(path, document.baseURI).toString(); } + function gql(query, variables) { + return fetch(url("graphql"), { + method: "POST", credentials: "same-origin", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ query: query, variables: variables }), + }).then(function (r) { return r.json(); }).then(function (j) { + if (j.errors && j.errors.length) throw new Error(j.errors.map(function (e) { return e.message; }).join("; ")); + return j.data; + }); + } + var ops = window.ImageMatcher.ops.createOps(gql, core); + + // thumbnail zoom is a per-browser convenience, so it lives in localStorage, not the plugin settings + var ZOOM_KEY = "imageMatcherZoom", ZOOM_MIN = 1, ZOOM_MAX = 4; + function readZoom() { + try { var z = Number(window.localStorage.getItem(ZOOM_KEY)); return z >= ZOOM_MIN && z <= ZOOM_MAX ? z : 1; } catch (err) { return 1; } + } + function writeZoom(z) { try { window.localStorage.setItem(ZOOM_KEY, String(z)); } catch (err) { /* private window: zoom just isn't remembered */ } } + + // Thumbnail check: draw the thumbnail small and measure it (in colour or not, and a brightness + // grid for comparing two look-alikes). Results are kept per image id; a few at a time. + var colorCache = {}, colorQueue = [], colorActive = 0; + function pumpColor() { + while (colorActive < 4 && colorQueue.length) { + colorActive++; + colorQueue.shift()(); + } + } + function colorOf(id) { + if (!colorCache[id]) colorCache[id] = new Promise(function (resolve) { + colorQueue.push(function () { + var img = new Image(), done = function (v) { colorActive--; resolve(v); pumpColor(); }; + img.onload = function () { + try { + var n = core.DIFF_N, c = document.createElement("canvas"); c.width = n; c.height = n; + var ctx = c.getContext("2d", { willReadFrequently: true }); ctx.drawImage(img, 0, 0, n, n); + var px = ctx.getImageData(0, 0, n, n).data; + done({ color: core.isColor(px), lum: core.luminance(px) }); + } catch (err) { done(null); } + }; + img.onerror = function () { done(null); }; + img.src = url("image/" + id + "/thumbnail"); + }); + pumpColor(); + }); + return colorCache[id]; + } + + // the comparison of two items, or null while either thumbnail is still being read (or could not be) + function diffOf(a, b) { return a.lum && b.lum ? core.localDiff(a.lum, b.lum) : null; } + function checking(list) { return list.filter(function (it) { return it.lum === undefined || it.lum === 0; }).length; } + var SPOT_TEXT = "differs in one area", DIFFERENT_TEXT = "looks like a different picture"; + function diffText(d) { return d.different ? DIFFERENT_TEXT : SPOT_TEXT; } + + function fmtSize(b) { + if (b >= 1048576) return (b / 1048576).toFixed(1) + " MB"; + if (b >= 1024) return Math.round(b / 1024) + " KB"; + return b + " B"; + } + function plural(n, one, many) { return n + " " + (n === 1 ? one : many || one + "s"); } + + // The picture sits in a box of its own shape inside the thumbnail frame, so a marker placed by + // fractions of the picture lands on the right spot at any zoom. + function Thumb(props) { + var it = props.item, box = props.box, pad = 0.03; + return h("a", { href: url("images/" + it.id), target: "_blank", rel: "noreferrer", className: "imt-thumb" + (props.small ? " imt-thumb-sm" : ""), title: "Open image " + it.id }, + h("span", { className: "imt-fit", style: { "--imt-ar": it.width && it.height ? it.width / it.height : 1 } }, + h("img", { src: url("image/" + it.id + "/thumbnail"), loading: "lazy", alt: "" }), + box ? h("span", { className: "imt-spot", style: { + left: Math.max(0, box.x - pad) * 100 + "%", top: Math.max(0, box.y - pad) * 100 + "%", + width: Math.min(1, box.w + pad * 2) * 100 + "%", height: Math.min(1, box.h + pad * 2) * 100 + "%" } }) : null)); + } + + // The guide: open on a first visit, closed once the reader has closed it (remembered per browser). + var HELP_KEY = "imageMatcherHelpSeen"; + function helpSeen() { try { return window.localStorage.getItem(HELP_KEY) === "1"; } catch (err) { return false; } } + var HELP = [ + ["What it finds", [ + "Images whose pictures look the same, using the perceptual hash Stash already stores. It finds resized and recompressed copies well. It mostly misses crops, rotations and mirror images.", + "Accuracy sets how close two hashes must be. Exact means identical hashes. High, Medium and Low allow 4, 8 and 10 bits of difference, and each step finds more copies and more false matches.", + "Images with no perceptual hash are skipped. Create them in Settings, Tasks, Generate, with image perceptual hashes ticked.", + ]], + ["The two tabs", [ + "Images lists each group of look-alikes on its own card.", + "Folder sets lists pairs of folders that share two or more look-alikes, so the same set saved in two places can be handled in one go. A folder set is a guess. Two folders of unrelated pictures can share look-alikes by chance. Use Show all to look at the pairs before acting on a set, and leave folders of loose pictures out under Folder rules.", + ]], + ["What the buttons do", [ + "Merge turns the look-alikes into one image in Stash. No file is deleted or moved. The kept copy is the one shown, and the image appears in each folder's gallery.", + "Delete keeps one copy and deletes the other files from disk for good. Tags, ratings and details move to the kept copy first.", + "Not a match hides the group or set from now on.", + "Leave out drops one pair or one image from the merge or delete. Put back undoes it.", + "There is no undo for a merge or a delete. One runs at a time.", + ]], + ["Which copy is kept", [ + "The green Keep badge marks the copy that stays, with the reason. The order is: a folder rule, then colour over black and white, then the larger resolution, then the larger file. Choose another with Keep this one, or the other folder on a set card.", + "Folder rules is an ordered list of folders that always win. The first one listed wins.", + ]], + ["The labels", [ + "Same hash: the two pictures hash identically. The files can still differ in size or quality.", + "N bits off: the hashes differ by that much. The larger the number, the more reason to look.", + "Differs in one area: the two thumbnails differ strongly in one place, marked with a red box. This is typical of two drawings with one changed detail. These are left out until you put them back. A very small or faint change can go unnoticed.", + "Looks like a different picture: the thumbnails differ nearly everywhere. Also left out.", + "Colour / B&W: one copy is in colour and the other is not.", + ]], + ["What is left out", [ + "Images inside zip and cbz archives, because a file inside an archive cannot be merged or deleted on its own.", + "An image that already holds more than one file is never deleted by Delete. Merge it instead.", + ]], + ]; + function Help(props) { + return h("div", { className: "card imt-card imt-help" }, + HELP.map(function (sec) { + return h("div", { key: sec[0] }, h("h5", null, sec[0]), h("ul", null, sec[1].map(function (line, i) { return h("li", { key: i }, line); }))); + }), + h("div", { className: "imt-actions" }, h("button", { className: "btn btn-secondary btn-sm", onClick: props.onClose }, "Close"))); + } + + function Pager(props) { + var pages = Math.ceil(props.total / props.per); + if (pages <= 1) return null; + var from = props.page * props.per + 1, to = Math.min(props.total, (props.page + 1) * props.per); + return h("div", { className: "imt-pager" }, + h("button", { className: "btn btn-secondary btn-sm", disabled: props.page === 0, onClick: function () { props.onPage(props.page - 1); } }, "Previous"), + h("span", null, from + "–" + to + " of " + props.total), + h("button", { className: "btn btn-secondary btn-sm", disabled: props.page >= pages - 1, onClick: function () { props.onPage(props.page + 1); } }, "Next")); + } + + function Actions(props) { + // shared footer: the three ways a card can end, plus the delete confirmation + var s = React.useState(false), confirm = s[0], setConfirm = s[1]; + if (props.busy) return h("div", { className: "imt-busy" }, h("span", { className: "imt-spinner" }), props.busy + "…"); + return h("div", null, + props.error ? h("div", { className: "imt-error" }, props.error) : null, + confirm + ? h("div", { className: "imt-confirm" }, + h("p", null, props.deleteWarning), + h("button", { className: "btn btn-danger btn-sm", onClick: function () { setConfirm(false); props.onDelete(); } }, props.deleteConfirmLabel), + h("button", { className: "btn btn-secondary btn-sm", onClick: function () { setConfirm(false); } }, "Cancel")) + : h("div", { className: "imt-actions" }, + h("button", { className: "btn btn-primary btn-sm", disabled: !!props.disabled, onClick: props.onMerge, title: "One image in Stash. Every file stays where it is; the kept copy is the one shown." }, props.mergeLabel), + h("button", { className: "btn btn-outline-danger btn-sm", disabled: !!props.disabled, onClick: function () { setConfirm(true); } }, props.deleteLabel), + props.onDismiss ? h("button", { className: "btn btn-link btn-sm", onClick: props.onDismiss }, props.dismissLabel) : null, + props.extra || null)); + } + + function GroupCard(props) { + var items = props.items, members = props.group.map(function (i) { return items[i]; }); + React.useEffect(function () { props.needColors(props.group); }, [props.group]); + var suggestion = React.useMemo(function () { return core.chooseKeeper(members, props.rules); }, [props.group, props.rules, props.colorTick]); + // until a copy is picked by hand, the card follows the suggestion (which can change once colour is known) + var k = React.useState(null), pickedId = k[0], setKeeperId = k[1]; + var keeperId = pickedId || suggestion.keeper.id; + var mixedColor = members.some(function (m) { return m.color === true; }) && members.some(function (m) { return m.color === false; }); + // the page runs one operation at a time and holds its progress, so a card that is redrawn + // mid-run still shows it, and every other card is locked meanwhile + var planKey = core.groupKey(members.map(function (m) { return m.id; })); + var busy = props.op && props.op.key === "group:" + planKey ? props.op.text : null, locked = !!props.op && !busy; + var e = React.useState(null), error = e[0], setError = e[1]; + var keeper = members.filter(function (m) { return m.id === keeperId; })[0] || suggestion.keeper; + // image ids left out of the action: true/false once chosen by hand; until then, an image that + // differs from the kept copy in one area is left out + var x = React.useState({}), picked = x[0], setOut = x[1]; + var diffs = {}, out = {}; + members.forEach(function (m) { + if (m === keeper) return; + diffs[m.id] = diffOf(keeper, m); + out[m.id] = picked[m.id] !== undefined ? picked[m.id] : !!(diffs[m.id] && diffs[m.id].flag); + }); + var waiting = checking(members); + var losers = members.filter(function (m) { return m !== keeper && !out[m.id]; }); + function toggleOut(id) { var next = Object.assign({}, picked); next[id] = !out[id]; setOut(next); } + + // tell the page what this card would do, for the "all on this page" buttons + React.useEffect(function () { + props.registerPlan(planKey, busy ? null : { waiting: waiting > 0, keeperId: keeper.id, + loserIds: losers.map(function (m) { return m.id; }), files: losers.reduce(function (t, m) { return t + m.files.length; }, 0) }); + }); + React.useEffect(function () { return function () { props.registerPlan(planKey, undefined); }; }, [planKey]); + + function run(mode) { + var plans = [{ keeperId: keeper.id, loserIds: losers.map(function (m) { return m.id; }) }]; + setError(null); + props.runOp("group:" + planKey, function (step) { return mode === "merge" ? ops.mergeImages(plans, step) : ops.dedupeImages(plans, step); }).then(function (res) { + props.onDone(res.plans || plans, mode, mode === "merge" + ? "Merged " + plural(res.merged, "file") + " into image " + keeper.id + "." + : "Deleted " + plural(res.deleted, "copy", "copies") + "; kept image " + keeper.id + "." + (res.skipped ? " Left " + plural(res.skipped, "image") + " alone because " + (res.skipped === 1 ? "it holds" : "they hold") + " more than one file; merge " + (res.skipped === 1 ? "it" : "those") + " instead." : "")); + }).catch(function (err) { setError(String(err.message || err)); }); + } + + return h("div", { className: "card imt-card" }, + h("div", { className: "imt-tiles" }, members.map(function (m) { + var isKeeper = m === keeper, dist = m.phash && keeper.phash ? core.hashDistance(m.phash, keeper.phash) : null; + return h("div", { key: m.id, className: "imt-tile" + (isKeeper ? " imt-keep" : "") + (!isKeeper && out[m.id] ? " imt-out" : "") }, + h(Thumb, { item: m, box: !isKeeper && diffs[m.id] && diffs[m.id].flag ? diffs[m.id].box : null }), + !isKeeper && diffs[m.id] && diffs[m.id].flag ? h("div", { className: "imt-diff" }, diffs[m.id].different ? DIFFERENT_TEXT + " than the kept copy" : SPOT_TEXT + " from the kept copy") : null, + h("div", { className: "imt-facts" }, + h("div", null, m.width + "×" + m.height + " · " + fmtSize(m.size) + (mixedColor ? (m.color === false ? " · black and white" : m.color === true ? " · colour" : "") : "")), + h("div", { className: "imt-path", title: m.folder }, m.folder), + h("div", { className: "imt-muted" }, + isKeeper ? "" : dist === 0 ? "same hash" : plural(dist, "bit") + " off", + m.files.length > 1 ? (isKeeper ? "" : " · ") + plural(m.files.length, "file") : "")), + isKeeper + ? h("span", { className: "imt-badge" }, "Keep" + (m === suggestion.keeper ? " · " + suggestion.reason : "")) + : h("div", { className: "imt-tile-btns" }, + h("button", { className: "btn btn-secondary btn-sm", disabled: !!busy, onClick: function () { setKeeperId(m.id); } }, "Keep this one"), + h("button", { className: "btn btn-link btn-sm", disabled: !!busy, onClick: function () { toggleOut(m.id); } }, out[m.id] ? "Put back" : "Leave out"))); + })), + h(Actions, { + busy: busy, error: error, disabled: losers.length === 0 || waiting > 0 || locked, + mergeLabel: waiting ? "Comparing pictures…" : "Merge into one", onMerge: function () { run("merge"); }, + deleteLabel: "Delete the other " + (losers.length === 1 ? "copy" : losers.length + " copies") + "…", + deleteWarning: "This deletes " + plural(losers.length, "image") + " and " + + plural(losers.reduce(function (t, m) { return t + m.files.length; }, 0), "file") + " from disk for good. Tags, ratings and details move to the kept copy first. There is no undo.", + deleteConfirmLabel: "Delete " + plural(losers.length, "copy", "copies"), onDelete: function () { run("dedupe"); }, + dismissLabel: "Not a match", onDismiss: function () { props.onDismiss(core.groupKey(members.map(function (m) { return m.id; }))); }, + })); + } + + function SetCard(props) { + var set = props.set, items = props.items; + React.useEffect(function () { + var all = []; + set.matches.forEach(function (m) { all = all.concat(m.a, m.b); }); + props.needColors(all); + }, [set]); + // until a side is picked by hand, the card follows the suggestion (which can change once colour is known) + var k = React.useState(null), pickedKeep = k[0], setKeep = k[1]; + var keep = pickedKeep || set.keep; + var r = React.useState(false), remember = r[0], setRemember = r[1]; + var busy = props.op && props.op.key === set.key ? props.op.text : null, locked = !!props.op && !busy; + var e = React.useState(null), error = e[0], setError = e[1]; + var keepFolder = keep === "A" ? set.folderA : set.folderB, otherFolder = keep === "A" ? set.folderB : set.folderA; + var otherTotal = keep === "A" ? set.countB : set.countA; + // pair keys left out of the action: true/false once chosen by hand; until then, a pair whose + // pictures differ in one area is left out + var x = React.useState({}), picked = x[0], setOut = x[1]; + function toggleOut(key) { var next = Object.assign({}, picked); next[key] = !out[key]; setOut(next); } + + function run(mode) { + var plans = core.setPlans(set, items, keep, out); + setError(null); + if (remember) props.onRemember(keepFolder, otherFolder); + props.runOp(set.key, function (step) { return mode === "merge" ? ops.mergeSet(plans, keepFolder, otherFolder, step) : ops.dedupeSet(plans, keepFolder, otherFolder, step); }).then(function (res) { + props.onDone(res.plans || plans, mode, mode === "merge" + ? "Merged " + plural(res.merged, "file") + " from " + otherFolder + " into " + plural(res.images, "image") + " kept in " + keepFolder + "." + : "Deleted " + plural(res.deleted, "copy", "copies") + " from " + otherFolder + "; kept " + keepFolder + "." + + (res.skipped ? " Left " + plural(res.skipped, "image") + " alone because " + (res.skipped === 1 ? "it holds" : "they hold") + " more than one file; merge " + (res.skipped === 1 ? "it" : "those") + " instead." : "")); + }).catch(function (err) { setError(String(err.message || err)); }); + } + + function side(which) { + var folder = which === "A" ? set.folderA : set.folderB, matched = which === "A" ? set.matchedA : set.matchedB, total = which === "A" ? set.countA : set.countB; + var only = Math.max(0, total - matched), id = "imt-" + set.key + which; + return h("label", { className: "imt-side" + (keep === which ? " imt-keep" : ""), htmlFor: id }, + h("input", { type: "radio", id: id, checked: keep === which, disabled: !!busy, onChange: function () { setKeep(which); } }), + h("div", null, + h("div", { className: "imt-path", title: folder }, folder), + h("div", { className: "imt-muted" }, matched + " of " + plural(total, "image") + " matched" + (only ? " · " + only + " only here" : "")), + keep === which ? h("span", { className: "imt-badge" }, "Keep" + (set.keep === which ? " · " + set.reason : "")) : null, + h("button", { className: "btn btn-link btn-sm imt-skip", disabled: !!busy, title: "Stop comparing this folder as a set. Its subfolders are still compared, and its images still show on the Images tab. Undo under Folder rules.", + onClick: function (ev) { ev.preventDefault(); ev.stopPropagation(); props.onSkipFolder(folder); } }, "Leave this folder out of sets"))); + } + + // every pair with how far apart its hashes are, furthest first, so the ones worth a look lead. + // A copy: set.matches itself is read by setPlans and must keep its order. + var sa = React.useState(false), showAll = sa[0], setShowAll = sa[1]; + var pairs = React.useMemo(function () { + return set.matches.map(function (m, i) { + var cols = m.a.concat(m.b).map(function (x) { return items[x].color; }); + // the clearest local difference between any picture on one side and any on the other + var diff = null; + m.a.forEach(function (ia) { m.b.forEach(function (ib) { + var d = diffOf(items[ia], items[ib]); + if (d && (!diff || (d.flag && !diff.flag) || (d.flag === diff.flag && d.peak > diff.peak))) diff = d; + }); }); + return { m: m, i: i, key: core.pairKey(items, m), dist: core.matchDistance(items, m), diff: diff, + mixed: cols.indexOf(true) >= 0 && cols.indexOf(false) >= 0 }; + }) + .sort(function (p, q) { return (q.diff && q.diff.flag ? 1 : 0) - (p.diff && p.diff.flag ? 1 : 0) || (q.dist || 0) - (p.dist || 0) || p.i - q.i; }); + }, [set, items, props.colorTick]); + var out = {}, spots = 0, others = 0; + pairs.forEach(function (p) { + var spot = !!(p.diff && p.diff.flag); + if (spot) spots++; + if (spot && p.diff.different) others++; + out[p.key] = picked[p.key] !== undefined ? picked[p.key] : spot; + }); + var waiting = 0; + set.matches.forEach(function (m) { waiting += checking(m.a.concat(m.b).map(function (x) { return items[x]; })); }); + var included = pairs.filter(function (p) { return !out[p.key]; }), leftOut = pairs.length - included.length; + var otherMatched = included.reduce(function (t, p) { return t + (keep === "A" ? p.m.b.length : p.m.a.length); }, 0); + var strays = Math.max(0, otherTotal - otherMatched); + var mix = core.distanceMix(included.map(function (p) { return p.dist; })); + var PREVIEW = 8, visible = showAll ? pairs : pairs.slice(0, PREVIEW); + var mixText = included.length === 0 ? "Every pair is left out" : (mix.near === 0 && mix.unknown === 0 + ? (included.length === 1 ? "The pair has" : included.length === 2 ? "Both pairs have" : "All " + included.length + " pairs have") + " the same hash" + : [mix.same ? mix.same + " same hash" : null, + mix.near ? mix.near + " within " + plural(mix.max, "bit") : null, + mix.unknown ? mix.unknown + " not measured" : null].filter(Boolean).join(" · ")) + + (leftOut && included.length ? " · " + leftOut + " left out" : "") + + (spots - others ? " · " + (spots - others) + " " + (spots - others === 1 ? "differs" : "differ") + " in one area" : "") + + (others ? " · " + others + " " + (others === 1 ? "looks like a different picture" : "look like different pictures") : "") + + (waiting ? " · comparing pictures…" : ""); + return h("div", { className: "card imt-card" }, + h("div", { className: "imt-sides" }, side("A"), side("B")), + // a pair of folders that mostly do not match is probably not one set saved twice + Math.max(set.countA ? set.matchedA / set.countA : 0, set.countB ? set.matchedB / set.countB : 0) < 0.5 + ? h("div", { className: "imt-caution" }, "Most of each folder has no match in the other, so these may be unrelated folders that share a few look-alikes. " + + "Look at the pairs before merging or deleting, or leave a folder of loose pictures out of sets.") + : null, + h("div", { className: "imt-pairs" }, visible.map(function (p) { + // every image of the match is drawn: the kept folder's on top, the other folder's below, + // each of those with its own marker against the first kept one + var m = p.m, tops = keep === "A" ? m.a : m.b, bottoms = keep === "A" ? m.b : m.a; + var boxes = bottoms.map(function (i) { var d = diffOf(items[tops[0]], items[i]); return d && d.flag ? d.box : null; }); + var box = boxes.filter(Boolean)[0] || null; + return h("div", { key: p.key, className: "imt-pair" + (out[p.key] ? " imt-out" : ""), + "data-ids": items[m.a[0]].id + "," + items[m.b[0]].id, "data-peak": p.diff ? p.diff.peak.toFixed(1) : "", "data-base": p.diff ? p.diff.base.toFixed(1) : "", "data-hot": p.diff ? p.diff.hot : "" }, + h("div", { className: "imt-pair-row" }, tops.map(function (i, n) { return h(Thumb, { key: i, item: items[i], small: true, box: n === 0 ? box : null }); })), + h("div", { className: "imt-pair-row" }, bottoms.map(function (i, n) { return h(Thumb, { key: i, item: items[i], small: true, box: boxes[n] }); })), + p.diff && p.diff.flag ? h("div", { className: "imt-pair-dist imt-diff" }, diffText(p.diff)) : null, + tops.length + bottoms.length > 2 ? h("div", { className: "imt-pair-dist" }, tops.length + bottoms.length + " images") : null, + h("div", { className: "imt-pair-dist" + (p.dist ? " imt-pair-off" : "") }, p.dist === null ? "not measured" : p.dist === 0 ? "same hash" : plural(p.dist, "bit") + " off"), + p.mixed ? h("div", { className: "imt-pair-dist imt-pair-off" }, "colour / B&W") : null, + h("button", { className: "btn btn-link btn-sm imt-pair-out", disabled: !!busy, title: "Leave this pair out of the merge or delete", + onClick: function () { toggleOut(p.key); } }, out[p.key] ? "Put back" : "Leave out")); + }), pairs.length > PREVIEW + ? h("button", { className: "btn btn-link btn-sm imt-more", onClick: function () { setShowAll(!showAll); } }, + showAll ? "Show fewer" : "Show all " + pairs.length + " pairs") + : null), + h("div", { className: "imt-mix", title: "The hash describes how a picture looks. Two files with the same hash look alike; the files themselves can still differ in size or quality." }, mixText), + h(Actions, { + busy: busy, error: error, disabled: included.length === 0 || waiting > 0 || locked, + mergeLabel: "Merge set (" + included.length + ")", onMerge: function () { run("merge"); }, + deleteLabel: "Delete duplicates from the other folder…", + deleteWarning: "This deletes " + plural(otherMatched, "file") + " from " + otherFolder + " for good, so they leave that folder's gallery. " + + "Tags, ratings and gallery details move to " + keepFolder + " first. " + + (strays ? plural(strays, "other image") + " in " + otherFolder + " " + (strays === 1 ? "stays" : "stay") + ". " : "") + "There is no undo.", + deleteConfirmLabel: "Delete " + plural(otherMatched, "file"), onDelete: function () { run("dedupe"); }, + dismissLabel: "Not the same set", onDismiss: function () { props.onDismiss(set.key); }, + extra: h("label", { className: "imt-remember" }, + h("input", { type: "checkbox", checked: remember, onChange: function (ev) { setRemember(ev.target.checked); } }), + " Always prefer this folder"), + })); + } + + function RulesPanel(props) { + var t = React.useState(props.rules.join("\n")), text = t[0], setText = t[1]; + var sk = React.useState(props.skip.join("\n")), skipText = sk[0], setSkipText = sk[1]; + return h("div", { className: "card imt-card imt-rules" }, + h("h5", null, "Folders that win"), + h("p", { className: "imt-muted" }, "One folder per line. The first folder listed wins, and a folder covers its subfolders. " + + "When no rule applies, the larger resolution wins, then the larger file."), + h("textarea", { className: "form-control text-input", rows: 6, value: text, spellCheck: false, onChange: function (ev) { setText(ev.target.value); } }), + h("h5", null, "Folders left out of folder sets"), + h("p", { className: "imt-muted" }, "One folder per line. A folder listed here is never compared as a set, which suits a folder of loose, unrelated images such as a year folder. " + + "Its subfolders are still compared; end the line with /* to leave those out too. Its images still show on the Images tab."), + h("textarea", { className: "form-control text-input imt-skip-list", rows: 4, value: skipText, spellCheck: false, onChange: function (ev) { setSkipText(ev.target.value); } }), + props.found && props.found.length ? h("div", null, + h("p", { className: "imt-muted" }, "Or tick them here. These are the folders in the sets found, the ones in the most sets first."), + h("div", { className: "imt-found" }, props.found.map(function (f) { + var lines = core.normalizeSkips(skipText.split("\n")), on = lines.indexOf(f.folder) >= 0; + return h("label", { key: f.folder, className: "imt-found-row" }, + h("input", { type: "checkbox", checked: on, onChange: function () { + setSkipText((on ? lines.filter(function (l) { return l !== f.folder; }) : lines.concat([f.folder])).join("\n")); + } }), + h("span", { className: "imt-path" }, f.folder), + h("span", { className: "imt-muted" }, "in " + plural(f.sets, "set"))); + }))) : null, + h("div", { className: "imt-actions" }, + h("button", { className: "btn btn-primary btn-sm", onClick: function () { props.onSave(text.split("\n"), skipText.split("\n")); } }, "Save"), + h("button", { className: "btn btn-secondary btn-sm", onClick: props.onClose }, "Close"))); + } + + function Page() { + var c = React.useState(null), cfg = c[0], setCfg = c[1]; + var p = React.useState({ status: "idle" }), phase = p[0], setPhase = p[1]; + var d = React.useState(null), data = d[0], setData = d[1]; + var t = React.useState("sets"), tab = t[0], setTab = t[1]; + var pg = React.useState(0), page = pg[0], setPage = pg[1]; + var sr = React.useState(false), showRules = sr[0], setShowRules = sr[1]; + var m = React.useState(null), message = m[0], setMessage = m[1]; + var z = React.useState(readZoom), zoom = z[0], setZoom = z[1]; + var hp = React.useState(function () { return !helpSeen(); }), showHelp = hp[0], setShowHelp = hp[1]; + function toggleHelp() { + if (showHelp) { try { window.localStorage.setItem(HELP_KEY, "1"); } catch (err) { /* stays open next visit */ } } + setShowHelp(!showHelp); + } + var ct = React.useState(0), colorTick = ct[0], setColorTick = ct[1]; + var tickTimer = React.useRef(null), itemsRef = React.useRef(null); + itemsRef.current = data ? data.items : null; + // cards ask for the colour check of the images they show; answers land on the items and the + // suggestions are worked out again (batched, so a card of many pairs redraws a few times, not hundreds) + function needColors(indices) { + var items = itemsRef.current; + if (!items) return; + indices.forEach(function (i) { + var it = items[i]; + if (!it || it.lum !== undefined) return; + it.lum = 0; // 0: being read; null: could not be read + colorOf(it.id).then(function (v) { + it.color = v ? v.color : null; it.lum = v ? v.lum : null; + if (tickTimer.current) return; + tickTimer.current = setTimeout(function () { tickTimer.current = null; setColorTick(function (n) { return n + 1; }); }, 150); + }); + }); + } + + React.useEffect(function () { + ops.loadConfig().then(setCfg).catch(function (err) { setPhase({ status: "error", text: String(err.message || err) }); }); + }, []); + + function saveCfg(next) { + setCfg(next); + ops.saveConfig(next).catch(function (err) { setMessage("Could not save settings: " + (err.message || err)); }); + } + + function find() { + setMessage(null); setPage(0); + setPhase({ status: "loading", text: "Reading the image list" }); + ops.loadImages(function (done, total) { setPhase({ status: "loading", text: "Reading the image list: " + done + " of " + total }); }).then(function (items) { + setPhase({ status: "loading", text: "Comparing " + items.length + " images" }); + setTimeout(function () { + var res = core.findGroups(items, cfg.distance); + setData({ items: items, groups: res.groups, skipped: res.skipped, archived: items.archived || 0 }); + setPhase({ status: "ready" }); + }, 30); + }).catch(function (err) { setPhase({ status: "error", text: String(err.message || err) }); }); + } + + var view = React.useMemo(function () { + if (!data || !cfg) return null; + var dismissed = {}; + cfg.dismissed.forEach(function (key) { dismissed[key] = true; }); + var groups = data.groups.filter(function (g) { return !dismissed[core.groupKey(g.map(function (i) { return data.items[i].id; }))]; }); + var sets = core.buildSets(data.items, groups, cfg.folderRules, 2, cfg.setSkip).filter(function (s) { return !dismissed[s.key]; }); + // every folder that is in a set when nothing is left out, for the tick list under Folder rules + var counts = {}; + core.buildSets(data.items, groups, cfg.folderRules, 2).forEach(function (s) { + if (dismissed[s.key]) return; + counts[s.folderA] = (counts[s.folderA] || 0) + 1; counts[s.folderB] = (counts[s.folderB] || 0) + 1; + }); + var found = Object.keys(counts).map(function (f) { return { folder: f, sets: counts[f] }; }) + .sort(function (p, q) { return q.sets - p.sets || (p.folder < q.folder ? -1 : 1); }); + return { groups: groups, sets: sets, found: found }; + }, [data, cfg, colorTick]); + + // Works from the list as it is when the operation ends, not as it was when it began. + function onDone(plans, mode, text) { + setData(function (d) { + if (!d) return d; + var gone = {}, byId = {}; + d.items.forEach(function (it) { byId[it.id] = it; }); + plans.forEach(function (pl) { + var keeper = byId[pl.keeperId]; + pl.loserIds.forEach(function (id) { + gone[id] = true; + // after a merge the kept image holds the other image's files too + if (mode === "merge" && keeper && byId[id]) { + keeper.files = keeper.files.concat(byId[id].files); + (byId[id].folders || []).forEach(function (f) { if (keeper.folders.indexOf(f) < 0) keeper.folders.push(f); }); + } + }); + }); + var groups = d.groups.map(function (g) { return g.filter(function (i) { return !gone[d.items[i].id]; }); }).filter(function (g) { return g.length > 1; }); + return { items: d.items, groups: groups, skipped: d.skipped, archived: d.archived }; + }); + setMessage(text); + } + // one merge or delete at a time, page-wide + var o = React.useState(null), op = o[0], setOp = o[1], opRef = React.useRef(null); + function runOp(key, start) { + if (opRef.current) return Promise.reject(new Error("Another merge or delete is still running. Wait for it to finish.")); + opRef.current = key; setOp({ key: key, text: "Starting" }); + var done = function () { opRef.current = null; setOp(null); }; + var p; + try { p = Promise.resolve(start(function (t) { if (opRef.current === key) setOp({ key: key, text: t }); })); } catch (err) { p = Promise.reject(err); } + return p.then(function (r) { done(); return r; }, function (err) { done(); throw err; }); + } + function onDismiss(key) { saveCfg(Object.assign({}, cfg, { dismissed: cfg.dismissed.concat([key]) })); } + function onRemember(keepFolder, otherFolder) { + saveCfg(Object.assign({}, cfg, { folderRules: core.rememberRule(cfg.folderRules, keepFolder, otherFolder) })); + } + function switchTab(next) { setTab(next); setPage(0); } + function onSkipFolder(folder) { + saveCfg(Object.assign({}, cfg, { setSkip: core.normalizeSkips(cfg.setSkip.concat([folder])) })); + setMessage(folder + " is left out of folder sets from now on. Undo under Folder rules."); + } + + // every group card on screen reports its plan here; the bar under the list acts on all of them + var pagePlans = React.useRef({}); + var pt = React.useState(0), setPlanTick = pt[1]; + var bulkBusy = op && op.key === "bulk" ? op.text : null; + var be = React.useState(null), bulkError = be[0], setBulkError = be[1]; + function registerPlan(key, plan) { + var was = JSON.stringify(pagePlans.current[key]), now = JSON.stringify(plan); + if (was === now) return; + if (plan === undefined) delete pagePlans.current[key]; else pagePlans.current[key] = plan; + setPlanTick(function (n) { return n + 1; }); + } + function runAll(mode, plans) { + setBulkError(null); + runOp("bulk", function (step) { return mode === "merge" ? ops.mergeImages(plans, step) : ops.dedupeImages(plans, step); }).then(function (res) { + var did = res.plans || plans; + onDone(did, mode, mode === "merge" + ? "Merged " + plural(res.merged, "file") + " into " + plural(did.length, "image") + " on this page." + : "Deleted " + plural(res.deleted, "copy", "copies") + " on this page; kept " + plural(did.length, "image") + "." + (res.skipped ? " Left " + plural(res.skipped, "image") + " alone because " + (res.skipped === 1 ? "it holds" : "they hold") + " more than one file; merge " + (res.skipped === 1 ? "it" : "those") + " instead." : "")); + goPage(page); + }).catch(function (err) { setBulkError(String(err.message || err)); }); + } + // a new page of cards starts at the top, whichever element Stash scrolls + function goPage(n) { + setPage(n); + try { + window.scrollTo(0, 0); + for (var el = document.querySelector(".imt-page"); el; el = el.parentElement) el.scrollTop = 0; + } catch (err) { /* scrolling is a convenience */ } + } + + if (!cfg) return h("div", { className: "imt-page" }, phase.status === "error" ? h("div", { className: "imt-error" }, phase.text) : "Loading…"); + + var list = view ? (tab === "sets" ? view.sets : view.groups) : []; + var per = tab === "sets" ? SETS_PER_PAGE : GROUPS_PER_PAGE; + var safePage = Math.min(page, Math.max(0, Math.ceil(list.length / per) - 1)); + var shown = list.slice(safePage * per, (safePage + 1) * per); + + return h("div", { className: "imt-page", style: { "--imt-zoom": zoom } }, + h("h3", null, "Image Matcher"), + h("p", { className: "imt-muted" }, "Finds images that look the same. Merge keeps every file and shows them as one image; delete removes the lesser copies from disk. Neither can be undone. ", + h("button", { className: "btn btn-link btn-sm imt-help-link", onClick: toggleHelp }, showHelp ? "Hide the guide" : "How this works")), + showHelp ? h(Help, { onClose: toggleHelp }) : null, + h("div", { className: "imt-toolbar" }, + h("label", null, "Accuracy ", + h("select", { className: "form-control input-control", value: cfg.distance, disabled: phase.status === "loading", + onChange: function (ev) { saveCfg(Object.assign({}, cfg, { distance: Number(ev.target.value) })); } }, + LEVELS.map(function (l) { return h("option", { key: l[0], value: l[0] }, l[1]); }))), + h("button", { className: "btn btn-primary", disabled: phase.status === "loading" || !!op, onClick: find }, data ? "Find matches again" : "Find matches"), + h("button", { className: "btn btn-secondary", onClick: function () { setShowRules(!showRules); } }, + "Folder rules" + (cfg.folderRules.length + cfg.setSkip.length ? " (" + (cfg.folderRules.length + cfg.setSkip.length) + ")" : "")), + h("label", { className: "imt-zoom" }, "Zoom", + h("input", { type: "range", min: ZOOM_MIN, max: ZOOM_MAX, step: 0.25, value: zoom, title: "Thumbnail size", + onChange: function (ev) { var v = Number(ev.target.value); setZoom(v); writeZoom(v); } }))), + showRules ? h(RulesPanel, { key: cfg.setSkip.join("\n"), rules: cfg.folderRules, skip: cfg.setSkip, found: view ? view.found : null, onClose: function () { setShowRules(false); }, + onSave: function (rules, skip) { saveCfg(Object.assign({}, cfg, { folderRules: core.normalizeRules(rules), setSkip: core.normalizeSkips(skip) })); setShowRules(false); } }) : null, + phase.status === "loading" ? h("div", { className: "imt-busy" }, h("span", { className: "imt-spinner" }), phase.text + "…") : null, + phase.status === "error" ? h("div", { className: "imt-error" }, phase.text) : null, + message ? h("div", { className: "imt-message" }, message) : null, + view ? h("div", null, + h("p", { className: "imt-summary" }, + plural(data.items.length, "image") + " checked" + + (data.archived ? " · " + data.archived + " inside archives (cbz, zip) left out" : "") + + (data.skipped ? " · " + data.skipped + " skipped (no perceptual hash yet; run Generate with image perceptual hashes on)" : "")), + h("div", { className: "imt-tabs" }, + h("button", { className: "btn btn-sm " + (tab === "sets" ? "btn-primary" : "btn-secondary"), onClick: function () { switchTab("sets"); } }, "Folder sets (" + view.sets.length + ")"), + h("button", { className: "btn btn-sm " + (tab === "images" ? "btn-primary" : "btn-secondary"), onClick: function () { switchTab("images"); } }, "Images (" + view.groups.length + ")")), + list.length === 0 ? h("p", { className: "imt-muted" }, tab === "sets" ? "No two folders share more than one look-alike image." : "No look-alike images left at this accuracy.") : null, + shown.map(function (entry) { + return tab === "sets" + ? h(SetCard, { key: entry.key, op: op, runOp: runOp, onSkipFolder: onSkipFolder, needColors: needColors, colorTick: colorTick, set: entry, items: data.items, onDone: onDone, onDismiss: onDismiss, onRemember: onRemember }) + : h(GroupCard, { key: core.groupKey(entry.map(function (i) { return data.items[i].id; })) + "|" + cfg.folderRules.join("\n"), op: op, runOp: runOp, registerPlan: registerPlan, needColors: needColors, colorTick: colorTick, group: entry, items: data.items, rules: cfg.folderRules, onDone: onDone, onDismiss: onDismiss }); + }), + tab === "images" && shown.length > 1 ? (function () { + // only the groups listed right now: a card that has just left the page may not have signed off yet + var all = shown.map(function (g) { return pagePlans.current[core.groupKey(g.map(function (i) { return data.items[i].id; }))]; }).filter(Boolean); + if (all.length < shown.length) all.push({ waiting: true, loserIds: [], files: 0 }); + var waiting = all.some(function (pl) { return pl.waiting; }); + var plans = all.filter(function (pl) { return pl.loserIds.length > 0; }).map(function (pl) { return { keeperId: pl.keeperId, loserIds: pl.loserIds }; }); + var images = plans.reduce(function (t, pl) { return t + pl.loserIds.length; }, 0); + var files = all.reduce(function (t, pl) { return t + (pl.loserIds.length ? pl.files : 0); }, 0); + return h("div", { className: "card imt-card imt-bulk" }, + h("div", { className: "imt-mix" }, waiting ? "Comparing pictures…" + : "Everything on this page, as the cards above are set: " + plural(plans.length, "group") + ", " + plural(images, "copy", "copies") + ". Images left out stay as they are."), + h(Actions, { + busy: bulkBusy, error: bulkError, disabled: waiting || plans.length === 0 || (!!op && !bulkBusy), + mergeLabel: "Merge all " + plans.length + " on this page", onMerge: function () { runAll("merge", plans); }, + deleteLabel: "Delete the other copies in all " + plans.length + "…", + deleteWarning: "This deletes " + plural(images, "image") + " and " + plural(files, "file") + " from disk for good, across " + plural(plans.length, "group") + + " on this page. Tags, ratings and details move to the kept copies first. There is no undo.", + deleteConfirmLabel: "Delete " + plural(images, "copy", "copies"), onDelete: function () { runAll("dedupe", plans); }, + })); + })() : null, + h(Pager, { page: safePage, per: per, total: list.length, onPage: goPage })) : null); + } + + PluginApi.register.route(ROUTE, Page); + + // a button in the top bar; wrapped so a change in Stash's nav can never break the bar itself + try { + PluginApi.patch.before("MainNavBar.UtilityItems", function (props) { + try { + var NavLink = PluginApi.libraries.ReactRouterDOM.NavLink, Button = PluginApi.libraries.Bootstrap.Button; + var Icon = PluginApi.components.Icon, faClone = PluginApi.libraries.FontAwesomeSolid.faClone; + return [{ children: h(React.Fragment, null, props.children, + h(NavLink, { className: "nav-utility", exact: true, to: ROUTE }, + h(Button, { className: "minimal d-flex align-items-center h-100", title: "Image Matcher" }, h(Icon, { icon: faClone })))) }]; + } catch (err) { return [props]; } + }); + } catch (err) { /* older Stash without this patch point: the page is still at /image-matcher */ } +})(); From f9f47db8fd1cf72baf29e5dd692f3e56aef81094 Mon Sep 17 00:00:00 2001 From: Speck Pratt Date: Sun, 15 Nov 2026 22:40:24 -0500 Subject: [PATCH 2/2] Image Matcher 0.11.0: long file names, multi-file delete, results on screen - Stash deletes a file by renaming it to .delete first (stashapp/stash#3664), so a name over 248 bytes cannot be deleted and one such file failed the whole delete. Delete now leaves those copies alone, labels them beforehand, and points to merge. - Delete works on an image that holds several files; its card lists every folder they are in, and warnings and results give the true file count. - Results show in a bar fixed to the bottom of the screen, so they are seen wherever the button was. - Choosing another copy to keep no longer leaves the old kept copy out. Co-Authored-By: Claude Opus 5.5 --- plugins/image-matcher/README.md | 6 +- plugins/image-matcher/image-matcher.css | 7 ++ plugins/image-matcher/image-matcher.yml | 2 +- plugins/image-matcher/imcore.js | 25 ++++- plugins/image-matcher/imops.js | 29 ++++-- plugins/image-matcher/imui.js | 116 ++++++++++++++++++------ 6 files changed, 141 insertions(+), 44 deletions(-) diff --git a/plugins/image-matcher/README.md b/plugins/image-matcher/README.md index 09dd6d4b..a66c83a6 100644 --- a/plugins/image-matcher/README.md +++ b/plugins/image-matcher/README.md @@ -44,18 +44,18 @@ Real content hashes (md5, oshash) are never touched. The `imagematch` stamp stay ## Limits - One merge or delete runs at a time; the other cards are locked until it finishes. -- A delete never removes an image that holds more than one file (one that was merged); it is left alone and counted, and merging is the right action for it. +- Deleting an image deletes every file it holds. A copy that holds several files (one that was merged) lists every folder they are in, and the delete warning names those folders and gives the true file count. +- Stash deletes a file by first renaming it to `.delete` ([stashapp/stash#3664](https://github.com/stashapp/stash/issues/3664)), so a file whose name is over 248 bytes cannot be deleted. Delete leaves those copies alone and labels them; merge them instead. - **No undo.** Stash has no "split image" call. A merge can only be reversed by hand; a delete cannot be reversed at all. - Matching uses the phash Stash already stores. It finds resized and recompressed copies well; crops, rotations and mirrors mostly not. - Groups are chains: if A looks like B and B looks like C, all three are shown together even when A and C are further apart. - Folder and zip galleries cannot be edited in Stash, so "merging galleries" means their details (tags, performers, rating, and so on) are synced; their contents converge because the merged images belong to both folders. -- In a folder-set delete, an image that also holds files in other folders is left alone; merge it instead. - Very loose accuracy on very large libraries is slow. At *High* a 200,000-image library compares in well under a second; at *Low* a 34,000-image library takes about a third of a second on a desktop. - `custom_fields` are not merged. ## How it was tested -- Unit tests of the matching and decision logic (17), end-to-end tests of merge, delete and folder sets against a throwaway Stash in Docker (12), and a scripted browser pass. All three pass on Stash v0.31.1 (release) and v0.32.0-12 (development image). +- Unit tests of the matching and decision logic (18), end-to-end tests of merge, delete and folder sets against a throwaway Stash in Docker (13), and a scripted browser pass. All three pass on Stash v0.31.1 (release) and v0.32.0-12 (development image). - Used by its author on a real library. - The test files are kept with the development copy, not shipped here, so they are not installed with the plugin. They can be added to this folder if that is preferred. diff --git a/plugins/image-matcher/image-matcher.css b/plugins/image-matcher/image-matcher.css index 88d464e4..056b013c 100644 --- a/plugins/image-matcher/image-matcher.css +++ b/plugins/image-matcher/image-matcher.css @@ -9,6 +9,11 @@ .imt-tabs { display: flex; gap: 0.5rem; margin: 0.75rem 0 1rem; } .imt-card { padding: 1rem; margin-bottom: 1rem; } .imt-message { padding: 0.5rem 0.75rem; margin: 0.5rem 0; border-left: 3px solid #4caf50; background: rgba(76, 175, 80, 0.12); } +.imt-toast { position: fixed; z-index: 1050; left: 50%; bottom: calc(1rem + env(safe-area-inset-bottom, 0px)); transform: translateX(-50%); width: min(40rem, calc(100vw - 2rem)); + display: flex; align-items: flex-start; gap: 0.25rem; border-radius: 4px; background: #202b33; box-shadow: 0 4px 16px rgba(0, 0, 0, 0.6); } +.imt-toast .imt-message { flex: 1; margin: 0; background: rgba(76, 175, 80, 0.18); } +.imt-toast-close { font-size: 1.25rem; line-height: 1; padding: 0.4rem 0.6rem; } +.imt-folders div:first-child { opacity: 0.7; } .imt-error { padding: 0.5rem 0.75rem; margin: 0.5rem 0; border-left: 3px solid #db3737; background: rgba(219, 55, 55, 0.12); word-break: break-word; } .imt-busy { display: flex; align-items: center; gap: 0.5rem; padding: 0.5rem 0; } .imt-spinner { width: 1rem; height: 1rem; border: 2px solid currentColor; border-right-color: transparent; border-radius: 50%; animation: imt-spin 0.8s linear infinite; } @@ -22,6 +27,7 @@ .imt-fit img { display: block; width: 100%; height: 100%; object-fit: contain; } .imt-spot { position: absolute; box-sizing: border-box; border: 2px solid #ff5252; border-radius: 3px; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.7); pointer-events: none; } .imt-diff { font-size: 0.8rem; color: #ff8a80; } +.imt-long { font-size: 0.8rem; color: #ffb74d; } .imt-pair-dist.imt-diff { opacity: 1; font-size: 0.75rem; } .imt-out .imt-pair-dist.imt-diff, .imt-out .imt-diff { text-decoration: none; opacity: 1; } .imt-thumb-sm { height: calc(72px * var(--imt-zoom)); width: calc(96px * var(--imt-zoom)); max-width: 100%; } @@ -72,4 +78,5 @@ .imt-thumb { height: calc(120px * var(--imt-zoom)); } .imt-zoom { margin-left: 0 !important; } .imt-remember { margin-left: 0; } + .imt-toast { bottom: calc(4.5rem + env(safe-area-inset-bottom, 0px)); } } diff --git a/plugins/image-matcher/image-matcher.yml b/plugins/image-matcher/image-matcher.yml index 0f0a6414..05e5a1e5 100644 --- a/plugins/image-matcher/image-matcher.yml +++ b/plugins/image-matcher/image-matcher.yml @@ -1,6 +1,6 @@ name: Image Matcher description: Finds images that look the same (perceptual hash). Merge them into one image with every file kept, or delete the lesser copies, one group or one folder pair at a time. -version: 0.9.0 +version: 0.11.0 url: https://github.com/stashapp/CommunityScripts/tree/main/plugins/image-matcher ui: javascript: diff --git a/plugins/image-matcher/imcore.js b/plugins/image-matcher/imcore.js index 283d696f..ab7ead8b 100644 --- a/plugins/image-matcher/imcore.js +++ b/plugins/image-matcher/imcore.js @@ -12,6 +12,26 @@ return i === 0 ? p.slice(0, 1) : p.slice(0, i); } + // Stash deletes a file by first renaming it to ".delete" (stashapp/stash#3664), so a file + // whose name is within 7 bytes of the usual 255-byte limit cannot be deleted through Stash. + var NAME_MAX = 255, DELETE_SUFFIX = ".delete"; + function utf8Length(s) { + var n = 0; + for (var i = 0; i < s.length; i++) { + var c = s.charCodeAt(i); + if (c < 0x80) n += 1; + else if (c < 0x800) n += 2; + else if (c >= 0xd800 && c < 0xdc00 && i + 1 < s.length) { n += 4; i++; } + else n += 3; + } + return n; + } + function tooLongToDelete(p) { + p = p || ""; + var name = p.slice(Math.max(p.lastIndexOf("/"), p.lastIndexOf("\\")) + 1); + return name.length > 0 && utf8Length(name) + DELETE_SUFFIX.length > NAME_MAX; + } + function popcount32(v) { v = v - ((v >>> 1) & 0x55555555); v = (v & 0x33333333) + ((v >>> 2) & 0x33333333); @@ -42,7 +62,7 @@ return { id: String(f.id), path: f.path || "", size: Number(f.size) || 0, width: f.width || 0, height: f.height || 0, phash: fingerprint(f, "phash"), - archived: !!(f.zip_file && f.zip_file.id), + archived: !!(f.zip_file && f.zip_file.id), longName: tooLongToDelete(f.path), }; }); var primary = files[0] || { path: "", size: 0, width: 0, height: 0 }; @@ -55,6 +75,7 @@ area: primary.width * primary.height, width: primary.width, height: primary.height, size: primary.size, phash: hash, archived: files.some(function (f) { return f.archived; }), // any file inside a cbz or zip + longName: files.some(function (f) { return f.longName; }), // a file Stash cannot delete }; } @@ -441,7 +462,7 @@ var api = { STAMP: STAMP, dirname: dirname, popcount32: popcount32, parseHash: parseHash, hashDistance: hashDistance, - fingerprint: fingerprint, toItem: toItem, findGroups: findGroups, bandKey: bandKey, + fingerprint: fingerprint, toItem: toItem, tooLongToDelete: tooLongToDelete, findGroups: findGroups, bandKey: bandKey, normalizeRules: normalizeRules, folderRank: folderRank, chooseKeeper: chooseKeeper, compareItems: compareItems, buildSets: buildSets, matchDistance: matchDistance, isColor: isColor, rememberRule: rememberRule, folderRank: folderRank, folderSkipped: folderSkipped, normalizeSkips: normalizeSkips, localDiff: localDiff, luminance: luminance, DIFF_N: DIFF_N, splitMatch: splitMatch, pairKey: pairKey, distanceMix: distanceMix, setKey: setKey, groupKey: groupKey, setPlans: setPlans, mergeMeta: mergeMeta, }; diff --git a/plugins/image-matcher/imops.js b/plugins/image-matcher/imops.js index 23cb10b9..a53f1691 100644 --- a/plugins/image-matcher/imops.js +++ b/plugins/image-matcher/imops.js @@ -308,11 +308,13 @@ } // ---- dedupe: keep one copy, delete the others from disk ---- - // Deleting an image deletes all of its files. A lesser image holding more than one file (one - // that was merged, here or elsewhere) is left alone and counted in `skipped`; merging is the - // right action for it. Works from what Stash says now, not from what the page loaded earlier. + // Deleting an image deletes all of its files, wherever they are: a lesser image that holds + // several files (one that was merged) loses every one of them, and `files` counts them. A + // lesser image with a file whose name is too long for Stash to delete (core.tooLongToDelete) + // is left alone and counted in `tooLong`: one such file would fail the whole delete call. + // Works from what Stash says now, not from what the page loaded earlier. function dedupeImages(plans, onStep) { - var step = onStep || function () {}, meta, skipped = 0; + var step = onStep || function () {}, meta, tooLong = 0; return Promise.resolve().then(function () { checkPlans(plans); return imageMeta(allIds(plans)); @@ -320,12 +322,12 @@ meta = m; plans = plans.map(function (p) { return { keeperId: p.keeperId, loserIds: p.loserIds.filter(function (id) { - if (meta[id].visual_files.length > 1) { skipped++; return false; } + if (meta[id].visual_files.some(function (f) { return core.tooLongToDelete(f.path); })) { tooLong++; return false; } return true; }) }; }).filter(function (p) { return p.loserIds.length; }); - if (!plans.length) return { deleted: 0, images: 0, skipped: skipped, plans: plans }; - return dedupeNow(plans, meta, step).then(function (res) { res.skipped = skipped; res.plans = plans; return res; }); + if (!plans.length) return { deleted: 0, files: 0, images: 0, tooLong: tooLong, plans: plans }; + return dedupeNow(plans, meta, step).then(function (res) { res.tooLong = tooLong; res.plans = plans; return res; }); }); } function dedupeNow(plans, meta, step) { @@ -336,7 +338,16 @@ step("Deleting the lesser copies"); var ids = plans.reduce(function (a, p) { return a.concat(p.loserIds); }, []); return gql("mutation($ids: [ID!]!) { imagesDestroy(input: { ids: $ids, delete_file: true, delete_generated: true }) }", { ids: ids }) - .then(function () { return { deleted: ids.length, images: plans.length }; }); + .then(function () { + var files = ids.reduce(function (t, id) { return t + meta[id].visual_files.length; }, 0); + return { deleted: ids.length, files: files, images: plans.length }; + }, function (err) { + // the check above should catch these; this covers a limit lower than 255 bytes + if (/file name too long/i.test(String(err && err.message))) { + throw new Error("Stash could not delete a file because its name is too long (Stash adds \".delete\" to the name while deleting). Merge instead: merging keeps every file, so nothing is renamed."); + } + throw err; + }); }); } @@ -387,7 +398,7 @@ otherG = folderGalleryIds(meta, otherFolder); return mode === "merge" ? mergeImages(plans, onStep) : dedupeImages(plans, onStep); }).then(function (res) { - res.skipped = res.skipped || 0; res.plans = res.plans || plans; + res.plans = res.plans || plans; if (onStep) onStep("Syncing gallery details"); return syncGalleries(keepG, otherG, mode === "merge").then(function (n) { res.galleriesUpdated = n; return res; }); }); diff --git a/plugins/image-matcher/imui.js b/plugins/image-matcher/imui.js index 52595a0a..5acbe1f8 100644 --- a/plugins/image-matcher/imui.js +++ b/plugins/image-matcher/imui.js @@ -73,6 +73,31 @@ return b + " B"; } function plural(n, one, many) { return n + " " + (n === 1 ? one : many || one + "s"); } + // what a delete left alone, and why; merging is the way out + function leftAlone(res) { + var t = ""; + if (res.tooLong) t += " Left " + plural(res.tooLong, "image") + " alone because " + (res.tooLong === 1 ? "its file name is" : "their file names are") + " too long for Stash to delete; merge " + (res.tooLong === 1 ? "it" : "those") + " instead."; + return t; + } + var LONG_TEXT = "Name too long for Stash to delete; merge instead"; + // every folder an image's files are in, with a count where a folder holds more than one + function folderList(m) { + var counts = {}, order = []; + m.files.forEach(function (f) { var d = core.dirname(f.path); if (!counts[d]) { counts[d] = 0; order.push(d); } counts[d]++; }); + return order.map(function (d) { return counts[d] > 1 ? d + " (" + counts[d] + " files)" : d; }); + } + // the folders of the images that hold several files, for a delete warning; "" when there are none + function severalFiles(images, except) { + var multi = images.filter(function (m) { return m.files.length > 1; }), dirs = []; + multi.forEach(function (m) { m.files.forEach(function (f) { var d = core.dirname(f.path); if (d !== except && dirs.indexOf(d) < 0) dirs.push(d); }); }); + if (!multi.length) return ""; + return (multi.length === 1 ? "One of these images holds" : plural(multi.length, "of these images", "of these images") + " hold") + " several files; all of them go" + + (dirs.length ? ", including files in " + dirs.join(", ") : "") + ". "; + } + // "Deleted 2 copies (3 files)" when merged images took extra files with them + function deletedText(res, where) { + return "Deleted " + plural(res.deleted, "copy", "copies") + (res.files > res.deleted ? " (" + plural(res.files, "file") + ")" : "") + where; + } // The picture sits in a box of its own shape inside the thumbnail frame, so a marker placed by // fractions of the picture lands on the right spot at any zoom. @@ -102,6 +127,7 @@ ["What the buttons do", [ "Merge turns the look-alikes into one image in Stash. No file is deleted or moved. The kept copy is the one shown, and the image appears in each folder's gallery.", "Delete keeps one copy and deletes the other files from disk for good. Tags, ratings and details move to the kept copy first.", + "A copy that holds several files (one merged earlier) lists every folder they are in. Delete removes all of them, and the warning names those folders.", "Not a match hides the group or set from now on.", "Leave out drops one pair or one image from the merge or delete. Put back undoes it.", "There is no undo for a merge or a delete. One runs at a time.", @@ -119,7 +145,7 @@ ]], ["What is left out", [ "Images inside zip and cbz archives, because a file inside an archive cannot be merged or deleted on its own.", - "An image that already holds more than one file is never deleted by Delete. Merge it instead.", + "A copy whose file name is too long for Stash to delete (over 248 bytes) is left out of Delete and labelled. Merge it instead.", ]], ]; function Help(props) { @@ -153,7 +179,7 @@ h("button", { className: "btn btn-secondary btn-sm", onClick: function () { setConfirm(false); } }, "Cancel")) : h("div", { className: "imt-actions" }, h("button", { className: "btn btn-primary btn-sm", disabled: !!props.disabled, onClick: props.onMerge, title: "One image in Stash. Every file stays where it is; the kept copy is the one shown." }, props.mergeLabel), - h("button", { className: "btn btn-outline-danger btn-sm", disabled: !!props.disabled, onClick: function () { setConfirm(true); } }, props.deleteLabel), + h("button", { className: "btn btn-outline-danger btn-sm", disabled: !!props.disabled || !!props.deleteDisabled, title: props.deleteTitle, onClick: function () { setConfirm(true); } }, props.deleteLabel), props.onDismiss ? h("button", { className: "btn btn-link btn-sm", onClick: props.onDismiss }, props.dismissLabel) : null, props.extra || null)); } @@ -183,12 +209,23 @@ }); var waiting = checking(members); var losers = members.filter(function (m) { return m !== keeper && !out[m.id]; }); + var deletable = losers.filter(function (m) { return !m.longName; }); function toggleOut(id) { var next = Object.assign({}, picked); next[id] = !out[id]; setOut(next); } + // choosing another copy to keep changes nothing else: every copy keeps its current in or out, + // and the copy that was kept joins the action. Without this, the old kept copy is compared + // against the new one, and a one-area difference leaves it out again. + function pickKeeper(id) { + var next = {}; + members.forEach(function (o) { if (o.id !== id) next[o.id] = o === keeper ? false : !!out[o.id]; }); + setOut(next); + setKeeperId(id); + } // tell the page what this card would do, for the "all on this page" buttons React.useEffect(function () { props.registerPlan(planKey, busy ? null : { waiting: waiting > 0, keeperId: keeper.id, - loserIds: losers.map(function (m) { return m.id; }), files: losers.reduce(function (t, m) { return t + m.files.length; }, 0) }); + loserIds: losers.map(function (m) { return m.id; }), deletable: deletable.map(function (m) { return m.id; }), + files: deletable.reduce(function (t, m) { return t + m.files.length; }, 0) }); }); React.useEffect(function () { return function () { props.registerPlan(planKey, undefined); }; }, [planKey]); @@ -198,7 +235,7 @@ props.runOp("group:" + planKey, function (step) { return mode === "merge" ? ops.mergeImages(plans, step) : ops.dedupeImages(plans, step); }).then(function (res) { props.onDone(res.plans || plans, mode, mode === "merge" ? "Merged " + plural(res.merged, "file") + " into image " + keeper.id + "." - : "Deleted " + plural(res.deleted, "copy", "copies") + "; kept image " + keeper.id + "." + (res.skipped ? " Left " + plural(res.skipped, "image") + " alone because " + (res.skipped === 1 ? "it holds" : "they hold") + " more than one file; merge " + (res.skipped === 1 ? "it" : "those") + " instead." : "")); + : deletedText(res, "; kept image " + keeper.id + ".") + leftAlone(res)); }).catch(function (err) { setError(String(err.message || err)); }); } @@ -208,25 +245,31 @@ return h("div", { key: m.id, className: "imt-tile" + (isKeeper ? " imt-keep" : "") + (!isKeeper && out[m.id] ? " imt-out" : "") }, h(Thumb, { item: m, box: !isKeeper && diffs[m.id] && diffs[m.id].flag ? diffs[m.id].box : null }), !isKeeper && diffs[m.id] && diffs[m.id].flag ? h("div", { className: "imt-diff" }, diffs[m.id].different ? DIFFERENT_TEXT + " than the kept copy" : SPOT_TEXT + " from the kept copy") : null, + !isKeeper && m.longName ? h("div", { className: "imt-long" }, LONG_TEXT) : null, h("div", { className: "imt-facts" }, h("div", null, m.width + "×" + m.height + " · " + fmtSize(m.size) + (mixedColor ? (m.color === false ? " · black and white" : m.color === true ? " · colour" : "") : "")), - h("div", { className: "imt-path", title: m.folder }, m.folder), - h("div", { className: "imt-muted" }, - isKeeper ? "" : dist === 0 ? "same hash" : plural(dist, "bit") + " off", - m.files.length > 1 ? (isKeeper ? "" : " · ") + plural(m.files.length, "file") : "")), + m.files.length > 1 + ? h("div", { className: "imt-path imt-folders" }, h("div", null, plural(m.files.length, "file") + " in:"), + folderList(m).map(function (t) { return h("div", { key: t }, t); })) + : h("div", { className: "imt-path", title: m.folder }, m.folder), + isKeeper ? null : h("div", { className: "imt-muted" }, dist === 0 ? "same hash" : plural(dist, "bit") + " off")), isKeeper ? h("span", { className: "imt-badge" }, "Keep" + (m === suggestion.keeper ? " · " + suggestion.reason : "")) : h("div", { className: "imt-tile-btns" }, - h("button", { className: "btn btn-secondary btn-sm", disabled: !!busy, onClick: function () { setKeeperId(m.id); } }, "Keep this one"), + h("button", { className: "btn btn-secondary btn-sm", disabled: !!busy, onClick: function () { pickKeeper(m.id); } }, "Keep this one"), h("button", { className: "btn btn-link btn-sm", disabled: !!busy, onClick: function () { toggleOut(m.id); } }, out[m.id] ? "Put back" : "Leave out"))); })), h(Actions, { busy: busy, error: error, disabled: losers.length === 0 || waiting > 0 || locked, mergeLabel: waiting ? "Comparing pictures…" : "Merge into one", onMerge: function () { run("merge"); }, - deleteLabel: "Delete the other " + (losers.length === 1 ? "copy" : losers.length + " copies") + "…", - deleteWarning: "This deletes " + plural(losers.length, "image") + " and " + - plural(losers.reduce(function (t, m) { return t + m.files.length; }, 0), "file") + " from disk for good. Tags, ratings and details move to the kept copy first. There is no undo.", - deleteConfirmLabel: "Delete " + plural(losers.length, "copy", "copies"), onDelete: function () { run("dedupe"); }, + deleteDisabled: deletable.length === 0, + deleteTitle: deletable.length === 0 && losers.length ? LONG_TEXT : undefined, + deleteLabel: losers.length === 0 ? "Delete the other copies…" : deletable.length === 0 ? "Can't delete: name too long" : "Delete the other " + (deletable.length === 1 ? "copy" : deletable.length + " copies") + "…", + deleteWarning: "This deletes " + plural(deletable.length, "image") + " and " + + plural(deletable.reduce(function (t, m) { return t + m.files.length; }, 0), "file") + " from disk for good. " + severalFiles(deletable) + "Tags, ratings and details move to the kept copy first. " + + (deletable.length < losers.length ? plural(losers.length - deletable.length, "copy", "copies") + " with a name too long for Stash to delete " + (losers.length - deletable.length === 1 ? "stays" : "stay") + "; merge to deal with " + (losers.length - deletable.length === 1 ? "it" : "them") + ". " : "") + + "There is no undo.", + deleteConfirmLabel: "Delete " + plural(deletable.length, "copy", "copies"), onDelete: function () { run("dedupe"); }, dismissLabel: "Not a match", onDismiss: function () { props.onDismiss(core.groupKey(members.map(function (m) { return m.id; }))); }, })); } @@ -258,8 +301,7 @@ props.runOp(set.key, function (step) { return mode === "merge" ? ops.mergeSet(plans, keepFolder, otherFolder, step) : ops.dedupeSet(plans, keepFolder, otherFolder, step); }).then(function (res) { props.onDone(res.plans || plans, mode, mode === "merge" ? "Merged " + plural(res.merged, "file") + " from " + otherFolder + " into " + plural(res.images, "image") + " kept in " + keepFolder + "." - : "Deleted " + plural(res.deleted, "copy", "copies") + " from " + otherFolder + "; kept " + keepFolder + "." + - (res.skipped ? " Left " + plural(res.skipped, "image") + " alone because " + (res.skipped === 1 ? "it holds" : "they hold") + " more than one file; merge " + (res.skipped === 1 ? "it" : "those") + " instead." : "")); + : deletedText(res, " from " + otherFolder + "; kept " + keepFolder + ".") + leftAlone(res)); }).catch(function (err) { setError(String(err.message || err)); }); } @@ -303,8 +345,10 @@ var waiting = 0; set.matches.forEach(function (m) { waiting += checking(m.a.concat(m.b).map(function (x) { return items[x]; })); }); var included = pairs.filter(function (p) { return !out[p.key]; }), leftOut = pairs.length - included.length; - var otherMatched = included.reduce(function (t, p) { return t + (keep === "A" ? p.m.b.length : p.m.a.length); }, 0); - var strays = Math.max(0, otherTotal - otherMatched); + var otherImgs = [], longOnes = 0; + included.forEach(function (p) { (keep === "A" ? p.m.b : p.m.a).forEach(function (i) { if (items[i].longName) longOnes++; else otherImgs.push(items[i]); }); }); + var otherMatched = otherImgs.length, otherFiles = otherImgs.reduce(function (t, m) { return t + m.files.length; }, 0); + var strays = Math.max(0, otherTotal - otherMatched - longOnes); var mix = core.distanceMix(included.map(function (p) { return p.dist; })); var PREVIEW = 8, visible = showAll ? pairs : pairs.slice(0, PREVIEW); var mixText = included.length === 0 ? "Every pair is left out" : (mix.near === 0 && mix.unknown === 0 @@ -337,6 +381,10 @@ tops.length + bottoms.length > 2 ? h("div", { className: "imt-pair-dist" }, tops.length + bottoms.length + " images") : null, h("div", { className: "imt-pair-dist" + (p.dist ? " imt-pair-off" : "") }, p.dist === null ? "not measured" : p.dist === 0 ? "same hash" : plural(p.dist, "bit") + " off"), p.mixed ? h("div", { className: "imt-pair-dist imt-pair-off" }, "colour / B&W") : null, + bottoms.some(function (i) { return items[i].longName; }) ? h("div", { className: "imt-pair-dist imt-pair-off", title: LONG_TEXT }, "name too long to delete") : null, + bottoms.filter(function (i) { return items[i].files.length > 1; }).map(function (i) { + return h("div", { key: "f" + i, className: "imt-pair-dist imt-pair-off imt-pair-files", title: folderList(items[i]).join("\n") }, plural(items[i].files.length, "file") + " in " + plural(folderList(items[i]).length, "folder")); + }), h("button", { className: "btn btn-link btn-sm imt-pair-out", disabled: !!busy, title: "Leave this pair out of the merge or delete", onClick: function () { toggleOut(p.key); } }, out[p.key] ? "Put back" : "Leave out")); }), pairs.length > PREVIEW @@ -348,10 +396,12 @@ busy: busy, error: error, disabled: included.length === 0 || waiting > 0 || locked, mergeLabel: "Merge set (" + included.length + ")", onMerge: function () { run("merge"); }, deleteLabel: "Delete duplicates from the other folder…", - deleteWarning: "This deletes " + plural(otherMatched, "file") + " from " + otherFolder + " for good, so they leave that folder's gallery. " + - "Tags, ratings and gallery details move to " + keepFolder + " first. " + - (strays ? plural(strays, "other image") + " in " + otherFolder + " " + (strays === 1 ? "stays" : "stay") + ". " : "") + "There is no undo.", - deleteConfirmLabel: "Delete " + plural(otherMatched, "file"), onDelete: function () { run("dedupe"); }, + deleteWarning: "This deletes " + plural(otherMatched, "image") + (otherFiles > otherMatched ? " (" + plural(otherFiles, "file") + ")" : "") + " from " + otherFolder + " for good, so they leave that folder's gallery. " + + severalFiles(otherImgs, otherFolder) + "Tags, ratings and gallery details move to " + keepFolder + " first. " + + (strays ? plural(strays, "other image") + " in " + otherFolder + " " + (strays === 1 ? "stays" : "stay") + ". " : "") + + (longOnes ? plural(longOnes, "file") + " with a name too long for Stash to delete " + (longOnes === 1 ? "stays" : "stay") + "; merge to deal with " + (longOnes === 1 ? "it" : "them") + ". " : "") + "There is no undo.", + deleteDisabled: otherMatched === 0, + deleteConfirmLabel: "Delete " + plural(otherFiles, "file"), onDelete: function () { run("dedupe"); }, dismissLabel: "Not the same set", onDismiss: function () { props.onDismiss(set.key); }, extra: h("label", { className: "imt-remember" }, h("input", { type: "checkbox", checked: remember, onChange: function (ev) { setRemember(ev.target.checked); } }), @@ -486,7 +536,7 @@ var o = React.useState(null), op = o[0], setOp = o[1], opRef = React.useRef(null); function runOp(key, start) { if (opRef.current) return Promise.reject(new Error("Another merge or delete is still running. Wait for it to finish.")); - opRef.current = key; setOp({ key: key, text: "Starting" }); + opRef.current = key; setOp({ key: key, text: "Starting" }); setMessage(null); var done = function () { opRef.current = null; setOp(null); }; var p; try { p = Promise.resolve(start(function (t) { if (opRef.current === key) setOp({ key: key, text: t }); })); } catch (err) { p = Promise.reject(err); } @@ -519,7 +569,7 @@ var did = res.plans || plans; onDone(did, mode, mode === "merge" ? "Merged " + plural(res.merged, "file") + " into " + plural(did.length, "image") + " on this page." - : "Deleted " + plural(res.deleted, "copy", "copies") + " on this page; kept " + plural(did.length, "image") + "." + (res.skipped ? " Left " + plural(res.skipped, "image") + " alone because " + (res.skipped === 1 ? "it holds" : "they hold") + " more than one file; merge " + (res.skipped === 1 ? "it" : "those") + " instead." : "")); + : deletedText(res, " on this page; kept " + plural(did.length, "image") + ".") + leftAlone(res)); goPage(page); }).catch(function (err) { setBulkError(String(err.message || err)); }); } @@ -559,7 +609,10 @@ onSave: function (rules, skip) { saveCfg(Object.assign({}, cfg, { folderRules: core.normalizeRules(rules), setSkip: core.normalizeSkips(skip) })); setShowRules(false); } }) : null, phase.status === "loading" ? h("div", { className: "imt-busy" }, h("span", { className: "imt-spinner" }), phase.text + "…") : null, phase.status === "error" ? h("div", { className: "imt-error" }, phase.text) : null, - message ? h("div", { className: "imt-message" }, message) : null, + // results show in a bar fixed to the bottom of the screen, so they are seen wherever the button was + message ? h("div", { className: "imt-toast", role: "status" }, + h("div", { className: "imt-message" }, message), + h("button", { className: "btn btn-link btn-sm imt-toast-close", "aria-label": "Close", onClick: function () { setMessage(null); } }, "×")) : null, view ? h("div", null, h("p", { className: "imt-summary" }, plural(data.items.length, "image") + " checked" + @@ -577,11 +630,14 @@ tab === "images" && shown.length > 1 ? (function () { // only the groups listed right now: a card that has just left the page may not have signed off yet var all = shown.map(function (g) { return pagePlans.current[core.groupKey(g.map(function (i) { return data.items[i].id; }))]; }).filter(Boolean); - if (all.length < shown.length) all.push({ waiting: true, loserIds: [], files: 0 }); + if (all.length < shown.length) all.push({ waiting: true, loserIds: [], deletable: [], files: 0 }); var waiting = all.some(function (pl) { return pl.waiting; }); var plans = all.filter(function (pl) { return pl.loserIds.length > 0; }).map(function (pl) { return { keeperId: pl.keeperId, loserIds: pl.loserIds }; }); var images = plans.reduce(function (t, pl) { return t + pl.loserIds.length; }, 0); - var files = all.reduce(function (t, pl) { return t + (pl.loserIds.length ? pl.files : 0); }, 0); + var delImages = []; + all.forEach(function (pl) { pl.deletable.forEach(function (id) { delImages.push(data.items.filter(function (it) { return it.id === id; })[0]); }); }); + delImages = delImages.filter(Boolean); + var files = delImages.reduce(function (t, m) { return t + m.files.length; }, 0), stuck = images - delImages.length; return h("div", { className: "card imt-card imt-bulk" }, h("div", { className: "imt-mix" }, waiting ? "Comparing pictures…" : "Everything on this page, as the cards above are set: " + plural(plans.length, "group") + ", " + plural(images, "copy", "copies") + ". Images left out stay as they are."), @@ -589,9 +645,11 @@ busy: bulkBusy, error: bulkError, disabled: waiting || plans.length === 0 || (!!op && !bulkBusy), mergeLabel: "Merge all " + plans.length + " on this page", onMerge: function () { runAll("merge", plans); }, deleteLabel: "Delete the other copies in all " + plans.length + "…", - deleteWarning: "This deletes " + plural(images, "image") + " and " + plural(files, "file") + " from disk for good, across " + plural(plans.length, "group") + - " on this page. Tags, ratings and details move to the kept copies first. There is no undo.", - deleteConfirmLabel: "Delete " + plural(images, "copy", "copies"), onDelete: function () { runAll("dedupe", plans); }, + deleteDisabled: delImages.length === 0, + deleteWarning: "This deletes " + plural(delImages.length, "image") + " and " + plural(files, "file") + " from disk for good, across " + plural(plans.length, "group") + + " on this page. " + severalFiles(delImages) + "Tags, ratings and details move to the kept copies first. " + + (stuck ? plural(stuck, "copy", "copies") + " with a name too long for Stash to delete " + (stuck === 1 ? "stays" : "stay") + "; merge to deal with " + (stuck === 1 ? "it" : "them") + ". " : "") + "There is no undo.", + deleteConfirmLabel: "Delete " + plural(delImages.length, "copy", "copies"), onDelete: function () { runAll("dedupe", plans); }, })); })() : null, h(Pager, { page: safePage, per: per, total: list.length, onPage: goPage })) : null);