From 82bb3be87596f4ad61c7e1d817be348be2ecddd7 Mon Sep 17 00:00:00 2001 From: Raz Ben Aharon <181751257+razbenaharon@users.noreply.github.com> Date: Sat, 14 Nov 2026 20:00:09 +0200 Subject: [PATCH 01/23] Plain copy: fail runs that end early, heal type swaps, add Repair, report copy limits A robocopy log without totals is a failed run. Each selected folder has its own log. A file and a folder that swapped kinds heal by moving the old item to the dated recovery folder. -Repair copies again what the last content check found different. Short-name skips and case collisions are reported. The same-disk guard works in both directions. Tests run in a temporary folder (-TestRoot). Co-Authored-By: Claude Fable 5.2 --- Backup.MirrorVerifier.cs | 186 +++++++++++++++-- Backup.Selection.psm1 | 181 ++++++++++++++-- Invoke-FastMirror.ps1 | 394 +++++++++++++++++++++++++++-------- Test-BackupCopyRules.ps1 | 28 ++- Test-BackupMirror.ps1 | 440 +++++++++++++++++++++++++++++++++++++++ Test-BackupSelection.ps1 | 8 +- 6 files changed, 1119 insertions(+), 118 deletions(-) create mode 100644 Test-BackupMirror.ps1 diff --git a/Backup.MirrorVerifier.cs b/Backup.MirrorVerifier.cs index 8bf50fd..14a5283 100644 --- a/Backup.MirrorVerifier.cs +++ b/Backup.MirrorVerifier.cs @@ -1,50 +1,200 @@ // Content check of a plain copy (Invoke-FastMirror.ps1 -Verify and -VerifyOnly). Both sides are walked // with the rules the copy uses (per-folder mode, skipped names and paths, links never followed), so a // selected source file that is missing from the copy is a difference, not only a file whose content differs. -using System; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Security.Cryptography; using System.Collections.Concurrent; using System.Collections.Generic; using System.Text.RegularExpressions; +// The same rules serve the repair of single files (-Repair) and the look for names robocopy treats +// differently from what the lists say (Survey). +using System; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Security.Cryptography; using System.Collections.Concurrent; using System.Collections.Generic; using System.Text.RegularExpressions; using System.Runtime.InteropServices; public static class MirrorVerifier { static string Hash(string p){using(var s=new FileStream(p,FileMode.Open,FileAccess.Read,FileShare.ReadWrite|FileShare.Delete,1<<20))using(var h=SHA256.Create())return Convert.ToHexString(h.ComputeHash(s));} + static bool Same(string a,string b){return string.Equals(a,b,StringComparison.OrdinalIgnoreCase);} static bool Under(string path,string parent){return path.StartsWith(parent+"\\",StringComparison.OrdinalIgnoreCase);} + // Every path is used in its \\?\ form: Windows otherwise drops a dot or space at the end of a name + // (and then looks for another file), and stops at 260 characters. + static string Long(string p){p=Path.GetFullPath(p).TrimEnd('\\');return p.StartsWith("\\\\?\\")?p:p.StartsWith("\\\\")?"\\\\?\\UNC\\"+p.Substring(2):"\\\\?\\"+p;} + static string Plain(string text){return text.Replace("\\\\?\\UNC\\","\\\\").Replace("\\\\?\\","");} // Mirror mode of a relative path: the longest entry that equals or contains it, else the default. static bool IsOn(string rel,string[] paths,bool[] on,bool defaultOn){ int best=-1; - for(int i=0;ipaths[best].Length))best=i; + for(int i=0;ipaths[best].Length))best=i; return best<0?defaultOn:on[best]; } // robocopy wildcards in /XD and /XF names: * and ?, not case-sensitive. static Regex[] Globs(string[] names){return names.Select(n=>new Regex("^"+Regex.Escape(n).Replace("\\*",".*").Replace("\\?",".")+"$",RegexOptions.IgnoreCase|RegexOptions.CultureInvariant)).ToArray();} - sealed class Rules {public string[] Paths;public bool[] On;public bool DefaultOn;public Regex[] DirNames,FileNames;public string[] SkipPaths;} + sealed class Rules {public string[] Paths;public bool[] On;public bool DefaultOn;public Regex[] DirNames,FileNames;public string[] SkipPaths,Roots;} + static Rules Make(string[] paths,bool[] on,bool defaultOn,string[] skipDirNames,string[] skipFileNames,string[] skipPaths,string[] roots){ + return new Rules{Paths=paths,On=on,DefaultOn=defaultOn,DirNames=Globs(skipDirNames),FileNames=Globs(skipFileNames),SkipPaths=skipPaths.Select(s=>s.Trim('\\')).Where(s=>s.Length>0).ToArray(),Roots=roots.Select(s=>s.Trim('\\')).Where(s=>s.Length>0).ToArray()}; + } + // robocopy skips names and paths only below the folder it copies, never that folder itself. So a + // folder or file that was copied on its own (one of Roots) is checked whatever is skipped above it + // or by its own name, and a skipped folder is still entered, without checking its files, where + // such a root lies inside it. + // What the copy does with a folder. 0: left out. 1: read. 2: skipped, and entered only for a root inside it. + static int Folder(string rel,string name,bool skipped,Rules r){ + if(r.Roots.Any(x=>Same(x,rel)))return 1; + if(skipped||r.SkipPaths.Any(s=>Same(rel,s))||r.DirNames.Any(g=>g.IsMatch(name)))return r.Roots.Any(x=>Under(x,rel))?2:0; + // Unselected folders are entered only when a selected entry lies inside them. + return IsOn(rel,r.Paths,r.On,r.DefaultOn)||Enumerable.Range(0,r.Paths.Length).Any(i=>r.On[i]&&Under(r.Paths[i],rel))?1:0; + } + // Whether the copy takes a file. + static bool Taken(string rel,string name,bool skipped,Rules r){ + return (r.Roots.Any(x=>Same(x,rel))||(!skipped&&!r.SkipPaths.Any(s=>Same(rel,s))&&!r.FileNames.Any(g=>g.IsMatch(name))))&&IsOn(rel,r.Paths,r.On,r.DefaultOn); + } + // The same for one file by its path alone, folder by folder, without reading the disk. + static bool Selected(string rel,Rules r){ + var parts=rel.Split('\\');string d="";bool skipped=false; + for(int i=0;i Files(string root,Rules r,ConcurrentBag bad){ - var stack=new Stack();stack.Push(root); - while(stack.Count>0){var d=stack.Pop();FileSystemInfo[] items; - try{items=new DirectoryInfo(d).GetFileSystemInfos();}catch(Exception ex){bad.Add("ERROR\t"+d.Substring(root.Length).TrimStart('\\')+"\t"+ex.Message);continue;} + var stack=new Stack>();stack.Push(new KeyValuePair("",false)); + while(stack.Count>0){var top=stack.Pop();string d=top.Key;bool skipped=top.Value;FileSystemInfo[] items; + try{items=new DirectoryInfo(d.Length==0?root+"\\":root+"\\"+d+"\\").GetFileSystemInfos();}catch(Exception ex){bad.Add("ERROR\t"+d+"\t"+Plain(ex.Message));continue;} foreach(var e in items){ if((e.Attributes&FileAttributes.ReparsePoint)!=0)continue; - string rel=e.FullName.Substring(root.Length).TrimStart('\\'); - if(r.SkipPaths.Any(s=>string.Equals(rel,s,StringComparison.OrdinalIgnoreCase)||Under(rel,s)))continue; - if((e.Attributes&FileAttributes.Directory)!=0){ - if(r.DirNames.Any(g=>g.IsMatch(e.Name)))continue; - // Unselected folders are entered only when a selected entry lies inside them. - if(IsOn(rel,r.Paths,r.On,r.DefaultOn)||Enumerable.Range(0,r.Paths.Length).Any(i=>r.On[i]&&Under(r.Paths[i],rel)))stack.Push(e.FullName); - }else if(!r.FileNames.Any(g=>g.IsMatch(e.Name))&&IsOn(rel,r.Paths,r.On,r.DefaultOn))yield return rel;}} + string rel=d.Length==0?e.Name:d+"\\"+e.Name; + if((e.Attributes&FileAttributes.Directory)!=0){int k=Folder(rel,e.Name,skipped,r);if(k>0)stack.Push(new KeyValuePair(rel,k==2));} + else if(Taken(rel,e.Name,skipped,r))yield return rel;}} } public static string[] Compare(string mirror,string source,int threads,string[] paths,bool[] on,bool defaultOn,string[] skipDirNames,string[] skipFileNames,string[] skipPaths,out long files,out long bytes){ - var bad=new ConcurrentBag();long f=0,b=0; - var r=new Rules{Paths=paths,On=on,DefaultOn=defaultOn,DirNames=Globs(skipDirNames),FileNames=Globs(skipFileNames),SkipPaths=skipPaths.Select(s=>s.Trim('\\')).Where(s=>s.Length>0).ToArray()}; + return Compare(mirror,source,threads,paths,on,defaultOn,skipDirNames,skipFileNames,skipPaths,new string[0],out files,out bytes); + } + // roots: the selected folders and files that were copied on their own ("" is the source root). + public static string[] Compare(string mirror,string source,int threads,string[] paths,bool[] on,bool defaultOn,string[] skipDirNames,string[] skipFileNames,string[] skipPaths,string[] roots,out long files,out long bytes){ + var bad=new ConcurrentBag();long f=0,b=0;mirror=Long(mirror);source=Long(source); + var r=Make(paths,on,defaultOn,skipDirNames,skipFileNames,skipPaths,roots); // Every selected source file must be in the copy with the same content. Parallel.ForEach(Files(source,r,bad),new ParallelOptions{MaxDegreeOfParallelism=threads},rel=>{ - string s=Path.Combine(source,rel),m=Path.Combine(mirror,rel); + string s=source+"\\"+rel,m=mirror+"\\"+rel; try{ var si=new FileInfo(s);var mi=new FileInfo(m); if(!mi.Exists){bad.Add("MISSING-IN-MIRROR\t"+rel);return;} if(mi.Length!=si.Length){bad.Add("SIZE\t"+rel);return;} if(Hash(m)!=Hash(s)){bad.Add("CONTENT\t"+rel);return;} Interlocked.Increment(ref f);Interlocked.Add(ref b,mi.Length); - }catch(Exception ex){bad.Add("ERROR\t"+rel+"\t"+ex.Message);} + }catch(Exception ex){bad.Add("ERROR\t"+rel+"\t"+Plain(ex.Message));} }); // Files only in the copy (deleted from the source) are listed, not read: a copy that never deletes keeps them. - foreach(var rel in Files(mirror,r,bad))if(!File.Exists(Path.Combine(source,rel)))bad.Add("MISSING-IN-SOURCE\t"+rel); + foreach(var rel in Files(mirror,r,bad))if(!File.Exists(source+"\\"+rel))bad.Add("MISSING-IN-SOURCE\t"+rel); files=f;bytes=b;return bad.OrderBy(x=>x,StringComparer.Ordinal).ToArray(); } + + // --- Repair of single files (Invoke-FastMirror.ps1 -Repair) --------------------------------------- + // Each file named by an earlier content check is looked at again and, where the copy still differs + // from the source, copied again. Nothing is deleted or overwritten: the new copy is first written + // beside the old one and read back, and only when it matches the source by SHA-256 is the old version + // moved to the recovery folder and the new one given its name. So a file that cannot be read, copied + // or matched leaves the copy as it was. One line per file, with tabs between: what was done, the + // path, where the old version went (or nothing), and a note. + // REPAIRED copied again, and the copy now matches the source + // MATCHES the copy matches the source already; nothing was done + // SKIPPED not a file this copy takes any more (no longer selected, gone from the source) + // STILL-DIFFERENT the new copy did not match the source (the file is changing, or the drive fails) + // FAILED could not be read, copied or moved + public static string[] Repair(string mirror,string source,string recovery,int threads,string[] rels,string[] paths,bool[] on,bool defaultOn,string[] skipDirNames,string[] skipFileNames,string[] skipPaths,string[] roots){ + var done=new ConcurrentBag();mirror=Long(mirror);source=Long(source);recovery=Long(recovery); + var r=Make(paths,on,defaultOn,skipDirNames,skipFileNames,skipPaths,roots); + Parallel.ForEach(rels.Distinct(StringComparer.OrdinalIgnoreCase),new ParallelOptions{MaxDegreeOfParallelism=Math.Max(1,threads)},rel=>done.Add(RepairOne(mirror,source,recovery,rel,r))); + return done.OrderBy(x=>x,StringComparer.Ordinal).ToArray(); + } + // A link or junction on the way from root down to rel, rel itself included. The copy never follows one. + static bool Link(string root,string rel){ + string p=root; + foreach(var part in rel.Split('\\')){p+="\\"+part; + try{if((File.GetAttributes(p)&FileAttributes.ReparsePoint)!=0)return true;}catch(FileNotFoundException){return false;}catch(DirectoryNotFoundException){return false;}} + return false; + } + // A working file of the repair is the repair's own; nothing else is ever deleted. + static void Drop(string p){try{File.SetAttributes(p,FileAttributes.Normal);File.Delete(p);}catch{}} + static string RepairOne(string mirror,string source,string recovery,string rel,Rules r){ + string s=source+"\\"+rel,m=mirror+"\\"+rel,old=recovery+"\\"+rel,work=null,moved=""; + try{ + // The list comes from a file; a path that could lead out of the copy is never followed. + if(rel.Length==0||rel.IndexOfAny(new[]{':','/'})>=0||rel.Split('\\').Any(p=>p.Length==0||p=="."||p==".."))return "SKIPPED\t"+rel+"\t\tnot the path of a file in the copy"; + if(!Selected(rel,r))return "SKIPPED\t"+rel+"\t\tno longer selected for the plain copy"; + if(Link(source,rel))return "SKIPPED\t"+rel+"\t\tbehind a link or junction in the source, which is never copied"; + if(Directory.Exists(s))return "SKIPPED\t"+rel+"\t\ta folder in the source, not a file"; + if(!File.Exists(s))return "SKIPPED\t"+rel+"\t\tno longer in the source"; + if(Link(mirror,rel))return "FAILED\t"+rel+"\t\tthe copy has a link or junction on the way to this file"; + if(Directory.Exists(m))return "FAILED\t"+rel+"\t\tthe copy has a folder with this name; back up, which moves that folder aside"; + bool had=File.Exists(m); + // Looked at again now: the file may have been put right since the check. A copy that cannot be + // read is a damaged copy; a source that cannot be read fails the file (below). + if(had&&new FileInfo(m).Length==new FileInfo(s).Length){ + string hs=Hash(s),hm=null;try{hm=Hash(m);}catch(IOException){} + if(hm==hs)return "MATCHES\t"+rel+"\t\t"; + } + string dir=Path.GetDirectoryName(m);Directory.CreateDirectory(dir); + work=dir+"\\~repair-"+Guid.NewGuid().ToString("N").Substring(0,12)+".tmp"; + File.Copy(s,work,false); + try{File.SetCreationTimeUtc(work,File.GetCreationTimeUtc(s));}catch{} + if(Hash(work)!=Hash(s)){Drop(work);return "STILL-DIFFERENT\t"+rel+"\t\tthe new copy did not match the source: the file is changing, or the drive is failing";} + if(had){Directory.CreateDirectory(Path.GetDirectoryName(old));File.Move(m,old);moved=Plain(old);} + File.Move(work,m); + return "REPAIRED\t"+rel+"\t"+moved+"\t"; + }catch(Exception ex){ + if(work!=null)Drop(work); + // The old version had left its place and the new one could not take it: it is put back. + if(moved.Length>0&&!File.Exists(m)){try{File.Move(old,m);moved="";}catch{}} + return "FAILED\t"+rel+"\t"+moved+"\t"+Plain(ex.Message).Replace("\r"," ").Replace("\n"," ").Replace("\t"," "); + } + } + + // --- Names robocopy treats differently from what the lists say -------------------------------------- + [StructLayout(LayoutKind.Sequential,CharSet=CharSet.Unicode)] + struct Entry {public uint Attributes,C1,C2,A1,A2,W1,W2,SizeHigh,SizeLow,R0,R1;[MarshalAs(UnmanagedType.ByValTStr,SizeConst=260)]public string Name;[MarshalAs(UnmanagedType.ByValTStr,SizeConst=14)]public string Short;} + [DllImport("kernel32.dll",CharSet=CharSet.Unicode,SetLastError=true)]static extern IntPtr FindFirstFileW(string pattern,out Entry e); + [DllImport("kernel32.dll",CharSet=CharSet.Unicode,SetLastError=true)]static extern bool FindNextFileW(IntPtr h,out Entry e); + [DllImport("kernel32.dll")]static extern bool FindClose(IntPtr h); + // The entries of one folder with their short (8.3) names, which .NET does not give. A folder that + // cannot be read gives nothing here; the copy reports it. + static List List(string dir){ + var l=new List();Entry e;IntPtr h=FindFirstFileW(dir+"\\*",out e); + if(h==new IntPtr(-1))return l; + try{do{if(e.Name!="."&&e.Name!="..")l.Add(e);}while(FindNextFileW(h,out e));}finally{FindClose(h);} + return l; + } + // The short (8.3) name Windows gave a file or folder; empty when it has none. + public static string ShortName(string path){ + Entry e;IntPtr h=FindFirstFileW(Long(path),out e);if(h==new IntPtr(-1))return ""; + FindClose(h);return e.Short??""; + } + // Names in one folder that differ only by the case of their letters, each paired with the first of + // its kind (first, tab, other). A folder that keeps case apart can hold both; the copy holds one. + public static string[] SameNames(IEnumerable names){ + var first=new Dictionary(StringComparer.OrdinalIgnoreCase);var pairs=new List(); + foreach(var n in names){string f;if(!first.TryGetValue(n,out f))first[n]=n;else if(f!=n)pairs.Add(f+"\t"+n);} + return pairs.ToArray(); + } + // One pass over the names (no file is read) in the folders the copy takes, for two things robocopy + // does that the lists do not say: + // SHORT path, short name, name: a file or folder (ending in \) that robocopy skips because its + // short name fits one of shortNames (the wildcard names of the personal Exclude list), + // while its own name fits no skipped name. Nothing inside such a folder is copied. + // CASE path, path: two names in one folder that differ only by letter case. + public static string[] Survey(string source,string[] paths,bool[] on,bool defaultOn,string[] skipDirNames,string[] skipFileNames,string[] skipPaths,string[] roots,string[] shortNames){ + var found=new List();source=Long(source);var r=Make(paths,on,defaultOn,skipDirNames,skipFileNames,skipPaths,roots);var globs=Globs(shortNames); + Func fits=name=>{if(string.IsNullOrEmpty(name))return -1;for(int i=0;i>();stack.Push(new KeyValuePair("",false)); + while(stack.Count>0){var top=stack.Pop();string d=top.Key;bool skipped=top.Value;var names=new List(); + foreach(var e in List(d.Length==0?source:source+"\\"+d)){ + if((e.Attributes&(uint)FileAttributes.ReparsePoint)!=0)continue; + string rel=d.Length==0?e.Name:d+"\\"+e.Name;bool root=r.Roots.Any(x=>Same(x,rel));int i; + if((e.Attributes&(uint)FileAttributes.Directory)!=0){ + int k=Folder(rel,e.Name,skipped,r);if(k==0)continue; + // robocopy reads this folder as part of a copied one (a folder entered only to reach a + // selection inside it is not read by robocopy, and one copied on its own is never skipped). + if(k==1&&!root&&IsOn(rel,r.Paths,r.On,r.DefaultOn)){ + if((i=fits(e.Short))>=0){found.Add("SHORT\t"+rel+"\\\t"+e.Short+"\t"+shortNames[i]);k=r.Roots.Any(x=>Under(x,rel))?2:0;if(k==0)continue;} + else names.Add(e.Name); + } + stack.Push(new KeyValuePair(rel,k==2)); + }else if(Taken(rel,e.Name,skipped,r)){ + if(!root&&(i=fits(e.Short))>=0)found.Add("SHORT\t"+rel+"\t"+e.Short+"\t"+shortNames[i]);else names.Add(e.Name); + } + } + foreach(var pair in SameNames(names)){var p=pair.Split('\t');string pre=d.Length==0?"":d+"\\";found.Add("CASE\t"+pre+p[0]+"\t"+pre+p[1]);} + } + return found.OrderBy(x=>x,StringComparer.Ordinal).ToArray(); + } } diff --git a/Backup.Selection.psm1 b/Backup.Selection.psm1 index 907b0c8..94ec31e 100644 --- a/Backup.Selection.psm1 +++ b/Backup.Selection.psm1 @@ -20,9 +20,17 @@ function Test-BackupModeIncludes([string]$Mode,[ValidateSet('Mirror','Versioned' $Mode -eq 'Both' -or $Mode -eq $Tool } +function Test-SameName([string]$A,[string]$B) { + # Names are compared the way Windows compares them: character by character, in any case. PowerShell's + # own -eq and -in compare the way a dictionary sorts and take an invisible character (a direction + # mark, a zero-width space) or another spelling of an accented letter as no difference, so two + # folders Windows keeps apart counted as one, and a name made only of such a character as empty. + [string]::Equals($A,$B,[StringComparison]::OrdinalIgnoreCase) +} + function ConvertTo-SelectionPath([string]$Path) { $p=$Path.Replace('/','\').Trim().Trim('\') - if(-not $p -or [IO.Path]::IsPathRooted($Path) -or $p.Contains(':') -or @($p.Split('\')|Where-Object {$_ -in @('','.','..')}).Count){throw "Choice path must be relative to the source root: '$Path'"} + if($p.Length -eq 0 -or [IO.Path]::IsPathRooted($Path) -or $p.Contains(':') -or @($p.Split('\')|Where-Object {$_.Length -eq 0 -or $_.Equals('.') -or $_.Equals('..')}).Count){throw "Choice path must be relative to the source root: '$Path'"} $p } @@ -33,11 +41,11 @@ function Get-BackupSelection($Config) { $entries=[Collections.Generic.List[object]]::new() if($hasModes){ $given=if($Config.PSObject.Properties['DefaultMode']){[string]$Config.DefaultMode}else{'Skip'} - $default=$script:Modes|Where-Object {$_ -ieq $given.Trim()} + $default=$script:Modes|Where-Object {Test-SameName $_ $given.Trim()} if(-not $default){$default=$given} foreach($c in $choices){ $given=if($c.PSObject.Properties['Mode']){[string]$c.Mode}else{''} - $mode=$script:Modes|Where-Object {$_ -ieq $given.Trim()} + $mode=$script:Modes|Where-Object {Test-SameName $_ $given.Trim()} if(-not $mode){throw "Choice '$($c.Path)' needs Mode Mirror, Versioned, Both or Skip (found '$given')"} $entries.Add([pscustomobject]@{Path=(ConvertTo-SelectionPath $c.Path);Mode=$mode}) } @@ -47,10 +55,10 @@ function Get-BackupSelection($Config) { foreach($p in @(if($Config.PSObject.Properties['Include']){$Config.Include})){if($p){$entries.Add([pscustomobject]@{Path=(ConvertTo-SelectionPath $p);Mode='Versioned'})}} } if($default -cnotin $script:Modes){throw "DefaultMode must be Mirror, Versioned, Both or Skip (found '$default')"} - $seen=@{} - foreach($e in $entries){if($seen.ContainsKey($e.Path.ToLowerInvariant())){throw "Choice path listed twice: $($e.Path)"};$seen[$e.Path.ToLowerInvariant()]=$true} + $seen=[Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + foreach($e in $entries){if(-not $seen.Add($e.Path)){throw "Choice path listed twice: $($e.Path)"}} foreach($never in @(if($Config.PSObject.Properties['NeverBackUp']){$Config.NeverBackUp|Where-Object {$_}|ForEach-Object {ConvertTo-SelectionPath $_}})){ - $hit=@($entries|Where-Object {$_.Path -ieq $never}) + $hit=@($entries|Where-Object {Test-SameName $_.Path $never}) if($hit.Count -and $hit[0].Mode -ne 'Skip'){throw "$never is listed in NeverBackUp; set its Mode to Skip"} if(-not $hit.Count){$entries.Add([pscustomobject]@{Path=$never;Mode='Skip'})} # A choice inside a NeverBackUp folder cannot bring part of it back: it is skipped too. @@ -61,7 +69,7 @@ function Get-BackupSelection($Config) { function Test-IsUnder([string]$Path,[string]$Parent) { # True when Path is strictly inside Parent; '' is the source root. - if($Parent -eq ''){return $Path -ne ''} + if($Parent.Length -eq 0){return $Path.Length -gt 0} $Path.StartsWith($Parent+'\',[StringComparison]::OrdinalIgnoreCase) } @@ -78,7 +86,7 @@ function Test-BackupPathInside([string]$Path,[string[]]$Roots) { function Get-BackupMode($Selection,[string]$Path) { $p=if($Path){ConvertTo-SelectionPath $Path}else{''} $best=$null - foreach($e in $Selection.Entries){if(($e.Path -ieq $p -or (Test-IsUnder $p $e.Path)) -and (-not $best -or $e.Path.Length -gt $best.Path.Length)){$best=$e}} + foreach($e in $Selection.Entries){if(((Test-SameName $e.Path $p) -or (Test-IsUnder $p $e.Path)) -and (-not $best -or $e.Path.Length -gt $best.Path.Length)){$best=$e}} if($best){$best.Mode}else{$Selection.DefaultMode} } @@ -103,18 +111,22 @@ function Get-BackupRegions($Selection,[ValidateSet('Mirror','Versioned')][string } } -function Get-RoboExtraPaths([string[]]$Lines,[string]$Root) { +function Get-RoboExtraPaths([string[]]$Lines,[string]$Root,[string[]]$OtherRoots=@()) { # The paths under Root that a robocopy /L /PURGE listing reports as extra. With /MT several threads # write into one log, so a record is not always one line: its path can land on the next line on its # own, and one line can hold the end of one record and the start of the next. Every tab-separated # field is therefore read, and a field is split again wherever a new path under Root starts. + # A path of the other side of the copy can follow at once as well (...\source\a\bX:\Copy\c\d), so + # the listing is read with that side's root in OtherRoots, and a field is split there too; a full + # path with its drive letter cannot stand inside a name. Only the paths under Root are returned. # Error lines are skipped: they can name a destination folder that is not extra. $prefix=$Root.TrimEnd('\')+'\' - $split='(?='+[regex]::Escape($prefix)+')' + $starts=@($prefix)+@($OtherRoots|Where-Object {$_}|ForEach-Object {$_.TrimEnd('\')+'\'}) + $split=[regex]::new('(?='+(@($starts|ForEach-Object {[regex]::Escape($_)}) -join '|')+')',[Text.RegularExpressions.RegexOptions]'IgnoreCase,CultureInvariant') foreach($line in $Lines){ if($line -match '\sERROR \d+ '){continue} foreach($field in ($line -split "`t")){ - foreach($p in ($field -split $split)){ + foreach($p in $split.Split($field)){ $p=$p.Trim() if($p.Length -gt $prefix.Length -and $p.StartsWith($prefix,[StringComparison]::OrdinalIgnoreCase)){$p} } @@ -122,4 +134,149 @@ function Get-RoboExtraPaths([string[]]$Lines,[string]$Root) { } } -Export-ModuleMember -Function Get-BackupModeNames,Test-BackupModeIncludes,ConvertTo-SelectionPath,Get-BackupSelection,Get-BackupMode,Get-BackupRegions,Get-RoboExtraPaths,Test-BackupPathInside +function Get-RoboTotals([string]$Log) { + # The summary rows (directories, files, bytes) each hold six numbers: total, copied, skipped, + # mismatch, failed, extras. Reading the numbers rather than the labels also works on Windows + # in other languages. + $rows=@(Get-Content -LiteralPath $Log -Encoding Unicode -Tail 14|ForEach-Object { + $m=[regex]::Match($_,'^\s*[^\d\s][^:]*:\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s*$') + if($m.Success){,@(1..6|ForEach-Object {[long]$m.Groups[$_].Value})} + }) + if($rows.Count -lt 3){return $null} + [pscustomobject]@{Dirs=$rows[0];Files=$rows[1];Bytes=$rows[2]} +} + +function Get-RoboRunResult([string]$Log,[int]$ExitCode) { + # What one robocopy run left uncopied. NotCopied adds up the files and folders in its totals that + # failed or are a mismatch (a folder in the source where the copy has a file of that name, or the + # reverse: robocopy copies nothing there and still exits below 8). A log without totals means + # robocopy was ended or crashed before it finished, whatever exit code that left: ending it from + # Task Manager leaves 1, which otherwise means "files were copied". And an exit code that reports + # failures or a mismatch never ends as nothing uncopied, whatever the totals read say. + # Complete: the run read the whole source, so a file the copy has beyond it is really gone from it. + $t=if(Test-Path -LiteralPath $Log){Get-RoboTotals $Log}else{$null} + $n=[long]0 + if($t){$n=$t.Files[3]+$t.Files[4]+$t.Dirs[3]+$t.Dirs[4]} + if($n -eq 0 -and (-not $t -or $ExitCode -lt 0 -or $ExitCode -ge 8 -or ($ExitCode -band 4))){$n=1} + [pscustomobject]@{Totals=$t;NotCopied=$n;Complete=([bool]$t -and $ExitCode -ge 0 -and $ExitCode -lt 8)} +} + +function Get-MirrorLogName([string]$Relative,[Collections.Generic.HashSet[string]]$Used) { + # The log file of one selected folder ('' is the source root). Characters a file name cannot hold + # become _, so two folders (a\b and a_b) can end up with one name; the later one then gets a number, + # or its log would replace the other's and hide what that run could not copy. + $base='copy-'+$(if($Relative){$Relative -replace '[\\/:*?"<>|]','_'}else{'main'}) + $name=$base + for($n=2;-not $Used.Add($name+'.log');$n++){$name=$base+'-'+$n} + $name+'.log' +} + +function Get-RecoveryFoldersToRemove([string[]]$Names,[int]$Keep,[string]$Own='') { + # The dated recovery folders to delete so that only the newest Keep stay. Their names are local + # times (yyyyMMdd-HHmmss), so the order of the names is the order of their age, except after the + # clock went back (another time zone, the end of daylight saving, a corrected clock): the folder + # this run just made (Own) can then sort as an old one. It holds the files moved a moment ago, so + # it always stays, and counts as one of the Keep. + if($Keep -le 0){return} + $others=[string[]]@($Names|Where-Object {-not $Own -or -not $_.Equals($Own)}) + [Array]::Sort($others,[StringComparer]::Ordinal);[Array]::Reverse($others) + $others|Select-Object -Skip $(if($Own -and $others.Count -lt @($Names).Count){$Keep-1}else{$Keep}) +} + +function Move-BackupTypeSwap([string]$Source,[string]$Copy,[string]$Target) { + # Moves the item at Copy to Target when it stands in the way of the item at Source: one of the two + # is a folder and the other a file, so robocopy can copy nothing there. Both are looked at again + # here, at the moment of the move: nothing is moved once the source item is gone or the two are the + # same kind again, and a link on either side is left alone. Nor where the source folder holds the + # name twice in different letter case (a file Build beside a folder build, in a folder that keeps + # case apart): the copy can hold only one of the two, and moving it would change which one on every + # run. Returns whether the item was moved. A move that fails (a file another program holds open) + # changes nothing. Paths are full paths; they are used in their \\?\ form, which keeps a name that + # ends in a dot and has no length limit. + $long={param($p)if($p.StartsWith('\\')){$p}else{'\\?\'+$p}} + $folder=[IO.FileAttributes]::Directory + # Whether the two are a file against a folder right now, neither of them a link. A missing item throws. + $swapped={$s=[IO.File]::GetAttributes((& $long $Source));$c=[IO.File]::GetAttributes((& $long $Copy));-not(($s -bor $c) -band [IO.FileAttributes]::ReparsePoint) -and [bool]($s -band $folder) -ne [bool]($c -band $folder)} + try{ + if(-not(& $swapped)){return $false} + $name=[IO.Path]::GetFileName($Source) + if(@([IO.Directory]::EnumerateFileSystemEntries((& $long ([IO.Path]::GetDirectoryName($Source))))|Where-Object {[string]::Equals([IO.Path]::GetFileName($_),$name,[StringComparison]::OrdinalIgnoreCase)}).Count -gt 1){return $false} + [IO.Directory]::CreateDirectory((& $long ([IO.Path]::GetDirectoryName($Target))))|Out-Null + # Looked at once more, as the last thing before the move: the steps above take a moment. + if(-not(& $swapped)){return $false} + if([IO.File]::GetAttributes((& $long $Copy)) -band $folder){[IO.Directory]::Move((& $long $Copy),(& $long $Target))}else{[IO.File]::Move((& $long $Copy),(& $long $Target))} + $true + }catch{$false} +} + +function Test-BackupLinkOnPath([string]$Root,[string]$Relative) { + # True when a folder on the way from Root down to the item at Relative (the item itself not + # counted) is a link or junction, or cannot be looked at. What lies behind a link is not part of + # the source or of the copy, so nothing is moved out of there. + $p=$Root.TrimEnd('\') + foreach($part in @($Relative.Split('\')|Where-Object {$_.Length}|Select-Object -SkipLast 1)){ + $p+='\'+$part + try{if([IO.File]::GetAttributes($(if($p.StartsWith('\\')){$p}else{'\\?\'+$p})) -band [IO.FileAttributes]::ReparsePoint){return $true}}catch{return $true} + } + $false +} + +function Get-BackupSameDiskReason($SourceDisk,$DestinationDisk,[scriptblock]$DiskOfLetter) { + # Why a copy on DestinationDisk would be lost together with the source, or nothing when the two + # are apart. A virtual disk (a VHDX, a Dev Drive) is a file on another disk, which can be virtual + # too; a disk that fails takes every disk stored on it with it. So each side is followed down to + # the disk it really lies on, and the two must not meet anywhere. DiskOfLetter returns the disk + # of a drive letter. + $chain={param($d)$n=@();for($i=0;$d -and $i -lt 8;$i++){$n+=[int]$d.Number;if([string]$d.BusType -ne 'File Backed Virtual' -or [string]$d.Location -notmatch '^([A-Za-z]):\\'){break};$d=& $DiskOfLetter $Matches[1]};$n} + if([int]$SourceDisk.Number -eq [int]$DestinationDisk.Number){return 'Destination is on the same physical disk as the source'} + $s=@(& $chain $SourceDisk);$d=@(& $chain $DestinationDisk) + if([int]$DestinationDisk.Number -in $s){return 'Destination is on the disk that holds the Dev Drive file'} + if(@($d|Where-Object {$_ -in $s}).Count){return 'Destination is a virtual disk whose file is on the same physical disk as the source'} +} + +function ConvertTo-NameRegex([string]$Name) { + # A name with the wildcards * and ? as a pattern for a whole name, in any case. + [regex]::new('^'+[regex]::Escape($Name).Replace('\*','.*').Replace('\?','.')+'$',[Text.RegularExpressions.RegexOptions]'IgnoreCase,CultureInvariant') +} + +function Get-BackupSkippedName([string]$Relative,[string[]]$FolderNames,[string[]]$FileNames,[bool]$IsFolder) { + # The first name along Relative (a path below a folder robocopy copies) that robocopy skips by + # name, or nothing: a folder name on FolderNames, or for a file its own name on FileNames. + $parts=@($Relative.Split('\')|Where-Object {$_.Length}) + $folders=@(if($IsFolder){$parts}elseif($parts.Count -gt 1){$parts[0..($parts.Count-2)]}) + $rx=@($FolderNames|Where-Object {$_}|ForEach-Object {ConvertTo-NameRegex $_}) + foreach($p in $folders){if(@($rx|Where-Object {$_.IsMatch($p)}).Count){return $p}} + if(-not $IsFolder -and $parts.Count -and @($FileNames|Where-Object {$_}|Where-Object {(ConvertTo-NameRegex $_).IsMatch($parts[-1])}).Count){return $parts[-1]} +} + +function Resolve-BackupExcludePath([string]$Root,[string]$Pattern) { + # The files and folders below Root that an Exclude path with wildcards names, relative to Root. + # robocopy takes wildcards in a name only and refuses them inside a path, while the versioned + # backup takes such a path as it is (for example */dist or logs/*.tmp). So the pattern is matched + # here, one path part at a time, against the names really there: * and ? within a name, ** for any + # number of folders. Square brackets are plain characters, as in every other Exclude path of the + # plain copy. Links are not followed. + $parts=@($Pattern.Replace('/','\').Split('\')|Where-Object {$_.Length}) + while($parts.Count -and $parts[-1].Equals('**')){$parts=@($parts|Select-Object -SkipLast 1)} # a\** is a itself + if(-not $parts.Count -or -not [IO.Directory]::Exists($Root)){return} + $found=[Collections.Generic.SortedSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + $isFolder={param($e)($e.Attributes -band [IO.FileAttributes]::Directory) -and -not($e.Attributes -band [IO.FileAttributes]::ReparsePoint)} + function Step([string]$Dir,[string]$Rel,[int]$I){ + # A folder that cannot be read matches nothing here; robocopy reports it when it copies. + $items=try{@([IO.DirectoryInfo]::new($Dir).GetFileSystemInfos())}catch{@()} + if($parts[$I].Equals('**')){ + Step $Dir $Rel ($I+1) + foreach($e in $items){if(& $isFolder $e){Step $e.FullName ($Rel+$e.Name+'\') $I}} + return + } + $rx=ConvertTo-NameRegex $parts[$I] + foreach($e in $items){ + if(-not $rx.IsMatch($e.Name)){continue} + if($I -eq $parts.Count-1){[void]$found.Add($Rel+$e.Name)}elseif(& $isFolder $e){Step $e.FullName ($Rel+$e.Name+'\') ($I+1)} + } + } + Step ($Root.TrimEnd('\')+'\') '' 0 + $found +} + +Export-ModuleMember -Function Get-BackupModeNames,Test-BackupModeIncludes,ConvertTo-SelectionPath,Get-BackupSelection,Get-BackupMode,Get-BackupRegions,Get-RoboExtraPaths,Test-BackupPathInside,Get-RoboTotals,Get-RoboRunResult,Get-MirrorLogName,Get-RecoveryFoldersToRemove,Move-BackupTypeSwap,Test-BackupLinkOnPath,Get-BackupSameDiskReason,Get-BackupSkippedName,Resolve-BackupExcludePath diff --git a/Invoke-FastMirror.ps1 b/Invoke-FastMirror.ps1 index f889b34..0bd4211 100644 --- a/Invoke-FastMirror.ps1 +++ b/Invoke-FastMirror.ps1 @@ -12,13 +12,26 @@ # the mirror is a difference. A selected folder that is missing or became a link fails the run. # DevDriveBackup's own password, key and state folders are never copied, even when one of them is # selected itself; an older copy of one found on the backup drive is reported. +# +# A folder in the source where the copy holds a file of the same name (or the reverse) heals itself +# in a backup: the old item is moved out of the copy into this run's dated recovery folder, with or +# without -MoveExtras, and the new one is copied in the same run (TypeSwapsHealed in the summary). +# +# Every content check also writes last-check--.json beside the run folders: when it +# ran, for which source, and the lines of verify-differences.txt that name a file of the source +# (CONTENT, SIZE, MISSING-IN-MIRROR, ERROR). -Repair reads that file and copies only those files +# again; see the Repair part below. It copies nothing else, deletes nothing and is not a backup. +# +# The summary also warns about names robocopy treats differently from what the lists say +# (SkippedByShortName, CaseCollisions); see the last part before the summary is written. param( [string]$ConfigPath=(Join-Path $PSScriptRoot 'backup.local.json'), [string]$Source, [Parameter(Mandatory)][ValidatePattern('^[A-Za-z]$')][string]$DestinationDrive, # One folder name, or two separated by a backslash (for example Drive-C-Mirror\Projects). [ValidatePattern('^[A-Za-z0-9][A-Za-z0-9._-]{0,79}(\\[A-Za-z0-9][A-Za-z0-9._-]{0,79})?$')][string]$Folder='DevDrive-Mirror', - [ValidateRange(1,64)][int]$Threads=16, + # Files copied at once (robocopy takes up to 128); the window calls it CopyThreads. + [ValidateRange(1,128)][Alias('CopyThreads')][int]$Threads=16, [switch]$MoveExtras, # With -MoveExtras: keep only this many dated recovery folders (the newest) and delete older # ones from the backup drive. 0 keeps every recovery folder. @@ -26,7 +39,16 @@ param( [switch]$Verify, [switch]$SkipVerify, # Compare the mirror with the source without copying anything. - [switch]$VerifyOnly + [switch]$VerifyOnly, + # Copy again the files the last content check of this copy found different, unreadable or missing + # in the copy, and nothing else. The old version of each is first moved to this run's dated + # recovery folder, and the new copy is compared with the source by SHA-256. Not a backup: the + # copy is not recorded as complete, and no recovery folder is cleaned up. + [switch]$Repair, + # For the tests only: copy into this folder instead of onto a drive and keep the run records there. + # It must lie below the temporary folder, so it can never stand in for a backup drive; the drive + # checks, which need two real disks, are skipped. -DestinationDrive is then not used. + [Parameter(DontShow)][string]$TestRoot ) $ErrorActionPreference='Stop' Set-StrictMode -Version Latest @@ -47,16 +69,21 @@ $skipPaths=@();$personalNames=@() # Personal skips come from the configuration's Exclude list, which the versioned backup also uses: # a bare name (wildcards allowed) skips that folder name anywhere; a path skips one folder, given # relative to the source root or as a full path below it (for example E:/Projects/app/build/cache). +# A path may hold wildcards, as the versioned backup allows (*/dist, logs/*.tmp, ** for any number of +# folders): robocopy refuses wildcards inside a path, so such a path is matched here against the +# names really in the source, and every match is skipped as a path of its own. $srcRoot=[IO.Path]::GetFullPath($(if($Source){$Source}else{[string]$config.Source})).TrimEnd('\')+'\' foreach($entry in @(if($config.PSObject.Properties['Exclude']){$config.Exclude})){ $e=([string]$entry).Trim();if(-not $e){continue} - if($e -notmatch '[\\/]'){if($e -notin $skipNames){$skipNames+=$e;$personalNames+=$e};continue} + if($e -match '^\*\*[\\/]([^\\/]+)$'){$e=$Matches[1]} # **/name is a name at any depth + if($e -notmatch '[\\/]'){if(-not @($skipNames|Where-Object {[string]::Equals($_,$e,[StringComparison]::OrdinalIgnoreCase)}).Count){$skipNames+=$e;$personalNames+=$e};continue} $e=$e.Replace('/','\').TrimEnd('\') if($e -match '^[A-Za-z]:\\'){ if(-not $e.StartsWith($srcRoot,[StringComparison]::OrdinalIgnoreCase)){continue} # belongs to another source $e=$e.Substring($srcRoot.Length) } - if($e){$skipPaths+=$e.TrimStart('\')} + $e=$e.TrimStart('\');if(-not $e){continue} + if($e -match '[*?]'){$skipPaths+=@(Resolve-BackupExcludePath $srcRoot $e)}else{$skipPaths+=$e} } # DevDriveBackup's private folders (protected passwords, run state, plaintext recovery keys) are never # copied, whatever the selection: the copy would not keep their private permissions. The job's own key @@ -70,30 +97,62 @@ foreach($p in $privateRoots){ } $src=[IO.Path]::GetFullPath($Source).TrimEnd('\')+'\' -$srcVolume=Get-Volume -DriveLetter $src.Substring(0,1) -$dstVolume=Get-Volume -DriveLetter $DestinationDrive -if($srcVolume.UniqueId -eq $dstVolume.UniqueId){throw 'Destination is the source volume'} -# The copy goes only to the backup drive this configuration was made for, not another drive that took its letter. -if($config.PSObject.Properties['DestinationVolumeId'] -and $config.DestinationVolumeId -and $dstVolume.UniqueId -ne $config.DestinationVolumeId){throw "Drive $($DestinationDrive.ToUpperInvariant()): is not the backup drive this configuration was made for (its volume identity differs). Check which drive has this letter now."} -# The configured source must still be the same volume, not another drive that took its letter. -$configured=if($config.PSObject.Properties['Source'] -and $config.Source){[IO.Path]::GetFullPath([string]$config.Source).TrimEnd('\')+'\'}else{''} -if($config.PSObject.Properties['SourceVolumeId'] -and $config.SourceVolumeId -and $configured -and $src.StartsWith($configured,[StringComparison]::OrdinalIgnoreCase) -and $srcVolume.UniqueId -ne $config.SourceVolumeId){throw 'The source drive is not the one this configuration was made for (its volume identity changed). Check which drive has this letter now.'} -$sourceDisk=Get-Partition -DriveLetter $src.Substring(0,1)|Get-Disk -$destDisk=Get-Partition -DriveLetter $DestinationDrive|Get-Disk -if($sourceDisk.Number -eq $destDisk.Number){throw 'Destination is on the same physical disk as the source'} -if($sourceDisk.BusType -eq 'File Backed Virtual' -and $sourceDisk.Location -match '^([A-Za-z]):\\'){ - if((Get-Partition -DriveLetter $Matches[1]|Get-Disk).Number -eq $destDisk.Number){throw 'Destination is on the disk that holds the Dev Drive file'} +if($TestRoot){ + $temp=[IO.Path]::GetTempPath().TrimEnd('\');$TestRoot=[IO.Path]::GetFullPath($TestRoot).TrimEnd('\') + if(-not $TestRoot.StartsWith($temp+'\',[StringComparison]::OrdinalIgnoreCase)){throw '-TestRoot is for the tests only and must be a folder below the temporary folder'} + # A link on the way could lead out of the temporary folder, onto a real drive. + for($p=$TestRoot;$p.Length -gt $temp.Length;$p=[IO.Path]::GetDirectoryName($p)){ + if((Test-Path -LiteralPath $p) -and ((Get-Item -LiteralPath $p -Force).Attributes -band [IO.FileAttributes]::ReparsePoint)){throw '-TestRoot is for the tests only and must not lie behind a link or junction'} + } + $volumeKey='test';$dst=Join-Path $TestRoot $Folder;$stateRoot=Join-Path $TestRoot 'state' + # The same for the folders written below it: a link standing in for one of them leads out as well. + foreach($inner in @($dst,($dst+'-recovery'),$stateRoot)){ + for($p=[IO.Path]::GetFullPath($inner).TrimEnd('\');$p.Length -gt $TestRoot.Length;$p=[IO.Path]::GetDirectoryName($p)){ + if((Test-Path -LiteralPath $p) -and ((Get-Item -LiteralPath $p -Force).Attributes -band [IO.FileAttributes]::ReparsePoint)){throw '-TestRoot is for the tests only and must not hold a link or junction where the copy is written'} + } + } +}else{ + $srcVolume=Get-Volume -DriveLetter $src.Substring(0,1) + $dstVolume=Get-Volume -DriveLetter $DestinationDrive + if($srcVolume.UniqueId -eq $dstVolume.UniqueId){throw 'Destination is the source volume'} + # The copy goes only to the backup drive this configuration was made for, not another drive that took its letter. + if($config.PSObject.Properties['DestinationVolumeId'] -and $config.DestinationVolumeId -and $dstVolume.UniqueId -ne $config.DestinationVolumeId){throw "Drive $($DestinationDrive.ToUpperInvariant()): is not the backup drive this configuration was made for (its volume identity differs). Check which drive has this letter now."} + # The configured source must still be the same volume, not another drive that took its letter. + $configured=if($config.PSObject.Properties['Source'] -and $config.Source){[IO.Path]::GetFullPath([string]$config.Source).TrimEnd('\')+'\'}else{''} + if($config.PSObject.Properties['SourceVolumeId'] -and $config.SourceVolumeId -and $configured -and $src.StartsWith($configured,[StringComparison]::OrdinalIgnoreCase) -and $srcVolume.UniqueId -ne $config.SourceVolumeId){throw 'The source drive is not the one this configuration was made for (its volume identity changed). Check which drive has this letter now.'} + # Never onto the disk the source lies on, in either direction: a Dev Drive is a file on another + # disk, and so is a destination that is a virtual disk. + $diskOf={param($letter)Get-Partition -DriveLetter $letter|Get-Disk} + $sameDisk=Get-BackupSameDiskReason (& $diskOf $src.Substring(0,1)) (& $diskOf $DestinationDrive) $diskOf + if($sameDisk){throw $sameDisk} + $dst=$DestinationDrive.ToUpperInvariant()+':\'+$Folder + # Keyed by the drive's volume identity, so another drive that takes this letter still gets its first full check. + $volumeKey=($dstVolume.UniqueId -replace '[^A-Za-z0-9-]+','_').Trim('_');$stateRoot=Join-Path $env:LOCALAPPDATA 'DevDriveMirror\fast-mirror' +} +$recoveryRoot=$dst+'-recovery' +$lastSuccess=Join-Path $stateRoot ('last-success-'+$volumeKey+'-'+($Folder -replace '\\','_')+'.json') +# What the last content check of this copy found; -Repair works from it. +$lastCheck=Join-Path $stateRoot ('last-check-'+$volumeKey+'-'+($Folder -replace '\\','_')+'.json') +if($Repair -and $VerifyOnly){throw 'Give one of -Repair and -VerifyOnly, not both'} +# A content check and a repair work on an existing copy: they never create one, never run robocopy +# and never count as a backup. +$noCopy=$VerifyOnly -or $Repair +if($noCopy -and -not(Test-Path -LiteralPath $dst -PathType Container)){throw "There is no plain copy at $dst to $(if($Repair){'repair'}else{'check'}). Back up first."} +$check=$null +if($Repair){ + if(-not(Test-Path -LiteralPath $lastCheck)){throw "There is no earlier content check of the plain copy at $dst to repair from. Check the content first, then repair."} + $check=Get-Content -LiteralPath $lastCheck -Raw|ConvertFrom-Json + if(-not [string]::Equals([string]$check.Source,$src,[StringComparison]::OrdinalIgnoreCase)){throw "The last content check of the plain copy at $dst was made for another source ($($check.Source)). Check the content first, then repair."} } -$dst=$DestinationDrive.ToUpperInvariant()+':\'+$Folder -$recoveryRoot=$DestinationDrive.ToUpperInvariant()+':\'+$Folder+'-recovery' -$run=Join-Path $env:LOCALAPPDATA ('DevDriveMirror\fast-mirror\'+(Get-Date -Format 'yyyyMMdd-HHmmss')) +# Named to the second; a run that starts in the same second as another gets a folder of its own. +$run=Join-Path $stateRoot (Get-Date -Format 'yyyyMMdd-HHmmss') +for($n=2;Test-Path -LiteralPath $run;$n++){$run=(Join-Path $stateRoot (Get-Date -Format 'yyyyMMdd-HHmmss'))+'-'+$n} New-Item -ItemType Directory -Path $run -Force|Out-Null -# A content check compares an existing copy; it never creates one. -if($VerifyOnly){if(-not(Test-Path -LiteralPath $dst -PathType Container)){throw "There is no plain copy at $dst to check. Back up first."}}else{New-Item -ItemType Directory -Path $dst -Force|Out-Null} -# Keyed by the drive's volume identity, so another drive that takes this letter still gets its first full check. -$lastSuccess=Join-Path $env:LOCALAPPDATA ('DevDriveMirror\fast-mirror\last-success-'+($dstVolume.UniqueId -replace '[^A-Za-z0-9-]+','_').Trim('_')+'-'+($Folder -replace '\\','_')+'.json') +if(-not $noCopy){New-Item -ItemType Directory -Path $dst -Force|Out-Null} $firstRun=-not(Test-Path -LiteralPath $lastSuccess) if(($firstRun -and -not $SkipVerify) -or $VerifyOnly){$Verify=$true} +# A repair compares each file it copies; the check of the whole copy is a run of its own. +if($Repair){$Verify=$false} function Join-Rel([string]$Root,[string]$Relative){if($Relative){Join-Path $Root $Relative}else{$Root.TrimEnd('\')}} # 'E:' alone means the current directory on E:, and 'E:\' would escape the closing quote; use 'E:\.' for a root. @@ -110,27 +169,68 @@ function Get-RoboArgs([string]$From,[string]$To,[string]$Log,[string[]]$Excluded function Invoke-Robo([string[]]$RoboArgs,[string]$Label){ # 8 means some files could not be copied (for example a file another program keeps locked); the # run goes on and they are counted below. 16 and above means robocopy could not work at all. + # Windows starts no program with a command line of more than about 32,000 characters, and every + # path skipped below this folder is on it four times. + if((($RoboArgs|Measure-Object -Property Length -Sum).Sum+3*$RoboArgs.Count) -gt 30000){throw "Too many folders are skipped by path for one copy of $Label. In Exclude, use a folder name instead of a path pattern, or skip a folder higher up."} & robocopy.exe @RoboArgs|Out-Null $code=$LASTEXITCODE if($code -ge 16){throw "robocopy $Label failed with exit code $code. See $($RoboArgs|Where-Object {$_ -like '/UNILOG:*'})"} $code } -function Get-RoboTotals([string]$Log){ - # The summary rows (directories, files, bytes) each hold six numbers: total, copied, skipped, - # mismatch, failed, extras. Reading the numbers rather than the labels also works on Windows - # in other languages. - $rows=@(Get-Content -LiteralPath $Log -Encoding Unicode -Tail 14|ForEach-Object { - $m=[regex]::Match($_,'^\s*[^\d\s][^:]*:\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s*$') - if($m.Success){,@(1..6|ForEach-Object {[long]$m.Groups[$_].Value})} - }) - if($rows.Count -lt 3){return $null} - [pscustomobject]@{Dirs=$rows[0];Files=$rows[1];Bytes=$rows[2]} +# A folder in the source where the copy has a file of the same name, or the reverse. robocopy copies +# nothing there and names at most one side, so each one found is kept with both paths. A backup then +# moves the item in the copy aside (Move-TypeSwap); what could not be moved stays on this list and is +# reported. Returns the pair, or nothing. +$conflicts=[Collections.Generic.List[object]]::new() +function Test-TypeConflict([string]$Relative){ + $s=Join-Rel $src $Relative;$m=Join-Rel $dst $Relative;$folder=[IO.Directory]::Exists($s) + if(-not(($folder -and [IO.File]::Exists($m)) -or ([IO.File]::Exists($s) -and [IO.Directory]::Exists($m)))){return} + $known=@($conflicts|Where-Object {[string]::Equals($_.Copy,$m,[StringComparison]::OrdinalIgnoreCase)}) + if($known.Count){return $known[0]} + $pair=[pscustomobject]@{Source=$s;Copy=$m;Relative=$Relative;SourceIsFolder=$folder};$conflicts.Add($pair);$pair +} +# The dated recovery folder of this run, beside the copy. Whatever a run moves out of the copy goes +# there under its own path, and nothing is ever deleted: files gone from the source (-MoveExtras), an +# item that changed between file and folder, the old version of a repaired file. It is named to the +# second, when the first item is about to be moved. A folder of that name left by a run a moment ago +# is not shared: this run waits for the next second, so the folder is this run's alone. +$script:recovery='';$script:recoveryMoves=0 +function Get-RunRecovery{ + while(-not $script:recovery){ + $p=Join-Path $recoveryRoot (Get-Date -Format 'yyyyMMdd-HHmmss') + if(Test-Path -LiteralPath $p){Start-Sleep -Milliseconds 200}else{$script:recovery=$p} + } + $script:recovery +} +# Heals one such pair in a backup: the item in the copy is moved to the recovery folder, so the next +# copy of that folder brings the new item. Never in a content check or a repair. Whether it was moved. +$healed=[Collections.Generic.List[object]]::new() +function Move-TypeSwap($Pair){ + if($noCopy -or -not $Pair -or -not $Pair.Relative){return $false} + # A folder in the copy can hold the old copy of a folder that is no longer selected. Such a copy + # stays where it is (as when its parent folder is deleted from the source), so the folder is not moved. + if([IO.Directory]::Exists($Pair.Copy) -and @($selection.Entries|Where-Object {-not(Test-BackupModeIncludes $_.Mode 'Mirror') -and $_.Path.StartsWith($Pair.Relative+'\',[StringComparison]::OrdinalIgnoreCase)}).Count){return $false} + # Nor anything that lies behind a link or junction, on either side. + if((Test-BackupLinkOnPath $src $Pair.Relative) -or (Test-BackupLinkOnPath $dst $Pair.Relative)){return $false} + $target=(Get-RunRecovery)+'\'+$Pair.Relative + if(-not(Move-BackupTypeSwap $Pair.Source $Pair.Copy $target)){return $false} + $script:recoveryMoves++;[void]$conflicts.Remove($Pair) + $healed.Add([ordered]@{Source=$Pair.Source;Copy=$Pair.Copy;MovedTo=$target});$true } -function Get-LogName([string]$Relative){'copy-'+$(if($Relative){$Relative -replace '[\\/:*?"<>|]','_'}else{'main'})+'.log'} $started=Get-Date $summary=[ordered]@{Source=$src;Destination=$dst;Started=$started.ToString('o');DefaultMode=$selection.DefaultMode;Selected=@($regions|ForEach-Object {if($_.Path){$_.Path}else{'(source root)'}});Runs=@();SelectionsMissing=@();SelectionsSkippedAsLinks=@();SelectionsSkippedAsPrivate=@()} -$copied=@() +$copied=@();$rootConflicts=0 +$logNames=[Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) +function Invoke-CopyRun([string[]]$RoboArgs,[string]$Part,[string]$Log){ + # One robocopy run, added to the summary. The totals are read as soon as the run ends, from the log + # of this run alone. Repeated marks a run whose folder was copied again after a file-or-folder swap + # in it was healed: the later run tells what is left uncopied there. + $code=Invoke-Robo $RoboArgs $Part;$result=Get-RoboRunResult $Log $code;$t=$result.Totals + $entry=[ordered]@{Part=$Part;ExitCode=$code;Log=$Log;Files=$(if($t){'Files : '+($t.Files -join ' ')}else{''});Bytes=$(if($t){'Bytes : '+($t.Bytes -join ' ')}else{''});NotCopied=$result.NotCopied;Complete=$result.Complete;Repeated=$false} + $summary.Runs+=$entry + [pscustomobject]@{Entry=$entry;Mismatch=([bool]($code -band 4) -or ($t -and ($t.Files[3]+$t.Dirs[3]) -gt 0))} +} foreach($region in $regions){ $from=Join-Rel $src $region.Path;$to=Join-Rel $dst $region.Path # /XD only skips folders below the one robocopy copies, never that folder itself, and a file selected @@ -140,16 +240,34 @@ foreach($region in $regions){ if(-not(Test-Path -LiteralPath $from)){$summary.SelectionsMissing+=$region.Path;continue} $item=Get-Item -LiteralPath $from -Force if($region.Path -and ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)){$summary.SelectionsSkippedAsLinks+=$region.Path;continue} - $log=Join-Path $run (Get-LogName $region.Path) + # A selected folder whose place in the copy is taken by a file: robocopy cannot even start there. + # A backup moves that file aside first. Where it cannot (or in a content check), the folder counts + # as one item not copied, and the other selections are still copied. (A selected file whose place + # is still taken by a folder is only noted here; robocopy reports that mismatch itself.) + $pair=Test-TypeConflict $region.Path + if($pair -and -not(Move-TypeSwap $pair) -and $item.PSIsContainer){$rootConflicts++;continue} + $log=Join-Path $run (Get-MirrorLogName $region.Path $logNames) if($item.PSIsContainer){$a=@(Get-RoboArgs $from $to $log $region.Excluded)} else{ # A single file selected on its own (for example README.md at the drive root). $a=@((Format-RoboDir (Split-Path -LiteralPath $from)),(Split-Path -LiteralPath $to),$item.Name,'/COPY:DAT','/R:1','/W:1','/NP','/FP','/BYTES',('/UNILOG:'+$log)) } $part=if($region.Path){$region.Path}else{'main'} - if(-not $VerifyOnly){$summary.Runs+=[ordered]@{Part=$part;ExitCode=(Invoke-Robo $a $part);Log=$log}} - $copied+=[pscustomobject]@{Region=$region;IsFolder=$item.PSIsContainer} + $copied+=[pscustomobject]@{Region=$region;IsFolder=$item.PSIsContainer;Log=$log;Part=$part;Args=$a;Run=$(if(-not $noCopy){Invoke-CopyRun $a $part $log})} } +# A choice inside a copied folder is left out by robocopy without a word when its own name, or a +# folder name on the way to it, is on the always-skipped list (a folder called venv, say). Such a +# choice is named, so a folder shown as selected is never missing from the copy unnoticed. +$rootPaths=@($copied|ForEach-Object {$_.Region.Path}) +$summary.SelectionsSkippedByName=@(foreach($e in $selection.Entries){ + if(-not(Test-BackupModeIncludes $e.Mode 'Mirror')){continue} + $root=$null + foreach($r in $rootPaths){if(($r.Length -eq 0 -or [string]::Equals($r,$e.Path,[StringComparison]::OrdinalIgnoreCase) -or $e.Path.StartsWith($r+'\',[StringComparison]::OrdinalIgnoreCase)) -and ($null -eq $root -or $r.Length -gt $root.Length)){$root=$r}} + if($null -eq $root -or $root.Length -eq $e.Path.Length){continue} # not in a copied folder, or copied on its own + $full=Join-Rel $src $e.Path;if(-not(Test-Path -LiteralPath $full)){continue} + $below=if($root){$e.Path.Substring($root.Length+1)}else{$e.Path} + if(Get-BackupSkippedName $below $skipNames $personalNames (Test-Path -LiteralPath $full -PathType Container)){$e.Path} +}) # An earlier version could copy a private folder. Such a copy holds readable recovery keys; it is # reported so it can be deleted by hand, never deleted here. $summary.PrivateCopiesOnDrive=@($privateInSource|Where-Object {$_ -and (Test-Path -LiteralPath (Join-Rel $dst $_))}) @@ -157,52 +275,130 @@ if($copied.Count -eq 0 -and $summary.SelectionsSkippedAsPrivate.Count){ $old=if($summary.PrivateCopiesOnDrive.Count){' An older copy of one is still on the backup drive and holds readable recovery keys; delete it: '+(@($summary.PrivateCopiesOnDrive|ForEach-Object {Join-Rel $dst $_}) -join ', ')+'.'}else{''} throw ("Everything selected is one of DevDriveBackup's private folders, which hold passwords and recovery keys and are never copied: "+($summary.SelectionsSkippedAsPrivate -join ', ')+'.'+$old) } -if($copied.Count -eq 0){throw ('None of the selected paths exist under '+$src+': '+($summary.Selected -join ', '))} +if($copied.Count -eq 0 -and -not $rootConflicts){throw ('None of the selected paths exist under '+$src+': '+($summary.Selected -join ', '))} # robocopy gives a folder copied from a drive root the root's hidden/system attributes. -if(-not $VerifyOnly){& attrib.exe -s -h $dst|Out-Null} +if(-not $noCopy){& attrib.exe -s -h $dst|Out-Null} # Files in the mirror that are gone from the source: list them, and optionally move them aside. -$extras=@() -foreach($c in @($copied|Where-Object {$_.IsFolder -and -not $VerifyOnly})){ - $extrasLog=Join-Path $run ('extras-'+(Get-LogName $c.Region.Path).Substring(5)) +# Kept in the order Windows compares names, not a dictionary's: two paths that differ by an invisible +# character are two paths, and a folder always comes before what is inside it. +$listed=[Collections.Generic.SortedSet[string]]::new([StringComparer]::OrdinalIgnoreCase) +foreach($c in @($copied|Where-Object {$_.IsFolder -and -not $noCopy})){ + $extrasLog=Join-Path $run ('extras-'+[IO.Path]::GetFileName($c.Log).Substring(5)) # Directory names must be listed here (no /NDL) so extra directories are reported too. - & robocopy.exe @(@(Get-RoboArgs (Join-Rel $src $c.Region.Path) (Join-Rel $dst $c.Region.Path) $extrasLog $c.Region.Excluded @('/L','/PURGE','/NJH','/NJS'))|Where-Object {$_ -ne '/NDL'})|Out-Null # With /MT, robocopy's records can be split over lines or share one; see Get-RoboExtraPaths. - $extras+=@(Get-RoboExtraPaths (Get-Content -LiteralPath $extrasLog -Encoding Unicode) $dst) + $list={ + & robocopy.exe @(@(Get-RoboArgs (Join-Rel $src $c.Region.Path) (Join-Rel $dst $c.Region.Path) $extrasLog $c.Region.Excluded @('/L','/PURGE','/NJH','/NJS'))|Where-Object {$_ -ne '/NDL'})|Out-Null + Get-Content -LiteralPath $extrasLog -Encoding Unicode + } + $lines=@(& $list) + # Where the copy run reported a mismatch, the listing shows which names: it holds every folder of + # the source and every file that differs. Each is looked up on both sides, so only a real + # folder-against-file pair counts. + $swaps={foreach($p in @(Get-RoboExtraPaths $lines $src $dst)){Test-TypeConflict $p.Substring($src.Length).Trim('\')}} + if($c.Run.Mismatch -and @(& $swaps|Where-Object {Move-TypeSwap $_}).Count){ + # The old items are out of the way now, so the folder is copied once more, which brings the new + # ones, and listed again. What is still a mismatch after that is reported. + $c.Run.Entry.Repeated=$true + $again=Join-Path $run (Get-MirrorLogName ($c.Part+' again') $logNames) + $c.Run=Invoke-CopyRun @($c.Args|ForEach-Object {if($_ -like '/UNILOG:*'){'/UNILOG:'+$again}else{$_}}) $c.Part $again + $lines=@(& $list) + } + if($c.Run.Mismatch){& $swaps|Out-Null} + foreach($p in @(Get-RoboExtraPaths $lines $dst $src)){[void]$listed.Add($p)} } # Only selected paths count as extras; old copies of unselected folders are reported separately below. # A path that still exists in the source is never an extra, whatever the log seems to say. -$extras=@($extras|Sort-Object -Unique|Where-Object { +$extras=@($listed|Where-Object { $rel=$_.Substring($dst.Length).Trim('\') $rel -and (Test-BackupModeIncludes (Get-BackupMode $selection $rel) 'Mirror') -and -not(Test-Path -LiteralPath (Join-Rel $src $rel)) }) -$summary.ExtrasFound=$extras.Count -# Files are moved aside only after a copy that could read the source: a run where robocopy failed -# (exit code 8 or more) may have missed files that still exist, so its extras are only reported. -$copyFailed=@($summary.Runs|Where-Object {$_.ExitCode -ge 8}).Count -gt 0 -if($MoveExtras -and $copyFailed){$summary.ExtrasNotMovedReason='Some files could not be copied in this run, so nothing was moved aside'} -if($MoveExtras -and $extras.Count -and -not $VerifyOnly -and -not $copyFailed){ - $stamp=Join-Path $recoveryRoot (Get-Date -Format 'yyyyMMdd-HHmmss') +# Counted as files: the listing names a deleted folder and every file in it. +$summary.ExtrasFound=@($extras|Where-Object {-not $_.EndsWith('\')}).Count +# Files are moved aside only after a copy that read the whole source: a run where robocopy failed +# (exit code 8 or more) or did not finish may have missed files that still exist, so its extras are +# only reported. +$copyFailed=@($summary.Runs|Where-Object {-not $_.Repeated -and -not $_.Complete}).Count -gt 0 +if($MoveExtras -and $copyFailed -and -not $noCopy){$summary.ExtrasNotMovedReason='Some files could not be copied in this run, so nothing was moved aside'} +if($MoveExtras -and $extras.Count -and -not $noCopy -and -not $copyFailed){ + $stamp=Get-RunRecovery + $moved=0;$movedItems=0;$notMoved=@() foreach($item in $extras){ if(-not(Test-Path -LiteralPath $item)){continue} $relative=$item.Substring($dst.Length).TrimStart('\') # Checked again here: a file put back in the source while earlier items were moved stays in the copy. if(Test-Path -LiteralPath (Join-Rel $src $relative.TrimEnd('\'))){continue} + # A deleted folder can still hold, in the copy, the old copy of a sub-folder that is no longer + # selected. Such a copy is kept, so the folder is not moved whole; its own files follow in the list. + if($item.EndsWith('\') -and @($selection.Entries|Where-Object {-not(Test-BackupModeIncludes $_.Mode 'Mirror') -and $_.Path.StartsWith($relative,[StringComparison]::OrdinalIgnoreCase)}).Count){continue} $target=Join-Path $stamp $relative - New-Item -ItemType Directory -Path ([IO.Path]::GetDirectoryName($target.TrimEnd('\'))) -Force|Out-Null - Move-Item -LiteralPath $item -Destination $target + # A file another program holds open cannot be moved. It stays in the copy and is named, and the + # run goes on. A folder that cannot be moved whole is left to its files, which follow in the list. + try{ + $count=if($item.EndsWith('\')){@(Get-ChildItem -LiteralPath $item -Recurse -File -Force).Count}else{1} + New-Item -ItemType Directory -Path ([IO.Path]::GetDirectoryName($target.TrimEnd('\'))) -Force|Out-Null + Move-Item -LiteralPath $item -Destination $target + $moved+=$count;$movedItems++;$script:recoveryMoves++ + }catch{if(-not $item.EndsWith('\')){$notMoved+=$item}} } - $summary.ExtrasMovedTo=$stamp - $summary.ExtrasMoved=$extras.Count + if($movedItems){$summary.ExtrasMovedTo=$stamp} + # Files really moved; a folder moved whole counts by the files in it. + $summary.ExtrasMoved=$moved + $summary.ExtrasNotMoved=$notMoved +} + +# The rules of the copy, as the content check, the repair and the look at names below take them: the +# same skips as the copy (names, personal names for files too, Exclude paths, the private folders). +# They apply below each folder robocopy copied, as with robocopy, so the copied folders are passed too. +# ForEach-Object rather than $selection.Entries.Path: under strict mode an empty list (a source +# with no subfolders listed) has no Path property. +$rulePaths=[string[]]@($selection.Entries|ForEach-Object {$_.Path}) +$ruleOn=[bool[]]@($selection.Entries|ForEach-Object {Test-BackupModeIncludes $_.Mode 'Mirror'}) +$ruleDefault=Test-BackupModeIncludes $selection.DefaultMode 'Mirror' +$ruleRoots=[string[]]@($copied|ForEach-Object {$_.Region.Path}) + +# --- Repair ----------------------------------------------------------------------------------------- +# The files the last content check found different, unreadable or missing in the copy are looked at +# again and, where the copy still differs, copied again (MirrorVerifier.Repair: the new copy is +# compared with the source by SHA-256 before the old version is moved to the recovery folder and the +# new one takes its name). Files only in the copy are not on the list; a file that is no longer +# selected, or gone from the source, is left alone; a file that changed in the source since the +# check is simply copied as it is now. What could not be put right counts as a copy error and stays +# in the record of the check, so the next repair tries it again; what was put right leaves the record. +if($Repair){ + Add-Type -Path (Join-Path $PSScriptRoot 'Backup.MirrorVerifier.cs') + $wanted=[string[]]@(@($check.Files)|ForEach-Object {$f=([string]$_).Split("`t");if($f.Count -ge 2 -and $f[1] -and $f[0] -cin 'CONTENT','SIZE','MISSING-IN-MIRROR','ERROR'){$f[1]}}) + $done=[string[]]@(if($wanted.Count){[MirrorVerifier]::Repair($dst,$src,(Get-RunRecovery),[math]::Min(8,$Threads),$wanted,$rulePaths,$ruleOn,$ruleDefault,[string[]]$skipNames,[string[]]$personalNames,[string[]]$skipPaths,$ruleRoots)}) + [IO.File]::WriteAllLines((Join-Path $run 'repair.txt'),$done) + $count={param($what)@($done|Where-Object {$_.StartsWith($what+"`t")}).Count} + $script:recoveryMoves+=@($done|Where-Object {$_.Split("`t")[2]}).Count + $when=$check.Checked;if($when -is [datetime]){$when=$when.ToString('o')} + $summary.Repair=[ordered]@{Requested=$done.Count;Repaired=(& $count 'REPAIRED');StillDifferent=(& $count 'STILL-DIFFERENT');Failed=(& $count 'FAILED');MovedTo='' + AlreadyMatching=(& $count 'MATCHES');Skipped=(& $count 'SKIPPED');CheckedAt=[string]$when;Details=(Join-Path $run 'repair.txt')} + $left=[Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + foreach($line in $done){if($line -cmatch "^(STILL-DIFFERENT|FAILED)`t"){[void]$left.Add($line.Split("`t")[1])}} + [ordered]@{Checked=[string]$when;Source=$src;Destination=$dst;RunFolder=[string]$check.RunFolder;Repaired=(Get-Date).ToString('o') + Files=@(@($check.Files)|Where-Object {$f=([string]$_).Split("`t");$f.Count -ge 2 -and $left.Contains($f[1])})}|ConvertTo-Json -Depth 3|Set-Content -LiteralPath $lastCheck +} +# A run that could move nothing made no recovery folder. The empty folders prepared for it are +# removed, or they would count as this run's folder and push an earlier one out of the kept ones. +if($script:recovery -and -not $script:recoveryMoves){ + if((Test-Path -LiteralPath $script:recovery) -and -not @(Get-ChildItem -LiteralPath $script:recovery -Recurse -File -Force).Count){Remove-Item -LiteralPath $script:recovery -Recurse -Force} + $script:recovery='' } +# What was healed, and where this run moved things to ('' when it moved nothing). +$summary.TypeSwapsHealed=@($healed);$summary.RecoveryFolder=$script:recovery +if($Repair){$summary.Repair.MovedTo=$script:recovery} # Old recovery folders: keep the newest KeepRecovery, delete the rest. Only folders this script -# names (yyyyMMdd-HHmmss) directly inside the recovery root are touched. +# names (yyyyMMdd-HHmmss) directly inside the recovery root are touched, and never the one this run +# made, whether it holds files gone from the source or an item moved aside by healing. Only a backup +# with -MoveExtras and a number to keep deletes anything; healing alone and a repair never do. $summary.RecoveryRemoved=@() -if($MoveExtras -and $KeepRecovery -gt 0 -and -not $VerifyOnly -and (Test-Path -LiteralPath $recoveryRoot -PathType Container)){ - $dated=@(Get-ChildItem -LiteralPath $recoveryRoot -Directory -Force|Where-Object {$_.Name -match '^\d{8}-\d{6}$' -and -not($_.Attributes -band [IO.FileAttributes]::ReparsePoint)}|Sort-Object Name -Descending) - foreach($old in @($dated|Select-Object -Skip $KeepRecovery)){ - Remove-Item -LiteralPath $old.FullName -Recurse -Force - $summary.RecoveryRemoved+=$old.Name +if($MoveExtras -and $KeepRecovery -gt 0 -and -not $noCopy -and (Test-Path -LiteralPath $recoveryRoot -PathType Container)){ + $dated=@(Get-ChildItem -LiteralPath $recoveryRoot -Directory -Force|Where-Object {$_.Name -match '^\d{8}-\d{6}$' -and -not($_.Attributes -band [IO.FileAttributes]::ReparsePoint)}|ForEach-Object {$_.Name}) + foreach($old in @(Get-RecoveryFoldersToRemove $dated $KeepRecovery $(if($script:recovery){[IO.Path]::GetFileName($script:recovery)}else{''}))){ + Remove-Item -LiteralPath (Join-Path $recoveryRoot $old) -Recurse -Force + $summary.RecoveryRemoved+=$old } } @@ -220,34 +416,61 @@ $summary.NotSelectedButInMirror=@($notSelected) # Files that still failed after the retry are counted from robocopy's totals; the ERROR lines # (one per attempt) are kept in copy-errors.txt for details. -$errors=@($summary.Runs|ForEach-Object {Get-Content -LiteralPath $_.Log -Encoding Unicode|Where-Object {$_ -match '\sERROR \d+ ' -or $_ -match '\*MISMATCH'}}) +# A run that was repeated after healing is left out: the later run of that folder tells what is left. +$lastRuns=@($summary.Runs|Where-Object {-not $_.Repeated}) +$errors=@($lastRuns|Where-Object {Test-Path -LiteralPath $_.Log}|ForEach-Object {Get-Content -LiteralPath $_.Log -Encoding Unicode|Where-Object {$_ -match '\sERROR \d+ ' -or $_ -match '\*MISMATCH'}}) +# A run without totals was ended or crashed; robocopy wrote no line about that itself. +$errors+=@($lastRuns|Where-Object {$_.Files -eq ''}|ForEach-Object {"NOT FINISHED`trobocopy ended before it finished copying $($_.Part) (exit code $($_.ExitCode)), so this copy is incomplete. Back up again. Log: $($_.Log)"}) +# The same name is a folder on one side and a file on the other: both paths, and what to do. +$errors+=@($conflicts|ForEach-Object { + $todo=if($noCopy){'The next backup moves it aside and copies the new one.'}else{'It could not be moved aside: a program may be using it, or it holds the old copy of a folder that is no longer selected. Move it yourself, then back up again.'} + if($_.SourceIsFolder){"TYPE CONFLICT`tThe source has a folder where the copy has a file of the same name, so nothing in that folder is copied. Source: $($_.Source) Copy: $($_.Copy) $todo"} + else{"TYPE CONFLICT`tThe source has a file where the copy has a folder of the same name, so the file is not copied. Source: $($_.Source) Copy: $($_.Copy) $todo"} +}) [IO.File]::WriteAllLines((Join-Path $run 'copy-errors.txt'),[string[]]$errors) -$failed=[long]0 -foreach($r in $summary.Runs){ - $t=Get-RoboTotals $r.Log - if($t){ - $r.Files='Files : '+($t.Files -join ' ');$r.Bytes='Bytes : '+($t.Bytes -join ' ') - # Mismatch (a folder in the source where the copy has a file of that name, or the reverse): robocopy - # copies nothing there and still exits below 8, so it counts as not copied, like a failed file. - $failed+=$t.Files[3]+$t.Files[4]+$t.Dirs[3]+$t.Dirs[4] - }else{$r.Files='';$r.Bytes='';if($r.ExitCode -ge 8 -or ($r.ExitCode -band 4)){$failed+=1}} -} -$summary.CopyErrors=$failed +$summary.TypeConflicts=@($conflicts|ForEach-Object {$_.Copy}) +# What each run left uncopied (see Get-RoboRunResult), plus each selected folder that could not be +# started because a file has its place in the copy. In a repair: each file it could not put right, +# and nothing else, so a repair fails only for what it was asked to do. +$summary.CopyErrors=if($Repair){[long]($summary.Repair.StillDifferent+$summary.Repair.Failed)}else{[long](@($lastRuns|ForEach-Object {$_.NotCopied})|Measure-Object -Sum).Sum+$rootConflicts} if($Verify){ Add-Type -Path (Join-Path $PSScriptRoot 'Backup.MirrorVerifier.cs') $vf=[long]0;$vb=[long]0 - # ForEach-Object rather than $selection.Entries.Path: under strict mode an empty list (a source - # with no subfolders listed) has no Path property. - $paths=[string[]]@($selection.Entries|ForEach-Object {$_.Path}) - $on=[bool[]]@($selection.Entries|ForEach-Object {Test-BackupModeIncludes $_.Mode 'Mirror'}) - # The same skips as the copy: names, personal names (files too), Exclude paths and the private folders. - $bad=[MirrorVerifier]::Compare($dst,$src,[math]::Min(8,[Environment]::ProcessorCount),$paths,$on,(Test-BackupModeIncludes $selection.DefaultMode 'Mirror'),[string[]]$skipNames,[string[]]$personalNames,[string[]]$skipPaths,[ref]$vf,[ref]$vb) + $bad=[MirrorVerifier]::Compare($dst,$src,[math]::Min(8,[Environment]::ProcessorCount),$rulePaths,$ruleOn,$ruleDefault,[string[]]$skipNames,[string[]]$personalNames,[string[]]$skipPaths,$ruleRoots,[ref]$vf,[ref]$vb) [IO.File]::WriteAllLines((Join-Path $run 'verify-differences.txt'),[string[]]$bad) # Files only in the mirror (deleted from the source, or now skipped) are expected in a copy that # never deletes; they are reported, but are not differences. Selected files missing from the mirror are. $onlyInMirror=@($bad|Where-Object {$_.StartsWith("MISSING-IN-SOURCE`t")}).Count $summary.Verify=[ordered]@{FilesMatched=$vf;GiBMatched=[math]::Round($vb/1GB,2);Differences=($bad.Count-$onlyInMirror);MissingInMirror=@($bad|Where-Object {$_.StartsWith("MISSING-IN-MIRROR`t")}).Count;OnlyInMirror=$onlyInMirror} + # What a later -Repair works from: the lines that name a file of the source. Files only in the + # copy are left out; a repair never touches them. Each check replaces the record of the one before. + [ordered]@{Checked=(Get-Date).ToString('o');Source=$src;Destination=$dst;RunFolder=$run;Files=@($bad|Where-Object {-not $_.StartsWith("MISSING-IN-SOURCE`t")})}|ConvertTo-Json -Depth 3|Set-Content -LiteralPath $lastCheck +} + +# --- Names robocopy treats differently from what the lists say ----------------------------------------- +# Two warnings; nothing is copied differently because of them, and neither is a copy error. +# SkippedByShortName: robocopy compares a wildcard name with the short (8.3) name of a file or folder +# too, so *.htm from Exclude also skips page.html, whose short name is PAGE~1.HTM. Only the personal +# names can do that (no short name fits a built-in one), and only where the drive gives short names: +# NTFS and FAT do, a Dev Drive (ReFS) does not. +# CaseCollisions: a folder that keeps letter case apart can hold Makefile and makefile; the backup +# drive holds one file for both, so one of them is not really in the copy. +# Both come from one pass over the names in the selected folders (no file is read), after a backup +# and in a content check. The summary holds the first 100 of each and the count of all; every one is +# in a file in the run folder. +$summary.SkippedByShortName=@();$summary.SkippedByShortNameCount=0;$summary.CaseCollisions=@();$summary.CaseCollisionsCount=0 +if(-not $Repair){ + Add-Type -Path (Join-Path $PSScriptRoot 'Backup.MirrorVerifier.cs') + $shortFs=try{[IO.DriveInfo]::new($src.Substring(0,3)).DriveFormat -in 'NTFS','FAT','FAT32'}catch{$true} + $shortNames=[string[]]@(if($shortFs){$personalNames|Where-Object {$_ -match '[*?~]'}}) + $seen=@([MirrorVerifier]::Survey($src,$rulePaths,$ruleOn,$ruleDefault,[string[]]$skipNames,[string[]]$personalNames,[string[]]$skipPaths,$ruleRoots,$shortNames)) + $short=@($seen|Where-Object {$_.StartsWith("SHORT`t")}|ForEach-Object {$_.Substring(6)}) + $case=@($seen|Where-Object {$_.StartsWith("CASE`t")}|ForEach-Object {$_.Substring(5).Replace("`t",' | ')}) + if($short.Count){[IO.File]::WriteAllLines((Join-Path $run 'skipped-by-short-name.txt'),[string[]]$short)} + if($case.Count){[IO.File]::WriteAllLines((Join-Path $run 'case-collisions.txt'),[string[]]$case)} + $summary.SkippedByShortName=@($short|Select-Object -First 100|ForEach-Object {$_.Split("`t")[0]});$summary.SkippedByShortNameCount=$short.Count + $summary.CaseCollisions=@($case|Select-Object -First 100);$summary.CaseCollisionsCount=$case.Count } $summary.Finished=(Get-Date).ToString('o') @@ -255,10 +478,13 @@ $summary.Minutes=[math]::Round(((Get-Date)-$started).TotalMinutes,1) $summary.RunFolder=$run $summary|ConvertTo-Json -Depth 6|Set-Content -LiteralPath (Join-Path $run 'summary.json') # A selected folder that is missing or became a link was not copied, so the run fails, as the versioned -# backup does. A content check only reads; it never marks the copy as complete. +# backup does. A content check only reads, and a repair copies single files; neither marks the copy +# as complete. $notCopied=@(@($summary.SelectionsMissing|ForEach-Object {"$_ (missing)"})+@($summary.SelectionsSkippedAsLinks|ForEach-Object {"$_ (now a link or junction)"})) -if(-not $VerifyOnly -and $summary.CopyErrors -eq 0 -and -not $notCopied.Count -and (-not $Verify -or $summary.Verify.Differences -eq 0)){ +if(-not $noCopy -and $summary.CopyErrors -eq 0 -and -not $notCopied.Count -and (-not $Verify -or $summary.Verify.Differences -eq 0)){ Copy-Item -LiteralPath (Join-Path $run 'summary.json') -Destination $lastSuccess -Force } if($notCopied.Count){throw ('Selected folders were not copied: '+($notCopied -join ', ')+'. Choose them again on What to back up, or set them to Skip. Details: '+(Join-Path $run 'summary.json'))} $summary|ConvertTo-Json -Depth 6 +# A repair that left a file unrepaired ends with an error code, after the summary, so the caller sees both. +if($Repair -and $summary.CopyErrors -gt 0){exit 1} diff --git a/Test-BackupCopyRules.ps1 b/Test-BackupCopyRules.ps1 index 43ec6be..a0e005e 100644 --- a/Test-BackupCopyRules.ps1 +++ b/Test-BackupCopyRules.ps1 @@ -45,6 +45,18 @@ try{ $bad=@([MirrorVerifier]::Compare((Join-Path $root 'flat-copy'),(Join-Path $root 'flat'),2,$ep,$eo,$true,[string[]]@(),[string[]]@(),[string[]]@(),[ref]$vf,[ref]$vb)) Assert ($bad.Count -eq 0 -and $vf -eq 1) 'a source with no subfolders listed is checked in full' + # --- Repair of single files: the same rules, one path at a time --------------------------------- + $rec=Join-Path $root 'recovery';Put (Join-Path $root 'outside.txt') 'outside the copy' + $left=@('work\node_modules\x.js','work\build\cache\c.bin','work\notes.secret','private\keys\k.txt','skipped\s.txt','work\deleted.txt','..\outside.txt','work\..\..\outside.txt',(Join-Path $root 'outside.txt')) + $done=@([MirrorVerifier]::Repair($dst,$src+'\',$rec,2,[string[]]@(@('work\changed.txt','work\resized.txt','work\not-copied.txt','work\same.txt')+$left),$paths,$on,$true,[string[]]@('node_modules'),[string[]]@('*.secret'),[string[]]@('work\build\cache'),[string[]]@())) + $what={param($rel)(@($done|Where-Object {$_.Split("`t")[1] -eq $rel})[0]).Split("`t")[0]} + Assert ((& $what 'work\changed.txt') -eq 'REPAIRED' -and (& $what 'work\resized.txt') -eq 'REPAIRED' -and (& $what 'work\not-copied.txt') -eq 'REPAIRED' -and (& $what 'work\same.txt') -eq 'MATCHES') 'a repair copies again the files that differ or are missing, and leaves a matching file alone' + Assert (@($left|Where-Object {(& $what $_) -ne 'SKIPPED'}).Count -eq 0) 'it leaves alone skipped names, personal names, Exclude paths, NeverBackUp and Skip folders, files only in the copy, and paths that lead out of the copy' + Assert ((@(Get-ChildItem -LiteralPath $rec -Recurse -File|ForEach-Object {$_.FullName.Substring($rec.Length+1)}|Sort-Object) -join '|') -eq 'work\changed.txt|work\resized.txt' -and [IO.File]::ReadAllText((Join-Path $rec 'work\changed.txt')) -ceq 'work\changed.tXt') 'the old versions are in the recovery folder, and nothing else is' + Assert (-not(Test-Path -LiteralPath (Join-Path $dst 'work\node_modules')) -and -not(Test-Path -LiteralPath (Join-Path $dst 'private')) -and [IO.File]::ReadAllText((Join-Path $root 'outside.txt')) -eq 'outside the copy' -and @(Get-ChildItem -LiteralPath (Join-Path $dst 'work') -File).Count -eq 5) 'nothing else was written, and no working file is left' + $bad=@([MirrorVerifier]::Compare($dst,$src+'\',2,$paths,$on,$true,[string[]]@('node_modules'),[string[]]@('*.secret'),[string[]]@('work\build\cache'),[ref]$vf,[ref]$vb)) + Assert (($bad -join '|') -eq "MISSING-IN-SOURCE`twork\deleted.txt" -and $vf -eq 4) 'after the repair the content check finds only the file that is only in the copy' + # --- Files gone from the source (robocopy /MT listing) -------------------------------------- # Line shapes seen in a real /MT log: a record's path on the next line, a path followed by the # next record, two records on one line, and an error line that names a folder that is not extra. @@ -64,6 +76,12 @@ try{ Assert (($found -join '|') -eq (($want|Sort-Object) -join '|')) 'every extra in a /MT listing is found, split lines included' Assert ('G:\Mirror\locked\' -notin $found) 'a folder named on an error line is not an extra' Assert ('G:\Mirror-recovery\elsewhere.txt' -notin $found) 'a path beside the mirror folder is not an extra' + # Also seen: a record that names a source path, followed at once by one that names a path in the copy + # (and the reverse). Both paths must come out whole, each for its own side. + $glued=@("`t *MISMATCH `t`t 15`tE:\Src\swap2\yG:\Mirror\swap\x","`t *EXTRA File `t`t 4`tg:\mirror\f\zE:\Src\q\w\","`t New File `t`t 17`tE:\Src\swap\x\new.txt") + Assert ((@(Get-RoboExtraPaths $glued 'G:\Mirror' 'E:\Src\'|Sort-Object) -join '|') -eq 'g:\mirror\f\z|G:\Mirror\swap\x') 'a path in the copy glued to a source path is still found whole' + Assert ((@(Get-RoboExtraPaths $glued 'E:\Src\' @('G:\Mirror','')|Sort-Object) -join '|') -eq 'E:\Src\q\w\|E:\Src\swap\x\new.txt|E:\Src\swap2\y') 'and so is the source path' + Assert ((@(Get-RoboExtraPaths @("`t *EXTRA File `t`t 1`tG:\x\a.txtE:\y\b.txt") 'G:\' 'E:\') -join '|') -eq 'G:\x\a.txt') 'also when both sides are drive roots' # --- Drive identity ----------------------------------------------------------------------- $letter=$src.Substring(0,1) @@ -71,8 +89,14 @@ try{ if($other.Count){ $job=Join-Path $root 'job.json' [pscustomobject]@{Source=$src+'\';SourceVolumeId=(Get-Volume -DriveLetter $letter).UniqueId;DestinationVolumeId='not-this-drive';DefaultMode='Mirror';Choices=@()}|ConvertTo-Json|Set-Content -LiteralPath $job - Assert (Throws {& (Join-Path $PSScriptRoot 'Invoke-FastMirror.ps1') -ConfigPath $job -DestinationDrive ([string]$other[0].DriveLetter) -Folder 'never-created'} 'not the backup drive') 'the plain copy refuses a drive that is not the one in the job' - Assert (-not(Test-Path -LiteralPath ([string]$other[0].DriveLetter+':\never-created'))) 'nothing was created on that drive' + # This runs the real tool against a real drive and relies on the very check it tests. So it only + # asks for a content check, which never creates a copy, and keeps the run records in the disposable + # folder: if the check ever let the drive through, nothing would be written to that drive or to + # the real state folder. + $realState=$env:LOCALAPPDATA;$env:LOCALAPPDATA=Join-Path $root 'state' + try{Assert (Throws {& (Join-Path $PSScriptRoot 'Invoke-FastMirror.ps1') -ConfigPath $job -DestinationDrive ([string]$other[0].DriveLetter) -Folder 'never-created' -VerifyOnly} 'not the backup drive') 'the plain copy refuses a drive that is not the one in the job'} + finally{$env:LOCALAPPDATA=$realState} + Assert (-not(Test-Path -LiteralPath ([string]$other[0].DriveLetter+':\never-created')) -and -not(Test-Path -LiteralPath (Join-Path $root 'state'))) 'nothing was created on that drive, and no run was recorded' }else{Write-Output 'SKIP drive identity: this computer has only one lettered volume'} # --- Plain copy folder names and job files ------------------------------------------------ diff --git a/Test-BackupMirror.ps1 b/Test-BackupMirror.ps1 new file mode 100644 index 0000000..c35359a --- /dev/null +++ b/Test-BackupMirror.ps1 @@ -0,0 +1,440 @@ +#Requires -Version 7.2 +# The plain copy (Invoke-FastMirror.ps1) without a backup drive: its rules as functions of +# Backup.Selection.psm1 (robocopy totals and exit codes, log names, clean-up of recovery folders, name +# comparison, Exclude patterns, the same-disk rule) and whole runs through the script's -TestRoot, which +# copies into a disposable folder below the temporary folder instead of onto a drive. Nothing outside +# that folder is written; it is removed at the end. -KeepGoing reports every failing check instead of +# stopping at the first. +param([switch]$KeepGoing) +$ErrorActionPreference='Stop' +Set-StrictMode -Version Latest +Import-Module (Join-Path $PSScriptRoot 'Backup.Selection.psm1') -Force +$tool=Join-Path $PSScriptRoot 'Invoke-FastMirror.ps1' +$script:failed=0 +function Assert($ok,$message){if(-not $ok){if($KeepGoing){$script:failed++;Write-Output ('FAIL '+$message);return};throw "FAIL $message"};Write-Output ('PASS '+$message)} +function Section([string]$Name,[scriptblock]$Body){try{& $Body}catch{if(-not $KeepGoing){throw};$script:failed++;Write-Output ("FAIL $Name stopped: "+($_.Exception.Message -replace '\s+',' '))}} +function Throws([scriptblock]$Block,[string]$Pattern){try{& $Block|Out-Null;$false}catch{$_.Exception.Message -match $Pattern}} +$root=Join-Path ([IO.Path]::GetTempPath()) ('dev-drive-backup-mirror-'+[guid]::NewGuid().ToString('N')) +# \\?\ keeps a name that ends in a dot as it is written. +function Put([string]$Path,[string]$Text){[IO.Directory]::CreateDirectory('\\?\'+[IO.Path]::GetDirectoryName($Path))|Out-Null;[IO.File]::WriteAllText('\\?\'+$Path,$Text)} +function New-Case([string]$Name,$Files,[hashtable]$Config){ + $c=Join-Path $root $Name;$s=Join-Path $c 'source' + foreach($k in $Files.Keys){Put ($s+'\'+$k) $Files[$k]} + [pscustomobject]@{Root=$c;Source=$s;Copy=(Join-Path $c 'Copy');Recovery=(Join-Path $c 'Copy-recovery');Job=(Join-Path $c 'job.json');Success=(Join-Path $c 'state\last-success-test-Copy.json');Config=$Config} +} +function Mirror($Case,[hashtable]$Extra=@{}){ + $job=[ordered]@{Source=$Case.Source+'\'};foreach($k in $Case.Config.Keys){$job[$k]=$Case.Config[$k]} + [IO.File]::WriteAllText($Case.Job,($job|ConvertTo-Json -Depth 8)) + $out=& $tool -ConfigPath $Case.Job -DestinationDrive Z -Folder Copy -TestRoot $Case.Root -Threads 4 @Extra + ($out -join "`n")|ConvertFrom-Json +} +function InCopy($Case,[string]$Rel){[IO.File]::Exists('\\?\'+$Case.Copy+'\'+$Rel)} +function Files([string]$Folder){if(-not(Test-Path -LiteralPath $Folder)){return @()};$f=[string[]]@(Get-ChildItem -LiteralPath $Folder -Recurse -File -Force|ForEach-Object {$_.FullName.Substring($Folder.Length+1)});[Array]::Sort($f,[StringComparer]::Ordinal);$f} +$rlm=[string][char]0x200F;$zw=[string][char]0x200B # a right-to-left mark and a zero-width space: invisible in a name + +try{ + [IO.Directory]::CreateDirectory($root)|Out-Null + + # --- robocopy totals and exit codes ------------------------------------------------------ + Section 'totals and exit codes' { + $totals="`r`n Total Copied Skipped Mismatch FAILED Extras`r`n {3} : 5 4 1 {0} 0 0`r`n {4} : 9 7 0 {1} {2} 0`r`n Bytes : 260 200 0 25 35 0`r`n Times : 0:00:00 0:00:00 0:00:00 0:00:00`r`n Ended : Thursday`r`n" + $log={param($name,$text)$p=Join-Path $root $name;[IO.File]::WriteAllText($p,$text,[Text.Encoding]::Unicode);$p} + $clean=& $log 'clean.log' ("header`r`n"+($totals -f 0,0,0,'Dirs','Files')) + $cut=& $log 'cut.log' "header`r`n`t New File `t`t 3`tC:\x\a.txt`r`n" + $mixed=& $log 'mixed.log' ("header`r`n"+($totals -f 1,1,2,'Dirs','Files')) + $other=& $log 'other.log' ("header`r`n"+($totals -f 0,0,3,'Verzeich.','Dateien')) + $r=Get-RoboRunResult $clean 1 + Assert ($r.NotCopied -eq 0 -and $r.Complete -and ($r.Totals.Files -join ' ') -eq '9 7 0 0 0 0') 'a finished run with clean totals has nothing left uncopied' + foreach($code in 0,1,3,7,-1,-1073741510){$r=Get-RoboRunResult $cut $code;Assert ($r.NotCopied -ge 1 -and -not $r.Complete) "a robocopy log without totals is a failed run (exit code $code)"} + $r=Get-RoboRunResult $clean 8 + Assert ($r.NotCopied -ge 1 -and -not $r.Complete) 'exit code 8 never ends as nothing uncopied, even when the log read has clean totals' + Assert ((Get-RoboRunResult $clean 4).NotCopied -ge 1) 'the mismatch exit code counts even with clean totals' + $r=Get-RoboRunResult $mixed 12 + Assert ($r.NotCopied -eq 4 -and -not $r.Complete) 'mismatched and failed files and folders are added up' + Assert ((Get-RoboRunResult $mixed 4).Complete) 'a run with only a file-or-folder mismatch still read the whole source' + Assert ((Get-RoboRunResult $other 8).NotCopied -eq 3) 'totals are read in another Windows language' + } + + # --- log names ---------------------------------------------------------------------------- + Section 'log names' { + $used=[Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + $names=@('','a\b','a_b','A_B','main'|ForEach-Object {Get-MirrorLogName $_ $used}) + Assert ($names[0] -eq 'copy-main.log' -and $names[1] -eq 'copy-a_b.log') 'log names stay as they were when nothing collides' + Assert (@($names|ForEach-Object {$_.ToLowerInvariant()}|Sort-Object -Unique).Count -eq 5) ('selected folders whose names turn into the same file name get separate logs ('+($names -join ', ')+')') + } + + # --- which recovery folders the clean-up deletes ------------------------------------------- + Section 'recovery clean-up rule' { + $old='20200101-000000';$mid='20260101-000000';$later='20991231-235959';$own='20261112-120000' + $del={param($names,$keep,$mine)(@(Get-RecoveryFoldersToRemove $names $keep $mine|Sort-Object) -join '|')} + Assert ((& $del @($old,$mid,$later) 2 '') -eq $old) 'the newest folders by name are kept' + Assert ((& $del @($old,$mid,$later,$own) 1 $own) -eq "$old|$mid|$later") "the folder this run made is kept even when another folder's name is later" + Assert ((& $del @($old,$mid,$later,$own) 2 $own) -eq "$old|$mid") 'the folder this run made counts as one of the kept' + Assert ((& $del @($old,$mid) 5 $own) -eq '' -and (& $del @($old,$mid,$later) 0 '') -eq '') 'nothing is deleted below the limit, or with no limit' + } + + # --- names are compared as Windows compares them -------------------------------------------- + Section 'name comparison' { + $choice={param($p,$m)[pscustomobject]@{Path=$p;Mode=$m}} + $one=Get-BackupSelection ([pscustomobject]@{DefaultMode='Mirror';Choices=@((& $choice 'data' 'Skip'))}) + Assert ((Get-BackupMode $one 'DATA\x') -eq 'Skip') 'folder names are still compared in any case' + Assert ((Get-BackupMode $one ('data'+$rlm)) -eq 'Mirror') 'a folder whose name differs by an invisible mark is another folder' + Assert ((Get-BackupMode $one $zw) -eq 'Mirror' -and (Get-BackupMode $one ($zw+'\in.txt')) -eq 'Mirror') 'a name made only of an invisible character is a name, not an empty path part' + $nfc='caf'+[char]0xE9;$nfd='cafe'+[char]0x301 + $acc=Get-BackupSelection ([pscustomobject]@{DefaultMode='Mirror';Choices=@((& $choice $nfc 'Skip'))}) + Assert ((Get-BackupMode $acc $nfc) -eq 'Skip' -and (Get-BackupMode $acc $nfd) -eq 'Mirror') 'two spellings of an accented name that Windows keeps apart are kept apart' + $two=Get-BackupSelection ([pscustomobject]@{DefaultMode='Mirror';NeverBackUp=@('data');Choices=@((& $choice 'data' 'Skip'),(& $choice ('data'+$rlm) 'Both'))}) + Assert ((Get-BackupMode $two 'data') -eq 'Skip' -and (Get-BackupMode $two ('data'+$rlm)) -eq 'Both') 'two such folders can both be listed, also beside a NeverBackUp folder' + Assert (Throws {Get-BackupSelection ([pscustomobject]@{DefaultMode='Mirror';Choices=@((& $choice 'data' 'Skip'),(& $choice 'DATA' 'Both'))})} 'twice') 'the same folder in another case is still refused as listed twice' + Assert (Throws {ConvertTo-SelectionPath 'a\.\b'} 'relative') 'a path part that is a dot is still refused' + } + + # --- Exclude paths with wildcards ----------------------------------------------------------- + Section 'Exclude path patterns' { + $t=Join-Path $root 'patterns' + foreach($f in 'app1\dist\o.js','app2\dist\o.js','app1\src\a.js','app1\src\b.tmp','deep\x\y\dist\z.js','br [x]\dist\q.js','logs\a.log','logs\b.txt'){Put ($t+'\'+$f) $f} + $find={param($p)(@(Resolve-BackupExcludePath ($t+'\') $p|Sort-Object) -join '|')} + Assert ((& $find '*\dist') -eq 'app1\dist|app2\dist|br [x]\dist') 'a star stands for one folder name' + Assert ((& $find 'app?\src\*.tmp') -eq 'app1\src\b.tmp' -and (& $find 'logs\*.log') -eq 'logs\a.log') 'a pattern can end in a file name' + Assert ((& $find 'deep\**\dist') -eq 'deep\x\y\dist' -and (& $find '**\y\dist') -eq 'deep\x\y\dist') 'two stars stand for any number of folders' + Assert ((& $find 'br [[]x]\dist') -eq '' -and (& $find 'nothing\*\here') -eq '') 'square brackets are plain characters, and no match is no path' + } + + # --- the same-disk rule ----------------------------------------------------------------------- + Section 'same-disk rule' { + $disk={param($n,$bus,$where)[pscustomobject]@{Number=$n;BusType=$bus;Location=$where}} + $disks=@{C=(& $disk 0 'NVMe' 'PCI Slot 1');G=(& $disk 2 'USB' 'USB port');E=(& $disk 1 'File Backed Virtual' 'C:\dev\drive.vhdx') + V=(& $disk 5 'File Backed Virtual' 'E:\nested.vhdx');W=(& $disk 6 'File Backed Virtual' 'G:\backup.vhdx')} + $of={param($letter)$disks[[string]$letter]} + $why={param($s,$d)[string](Get-BackupSameDiskReason $disks[$s] $disks[$d] $of)} + Assert (-not (& $why 'C' 'G') -and -not (& $why 'E' 'G') -and -not (& $why 'C' 'W') -and -not (& $why 'E' 'W')) 'separate physical disks are accepted, virtual or not' + Assert ((& $why 'C' 'C') -match 'same physical disk') 'the source disk itself is refused' + Assert ((& $why 'E' 'C') -match 'Dev Drive file') 'the disk that holds the Dev Drive file is refused' + Assert ((& $why 'C' 'E') -match 'virtual disk') 'a virtual disk whose file is on the source disk is refused' + Assert ((& $why 'C' 'V') -match 'virtual disk' -and (& $why 'E' 'V') -match 'virtual disk') 'also through a virtual disk inside a virtual disk' + } + + # --- names on the always-skipped list --------------------------------------------------------- + Section 'skipped names' { + Assert ((Get-BackupSkippedName 'tools\venv\keep' @('venv','pytest-of-*') @() $true) -eq 'venv' -and (Get-BackupSkippedName 'pytest-of-me' @('venv','pytest-of-*') @() $true) -eq 'pytest-of-me') 'a folder name on the list is found anywhere along a path, wildcards included' + Assert ((Get-BackupSkippedName 'docs\secret.key' @('venv') @('*.key') $false) -eq 'secret.key' -and -not (Get-BackupSkippedName 'docs\venv' @('venv') @() $false)) 'file names are checked against the personal names only' + Assert (-not (Get-BackupSkippedName 'src\app\environment' @('venv','.next') @('*.key') $true)) 'other names are not skipped' + } + + # --- whole runs ------------------------------------------------------------------------------- + Section 'test folder rule' { + $c=New-Case 'seam' @{'keep\a.txt'='a'} @{DefaultMode='Mirror';Choices=@()} + [void](Mirror $c) + Assert (Throws {& $tool -ConfigPath $c.Job -DestinationDrive Z -Folder Copy -TestRoot (Join-Path $PSScriptRoot 'not-below-temp')} 'temporary folder') '-TestRoot is refused outside the temporary folder' + Assert (-not(Test-Path -LiteralPath (Join-Path $PSScriptRoot 'not-below-temp'))) 'and nothing was created there' + [IO.Directory]::CreateDirectory((Join-Path $root 'elsewhere'))|Out-Null + New-Item -ItemType Junction -Path (Join-Path $root 'link') -Target (Join-Path $root 'elsewhere')|Out-Null + Assert (Throws {& $tool -ConfigPath $c.Job -DestinationDrive Z -Folder Copy -TestRoot (Join-Path $root 'link\case')} 'link') '-TestRoot is refused when a link on the way could lead out of the temporary folder' + Assert (@(Get-ChildItem -LiteralPath (Join-Path $root 'elsewhere') -Force).Count -eq 0) 'and nothing was created behind the link' + # A link standing where the copy itself is written leads out just the same. + $inner=Join-Path $root 'inner';[IO.Directory]::CreateDirectory($inner)|Out-Null + New-Item -ItemType Junction -Path (Join-Path $inner 'Copy') -Target (Join-Path $root 'elsewhere')|Out-Null + Assert (Throws {& $tool -ConfigPath $c.Job -DestinationDrive Z -Folder Copy -TestRoot $inner} 'link') '-TestRoot is refused when the copy folder inside it is a link' + Assert (@(Get-ChildItem -LiteralPath (Join-Path $root 'elsewhere') -Force).Count -eq 0) 'and nothing was copied through it' + [IO.Directory]::Delete((Join-Path $inner 'Copy')) # the junction only + } + + Section 'copy, check, deleted files' { + $c=New-Case 'base' ([ordered]@{'both\a.txt'='a';'both\gone.txt'='deleted later';'both\inner [skip]\f.txt'='skipped';'both\inner [skip]\keep\k.txt'='kept';'both\node_modules\x.js'='cache' + 'skipped\d.txt'='d';'skipped\evidence\e.txt'='e';'unlisted\g.txt'='g';'credentials\secret.txt'='never';'note.md'='single file';'empty.bin'='';'עברית\ש.txt'='hebrew';'old\one.txt'='1';'old\sub\two.txt'='2'}) @{ + DefaultMode='Mirror';NeverBackUp=@('credentials');Exclude=@('node_modules');Choices=@(@{Path='both';Mode='Both'},@{Path='both\inner [skip]';Mode='Skip'},@{Path='both\inner [skip]\keep';Mode='Both'},@{Path='skipped';Mode='Skip'},@{Path='skipped\evidence';Mode='Both'},@{Path='note.md';Mode='Versioned'})} + $r=Mirror $c + Assert ($r.CopyErrors -eq 0 -and $r.Verify.Differences -eq 0 -and $r.Verify.FilesMatched -eq 9 -and (Test-Path -LiteralPath $c.Success)) 'the first copy is checked in full and recorded as complete' + Assert (((Files $c.Copy) -join '|') -eq 'both\a.txt|both\gone.txt|both\inner [skip]\keep\k.txt|empty.bin|old\one.txt|old\sub\two.txt|skipped\evidence\e.txt|unlisted\g.txt|עברית\ש.txt') 'only the selected files are in the copy' + [IO.File]::Delete((Join-Path $c.Source 'both\gone.txt'));[IO.Directory]::Delete((Join-Path $c.Source 'old'),$true) + $r=Mirror $c @{MoveExtras=$true;Verify=$true} + Assert ($r.ExtrasFound -eq 3 -and $r.ExtrasMoved -eq 3 -and @($r.ExtrasNotMoved).Count -eq 0) 'files gone from the source are counted as files, a deleted folder by its files' + Assert (((Files $r.ExtrasMovedTo) -join '|') -eq 'both\gone.txt|old\one.txt|old\sub\two.txt' -and -not(Test-Path -LiteralPath (Join-Path $c.Copy 'old')) -and (InCopy $c 'both\a.txt')) 'they are moved to the recovery folder, and nothing else is' + Assert ($r.Verify.Differences -eq 0 -and $r.Verify.FilesMatched -eq 6) 'the copy still matches the source' + } + + Section 'old copy of a skipped folder inside a deleted folder' { + $c=New-Case 'skipped-inside' @{'keep\a.txt'='a';'old\one.txt'='1';'old\sub\two.txt'='2';'old\private\p.txt'='stops being copied'} @{DefaultMode='Mirror';Choices=@()} + [void](Mirror $c) + $c.Config=@{DefaultMode='Mirror';Choices=@(@{Path='old\private';Mode='Skip'})} + [IO.Directory]::Delete((Join-Path $c.Source 'old'),$true) + $r=Mirror $c @{MoveExtras=$true} + Assert ((InCopy $c 'old\private\p.txt') -and ((Files $c.Recovery) -join '|') -notmatch 'private') 'the old copy of a folder that is no longer selected stays in the copy when its parent folder is deleted from the source' + Assert ($r.ExtrasFound -eq 2 -and $r.ExtrasMoved -eq 2 -and ((Files $r.ExtrasMovedTo) -join '|') -eq 'old\one.txt|old\sub\two.txt') 'the selected files of that folder are still moved aside' + } + + Section 'two selected folders, one log name' { + $c=New-Case 'logs' @{'a\b\db.bin'='v1';'a\b\ok.txt'='ok';'a_b\fine.txt'='fine'} @{DefaultMode='Skip';Choices=@(@{Path='a\b';Mode='Mirror'},@{Path='a_b';Mode='Mirror'})} + $r=Mirror $c + Assert ($r.CopyErrors -eq 0 -and $r.Verify.Differences -eq 0) 'both folders are copied' + $before=(Get-FileHash -LiteralPath $c.Success).Hash + [IO.File]::WriteAllText((Join-Path $c.Source 'a\b\db.bin'),'v2, changed and held open by another program') + $hold=[IO.File]::Open((Join-Path $c.Source 'a\b\db.bin'),'Open','ReadWrite','None') + try{$r=Mirror $c}finally{$hold.Dispose()} + Assert (@($r.Runs|ForEach-Object {$_.Log}|Sort-Object -Unique).Count -eq 2) 'each selected folder has its own log' + Assert ($r.CopyErrors -ge 1) 'a file that could not be copied is counted, whatever the other folder reports' + Assert ((Get-FileHash -LiteralPath $c.Success).Hash -eq $before) 'and the run is not recorded as a complete copy' + } + + Section 'clean-up of recovery folders' { + $c=New-Case 'prune' @{'keep\a.txt'='a';'keep\gone.txt'='the only copy left'} @{DefaultMode='Mirror';Choices=@()} + [void](Mirror $c) + Put (Join-Path $c.Recovery '20200101-000000\old.txt') 'old';Put (Join-Path $c.Recovery '20991231-235959\later.txt') 'named later than now';Put (Join-Path $c.Recovery 'my-notes\n.txt') 'not ours' + [IO.File]::Delete((Join-Path $c.Source 'keep\gone.txt')) + $r=Mirror $c @{MoveExtras=$true;KeepRecovery=1} + Assert (Test-Path -LiteralPath (Join-Path $r.ExtrasMovedTo 'keep\gone.txt')) 'the recovery folder made in this run is not deleted by its own clean-up' + $dated=@(Get-ChildItem -LiteralPath $c.Recovery -Directory|Where-Object Name -match '^\d{8}-\d{6}$') + Assert ($dated.Count -eq 1 -and $dated[0].FullName -eq $r.ExtrasMovedTo -and @($r.RecoveryRemoved).Count -eq 2 -and (Test-Path -LiteralPath (Join-Path $c.Recovery 'my-notes\n.txt'))) 'it is the one kept; older folders go, other folders are left alone' + } + + Section 'file and folder that swapped kinds' { + $c=New-Case 'swap' @{'ok\a.txt'='a';'swap\x'='x is a file';'swap2\y\inner.txt'='y is a folder';'swap2\y\deep\d.txt'='d'} @{DefaultMode='Mirror';Choices=@()} + [void](Mirror $c) + $before=(Get-FileHash -LiteralPath $c.Success).Hash + [IO.File]::Delete((Join-Path $c.Source 'swap\x'));Put (Join-Path $c.Source 'swap\x\new.txt') 'x is now a folder' + [IO.Directory]::Delete((Join-Path $c.Source 'swap2\y'),$true);Put (Join-Path $c.Source 'swap2\y') 'y is now a file' + $r=Mirror $c @{VerifyOnly=$true} + Assert ($r.Verify.Differences -eq 2 -and @($r.TypeSwapsHealed).Count -eq 0 -and (InCopy $c 'swap\x') -and (InCopy $c 'swap2\y\inner.txt') -and -not(Test-Path -LiteralPath $c.Recovery)) 'a content check reports the two as differences and moves nothing' + $r=Mirror $c @{Verify=$true} + Assert ($r.CopyErrors -eq 0 -and @($r.TypeConflicts).Count -eq 0 -and (InCopy $c 'swap\x\new.txt') -and (InCopy $c 'swap2\y')) 'a backup copies the new folder and the new file in the same run, without -MoveExtras' + Assert ((@($r.TypeSwapsHealed|ForEach-Object {$_.Source}|Sort-Object) -join '|') -eq ((Join-Path $c.Source 'swap\x')+'|'+(Join-Path $c.Source 'swap2\y'))) 'the summary names both' + $to=@{};foreach($h in $r.TypeSwapsHealed){$to[$h.Copy.Substring($c.Copy.Length+1)]=$h.MovedTo} + Assert ($to['swap\x'] -eq (Join-Path $r.RecoveryFolder 'swap\x') -and [IO.Path]::GetDirectoryName($r.RecoveryFolder) -eq $c.Recovery -and [IO.Path]::GetFileName($r.RecoveryFolder) -match '^\d{8}-\d{6}$') 'the old items went to the dated recovery folder of this run, under their own paths' + Assert ([IO.File]::ReadAllText($to['swap\x']) -eq 'x is a file' -and ((Files $to['swap2\y']) -join '|') -eq 'deep\d.txt|inner.txt') 'and nothing of them is lost' + Assert ($r.Verify.Differences -eq 0 -and $r.Verify.FilesMatched -eq 3 -and (Get-FileHash -LiteralPath $c.Success).Hash -ne $before) 'the copy matches the source again and the run counts as a complete backup' + Assert (@($r.Runs).Count -eq 2 -and @($r.Runs|Where-Object {$_.Repeated}).Count -eq 1 -and $r.Runs[0].Repeated -and $r.Runs[1].NotCopied -eq 0 -and $r.Runs[0].Log -ne $r.Runs[1].Log) 'the folder was copied a second time after healing, and that run tells what is left uncopied' + # Together with -MoveExtras: one recovery folder for the run. + [IO.File]::Delete((Join-Path $c.Source 'swap\x\new.txt'));[IO.Directory]::Delete((Join-Path $c.Source 'swap\x'));Put (Join-Path $c.Source 'swap\x') 'x is a file again';[IO.File]::Delete((Join-Path $c.Source 'ok\a.txt')) + $r=Mirror $c @{MoveExtras=$true;Verify=$true} + Assert ($r.CopyErrors -eq 0 -and @($r.TypeSwapsHealed).Count -eq 1 -and $r.ExtrasMoved -eq 1 -and $r.ExtrasMovedTo -eq $r.RecoveryFolder -and ((Files $r.RecoveryFolder) -join '|') -eq 'ok\a.txt|swap\x\new.txt' -and $r.Verify.Differences -eq 0) 'with -MoveExtras the healed item and the files gone from the source share the recovery folder of the run' + # The same at a selected folder or file itself. + $c=New-Case 'swap-root' @{'note.md'='n';'dir\f.txt'='f';'keep\a.txt'='a'} @{DefaultMode='Skip';Choices=@(@{Path='note.md';Mode='Mirror'},@{Path='dir';Mode='Mirror'},@{Path='keep';Mode='Mirror'})} + [IO.Directory]::CreateDirectory((Join-Path $c.Copy 'note.md\old'))|Out-Null;Put (Join-Path $c.Copy 'dir') 'a file where the selected folder belongs' + $r=Mirror $c + Assert ($r.CopyErrors -eq 0 -and @($r.TypeSwapsHealed).Count -eq 2 -and @($r.TypeConflicts).Count -eq 0 -and (InCopy $c 'note.md') -and (InCopy $c 'dir\f.txt') -and (InCopy $c 'keep\a.txt') -and $r.Verify.Differences -eq 0) 'a selected folder or file whose place in the copy is taken by the other kind is healed the same way' + Assert ([IO.File]::ReadAllText((Join-Path $r.RecoveryFolder 'dir')) -eq 'a file where the selected folder belongs' -and (Test-Path -LiteralPath (Join-Path $r.RecoveryFolder 'note.md\old') -PathType Container)) 'and the old items are in the recovery folder' + # An old item that cannot be moved (another program holds it open) is reported as before. + $c=New-Case 'swap-held' @{'s\x\new.txt'='a folder now';'keep\a.txt'='a'} @{DefaultMode='Mirror';Choices=@()} + Put (Join-Path $c.Copy 's\x') 'the old file' + $hold=[IO.File]::Open((Join-Path $c.Copy 's\x'),'Open','Read','None') + try{$r=Mirror $c @{SkipVerify=$true}}finally{$hold.Dispose()} + $text=[IO.File]::ReadAllText((Join-Path $r.RunFolder 'copy-errors.txt')) + Assert ($r.CopyErrors -ge 1 -and @($r.TypeSwapsHealed).Count -eq 0 -and (@($r.TypeConflicts) -join '|') -eq (Join-Path $c.Copy 's\x') -and $text.Contains((Join-Path $c.Copy 's\x')) -and $text.Contains((Join-Path $c.Source 's\x'))) 'an old item that cannot be moved stays a counted error, named with both paths' + Assert ((InCopy $c 's\x') -and (InCopy $c 'keep\a.txt') -and -not $r.RecoveryFolder -and @(Get-ChildItem -LiteralPath $c.Recovery -Directory -ErrorAction SilentlyContinue).Count -eq 0 -and -not(Test-Path -LiteralPath $c.Success)) 'it stays in the copy, the rest is copied, no empty recovery folder is left and the run is not complete' + $r=Mirror $c @{SkipVerify=$true} + Assert ($r.CopyErrors -eq 0 -and @($r.TypeSwapsHealed).Count -eq 1 -and (InCopy $c 's\x\new.txt') -and (Test-Path -LiteralPath $c.Success)) 'the next run heals it' + # A folder in the copy that holds the old copy of a folder no longer selected is kept where it is. + $c=New-Case 'swap-kept' @{'p\x'='a file now';'keep\a.txt'='a'} @{DefaultMode='Mirror';Choices=@(@{Path='p\x\private';Mode='Skip'})} + Put (Join-Path $c.Copy 'p\x\private\old.txt') 'old copy of a folder that is no longer selected' + $r=Mirror $c @{SkipVerify=$true} + Assert ($r.CopyErrors -ge 1 -and @($r.TypeSwapsHealed).Count -eq 0 -and (InCopy $c 'p\x\private\old.txt') -and (@($r.TypeConflicts) -join '|') -eq (Join-Path $c.Copy 'p\x')) 'a folder in the copy that holds the old copy of a folder no longer selected is not moved, and is named as before' + } + + Section 'moving a swapped item aside' { + $t=Join-Path $root 'move-rule';$s=Join-Path $t 'source';$m=Join-Path $t 'copy';$rec=Join-Path $t 'rec' + Put "$s\a\in.txt" 'folder';Put "$m\a" 'file';Put "$m\gone" 'file';Put "$s\same" 's';Put "$m\same" 'm';Put "$s\b" 'file';Put "$m\b\in.txt" 'folder';Put "$m\link" 'file' + New-Item -ItemType Junction -Path "$s\link" -Target "$s\a"|Out-Null + Assert (-not(Move-BackupTypeSwap "$s\gone" "$m\gone" "$rec\gone") -and (Test-Path -LiteralPath "$m\gone") -and -not(Test-Path -LiteralPath $rec)) 'nothing is moved when the source item no longer exists' + Assert (-not(Move-BackupTypeSwap "$s\same" "$m\same" "$rec\same") -and (Test-Path -LiteralPath "$m\same")) 'nor when both are the same kind' + Assert (-not(Move-BackupTypeSwap "$s\link" "$m\link" "$rec\link") -and (Test-Path -LiteralPath "$m\link")) 'nor when the source item is a link' + Assert ((Move-BackupTypeSwap "$s\a" "$m\a" "$rec\a") -and [IO.File]::ReadAllText("$rec\a") -eq 'file' -and -not(Test-Path -LiteralPath "$m\a")) 'a file in the way of a folder is moved' + Assert ((Move-BackupTypeSwap "$s\b" "$m\b" "$rec\deep\b") -and (Test-Path -LiteralPath "$rec\deep\b\in.txt") -and -not(Test-Path -LiteralPath "$m\b")) 'a folder in the way of a file is moved whole' + # What lies behind a link is not part of the source or of the copy: nothing is moved from there. + [IO.Directory]::CreateDirectory("$t\elsewhere\sub")|Out-Null;New-Item -ItemType Junction -Path "$m\j" -Target "$t\elsewhere"|Out-Null + Assert ((Test-BackupLinkOnPath $m 'j\sub\x') -and (Test-BackupLinkOnPath $m 'j\x') -and (Test-BackupLinkOnPath ($s+'\') 'link\in.txt')) 'a link or junction on the way to an item is found' + Assert (-not(Test-BackupLinkOnPath $m 'j') -and -not(Test-BackupLinkOnPath $s 'a\in.txt') -and -not(Test-BackupLinkOnPath $s 'x')) 'the item itself and ordinary folders are not' + # A source folder that keeps letter case apart holds a file and a folder of one name: the copy can + # hold only one of them, so neither is moved out of the way of the other. + [IO.Directory]::CreateDirectory("$s\cs")|Out-Null;& fsutil.exe file setCaseSensitiveInfo "$s\cs" enable 2>&1|Out-Null + $made=$LASTEXITCODE -eq 0 + if($made){try{[IO.File]::WriteAllText("$s\cs\Both",'file');[IO.Directory]::CreateDirectory("$s\cs\both")|Out-Null;$made=@(Get-ChildItem -LiteralPath "$s\cs" -Force).Count -eq 2}catch{$made=$false}} + if($made){ + Put "$m\cs\Both" 'the copy holds the file' + Assert (-not(Move-BackupTypeSwap "$s\cs\both" "$m\cs\both" "$rec\cs\both") -and (Test-Path -LiteralPath "$m\cs\Both" -PathType Leaf)) 'nothing is moved where the source holds the name twice in different letter case' + }else{Write-Output 'SKIP one name as file and folder: this Windows cannot make a folder that keeps letter case apart here'} + } + + Section 'healing and the clean-up of recovery folders' { + $c=New-Case 'heal-prune' @{'a\one\n.txt'='1';'a\two'='2';'a\three'='3';'keep\k.txt'='k'} @{DefaultMode='Mirror';Choices=@()} + Put (Join-Path $c.Copy 'a\one') 'an old file' + Put (Join-Path $c.Recovery '20200101-000000\old.txt') 'old';Put (Join-Path $c.Recovery '20991231-235959\later.txt') 'named later than now' + $swap={param($name)[IO.File]::Delete((Join-Path $c.Source "a\$name"));Put (Join-Path $c.Source "a\$name\n.txt") $name} + $dated={@(Get-ChildItem -LiteralPath $c.Recovery -Directory|ForEach-Object {$_.Name}|Sort-Object)} + $r=Mirror $c @{SkipVerify=$true};$h1=[IO.Path]::GetFileName($r.RecoveryFolder) + Assert (@($r.TypeSwapsHealed).Count -eq 1 -and @($r.RecoveryRemoved).Count -eq 0 -and @(& $dated).Count -eq 3) 'healing alone deletes no recovery folder' + & $swap 'two' + $r=Mirror $c @{MoveExtras=$true};$h2=[IO.Path]::GetFileName($r.RecoveryFolder) + Assert (@($r.TypeSwapsHealed).Count -eq 1 -and $h2 -ne $h1 -and @($r.RecoveryRemoved).Count -eq 0 -and @(& $dated).Count -eq 4) 'nor with -MoveExtras when no number of folders to keep is given' + $r=Mirror $c @{KeepRecovery=1} + Assert (@($r.RecoveryRemoved).Count -eq 0 -and @(& $dated).Count -eq 4) 'a number to keep without -MoveExtras deletes nothing either, as before' + & $swap 'three' + $r=Mirror $c @{MoveExtras=$true;KeepRecovery=2};$h3=[IO.Path]::GetFileName($r.RecoveryFolder) + Assert ((Test-Path -LiteralPath (Join-Path $r.RecoveryFolder 'a\three')) -and (@(& $dated) -join '|') -eq (@($h3,'20991231-235959'|Sort-Object) -join '|')) 'with the keep rule, the folder this run healed into is never deleted and counts as one of the kept' + Assert ((@($r.RecoveryRemoved|Sort-Object) -join '|') -eq (@('20200101-000000',$h1,$h2|Sort-Object) -join '|')) 'and folders made by earlier healing are cleaned up like any other recovery folder' + } + + Section 'repair from the last content check' { + $c=New-Case 'repair' ([ordered]@{'d\good.txt'='good';'d\damaged.txt'='the right content';'d\resized.txt'='the right size';'d\missing.txt'='was lost from the copy';'d\fixed.txt'='put right by hand';'d\gone.txt'='deleted from the source later';'later\skipped.txt'='its folder is set to skip later'}) @{DefaultMode='Mirror';Choices=@()} + Assert (Throws {Mirror $c @{Repair=$true}} 'no plain copy') 'a repair without a copy says so' + [void](Mirror $c @{SkipVerify=$true}) + Assert (Throws {Mirror $c @{Repair=$true}} 'no earlier content check') 'a repair without an earlier content check says so' + Assert (Throws {Mirror $c @{Repair=$true;VerifyOnly=$true}} 'one of') 'a repair and a content check are not asked for together' + $before=(Get-FileHash -LiteralPath $c.Success).Hash + foreach($f in 'd\damaged.txt','d\fixed.txt','d\gone.txt','later\skipped.txt'){[IO.File]::WriteAllText((Join-Path $c.Copy $f),[IO.File]::ReadAllText((Join-Path $c.Copy $f)).ToUpperInvariant())} + [IO.File]::WriteAllText((Join-Path $c.Copy 'd\resized.txt'),'short');[IO.File]::Delete((Join-Path $c.Copy 'd\missing.txt'));Put (Join-Path $c.Copy 'd\only-in-copy.txt') 'only in the copy' + $r=Mirror $c @{VerifyOnly=$true} + Assert ($r.Verify.Differences -eq 6 -and $r.Verify.OnlyInMirror -eq 1) 'the content check finds the damaged, resized and missing files' + # Between the check and the repair: one file is put right by hand, one is deleted from the source, + # one changes in the source, and one folder is set to Skip. + [IO.File]::WriteAllText((Join-Path $c.Copy 'd\fixed.txt'),'put right by hand');[IO.File]::Delete((Join-Path $c.Source 'd\gone.txt'));[IO.File]::WriteAllText((Join-Path $c.Source 'd\resized.txt'),'changed in the source since the check') + $c.Config=@{DefaultMode='Mirror';Choices=@(@{Path='later';Mode='Skip'})} + Put (Join-Path $c.Recovery '20200101-000000\old.txt') 'old';Put (Join-Path $c.Recovery '20200102-000000\old.txt') 'old' + $global:LASTEXITCODE=0 + $r=Mirror $c @{Repair=$true;MoveExtras=$true;KeepRecovery=1};$p=$r.Repair + Assert ($p.Requested -eq 6 -and $p.Repaired -eq 3 -and $p.StillDifferent -eq 0 -and $p.Failed -eq 0 -and $p.AlreadyMatching -eq 1 -and $p.Skipped -eq 2 -and $r.CopyErrors -eq 0 -and $LASTEXITCODE -eq 0) 'the repair copies again what still differs, and only that' + Assert (@('d\damaged.txt','d\resized.txt','d\missing.txt'|Where-Object {[IO.File]::ReadAllText((Join-Path $c.Copy $_)) -cne [IO.File]::ReadAllText((Join-Path $c.Source $_))}).Count -eq 0) 'the repaired files match the source, a file changed since the check included' + Assert (((Files $p.MovedTo) -join '|') -eq 'd\damaged.txt|d\resized.txt' -and [IO.File]::ReadAllText((Join-Path $p.MovedTo 'd\damaged.txt')) -ceq 'THE RIGHT CONTENT' -and [IO.Path]::GetDirectoryName($p.MovedTo) -eq $c.Recovery -and [IO.Path]::GetFileName($p.MovedTo) -match '^\d{8}-\d{6}$') 'the old versions were moved to the dated recovery folder of this run' + Assert ((InCopy $c 'd\only-in-copy.txt') -and [IO.File]::ReadAllText((Join-Path $c.Copy 'd\gone.txt')) -ceq 'DELETED FROM THE SOURCE LATER' -and [IO.File]::ReadAllText((Join-Path $c.Copy 'later\skipped.txt')) -ceq 'ITS FOLDER IS SET TO SKIP LATER') 'files only in the copy and files no longer selected are not touched' + Assert (@(Get-ChildItem -LiteralPath (Join-Path $c.Copy 'd') -Force).Count -eq 7) 'and no working file is left in the copy' + Assert ((Get-FileHash -LiteralPath $c.Success).Hash -eq $before -and -not $r.PSObject.Properties['Verify'] -and @($r.Runs).Count -eq 0) 'a repair is not recorded as a complete backup' + Assert (@($r.RecoveryRemoved).Count -eq 0 -and (Test-Path -LiteralPath (Join-Path $c.Recovery '20200101-000000\old.txt')) -and (Test-Path -LiteralPath (Join-Path $c.Recovery '20200102-000000\old.txt'))) 'a repair deletes no recovery folder, whatever it is told to keep' + $r=Mirror $c @{Repair=$true} + Assert ($r.Repair.Requested -eq 0 -and $r.CopyErrors -eq 0 -and -not $r.Repair.MovedTo) 'a second repair finds nothing left to do' + $r=Mirror $c @{VerifyOnly=$true} + Assert ($r.Verify.Differences -eq 0) 'and the next content check finds no difference' + # A file that cannot be repaired, run the way the window runs the tool. + [IO.File]::WriteAllText((Join-Path $c.Copy 'd\good.txt'),'GOOD') + [void](Mirror $c @{VerifyOnly=$true}) + $hold=[IO.File]::Open((Join-Path $c.Source 'd\good.txt'),'Open','ReadWrite','None') + try{$out=& (Get-Process -Id $PID).Path -NoProfile -File $tool -ConfigPath $c.Job -DestinationDrive Z -Folder Copy -Repair -CopyThreads 2 -TestRoot $c.Root;$code=$LASTEXITCODE}finally{$hold.Dispose()} + $r=($out -join "`n")|ConvertFrom-Json + Assert ($code -ne 0 -and $r.CopyErrors -ge 1 -and $r.Repair.Requested -eq 1 -and $r.Repair.Failed -eq 1 -and $r.Repair.Repaired -eq 0 -and -not $r.Repair.MovedTo) 'a file that cannot be repaired is counted, and the tool ends with an error code' + Assert ([IO.File]::ReadAllText((Join-Path $c.Copy 'd\good.txt')) -ceq 'GOOD' -and @(Get-ChildItem -LiteralPath (Join-Path $c.Copy 'd') -Force).Count -eq 7 -and @(Get-ChildItem -LiteralPath $c.Recovery -Directory).Count -eq 3) 'its old version stays in the copy and nothing is left half done' + $r=Mirror $c @{Repair=$true} + Assert ($r.Repair.Repaired -eq 1 -and $r.CopyErrors -eq 0 -and [IO.File]::ReadAllText((Join-Path $c.Copy 'd\good.txt')) -ceq 'good') 'the next repair, from the same check, puts it right' + # A file of the copy that the check could not read is on the list as well. + $hold=[IO.File]::Open((Join-Path $c.Copy 'd\good.txt'),'Open','Read','None') + try{ + $r=Mirror $c @{VerifyOnly=$true} + Assert ($r.Verify.Differences -eq 1) 'a file of the copy that cannot be read is a difference' + $r=Mirror $c @{Repair=$true} + Assert ($r.Repair.Requested -eq 1 -and $r.Repair.Failed -eq 1 -and $r.CopyErrors -eq 1 -and -not $r.Repair.MovedTo -and @(Get-ChildItem -LiteralPath (Join-Path $c.Copy 'd') -Force).Count -eq 7) 'while it is still held, the repair fails for it and leaves the copy as it was' + }finally{$hold.Dispose()} + $r=Mirror $c @{Repair=$true} + Assert ($r.Repair.Requested -eq 1 -and $r.Repair.AlreadyMatching -eq 1 -and $r.Repair.Repaired -eq 0 -and $r.CopyErrors -eq 0) 'once it can be read and matches, the repair leaves it alone' + } + + Section 'files copied at once' { + $c=New-Case 'threads' @{'keep\a.txt'='a'} @{DefaultMode='Mirror';Choices=@()} + [IO.File]::WriteAllText($c.Job,([ordered]@{Source=$c.Source+'\';DefaultMode='Mirror';Choices=@()}|ConvertTo-Json -Depth 8)) + $r=((& $tool -ConfigPath $c.Job -DestinationDrive Z -Folder Copy -TestRoot $c.Root -CopyThreads 128 -SkipVerify) -join "`n")|ConvertFrom-Json + Assert ($r.CopyErrors -eq 0 -and (InCopy $c 'keep\a.txt')) 'the number of files copied at once can be given as -CopyThreads, up to the 128 the window offers' + } + + Section 'names skipped through their short name' { + Add-Type -Path (Join-Path $PSScriptRoot 'Backup.MirrorVerifier.cs') + $c=New-Case 'short' @{'web\page.html'='its short name ends in .HTM';'web\index.htm'='skipped by its own name';'web\notes.txt'='n';'web\old.html\in.txt'='a folder whose short name ends in .HTM'} @{DefaultMode='Mirror';Choices=@();Exclude=@('*.htm')} + $short=[MirrorVerifier]::ShortName((Join-Path $c.Source 'web\page.html')) + $r=Mirror $c @{SkipVerify=$true} + Assert ((InCopy $c 'web\notes.txt') -and -not(InCopy $c 'web\index.htm') -and $r.CopyErrors -eq 0) 'a wildcard name from Exclude skips the files it names' + if($short){ + Assert (-not(InCopy $c 'web\page.html') -and -not(InCopy $c 'web\old.html\in.txt')) "robocopy also skips a file and a folder whose short name ($short) fits the wildcard" + Assert ((@($r.SkippedByShortName) -join '|') -eq 'web\old.html\|web\page.html' -and $r.SkippedByShortNameCount -eq 2 -and $r.CopyErrors -eq 0) 'the summary names them as a warning, not as a copy error, and leaves out what its own name skips' + Assert ([IO.File]::ReadAllText((Join-Path $r.RunFolder 'skipped-by-short-name.txt')).Contains("web\page.html`t$short`t*.htm")) 'the run folder keeps the short name and the Exclude name of each' + }else{ + Write-Output 'SKIP short names: this drive gives files no short (8.3) names, so robocopy cannot skip a file through one' + Assert ((InCopy $c 'web\page.html') -and @($r.SkippedByShortName).Count -eq 0) 'without short names nothing is skipped through one, and nothing is reported' + } + $c=New-Case 'short-none' @{'web\page.html'='p';'web\notes.txt'='n'} @{DefaultMode='Mirror';Choices=@();Exclude=@('notes.txt')} + $r=Mirror $c @{SkipVerify=$true} + Assert ((InCopy $c 'web\page.html') -and @($r.SkippedByShortName).Count -eq 0 -and $r.SkippedByShortNameCount -eq 0) 'without a wildcard name in Exclude nothing is reported' + } + + Section 'names that differ only by letter case' { + Add-Type -Path (Join-Path $PSScriptRoot 'Backup.MirrorVerifier.cs') + Assert ((@([MirrorVerifier]::SameNames([string[]]@('Makefile','b.txt','makefile','MAKEFILE','B.TXT','c'))) -join '|') -eq "Makefile`tmakefile|Makefile`tMAKEFILE|b.txt`tB.TXT") 'names that differ only by the case of their letters are paired with the first of them' + Assert (@([MirrorVerifier]::SameNames([string[]]@('a','b','a1',('a'+$zw)))).Count -eq 0) 'other names are not' + $c=New-Case 'case' @{'plain\a.txt'='a'} @{DefaultMode='Mirror';Choices=@()} + $dir=Join-Path $c.Source 'cs';[IO.Directory]::CreateDirectory($dir)|Out-Null + & fsutil.exe file setCaseSensitiveInfo $dir enable 2>&1|Out-Null + $made=$LASTEXITCODE -eq 0 + if($made){try{[IO.File]::WriteAllText("$dir\Makefile",'upper');[IO.File]::WriteAllText("$dir\makefile",'lower');$made=@(Get-ChildItem -LiteralPath $dir -Force).Count -eq 2}catch{$made=$false}} + if($made){ + $r=Mirror $c @{SkipVerify=$true} + Assert ((@($r.CaseCollisions) -join '|') -eq 'cs\Makefile | cs\makefile' -and $r.CaseCollisionsCount -eq 1) 'a folder that holds two names differing only by case is named in the summary' + }else{ + Write-Output 'SKIP case-sensitive folder: this Windows cannot make one here, so only the rule is tested' + } + $c=New-Case 'case-none' @{'plain\a.txt'='a';'plain\b.txt'='b'} @{DefaultMode='Mirror';Choices=@()} + $r=Mirror $c @{SkipVerify=$true} + Assert (@($r.CaseCollisions).Count -eq 0 -and $r.CaseCollisionsCount -eq 0) 'an ordinary folder is not' + } + + Section 'selected folder with a cache name' { + $files=@{'venv\important.txt'='not a cache';'keep\a.txt'='a';'tools\node_modules\pkg\x.js'='x'} + $choices=@(@{Path='venv';Mode='Both'},@{Path='keep';Mode='Both'},@{Path='tools\node_modules\pkg';Mode='Mirror'}) + $c=New-Case 'cache-inside' $files @{DefaultMode='Mirror';Choices=$choices} + $r=Mirror $c + Assert (-not(InCopy $c 'venv\important.txt') -and (@($r.SelectionsSkippedByName|Sort-Object) -join '|') -eq 'tools\node_modules\pkg|venv') 'a selected folder that is left out because of its name is named in the summary' + $c=New-Case 'cache-root' $files @{DefaultMode='Skip';Exclude=@('tools/node_modules');Choices=$choices} + $r=Mirror $c + Assert ((InCopy $c 'venv\important.txt') -and (InCopy $c 'tools\node_modules\pkg\x.js') -and @($r.SelectionsSkippedByName).Count -eq 0) 'a folder selected on its own is copied whatever its name' + Assert ($r.Verify.FilesMatched -eq 3 -and $r.Verify.Differences -eq 0) 'and everything copied is checked' + [IO.File]::WriteAllText((Join-Path $c.Copy 'venv\important.txt'),'NOT A CACHE') + $r=Mirror $c @{VerifyOnly=$true} + Assert ($r.Verify.Differences -eq 1) 'a damaged file in such a folder is found by the content check' + } + + Section 'names ending in a dot' { + $c=New-Case 'dots' @{'odd\normal.txt'='n';'odd\trail.'='file name ends in a dot';'odd\dir.\in.txt'='folder name ends in a dot'} @{DefaultMode='Mirror';Choices=@()} + $r=Mirror $c + Assert ((InCopy $c 'odd\trail.') -and (InCopy $c 'odd\dir.\in.txt') -and $r.CopyErrors -eq 0) 'robocopy copies names that end in a dot' + Assert ($r.Verify.Differences -eq 0 -and $r.Verify.FilesMatched -eq 3) 'and the check finds them on both sides' + } + + Section 'invisible characters in names' { + $c=New-Case 'marks' @{'data\skip.txt'='s';('data'+$rlm+'\copied.txt')='c';($zw+'\inside.txt')='i';'keep\a.txt'='a'} @{DefaultMode='Mirror';Choices=@(@{Path='data';Mode='Skip'})} + $r=Mirror $c + Assert ($r.CopyErrors -eq 0 -and $r.Verify.FilesMatched -eq 3 -and $r.Verify.Differences -eq 0) 'a folder named with an invisible character does not stop the run' + Assert (@($r.NotSelectedButInMirror).Count -eq 0) 'a copied folder is not reported as no longer selected' + } + + Section 'Exclude with wildcards in a path' { + $c=New-Case 'patterns-run' @{'app1\dist\o.js'='1';'app2\dist\o.js'='2';'app1\src\a.js'='a';'deep\x\dist\o.js'='3';'logs\run.tmp'='t';'logs\keep.txt'='k';'any\where\cache2\c.bin'='c'} @{DefaultMode='Mirror';Choices=@();Exclude=@('*/dist','logs/*.tmp','**/cache2')} + $r=Mirror $c + Assert (((Files $c.Copy) -join '|') -eq 'app1\src\a.js|deep\x\dist\o.js|logs\keep.txt') 'the plain copy takes the path patterns the versioned backup takes' + Assert ($r.CopyErrors -eq 0 -and $r.Verify.Differences -eq 0 -and $r.Verify.FilesMatched -eq 3) 'and the check skips the same paths' + } + + Section 'more skipped paths than one command can hold' { + $files=[ordered]@{'keep\a.txt'='a'};foreach($i in 1..120){$files["p$i\"+('n'*60)+'\dist\o.js']='o'} + $c=New-Case 'many' $files @{DefaultMode='Mirror';Choices=@();Exclude=@('*/*/dist')} + Assert (Throws {Mirror $c} 'Too many folders are skipped by path') 'a pattern that matches more paths than one robocopy command can hold is refused in plain words' + Assert (-not(Test-Path -LiteralPath (Join-Path $c.Copy 'keep'))) 'before anything is copied' + } + + Section 'a file in the copy that cannot be moved aside' { + $c=New-Case 'held' @{'keep\a.txt'='a';'keep\single.txt'='s';'keep\held.txt'='h';'gone\one.txt'='1';'gone\sub\two.txt'='2'} @{DefaultMode='Mirror';Choices=@()} + [void](Mirror $c) + foreach($f in 'keep\single.txt','keep\held.txt'){[IO.File]::Delete((Join-Path $c.Source $f))};[IO.Directory]::Delete((Join-Path $c.Source 'gone'),$true) + $hold=[IO.File]::Open((Join-Path $c.Copy 'keep\held.txt'),'Open','Read','None') + try{$r=Mirror $c @{MoveExtras=$true}}finally{$hold.Dispose()} + Assert ($r.ExtrasFound -eq 4 -and $r.ExtrasMoved -eq 3) 'the run goes on and reports how many files were really moved' + Assert (@($r.ExtrasNotMoved).Count -eq 1 -and $r.ExtrasNotMoved[0] -eq (Join-Path $c.Copy 'keep\held.txt') -and (InCopy $c 'keep\held.txt')) 'the file that could not be moved is named and stays in the copy' + Assert (((Files $r.ExtrasMovedTo) -join '|') -eq 'gone\one.txt|gone\sub\two.txt|keep\single.txt' -and (Test-Path -LiteralPath (Join-Path $r.RunFolder 'summary.json'))) 'the others are in the recovery folder and the run has its summary' + $r=Mirror $c @{MoveExtras=$true} + Assert ($r.ExtrasMoved -eq 1 -and -not(InCopy $c 'keep\held.txt')) 'the next run moves it' + # A run that could move nothing aside made no recovery folder, so it must not push an earlier one out. + $c=New-Case 'held-only' @{'keep\a.txt'='a';'keep\held.txt'='h'} @{DefaultMode='Mirror';Choices=@()} + [void](Mirror $c) + Put (Join-Path $c.Recovery '20200101-000000\earlier.txt') 'files moved aside by an earlier run' + [IO.File]::Delete((Join-Path $c.Source 'keep\held.txt')) + $hold=[IO.File]::Open((Join-Path $c.Copy 'keep\held.txt'),'Open','Read','None') + try{$r=Mirror $c @{MoveExtras=$true;KeepRecovery=1}}finally{$hold.Dispose()} + Assert ((Test-Path -LiteralPath (Join-Path $c.Recovery '20200101-000000\earlier.txt')) -and @($r.RecoveryRemoved).Count -eq 0) 'a run that could move nothing aside deletes no earlier recovery folder' + Assert ($r.ExtrasMoved -eq 0 -and @(Get-ChildItem -LiteralPath $c.Recovery -Directory).Count -eq 1) 'and leaves no empty recovery folder behind' + } +}finally{ + if([IO.Directory]::Exists($root)){[IO.Directory]::Delete('\\?\'+$root,$true)} +} +if($script:failed){Write-Output "$($script:failed) check(s) failed";exit 1} diff --git a/Test-BackupSelection.ps1 b/Test-BackupSelection.ps1 index bd8aaef..d071c1b 100644 --- a/Test-BackupSelection.ps1 +++ b/Test-BackupSelection.ps1 @@ -3,11 +3,14 @@ # a small plain mirror (Invoke-FastMirror.ps1) and a small versioned backup (Backup-Worker.ps1) # of a disposable source. Everything it creates carries one random id and is removed at the # end unless -KeepArtifacts is given. The configured real mirror and repository are never used. -param([Parameter(Mandatory)][ValidatePattern('^[A-Za-z]$')][string]$DestinationDrive,[switch]$KeepArtifacts) +# Without -DestinationDrive only the rules are checked: that part touches no disk and needs no drive. +param([ValidatePattern('^[A-Za-z]$')][string]$DestinationDrive,[switch]$KeepArtifacts) $ErrorActionPreference='Stop' Import-Module (Join-Path $PSScriptRoot 'Backup.Selection.psm1') -Force function Assert($ok,$message){if(-not $ok){throw "FAIL $message"};Write-Output ('PASS '+$message)} -function Throws([scriptblock]$Block,[string]$Pattern){try{& $Block;$false}catch{$_.Exception.Message -match $Pattern}} +# The block's own output is thrown away: a block that returned something instead of failing would +# otherwise hand Assert that output together with False, and a list of two counts as true. +function Throws([scriptblock]$Block,[string]$Pattern){try{& $Block|Out-Null;$false}catch{$_.Exception.Message -match $Pattern}} function Config($default,[object[]]$choices){[pscustomobject]@{DefaultMode=$default;NeverBackUp=@('credentials-folder');Choices=@($choices|ForEach-Object {[pscustomobject]@{Path=$_[0];Mode=$_[1];Reason='test'}})}} # --- Rules, without touching any disk ------------------------------------------------- @@ -42,6 +45,7 @@ Assert (-not @(@(Get-BackupRegions $nested 'Mirror')+@(Get-BackupRegions $nested Assert ($empty.HasModes -and $roots.Count -eq 1 -and $roots[0].Path -eq '' -and @($roots[0].Excluded).Count -eq 0 -and (Get-BackupMode $empty 'any\folder') -eq 'Mirror') 'an empty Choices list mirrors the whole source' Assert (@($empty.Entries|ForEach-Object {$_.Path}).Count -eq 0 -and @(Get-BackupRegions $empty 'Versioned').Count -eq 0) 'an empty Choices list gives the check an empty list, not an error' } +if(-not $DestinationDrive){Write-Output 'SKIP the plain mirror and the versioned backup of a disposable source: they need -DestinationDrive';return} $real=Join-Path $PSScriptRoot 'backup.local.json' if(Test-Path -LiteralPath $real){ $realConfig=Get-Content -LiteralPath $real -Raw|ConvertFrom-Json From 276afa263b731d58aadf1356e2e9086963bc544f Mon Sep 17 00:00:00 2001 From: Raz Ben Aharon <181751257+razbenaharon@users.noreply.github.com> Date: Sat, 14 Nov 2026 20:00:14 +0200 Subject: [PATCH 02/23] Versions: mark partial versions, release dead locks, batch deletions A backup that could not read every file is tagged as partial and never counted or deleted by Keep versions. A lock left by a run that ended is removed only when it was made on this computer and its process is gone; the step is retried once. Clean-up is limited to this computer, deletes in batches, and restore below protected folders reports that content was not verified. Co-Authored-By: Claude Fable 5.2 --- Backup-History.ps1 | 10 +- Backup-Worker.ps1 | 209 ++++++++++++++++++++++++++------ Backup.History.psm1 | 124 +++++++++++++++++-- Backup.Security.psm1 | 18 ++- Backup.Status.psm1 | 16 ++- Test-BackupHistory.ps1 | 19 +-- Test-BackupRetention.ps1 | 9 +- Test-BackupSecurity.ps1 | 5 +- Test-BackupSecuritySafety.ps1 | 23 ++-- Test-BackupSystem.ps1 | 5 +- Test-BackupVersions.ps1 | 222 ++++++++++++++++++++++++++++++++++ Test-BackupVersionsRules.ps1 | 133 ++++++++++++++++++++ 12 files changed, 714 insertions(+), 79 deletions(-) create mode 100644 Test-BackupVersions.ps1 create mode 100644 Test-BackupVersionsRules.ps1 diff --git a/Backup-History.ps1 b/Backup-History.ps1 index aa7141d..e822c34 100644 --- a/Backup-History.ps1 +++ b/Backup-History.ps1 @@ -36,7 +36,7 @@ $add=C Button 520 648 230 32 'Add selected files / folders' $clear=C Button 765 648 165 32 'Clear selection' $restore=C Button 945 648 185 32 'Restore selected...' $selectionLabel=C Label 20 691 1110 24 'Restore selection: 0 items. Folders include their contents. Double-click a folder to open it.' -$status=C Label 20 725 1110 48 'Ready. Refresh history to load saved versions. The newest version of each computer/source set is protected from deletion.' +$status=C Label 20 725 1110 48 'Ready. Refresh history to load saved versions. The newest version and the newest complete version of each computer/source set are protected from deletion.' $logs=C LinkLabel 20 782 1110 25 '' $script:history=@();$script:entries=@();$script:page=0;$script:pageSize=500;$script:currentPath='/';$script:snapshot=$null;$script:worker=$null;$script:run=$null;$script:operation=$null;$script:target=$null $script:basket=[Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) @@ -49,7 +49,8 @@ function Show-Versions { foreach($snapshot in @($script:history|Sort-Object {(ConvertTo-HistoryDate $_.time).UtcTicks} -Descending)){ $date=(ConvertTo-HistoryDate $snapshot.time).LocalDateTime if($from.Checked -and $date.Date -lt $from.Value.Date){continue};if($to.Checked -and $date.Date -gt $to.Value.Date){continue} - $i=$versions.Rows.Add($false,$date.ToString('yyyy-MM-dd HH:mm:ss'),$snapshot.hostname,(@($snapshot.paths)-join '; '),$snapshot.id.Substring(0,12));$versions.Rows[$i].Tag=$snapshot + # A partial version (a backup that could not read every file) is labelled, so it is not taken for a complete one. + $i=$versions.Rows.Add($false,$date.ToString('yyyy-MM-dd HH:mm:ss')+$(if(Test-PartialSnapshot $snapshot){' (partial)'}else{''}),$snapshot.hostname,(@($snapshot.paths)-join '; '),$snapshot.id.Substring(0,12));$versions.Rows[$i].Tag=$snapshot } } function Show-Files { @@ -73,7 +74,7 @@ function Open-Folder([string]$path) {if(-not $script:snapshot){throw 'Browse a b function Confirm-Deletion($plan) { $dialog=[Windows.Forms.Form]::new();$dialog.Text='Delete backup history';$dialog.ClientSize=[Drawing.Size]::new(740,470);$dialog.StartPosition='CenterParent';$dialog.Font=$form.Font;$dialog.RightToLeft='No';$dialog.FormBorderStyle='FixedDialog';$dialog.MaximizeBox=$false $label=[Windows.Forms.Label]::new();$label.SetBounds(20,15,700,58);$label.Text='This permanently removes the listed backup versions. Their unique file versions will no longer be recoverable. Your source files are not deleted.';$dialog.Controls.Add($label) - $list=[Windows.Forms.TextBox]::new();$list.SetBounds(20,80,700,180);$list.Multiline=$true;$list.ReadOnly=$true;$list.ScrollBars='Both';$list.WordWrap=$false;$list.Text=(@($plan|ForEach-Object {(ConvertTo-HistoryDate $_.time).LocalDateTime.ToString('yyyy-MM-dd HH:mm:ss')+' '+$_.id})-join [Environment]::NewLine);$dialog.Controls.Add($list) + $list=[Windows.Forms.TextBox]::new();$list.SetBounds(20,80,700,180);$list.Multiline=$true;$list.ReadOnly=$true;$list.ScrollBars='Both';$list.WordWrap=$false;$list.Text=(@($plan|ForEach-Object {(ConvertTo-HistoryDate $_.time).LocalDateTime.ToString('yyyy-MM-dd HH:mm:ss')+' '+$_.id+$(if(Test-PartialSnapshot $_){' (partial)'}else{''})})-join [Environment]::NewLine);$dialog.Controls.Add($list) $reclaim=[Windows.Forms.CheckBox]::new();$reclaim.SetBounds(20,274,700,28);$reclaim.Text='Also reclaim unused storage (prune; may take time and temporarily need free space)';$dialog.Controls.Add($reclaim) $hint=[Windows.Forms.Label]::new();$hint.SetBounds(20,311,700,25);$hint.Text='Type DELETE to confirm deletion of '+$plan.Count+' selected version(s):';$dialog.Controls.Add($hint) $input=[Windows.Forms.TextBox]::new();$input.SetBounds(20,341,700,28);$dialog.Controls.Add($input) @@ -108,7 +109,8 @@ $timer.Add_Tick({try{ switch($script:operation){ 'Snapshots' {$script:history=@(Get-Content -LiteralPath (Join-Path $script:run 'ListingSnapshots.jsonl') -Raw|ConvertFrom-Json);Show-Versions;$status.Text='Loaded '+$script:history.Count+' backup version(s). Select a row and choose Browse selected version.'} 'Files' {$items=[Collections.Generic.List[object]]::new();foreach($line in [IO.File]::ReadLines((Join-Path $script:run 'ListingFiles.jsonl'))){$item=$line|ConvertFrom-Json;if($item.type -and $item.path -ne $script:currentPath){$items.Add($item)}};$script:entries=@($items|Sort-Object @{Expression={if($_.type -eq 'dir'){0}else{1}}},name);$script:page=0;Show-Files;$status.Text='Browsing snapshot '+$script:snapshot.Substring(0,12)+'. Files are loaded one folder at a time.'} - 'Restore' {$status.Text='Restore completed and contents verified. Restored to: '+$script:target;[void][Windows.Forms.MessageBox]::Show($form,$status.Text,'Restore completed')} + # restic skips its content check when it could not set the date of a protected parent folder: say so. + 'Restore' {$status.Text=if($s.PSObject.Properties['ContentVerified'] -and $s.ContentVerified -eq $false){'Restore completed, but the content of the restored files was NOT verified: the dates of '+@($s.Warnings).Count+' protected parent folder(s) could not be set, and the check is skipped then. The files are in: '+$script:target}else{'Restore completed and contents verified. Restored to: '+$script:target};[void][Windows.Forms.MessageBox]::Show($form,$status.Text,'Restore completed')} 'Forget' {$script:entries=@();$script:snapshot=$null;Reset-Selection;Show-Files;Start-HistoryAction 'Snapshots'} } }catch{$timer.Stop();foreach($control in $busyControls){$control.Enabled=$true};Error-Message $_.Exception.Message}}) diff --git a/Backup-Worker.ps1 b/Backup-Worker.ps1 index 82bf6e1..85e0aaf 100644 --- a/Backup-Worker.ps1 +++ b/Backup-Worker.ps1 @@ -13,7 +13,11 @@ param( # (all 0: keep every version). [ValidateRange(0,3650)][int]$KeepDays=0, [ValidateRange(0,520)][int]$KeepWeeks=0, - [ValidateRange(0,240)][int]$KeepMonths=0 + [ValidateRange(0,240)][int]$KeepMonths=0, + # Test suite only: an existing folder under the temporary directory that stands in for the backup + # drive (repository, engine cache, run folders), so the tests need no second drive and never touch + # the real state. A real run never passes it and keeps every volume check. + [string]$TestRoot ) $ErrorActionPreference='Stop' Set-StrictMode -Version Latest @@ -21,9 +25,17 @@ Import-Module (Join-Path $PSScriptRoot 'Backup.Status.psm1') -Force Import-Module (Join-Path $PSScriptRoot 'Backup.History.psm1') -Force Import-Module (Join-Path $PSScriptRoot 'Backup.Security.psm1') -Force Import-Module (Join-Path $PSScriptRoot 'Backup.Selection.psm1') -Force -if(-not $RunDirectory){$RunDirectory=Join-Path $env:LOCALAPPDATA ('DevDriveMirror\runs\'+(Get-Date -Format 'yyyyMMdd-HHmmss')+'-'+[guid]::NewGuid().ToString('N'))} +if($TestRoot){ + $TestRoot=[IO.Path]::GetFullPath($TestRoot).TrimEnd('\') + if(-not ($TestRoot+'\').StartsWith([IO.Path]::GetFullPath([IO.Path]::GetTempPath()).TrimEnd('\')+'\',[StringComparison]::OrdinalIgnoreCase) -or -not(Test-Path -LiteralPath $TestRoot -PathType Container)){throw 'TestRoot is for the test suite only: an existing folder under the temporary directory'} +} +if(-not $RunDirectory){$RunDirectory=Join-Path $(if($TestRoot){$TestRoot}else{Join-Path $env:LOCALAPPDATA 'DevDriveMirror'}) ('runs\'+(Get-Date -Format 'yyyyMMdd-HHmmss')+'-'+[guid]::NewGuid().ToString('N'))} New-Item -ItemType Directory -Path $RunDirectory -Force | Out-Null -$script:status=[ordered]@{State='Starting';Action=$Action;WorkerPid=$PID;StartedUtc=[DateTime]::UtcNow.ToString('o');UpdatedUtc='';Progress=$null;MemoryMiB=0;PeakMemoryMiB=0;CpuPercent=0;Error=$null;Snapshot=$null;RunDirectory=$RunDirectory} +# BackupSummary: restic's own summary of the backup step (Progress ends up holding the summary of the +# check that follows). Partial: the backup saved a version but could not read every file. LockReleased: +# nothing, or the locks of an earlier run that this run removed before trying a step again (Run-Engine). +$script:lockExamined=$false +$script:status=[ordered]@{State='Starting';Action=$Action;WorkerPid=$PID;StartedUtc=[DateTime]::UtcNow.ToString('o');UpdatedUtc='';Progress=$null;MemoryMiB=0;PeakMemoryMiB=0;CpuPercent=0;Error=$null;Snapshot=$null;RunDirectory=$RunDirectory;BackupSummary=$null;Partial=$false;LockReleased=$null} function Save-Status { $script:status.UpdatedUtc=[DateTime]::UtcNow.ToString('o') [void](Write-BackupStatus $script:status (Join-Path $RunDirectory 'status.json') -BestEffort) @@ -32,7 +44,7 @@ function Assert-PlainAncestors([string]$Path) { $p=[IO.Path]::GetFullPath($Path) while($p){if(Test-Path -LiteralPath $p){if((Get-Item -LiteralPath $p -Force).Attributes -band [IO.FileAttributes]::ReparsePoint){throw "Linked repository/root/restore path refused: $p"}};$parent=[IO.Directory]::GetParent($p);if(-not $parent){break};$p=$parent.FullName} } -function Run-Engine([string[]]$EngineArguments,[string]$Stage,[string]$InputSecret) { +function Invoke-EngineStep([string[]]$EngineArguments,[string]$Stage,[string]$InputSecret) { $script:status.State=$Stage;Save-Status $psi=[Diagnostics.ProcessStartInfo]::new($script:engine);$psi.UseShellExecute=$false;$psi.CreateNoWindow=$true;$psi.RedirectStandardOutput=$true;$psi.RedirectStandardError=$true # Pruning needs at least two connections, whether it runs with forget or on its own. @@ -60,7 +72,8 @@ function Run-Engine([string[]]$EngineArguments,[string]$Stage,[string]$InputSecr $line=$lineTask.Result if($null -eq $line){break} $output.WriteLine($line);$output.Flush() - try{$event=$line|ConvertFrom-Json;if($event.message_type -eq 'status'){$script:status.Progress=$event};if($event.message_type -eq 'summary'){$script:status.Progress=$event;if($event.PSObject.Properties['snapshot_id']){$script:status.Snapshot=$event.snapshot_id}}}catch{} + try{$event=$line|ConvertFrom-Json;if($event.message_type -eq 'status'){$script:status.Progress=$event};if($event.message_type -eq 'summary'){$script:status.Progress=$event;if($event.PSObject.Properties['snapshot_id']){$script:status.Snapshot=$event.snapshot_id} + if($Stage -eq 'BackingUp'){$own=[ordered]@{};foreach($name in @('files_new','files_changed','data_added','total_files_processed','total_bytes_processed','snapshot_id')){$own[$name]=if($event.PSObject.Properties[$name]){$event.$name}else{$null}};$script:status.BackupSummary=$own}}}catch{} $lineTask=$process.StandardOutput.ReadLineAsync() } if($watch.Elapsed.TotalSeconds-$lastSave -ge 2){ @@ -71,23 +84,96 @@ function Run-Engine([string[]]$EngineArguments,[string]$Stage,[string]$InputSecr } } $process.WaitForExit();$errorText=$errors.GetAwaiter().GetResult();[IO.File]::WriteAllText((Join-Path $RunDirectory ($Stage+'.errors.log')),$errorText) - # A restore below a protected folder (for example C:\Users) can restore every file but fail to set - # the timestamps of the recreated parent folders. That is a warning, not a failed restore. - $ignored=@($errorText -split "`r?`n"|Where-Object {$_ -match '^ignoring error for '}) - if($Stage -eq 'Restoring' -and $process.ExitCode -eq 1 -and $ignored.Count -and -not @($ignored|Where-Object {$_ -notmatch 'failed to restore (timestamp|attributes|security descriptor)'}).Count){ - $script:status|Add-Member -NotePropertyName Warnings -NotePropertyValue @($ignored) -Force - }elseif($process.ExitCode -ne 0){ - $hint=if($Stage -eq 'DeletingHistory'){'History deletion may be partial. Refresh history before retrying.'}elseif($Stage -eq 'BackingUp'){'Code 3 means an incomplete snapshot.'}else{'The operation did not complete successfully.'} - $engineError=[InvalidOperationException]::new("Restic $Stage returned exit code $($process.ExitCode). $hint See $Stage.errors.log. A memory allocation failure requires a higher memory budget.") + if($process.ExitCode -ne 0){ + $engineError=[InvalidOperationException]::new((Get-EngineFailureText $Stage $process.ExitCode)) $engineError.Data['ExitCode']=$process.ExitCode;throw $engineError } }finally{if($started -and -not $process.HasExited){$process.Kill($true)};$output.Dispose();$job.Dispose();$process.Dispose()} } +function Get-LockProcess($Id) { + # What runs on this computer under a lock's process number. Nothing: no process has that number. + # Without a start time: there is one, but it cannot be examined (for example another account's). + try{$p=[Diagnostics.Process]::GetProcessById([int]$Id)}catch [ArgumentException]{return}catch{return [pscustomobject]@{Name='';StartedUtc=$null}} + try{[pscustomobject]@{Name=$p.ProcessName;StartedUtc=$p.StartTime.ToUniversalTime()}} + catch{try{[void][Diagnostics.Process]::GetProcessById([int]$Id)}catch [ArgumentException]{return}catch{};[pscustomobject]@{Name='';StartedUtc=$null}} + finally{$p.Dispose()} +} +function Read-RepositoryLockIds([string]$Stage) { + Invoke-EngineStep @('list','locks','--no-lock') $Stage + @([IO.File]::ReadAllLines((Join-Path $RunDirectory ($Stage+'.jsonl')))|ForEach-Object {$_.Trim()}|Where-Object {$_}) +} +function Clear-EndedLocks { + # Reads every lock in the repository and removes them only when all of them were left by runs on this + # computer that have ended (Get-LockReleaseDecision). Removing is left to restic's own "unlock", never + # with --remove-all: it lets go of a lock of this computer whose process is gone, or of one that is + # over 30 minutes old, and keeps every other one, so a lock goes only when both tests agree. Returns + # the locks that are gone afterwards and, when none is, why. The worker's own run lock is held here. + $found=@(foreach($id in @(Read-RepositoryLockIds 'ReadingLocks')){ + $record=$null;$process=$null + if($id -cmatch '^[0-9a-f]{64}$'){try{Invoke-EngineStep @('cat','lock',$id,'--no-lock') 'ReadingLock';$record=Get-Content -LiteralPath (Join-Path $RunDirectory 'ReadingLock.jsonl') -Raw|ConvertFrom-Json}catch{$record=$null}} + if($record -is [pscustomobject] -and $record.PSObject.Properties['pid']){$process=Get-LockProcess $record.pid} + [pscustomobject]@{Id=$id;Lock=$record;Process=$process} + }) + # restic records the computer's DNS host name, as this returns it. + $decision=Get-LockReleaseDecision $found ([Net.Dns]::GetHostName()) + if(-not $decision.Release){return [pscustomobject]@{Released=@();Reason=$decision.Reason}} + Invoke-EngineStep @('unlock') 'ReleasingLock' + $left=@(Read-RepositoryLockIds 'ReadingLocksAgain') + $gone=@($decision.Locks|Where-Object {$_.Id -cnotin $left}) + $kept=@($decision.Locks|Where-Object {$_.Id -cin $left}) + [pscustomobject]@{Released=$gone;Reason=$(if($kept.Count){"the run that made the lock has ended, but restic still sees a process under its number ($($kept[0].Pid)) and keeps the lock until $(([DateTimeOffset]::Parse($kept[0].CreatedUtc,[Globalization.CultureInfo]::InvariantCulture)).ToLocalTime().AddMinutes(30).ToString('HH:mm')), 30 minutes after it was made. Try again after that."}else{''})} +} +function Run-Engine([string[]]$EngineArguments,[string]$Stage,[string]$InputSecret) { + # One restic step. A run that was stopped, crashed or was ended by the memory cap leaves its lock in the + # repository, and every later step that needs the repository then ends with exit code 11. The first + # time that happens in a run the locks are examined; when they were all left by ended runs of this + # computer they are removed, the result records it (LockReleased) and the step is tried once more. + # Otherwise nothing is removed and the failure says whose lock it is. A step that is locked out later + # in the same run fails as it is: one removal and one second try per run. + try{Invoke-EngineStep @PSBoundParameters;return}catch{if($_.Exception.Data['ExitCode'] -ne 11 -or $script:lockExamined){throw};$locked=$_} + $script:lockExamined=$true;$outcome=$null + try{$outcome=Clear-EndedLocks}catch{} + if(-not $outcome -or (-not @($outcome.Released).Count -and -not $outcome.Reason)){throw $locked} + if(-not @($outcome.Released).Count){ + $stillLocked=[InvalidOperationException]::new((Get-EngineFailureText $Stage 11 $outcome.Reason));$stillLocked.Data['ExitCode']=11;throw $stillLocked + } + $script:status.LockReleased=[ordered]@{Stage=$Stage;Locks=@($outcome.Released|ForEach-Object {[ordered]@{Id=$_.Id;Pid=$_.Pid;CreatedUtc=$_.CreatedUtc;Exclusive=$_.Exclusive}})} + # What the first try reported stays beside the second try's files. + try{[IO.File]::Move((Join-Path $RunDirectory ($Stage+'.errors.log')),(Join-Path $RunDirectory ($Stage+'.locked.errors.log')),$true)}catch{} + Invoke-EngineStep @PSBoundParameters +} function Save-RecoveryLocation([string]$Path) { $fresh=Get-Content -LiteralPath $ConfigPath -Raw|ConvertFrom-Json if($fresh.PasswordDpapiPath -ine $script:config.PasswordDpapiPath -or $fresh.DestinationVolumeId -cne $script:config.DestinationVolumeId -or $fresh.RepositoryFolder -cne $script:config.RepositoryFolder){throw 'Configuration changed during key management. Resume with the original repository configuration.'} $fresh.RecoveryKeyPath=$Path;[void](Write-BackupStatus $fresh $ConfigPath);$script:config=$fresh } +function Set-PartialMark([string]$Id) { + # Marks a version partial and returns its new ID (changing the tag changes the ID). + Run-Engine (Get-PartialMarkArguments $Id) 'MarkingPartial' + $changed=@([IO.File]::ReadAllLines((Join-Path $RunDirectory 'MarkingPartial.jsonl'))|ForEach-Object {try{$_|ConvertFrom-Json}catch{}}|Where-Object {$_ -and $_ -is [pscustomobject] -and $_.PSObject.Properties['message_type'] -and $_.message_type -eq 'changed'}) + if($changed.Count -ne 1 -or $changed[0].new_snapshot_id -cnotmatch '^[0-9a-f]{64}$'){throw 'the engine did not report the marked version'} + $changed[0].new_snapshot_id +} +# A partial version that could not be marked (for example while the repository is locked) still looks +# like a complete one. It is noted beside the password file and marked on a later run, before anything +# is deleted: until then neither the clean-up nor a deletion by hand goes ahead. +function Save-PendingPartialMark([string]$Id) { + $path=$keyPath+'.partial-pending.local.json' + $ids=@(if(Test-Path -LiteralPath $path){Get-Content -LiteralPath $path -Raw|ConvertFrom-Json})+$Id + [void](Write-BackupStatus @($ids|Select-Object -Unique) $path) +} +function Complete-PendingPartialMarks { + $path=$keyPath+'.partial-pending.local.json' + if(-not(Test-Path -LiteralPath $path)){return} + $pending=@(Get-Content -LiteralPath $path -Raw|ConvertFrom-Json) + Run-Engine @('snapshots') 'ReadingPartialMarks' + $saved=@(Get-Content -LiteralPath (Join-Path $RunDirectory 'ReadingPartialMarks.jsonl') -Raw|ConvertFrom-Json) + foreach($id in $pending){ + $version=@($saved|Where-Object {$_.id -ceq $id}) + if($version.Count -and -not (Test-PartialSnapshot $version[0])){[void](Set-PartialMark $id)} + } + [IO.File]::Delete($path) +} function Read-RepositoryKeys([string]$Stage) { Run-Engine @('key','list') $Stage @(Get-Content -LiteralPath (Join-Path $RunDirectory ($Stage+'.jsonl')) -Raw|ConvertFrom-Json) @@ -98,7 +184,7 @@ function Invoke-KeyManagement { if(-not(Test-Path -LiteralPath $journalPath)){throw 'There is no interrupted password change to resume'} $journal=Get-Content -LiteralPath $journalPath -Raw|ConvertFrom-Json if($journal.Version -ne 1 -or $journal.OldKeyId -cnotmatch '^[0-9a-f]{64}$'){throw 'Password-change journal is invalid; keep the existing protected credential and recovery files'} - if($journal.Repository -ine $script:repository -or $journal.ConfigPath -ine [IO.Path]::GetFullPath($ConfigPath)){throw 'Password-change journal belongs to another configuration or repository'} + Assert-RotationJournalTarget $journal ([string]$script:config.DestinationVolumeId) ([string]$script:config.RepositoryFolder) $script:repository if($RequestPath){ $request=Get-Content -LiteralPath $RequestPath -Raw|ConvertFrom-Json if($request.RecoveryKeyPath -ine $journal.RecoveryKeyPath){ @@ -113,6 +199,8 @@ function Invoke-KeyManagement { $export=Assert-PrivateExportPath $request.RecoveryKeyPath @($source,$script:repository,$PSScriptRoot) @($keyPath,$ConfigPath,$journalPath) if(Test-Path -LiteralPath $export){throw 'The recovery-key destination already exists. Choose a new file; existing files are never overwritten.'} if($Action -eq 'ExportKey'){ + # A recovery key that does not open the repository is worse than none: check the stored password first. + try{[void](Read-RepositoryKeys 'CheckingPassword')}catch{if($_.Exception.Data['ExitCode'] -eq 12){throw 'The password stored on this computer does not open this backup repository, so no recovery key was saved. Use an earlier recovery key file to regain access.'};throw} New-PrivateTextFile $export (Get-RecoveryKeyText $script:repository $script:password) Save-RecoveryLocation $export $script:status|Add-Member -NotePropertyName RecoveryKeyPath -NotePropertyValue $export -Force @@ -124,7 +212,7 @@ function Invoke-KeyManagement { if($newPassword -ceq $script:password){throw 'The new password must differ from the current password'} $keys=@(Read-RepositoryKeys 'ReadingAccessKeys');$current=@($keys|Where-Object {$_.current -eq $true}) if($current.Count -ne 1 -or $current[0].id -cnotmatch '^[0-9a-f]{64}$'){throw 'Cannot identify the currently used repository access key'} - $journal=[pscustomobject]@{Version=1;Repository=$script:repository;ConfigPath=[IO.Path]::GetFullPath($ConfigPath);RecoveryKeyPath=$export;OldKeyId=$current[0].id;NewKeyId=$null;OldPasswordDpapi=([IO.File]::ReadAllText($keyPath)).Trim();NewPasswordDpapi=$protectedNew.Trim();Phase='Prepared';CreatedUtc=[DateTime]::UtcNow.ToString('o')} + $journal=[pscustomobject]@{Version=1;Repository=$script:repository;ConfigPath=[IO.Path]::GetFullPath($ConfigPath);DestinationVolumeId=[string]$script:config.DestinationVolumeId;RepositoryFolder=[string]$script:config.RepositoryFolder;RecoveryKeyPath=$export;OldKeyId=$current[0].id;NewKeyId=$null;OldPasswordDpapi=([IO.File]::ReadAllText($keyPath)).Trim();NewPasswordDpapi=$protectedNew.Trim();Phase='Prepared';CreatedUtc=[DateTime]::UtcNow.ToString('o')} New-PrivateTextFile $journalPath ($journal|ConvertTo-Json -Depth 6) } $oldPassword=[Net.NetworkCredential]::new('',($journal.OldPasswordDpapi|ConvertTo-SecureString)).Password @@ -158,6 +246,8 @@ function Invoke-KeyManagement { try { Save-Status $script:config=Get-Content -LiteralPath $ConfigPath -Raw|ConvertFrom-Json + # The test folder never goes together with real settings or key files. + if($TestRoot){$temporary=[IO.Path]::GetFullPath([IO.Path]::GetTempPath()).TrimEnd('\')+'\';foreach($testPath in @($ConfigPath,$script:config.PasswordDpapiPath,$script:config.RecoveryKeyPath)){if(-not [IO.Path]::GetFullPath([string]$testPath).StartsWith($temporary,[StringComparison]::OrdinalIgnoreCase)){throw 'TestRoot is for the test suite only: the settings and key files must be under the temporary directory too'}}} $script:engine=Join-Path $PSScriptRoot 'bin\restic_0.19.1_windows_amd64.exe' if(-not(Test-Path -LiteralPath $script:engine)){throw 'Run Install-BackupEngine.ps1 first'} if($script:config.MemoryMiB -lt 512 -or $script:config.MemoryMiB -gt 8192){throw 'Memory budget must be 512..8192 MiB'} @@ -169,14 +259,18 @@ try { $script:backendConnections=if($script:config.PSObject.Properties['BackendConnections']){[int]$script:config.BackendConnections}else{1} if($script:backendConnections -lt 1 -or $script:backendConnections -gt 8){throw 'BackendConnections must be 1..8'} $source=[IO.Path]::GetFullPath($script:config.Source).TrimEnd('\')+'\' - if($Action -in @('Backup','Preview')){Assert-PlainAncestors $source;if((Get-Volume -DriveLetter $source.Substring(0,1)).UniqueId -ne $script:config.SourceVolumeId){throw 'Source volume identity changed'}} - $volume=@(Get-Volume -UniqueId $script:config.DestinationVolumeId|Where-Object {$_.DriveLetter}) - if($volume.Count -ne 1){throw 'Expected destination volume is missing or ambiguous'} - if($volume[0].UniqueId -eq $script:config.SourceVolumeId){throw 'Backup must use a separate volume'} + if($Action -in @('Backup','Preview')){Assert-PlainAncestors $source;if(-not $TestRoot -and (Get-Volume -DriveLetter $source.Substring(0,1)).UniqueId -ne $script:config.SourceVolumeId){throw 'Source volume identity changed'}} + if($TestRoot){$destinationRoot=$TestRoot;$free=[IO.DriveInfo]::new([IO.Path]::GetPathRoot($TestRoot)).AvailableFreeSpace} + else{ + $volume=@(Get-Volume -UniqueId $script:config.DestinationVolumeId|Where-Object {$_.DriveLetter}) + if($volume.Count -ne 1){throw 'Expected destination volume is missing or ambiguous'} + if($volume[0].UniqueId -eq $script:config.SourceVolumeId){throw 'Backup must use a separate volume'} + $destinationRoot=$volume[0].DriveLetter+':\';$free=$volume[0].SizeRemaining + } $folder=[string]$script:config.RepositoryFolder if($folder -notmatch '^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$' -or $folder.EndsWith('.')){throw 'RepositoryFolder must be a single plain folder name'} - $script:repository=Join-Path ($volume[0].DriveLetter+':\') $folder;Assert-PlainAncestors $script:repository - if($Action -in @('Backup','Preview') -and $volume[0].SizeRemaining -lt 5GB){throw 'Less than 5 GiB free on the backup volume'} + $script:repository=Join-Path $destinationRoot $folder;Assert-PlainAncestors $script:repository + if($Action -in @('Backup','Preview') -and $free -lt 5GB){throw 'Less than 5 GiB free on the backup volume'} # What to back up comes from the per-folder Mode shared with the plain mirror (Versioned or Both). $sources=@();$selectionExcludes=@();$privateExcludes=@() if($Action -in @('Backup','Preview')){ @@ -213,7 +307,7 @@ try { }) $script:status.Sources=$sources;$script:status.SelectionExcludes=$selectionExcludes } - $script:cache=Join-Path $env:LOCALAPPDATA 'DevDriveMirror\restic-cache';New-Item -ItemType Directory -Path $script:cache -Force|Out-Null + $script:cache=if($TestRoot){Join-Path $TestRoot 'engine-cache'}else{Join-Path $env:LOCALAPPDATA 'DevDriveMirror\restic-cache'};New-Item -ItemType Directory -Path $script:cache -Force|Out-Null $keyPath=[IO.Path]::GetFullPath($script:config.PasswordDpapiPath);$exists=Test-Path -LiteralPath (Join-Path $script:repository 'config') $recoveryPath=[IO.Path]::GetFullPath($script:config.RecoveryKeyPath) if($keyPath -ieq $recoveryPath){throw 'Protected password and portable recovery key must be different files'} @@ -251,7 +345,23 @@ try { switch($Action){ {$_ -in @('ExportKey','ChangePassword','ResumePasswordChange')} {Invoke-KeyManagement} 'Backup' { - Run-Engine ($base+@('--')+$sources) 'BackingUp';Run-Engine @('check') 'CheckingStructure' + try{Run-Engine ($base+@('--')+$sources) 'BackingUp'} + catch{ + # Exit code 3: restic saved a version although some files could not be read. Mark it (the tag + # changes, and with it the version's ID) so the clean-up rule and the newest-complete protection + # never treat it as a complete version, then fail the run as before. + if($_.Exception.Data['ExitCode'] -ne 3 -or -not $script:status.Snapshot){throw} + $script:status.Partial=$true;$markError=$null + try{ + $marked=Set-PartialMark $script:status.Snapshot + $script:status.Snapshot=$marked;if($script:status.BackupSummary){$script:status.BackupSummary.snapshot_id=$marked} + }catch{$markError=$_.Exception.Message;try{Save-PendingPartialMark $script:status.Snapshot}catch{$markError+=' It could not be noted for later either: '+$_.Exception.Message}} + $partialText='Some files could not be read, so this version is incomplete (restic exit code 3). ' + if($markError){throw ($partialText+"It was saved, but could not be marked as partial yet, so for now the history shows it like a complete version; it is marked on a later run, before any version is deleted. See BackingUp.errors.log for the files that were skipped, then back up again. ($markError)")} + throw ($partialText+'It was saved and marked as partial: the history shows it as partial and the automatic clean-up of old versions leaves it alone. See BackingUp.errors.log for the files that were skipped (for example files another program keeps open), then back up again.') + } + try{Run-Engine @('check') 'CheckingStructure'} + catch{if($_.Exception.Data['ExitCode'] -eq 11){throw ('The new version was saved, but it could not be checked. '+$_.Exception.Message)};throw} if($KeepDays+$KeepWeeks+$KeepMonths -gt 0){ # The backup itself has succeeded and been checked. A clean-up that fails is reported, not a failed # backup. Old versions are deleted first and their space freed in a second step, so the result says @@ -259,29 +369,35 @@ try { # a repository that fails this check fails the run, since it may be damaged. $new=$script:status.Snapshot $retention=[ordered]@{Days=$KeepDays;Weeks=$KeepWeeks;Months=$KeepMonths;Removed=0;Kept=$null;SpaceFreed=$false} - $remove=@();$deleting=$false + $remove=@();$deleting=$false;$computer='' try{ - Run-Engine (Get-RetentionArguments $KeepDays $KeepWeeks $KeepMonths) 'PlanningCleanUp' - $plan=@(Get-Content -LiteralPath (Join-Path $RunDirectory 'PlanningCleanUp.jsonl') -Raw|ConvertFrom-Json) + # The rule covers this computer's complete versions only (tag devdrive: a partial version carries + # another tag). The computer name is read from the version just made, as restic recorded it. + try{Complete-PendingPartialMarks}catch{throw "An earlier partial version is not marked yet, so nothing was cleaned up. $($_.Exception.Message)"} Run-Engine @('snapshots','--tag','devdrive') 'ReadingHistory' $available=@(Get-Content -LiteralPath (Join-Path $RunDirectory 'ReadingHistory.jsonl') -Raw|ConvertFrom-Json) - $remove=@(Get-RetentionRemovals $plan $new $available) + $made=@($available|Where-Object {$new -and $_.id -ceq $new}) + if($made.Count -ne 1 -or -not $made[0].hostname){throw 'The version this backup just made was not found among the saved versions, so the clean-up cannot protect it. Nothing was deleted.'} + $computer=[string]$made[0].hostname + Run-Engine (Get-RetentionArguments $KeepDays $KeepWeeks $KeepMonths $computer) 'PlanningCleanUp' + $plan=@(Get-Content -LiteralPath (Join-Path $RunDirectory 'PlanningCleanUp.jsonl') -Raw|ConvertFrom-Json) + $remove=@(Get-RetentionRemovals $plan $new $available $computer) [IO.File]::WriteAllText((Join-Path $RunDirectory 'clean-up-plan.json'),(ConvertTo-Json @($remove) -Depth 8)) - $retention.Kept=$available.Count-$remove.Count + $retention.Kept=@($available|Where-Object {$_.hostname -ceq $computer}).Count-$remove.Count if($remove.Count){ $deleting=$true - Run-Engine (@('forget')+@($remove|ForEach-Object {$_.id})) 'DeletingOldVersions' - $retention.Removed=$remove.Count + # In batches: one command line cannot hold more than about 500 IDs. + foreach($batch in @(Split-SnapshotIdBatches @($remove|ForEach-Object {$_.id}))){Run-Engine (@('forget')+@($batch)) 'DeletingOldVersions';$retention.Removed+=@($batch).Count} Run-Engine @('prune') 'FreeingSpace' $retention.SpaceFreed=$true } }catch{ $retention.Error=$_.Exception.Message # Deleting may have stopped part way: count the versions that are really gone. - if($remove.Count -and -not $retention.Removed){ + if($remove.Count -and $retention.Removed -lt $remove.Count){ try{ Run-Engine @('snapshots','--tag','devdrive') 'ReadingHistoryAgain' - $still=@(Get-Content -LiteralPath (Join-Path $RunDirectory 'ReadingHistoryAgain.jsonl') -Raw|ConvertFrom-Json|ForEach-Object {$_.id}) + $still=@(Get-Content -LiteralPath (Join-Path $RunDirectory 'ReadingHistoryAgain.jsonl') -Raw|ConvertFrom-Json|Where-Object {$_.hostname -ceq $computer}|ForEach-Object {$_.id}) $retention.Removed=@($remove|Where-Object {$_.id -notin $still}).Count $retention.Kept=@($still).Count }catch{} @@ -308,13 +424,18 @@ try { if(-not $RequestPath){throw 'An explicit history deletion request is required'} $request=Get-Content -LiteralPath $RequestPath -Raw|ConvertFrom-Json if($request.Confirmation -cne 'DELETE SELECTED HISTORY'){throw 'History deletion was not confirmed'} + try{Complete-PendingPartialMarks}catch{throw "An earlier partial version is not marked yet, so nothing was deleted. $($_.Exception.Message)"} Run-Engine @('snapshots') 'ReadingHistory' $available=Get-Content -LiteralPath (Join-Path $RunDirectory 'ReadingHistory.jsonl') -Raw|ConvertFrom-Json $plan=@(Get-HistoryDeletionPlan $available @($request.SnapshotIds)) [IO.File]::WriteAllText((Join-Path $RunDirectory 'deletion-plan.json'),($plan|ConvertTo-Json -Depth 8)) - $forgetArgs=@('forget')+@($plan.id) - if($request.ReclaimSpace -eq $true){$forgetArgs+='--prune'} - Run-Engine $forgetArgs 'DeletingHistory' + # In batches: one command line cannot hold more than about 500 IDs. Space is reclaimed with the last. + $batches=@(Split-SnapshotIdBatches @($plan|ForEach-Object {$_.id})) + for($i=0;$i -lt $batches.Count;$i++){ + $forgetArgs=@('forget')+@($batches[$i]) + if($request.ReclaimSpace -eq $true -and $i -eq $batches.Count-1){$forgetArgs+='--prune'} + Run-Engine $forgetArgs 'DeletingHistory' + } Run-Engine @('check') 'CheckingStructure' } 'Restore' { @@ -338,7 +459,23 @@ try { $restoreArguments+=@('--include-file',$includeFile) } foreach($path in $RestoreInclude){$restoreArguments+=@('--include',$path)} - Run-Engine $restoreArguments 'Restoring' + try{Run-Engine $restoreArguments 'Restoring'} + catch{ + # A restore below a protected folder (for example C:\Users) can restore every file but fail to set + # the dates of the recreated parent folders; restic then ends with exit code 1 and skips its content + # check. That is a warning, not a failed restore, but the restore is reported as not verified. Which + # folders are parents is read from the restored version itself. + if($_.Exception.Data['ExitCode'] -ne 1){throw} + $failure=$_;$summary=$script:status.Progress;$warnings=@() + try{ + Run-Engine @('snapshots',$Snapshot) 'ReadingVersion' + $version=@(Get-Content -LiteralPath (Join-Path $RunDirectory 'ReadingVersion.jsonl') -Raw|ConvertFrom-Json) + if($version.Count -eq 1){$warnings=@(Get-RestoreParentWarnings ([IO.File]::ReadAllText((Join-Path $RunDirectory 'Restoring.errors.log'))) @($version[0].paths) $summary)} + }catch{} + if(-not $warnings.Count){throw $failure} + $script:status.Warnings=$warnings + } + $script:status.ContentVerified=-not $script:status.Contains('Warnings') } } $script:status.State='Completed';$script:status.UpdatedUtc=[DateTime]::UtcNow.ToString('o');[void](Write-BackupStatus $script:status (Join-Path $RunDirectory 'result.json'));Save-Status diff --git a/Backup.History.psm1 b/Backup.History.psm1 index 4e4238e..33117f6 100644 --- a/Backup.History.psm1 +++ b/Backup.History.psm1 @@ -15,34 +15,90 @@ function ConvertTo-HistoryDate($Value) { if($Value -is [DateTime]){return [DateTimeOffset]::new($Value)} [DateTimeOffset]::Parse([string]$Value,[Globalization.CultureInfo]::InvariantCulture,[Globalization.DateTimeStyles]::RoundtripKind) } +# A backup that could not read every file (restic exit code 3) still saves a version. The worker marks +# it by replacing the tag devdrive with this one, so the clean-up rule (which only sees devdrive) never +# counts, keeps or deletes it, and history can label it. Versions saved before this mark existed cannot +# be recognised. +$script:PartialTag='devdrive-partial' +function Test-PartialSnapshot($Snapshot) { + [bool]($Snapshot -and $Snapshot.PSObject.Properties['tags'] -and @($Snapshot.tags) -ccontains $script:PartialTag) +} +function Get-PartialMarkArguments([string]$Id) { + Assert-SnapshotId $Id + @('tag','--remove','devdrive','--add',$script:PartialTag,$Id) +} +function Split-SnapshotIdBatches([string[]]$Ids,[int]$Size=200) { + # Versions are deleted by exact ID on the command line, and Windows refuses a command line over 32,767 + # characters (about 500 IDs). 200 IDs are 13,000 characters. + if($Size -lt 1){throw 'Batch size must be at least 1'} + $all=@($Ids|Where-Object {$_}) + for($i=0;$i -lt $all.Count;$i+=$Size){,@($all[$i..([math]::Min($i+$Size,$all.Count)-1)])} +} +function Get-RestoreParentWarnings([string]$ErrorText,[string[]]$SourcePaths,$Summary=$null) { + # restic ends a restore with exit code 1 when any item reported an error, and then skips checking the + # restored content. The only errors accepted as warnings are dates or attributes that could not be set + # on the folders ABOVE what the version backed up ($SourcePaths: the version's own source paths, not + # today's settings): restic recreates them inside the restore folder with their original, protected + # permissions (for example C:\Users). Returns those messages; nothing when anything else went wrong, + # when the output cannot be understood, or when more items are missing than there are warnings. + $roots=@($SourcePaths|Where-Object {$_}|ForEach-Object {'/'+([string]$_).TrimEnd('\','/').Replace(':','').Replace('\','/')}) + if(-not $roots.Count){return} + $warnings=@() + foreach($line in @($ErrorText -split "`r?`n"|Where-Object {$_.Trim()})){ + $e=$null;try{$e=$line|ConvertFrom-Json}catch{return} + if(-not $e -or $e -isnot [pscustomobject] -or -not $e.PSObject.Properties['message_type']){return} + if($e.message_type -eq 'exit_error'){continue} + if($e.message_type -ne 'error' -or -not $e.PSObject.Properties['during'] -or $e.during -ne 'restore' -or -not $e.PSObject.Properties['item'] -or -not $e.PSObject.Properties['error']){return} + $item=([string]$e.item).Replace('\','/').TrimEnd('/');$message=[string]$e.error.message + if($message -notmatch '^failed to restore (timestamp|attributes) of ' -or -not $item){return} + # Above at least one backed-up path, and neither one of them nor inside one. + $above=@($roots|Where-Object {$_.StartsWith($item+'/',[StringComparison]::OrdinalIgnoreCase)}).Count + $content=@($roots|Where-Object {$item -ieq $_ -or $item.StartsWith($_+'/',[StringComparison]::OrdinalIgnoreCase)}).Count + if(-not $above -or $content){return} + $warnings+=$message + } + if($Summary -and $Summary.PSObject.Properties['total_files'] -and $Summary.PSObject.Properties['files_restored'] -and ([long]$Summary.total_files-[long]$Summary.files_restored) -gt $warnings.Count){return} + $warnings +} function Get-HistoryDeletionPlan($Snapshots,[string[]]$Ids) { $all=@($Snapshots);$selected=@($Ids|Select-Object -Unique) if($selected.Count -eq 0){throw 'Select at least one older snapshot'} foreach($id in $selected){Assert-SnapshotId $id;if($id -cnotin @($all.id)){throw 'A selected snapshot no longer exists. Refresh history.'}} - # Preserve the newest snapshot of each host/source set, including equal-time ties. + # Preserve the newest snapshot of each host/source set, including equal-time ties, and also the newest + # complete one: a newer partial version must never be what is left of a set. foreach($group in @($all|Group-Object {([string]$_.hostname)+'|'+((@($_.paths)|Sort-Object)-join '|')})){ $latest=$group.Group|ForEach-Object {(ConvertTo-HistoryDate $_.time).UtcTicks}|Sort-Object -Descending|Select-Object -First 1 - foreach($snapshot in $group.Group){if((ConvertTo-HistoryDate $snapshot.time).UtcTicks -eq $latest -and $snapshot.id -cin $selected){throw 'The newest snapshot for each computer/source set is protected. Select older versions.'}} + $latestComplete=$group.Group|Where-Object {-not (Test-PartialSnapshot $_)}|ForEach-Object {(ConvertTo-HistoryDate $_.time).UtcTicks}|Sort-Object -Descending|Select-Object -First 1 + foreach($snapshot in $group.Group){ + if($snapshot.id -cnotin $selected){continue} + $ticks=(ConvertTo-HistoryDate $snapshot.time).UtcTicks + if($ticks -eq $latest){throw 'The newest snapshot for each computer/source set is protected. Select older versions.'} + if($null -ne $latestComplete -and $ticks -eq $latestComplete -and -not (Test-PartialSnapshot $snapshot)){throw 'The newest complete snapshot for each computer/source set is protected: the newer ones are partial. Select older versions.'} + } } @($all|Where-Object {$_.id -cin $selected}) } -function Get-RetentionArguments([int]$Days,[int]$Weeks,[int]$Months) { +function Get-RetentionArguments([int]$Days,[int]$Weeks,[int]$Months,[string]$HostName='') { # Automatic clean-up after a backup: one version per day for $Days days, per week for $Weeks weeks and # per month for $Months months. restic counts these windows back from the newest version, not from # today, so a long pause between backups never empties the history; --keep-last 1 keeps the newest - # version whatever the numbers. Only this tool's own versions (tag devdrive) are considered. Always a - # dry run: the versions to delete are checked first and then deleted by exact ID. + # version whatever the numbers. Only this tool's own complete versions (tag devdrive) are considered, + # and with $HostName only this computer's: another computer sharing the repository has its own rule. + # Always a dry run: the versions to delete are checked first and then deleted by exact ID. if($Days -lt 0 -or $Weeks -lt 0 -or $Months -lt 0 -or $Days -gt 3650 -or $Weeks -gt 520 -or $Months -gt 240){throw 'Retention numbers are out of range'} if($Days+$Weeks+$Months -eq 0){throw 'Retention needs at least one of days, weeks or months'} - $a=@('forget','--dry-run','--tag','devdrive','--group-by','host,paths','--keep-last','1') + $a=@('forget','--dry-run','--tag','devdrive') + if($HostName){$a+=@('--host',$HostName)} + $a+=@('--group-by','host,paths','--keep-last','1') if($Days){$a+=@('--keep-within-daily',"$($Days)d")} if($Weeks){$a+=@('--keep-within-weekly',"$($Weeks*7)d")} if($Months){$a+=@('--keep-within-monthly',"$($Months)m")} $a } -function Get-RetentionRemovals($Plan,[string]$NewSnapshot,$Snapshots) { +function Get-RetentionRemovals($Plan,[string]$NewSnapshot,$Snapshots,[string]$HostName='') { # The versions a retention dry run would delete, after our own checks: the version this backup just - # made must be kept, every group keeps at least one version, and the newest version of each + # made must be known, complete and kept, every group keeps at least one version, no partial version + # and (with $HostName) no other computer's version is deleted, and the newest version of each # computer/source set stays protected (Get-HistoryDeletionPlan). Returns the snapshots to delete. $groups=@($Plan|Where-Object {$_}) $remove=@();$keep=@() @@ -51,10 +107,58 @@ function Get-RetentionRemovals($Plan,[string]$NewSnapshot,$Snapshots) { if($r.Count -and -not $k.Count){throw 'The clean-up would delete every version of a folder set. Nothing was deleted.'} $keep+=@($k|ForEach-Object {[string]$_.id});$remove+=@($r|ForEach-Object {[string]$_.id}) } - if($NewSnapshot -and $NewSnapshot -cnotin $keep){throw 'The clean-up would not keep the version this backup just made. Nothing was deleted.'} + if(-not $NewSnapshot){throw 'The version this backup just made is not known, so the clean-up cannot protect it. Nothing was deleted.'} + if($NewSnapshot -cnotin $keep){throw 'The clean-up would not keep the version this backup just made. Nothing was deleted.'} + $known=@{};foreach($s in @($Snapshots)){$known[[string]$s.id]=$s} + if($known.ContainsKey($NewSnapshot) -and (Test-PartialSnapshot $known[$NewSnapshot])){throw 'The version this backup just made is partial, so nothing is cleaned up on its account. Nothing was deleted.'} $remove=@($remove|Where-Object {$_}|Select-Object -Unique) if(-not $remove.Count){return @()} if(@($remove|Where-Object {$_ -cin $keep}).Count){throw 'The clean-up plan both keeps and deletes one version. Nothing was deleted.'} + foreach($id in $remove){ + if(-not $known.ContainsKey($id)){continue} + if(Test-PartialSnapshot $known[$id]){throw 'The clean-up plan would delete a partial version. Nothing was deleted.'} + if($HostName -and [string]$known[$id].hostname -cne $HostName){throw 'The clean-up plan would delete a version of another computer. Nothing was deleted.'} + } @(Get-HistoryDeletionPlan $Snapshots $remove) } -Export-ModuleMember -Function Assert-SnapshotId,Assert-SnapshotPath,ConvertTo-ResticLiteralPattern,ConvertTo-HistoryDate,Get-HistoryDeletionPlan,Get-RetentionArguments,Get-RetentionRemovals +function Get-LockReleaseDecision($Locks,[string]$ComputerName) { + # Which locks in the repository may be removed without asking: only when EVERY lock present was made on + # this computer by a process that has ended. A run that was stopped, crashed or was ended by the memory + # cap leaves such a lock behind, and restic then refuses its next steps (exit code 11). + # $Locks: one record per lock file, with Id, Lock (what restic's "cat lock" printed: time, hostname, + # pid, exclusive; nothing when it could not be read, and a record that lacks one of them counts as + # unreadable) and Process (what runs on this computer under that process number now: nothing, or its + # Name and StartedUtc; no start time when it cannot be examined). + # The process has ended when nothing runs under its number, or when what does started after the lock was + # made and is not restic (Windows hands process numbers out again). A restic process that started before + # the lock is its maker and still running. Everything else is unclear and stays: another program that + # was already running under that number, a restic process that seems to have started after the lock + # (the clock may have been set back), a process or a lock that cannot be read. + $judged=@(foreach($l in @($Locks|Where-Object {$_})){ + $r=if($l.PSObject.Properties['Lock']){$l.Lock}else{$null};$p=$null;if($r -is [pscustomobject]){$p=$r.PSObject.Properties} + $state='Unclear';$why='the lock in the repository could not be read, so it was left in place.';$number=$null;$created=$null;$computer=$null;$exclusive=$null + if([string]$l.Id -cmatch '^[0-9a-f]{64}$' -and $l.PSObject.Properties['Process'] -and $ComputerName -and $p -and $p['hostname'] -and $p['pid'] -and $p['time'] -and $p['exclusive'] -and $r.exclusive -is [bool] -and ([string]$r.hostname).Trim() -and ($r.pid -is [int] -or $r.pid -is [long]) -and $r.pid -gt 0 -and $r.pid -le [int]::MaxValue){ + try{$created=(ConvertTo-HistoryDate $r.time).UtcDateTime}catch{$created=$null} + } + if($null -ne $created){ + $number=[int]$r.pid;$computer=[string]$r.hostname;$exclusive=$r.exclusive;$process=$l.Process + if($computer -ine $ComputerName){$state='OtherComputer';$why="another computer ($computer) is using this backup, or stopped while it was using it. This computer cannot tell which, so the lock was left in place."} + elseif($null -eq $process){$state='Ended';$why=''} + else{ + $started=$null;try{if($null -ne $process.StartedUtc){$started=(ConvertTo-HistoryDate $process.StartedUtc).UtcDateTime}}catch{$started=$null} + $engine=([string]$process.Name) -like 'restic*' + if($null -eq $started){$why="process $number on this computer may still be using this backup: it could not be examined, so the lock was left in place."} + elseif($started -le $created){ + if($engine){$state='Running';$why="a backup is still running on this computer (process $number). Wait until it has finished, then try again."} + else{$why="process $number on this computer was already running when the lock was made and may be using this backup, so the lock was left in place."} + } + elseif($engine){$why="another backup program now runs on this computer under the process number of the run that made the lock ($number), and the two cannot be told apart, so the lock was left in place."} + else{$state='Ended';$why=''} + } + } + [pscustomobject]@{Id=[string]$l.Id;Pid=$number;CreatedUtc=$(if($null -ne $created){$created.ToString('o')}else{$null});Exclusive=$exclusive;Computer=$computer;State=$state;Why=$why} + }) + $blocking=@($judged|Where-Object {$_.State -ne 'Ended'}) + [pscustomobject]@{Release=[bool]($judged.Count -and -not $blocking.Count);Reason=$(if($blocking.Count){$blocking[0].Why}else{''});Locks=$judged} +} +Export-ModuleMember -Function Assert-SnapshotId,Assert-SnapshotPath,ConvertTo-ResticLiteralPattern,ConvertTo-HistoryDate,Test-PartialSnapshot,Get-PartialMarkArguments,Split-SnapshotIdBatches,Get-RestoreParentWarnings,Get-HistoryDeletionPlan,Get-RetentionArguments,Get-RetentionRemovals,Get-LockReleaseDecision diff --git a/Backup.Security.psm1 b/Backup.Security.psm1 index 755cdba..b27a9fe 100644 --- a/Backup.Security.psm1 +++ b/Backup.Security.psm1 @@ -28,4 +28,20 @@ function Assert-PrivateExportPath([string]$Path,[string[]]$ForbiddenRoots,[strin function Get-RecoveryKeyText([string]$Repository,[string]$Password) { "Restic repository recovery key`r`nRepository: $Repository`r`nPassword: $Password`r`nKeep this file in a password manager or separate offline location. Anyone with this password can decrypt the backup.`r`n" } -Export-ModuleMember -Function New-PrivateTextFile,Set-PrivateTextFile,Assert-NewBackupPassword,Assert-PrivateExportPath,Get-RecoveryKeyText +function Assert-RotationJournalTarget($Journal,[string]$VolumeId,[string]$Folder,[string]$Repository) { + # A pending password change belongs to one repository. The backup drive can come back under another + # letter between the interruption and the resume, so the repository is recognised by its volume and + # folder. A journal written before those were recorded holds only the path: the same path, or the + # same path under another drive letter, is accepted (the old password in the journal must still open + # the repository before anything changes). + $p=$Journal.PSObject.Properties + if($p['DestinationVolumeId'] -and $p['RepositoryFolder'] -and $Journal.DestinationVolumeId -and $Journal.RepositoryFolder){ + if([string]$Journal.DestinationVolumeId -cne $VolumeId -or [string]$Journal.RepositoryFolder -ine $Folder){throw 'Password-change journal belongs to another configuration or repository'} + return + } + $old=[string]$Journal.Repository + if($old -and $old -ieq $Repository){return} + if($old -match '^[A-Za-z]:\\.' -and $Repository -match '^[A-Za-z]:\\.' -and $old.Substring(1) -ieq $Repository.Substring(1)){return} + throw 'Password-change journal belongs to another configuration or repository' +} +Export-ModuleMember -Function New-PrivateTextFile,Set-PrivateTextFile,Assert-NewBackupPassword,Assert-PrivateExportPath,Assert-RotationJournalTarget,Get-RecoveryKeyText diff --git a/Backup.Status.psm1 b/Backup.Status.psm1 index 00ae23f..a6f0e81 100644 --- a/Backup.Status.psm1 +++ b/Backup.Status.psm1 @@ -24,4 +24,18 @@ function Write-BackupStatus { }catch{if($BestEffort){return $false};throw} finally{if([IO.File]::Exists($temp)){try{[IO.File]::Delete($temp)}catch{}}} } -Export-ModuleMember -Function Read-BackupStatus,Write-BackupStatus +function Get-EngineFailureText([string]$Stage,[int]$ExitCode,[string]$LockDetail='') { + # What a failed restic step means, in plain words. Exit code 11 is a repository lock: a run that was + # stopped, crashed or ran out of memory leaves its lock file behind and restic then refuses every + # step that needs the repository to itself. This only explains it. The worker removes a lock that a + # run on this computer left behind; $LockDetail is why it left this one in place (another computer, + # a backup that is still running, a lock it could not judge), when it found out. + $hint=if($ExitCode -eq 11 -and $LockDetail){"The backup repository is locked: $LockDetail Nothing is damaged and no saved version was lost, but this step could not run."} + elseif($ExitCode -eq 11){'The backup repository is locked. An earlier run was stopped or crashed before it finished and left its lock behind, or another backup program is using this repository right now. Nothing is damaged and no saved version was lost, but this step could not run. Until the lock is removed, backups, checks and clean-up of this source will stop here.'} + elseif($ExitCode -eq 12){'The stored password does not open this backup repository.'} + elseif($Stage -eq 'DeletingHistory'){'History deletion may be partial. Refresh history before retrying.'} + elseif($Stage -eq 'BackingUp'){'Code 3 means an incomplete snapshot.'} + else{'The operation did not complete successfully.'} + "Restic $Stage returned exit code $ExitCode. $hint See $Stage.errors.log."+$(if($ExitCode -notin 11,12){' A memory allocation failure requires a higher memory budget.'}) +} +Export-ModuleMember -Function Read-BackupStatus,Write-BackupStatus,Get-EngineFailureText diff --git a/Test-BackupHistory.ps1 b/Test-BackupHistory.ps1 index 6f6bef1..60e2ae2 100644 --- a/Test-BackupHistory.ps1 +++ b/Test-BackupHistory.ps1 @@ -2,19 +2,20 @@ $ErrorActionPreference='Stop' Import-Module (Join-Path $PSScriptRoot 'Backup.History.psm1') -Force function Assert($ok,$message){if(-not $ok){throw $message};Write-Output ('PASS '+$message)} -function Refuses($action,$message){$refused=$false;try{& $action}catch{$refused=$true};Assert $refused $message} +# A refusal counts only when it is the expected one: any other error (a typo, a missing function) fails the test. +function Refuses($action,[string]$like,$message){$refused=$false;try{& $action}catch{if($_.Exception.Message -notlike $like){throw "$message (unexpected error: $($_.Exception.Message))"};$refused=$true};Assert $refused $message} $old=[pscustomobject]@{id=('a'*64);time='2026-10-16T10:00:00Z';hostname='test';paths=@('X:\source')} $new=[pscustomobject]@{id=('b'*64);time='2026-10-17T10:00:00Z';hostname='test';paths=@('X:\source')} $other=[pscustomobject]@{id=('c'*64);time='2026-10-15T10:00:00Z';hostname='other';paths=@('X:\source')} $snapshots=@($old,$new,$other) Assert (@(Get-HistoryDeletionPlan $snapshots @($old.id)).Count -eq 1) 'older selected version can be deleted' -Refuses {Get-HistoryDeletionPlan $snapshots @($new.id)} 'newest version is protected' -Refuses {Get-HistoryDeletionPlan $snapshots @($other.id)} 'newest version of another source set is also protected' -Refuses {Get-HistoryDeletionPlan $snapshots @()} 'empty selection is rejected' -Refuses {Get-HistoryDeletionPlan $snapshots @('latest')} 'aliases are rejected for deletion' -Refuses {Get-HistoryDeletionPlan $snapshots @('d'*64)} 'stale snapshot selection is rejected' -Refuses {Assert-SnapshotPath '/E/../source'} 'parent traversal is rejected' -Refuses {Assert-SnapshotPath "--flag"} 'relative and option-like paths are rejected' +Refuses {Get-HistoryDeletionPlan $snapshots @($new.id)} '*protected*' 'newest version is protected' +Refuses {Get-HistoryDeletionPlan $snapshots @($other.id)} '*protected*' 'newest version of another source set is also protected' +Refuses {Get-HistoryDeletionPlan $snapshots @()} '*Select at least*' 'empty selection is rejected' +Refuses {Get-HistoryDeletionPlan $snapshots @('latest')} '*exact snapshot ID*' 'aliases are rejected for deletion' +Refuses {Get-HistoryDeletionPlan $snapshots @('d'*64)} '*no longer exists*' 'stale snapshot selection is rejected' +Refuses {Assert-SnapshotPath '/E/../source'} '*Invalid absolute snapshot path*' 'parent traversal is rejected' +Refuses {Assert-SnapshotPath "--flag"} '*Invalid absolute snapshot path*' 'relative and option-like paths are rejected' Assert ((ConvertTo-ResticLiteralPattern '/E/a [sample].txt') -ceq '/E/a[ ][[]sample].txt') 'bracket filename is escaped for literal restore' Assert ((ConvertTo-ResticLiteralPattern '/E/a*?.txt') -ceq '/E/a[*][?].txt') 'wildcard characters are escaped for literal restore' $previousCulture=[Globalization.CultureInfo]::CurrentCulture @@ -22,5 +23,5 @@ try{ [Globalization.CultureInfo]::CurrentCulture=[Globalization.CultureInfo]::GetCultureInfo('he-IL') $typed=@($snapshots|ConvertTo-Json -Depth 5|ConvertFrom-Json) Assert (@(Get-HistoryDeletionPlan $typed @($old.id)).Count -eq 1) 'JSON dates work with non-English Windows culture' - Refuses {Get-HistoryDeletionPlan $typed @($new.id)} 'newest protection survives JSON date conversion' + Refuses {Get-HistoryDeletionPlan $typed @($new.id)} '*protected*' 'newest protection survives JSON date conversion' }finally{[Globalization.CultureInfo]::CurrentCulture=$previousCulture} diff --git a/Test-BackupRetention.ps1 b/Test-BackupRetention.ps1 index d778e91..5883c49 100644 --- a/Test-BackupRetention.ps1 +++ b/Test-BackupRetention.ps1 @@ -52,10 +52,13 @@ try{ } # Another program's version (no devdrive tag) must never be touched. [void](Invoke-Restic @('backup','--json','--host','pc','--time',$newest.AddDays(-300).ToString('yyyy-MM-dd HH:mm:ss'),$src)) - $planJson=(Invoke-Restic ((Get-RetentionArguments 7 4 3)+'--json'))|Where-Object {"$_".StartsWith('[')}|Select-Object -First 1 + # Another computer sharing the repository: its versions are its own rule's business, also the one far outside this rule. + foreach($d in 400,250){[void](Invoke-Restic @('backup','--json','--host','laptop','--tag','devdrive','--time',$newest.AddDays(-$d).ToString('yyyy-MM-dd HH:mm:ss'),$src))} + $planJson=(Invoke-Restic ((Get-RetentionArguments 7 4 3 'pc')+'--json'))|Where-Object {"$_".StartsWith('[')}|Select-Object -First 1 $plan=@($planJson|ConvertFrom-Json) $snapshots=@((Invoke-Restic @('snapshots','--json','--tag','devdrive'))|Where-Object {"$_".StartsWith('[')}|Select-Object -First 1|ConvertFrom-Json) - $remove=@(Get-RetentionRemovals $plan $ids[0] $snapshots) + $remove=@(Get-RetentionRemovals $plan $ids[0] $snapshots 'pc') + Assert (@($remove|Where-Object {$_.hostname -ne 'pc'}).Count -eq 0 -and @($snapshots|Where-Object {$_.hostname -eq 'laptop'}).Count -eq 2) 'another computer''s versions are never in this computer''s clean-up' $gone=@($remove|ForEach-Object {$_.id}) $keptAges=@($ages|Where-Object {$ids[$_] -notin $gone}) Write-Output (" kept, days before the newest: "+(($keptAges|Sort-Object) -join ', ')) @@ -68,7 +71,7 @@ try{ [void](Invoke-Restic (@('forget')+$gone+'--prune')) [void](Invoke-Restic @('check')) $left=@((Invoke-Restic @('snapshots','--json'))|Where-Object {"$_".StartsWith('[')}|Select-Object -First 1|ConvertFrom-Json) - Assert ($left.Count -eq ($keptAges.Count+1) -and @($left|Where-Object {-not $_.tags}).Count -eq 1) 'after deleting and freeing space the repository checks clean, and the untagged version is untouched' + Assert ($left.Count -eq ($keptAges.Count+3) -and @($left|Where-Object {-not $_.tags}).Count -eq 1 -and @($left|Where-Object {$_.hostname -eq 'laptop'}).Count -eq 2) 'after deleting and freeing space the repository checks clean, and the untagged version and the other computer''s versions are untouched' # Read the file straight out of the newest version (a restore would recreate the folder permissions too). $inside='/'+$src.Replace(':','').Replace('\','/')+'/file.txt' Assert ((@(Invoke-Restic @('dump',$ids[0],$inside)) -join '') -eq 'version from 0 days before') 'the newest version still reads back intact' diff --git a/Test-BackupSecurity.ps1 b/Test-BackupSecurity.ps1 index efac668..119dcc8 100644 --- a/Test-BackupSecurity.ps1 +++ b/Test-BackupSecurity.ps1 @@ -68,6 +68,7 @@ Assert ([IO.File]::ReadAllText($occupied) -ceq 'Preserve this unrelated file.' - [void](Run 'Restore' $null @{Snapshot=$first.Snapshot;RestoreTarget=(Join-Path $private 'restored-after-resume')}) $second=Run 'Backup';Assert ($second.Snapshot) 'incremental backup still works after password rotation and resume' $secretLeak=$false -foreach($log in Get-ChildItem -LiteralPath $private -Recurse -File|Where-Object {$_.Extension -in @('.json','.jsonl','.log') -or $_.Name -eq 'worker-error.txt'}){$body=[IO.File]::ReadAllText($log.FullName);if($body.Contains($new) -or $body.Contains($next) -or $body.Contains($old)){$secretLeak=$true}} -Assert (-not $secretLeak) 'passwords never appear in operation logs or status results' +# Every file of any kind, as UTF-8 and as UTF-16; only the recovery key files may hold a password. +foreach($log in Get-ChildItem -LiteralPath $private -Recurse -File -Force|Where-Object {$_.Name -notlike '*recovery*.txt'}){$body=[Text.Encoding]::Latin1.GetString([IO.File]::ReadAllBytes($log.FullName));foreach($secret in @($new,$next,$old)){foreach($encoding in @([Text.Encoding]::UTF8,[Text.Encoding]::Unicode)){if($body.Contains([Text.Encoding]::Latin1.GetString($encoding.GetBytes($secret)))){$secretLeak=$true}}}} +Assert (-not $secretLeak) 'passwords never appear in plain text in any file written, recovery keys aside' Write-Output ('Artifacts: '+$private) diff --git a/Test-BackupSecuritySafety.ps1 b/Test-BackupSecuritySafety.ps1 index 1790d3e..f4c86f7 100644 --- a/Test-BackupSecuritySafety.ps1 +++ b/Test-BackupSecuritySafety.ps1 @@ -1,26 +1,27 @@ -#Requires -Version 7.2 +#Requires -Version 7.2 $ErrorActionPreference='Stop' Import-Module (Join-Path $PSScriptRoot 'Backup.Security.psm1') -Force function Assert($ok,$message){if(-not $ok){throw $message};Write-Output ('PASS '+$message)} -function Refuses($action,$message){$refused=$false;try{& $action}catch{$refused=$true};Assert $refused $message} +# A refusal counts only when it is the expected one: any other error (a typo, a missing function) fails the test. +function Refuses($action,[string]$like,$message){$refused=$false;try{& $action}catch{if($_.Exception.Message -notlike $like){throw "$message (unexpected error: $($_.Exception.Message))"};$refused=$true};Assert $refused $message} $root=Join-Path $env:TEMP ('DevDriveSecuritySafety-'+[guid]::NewGuid().ToString('N'));$source=Join-Path $root 'source';$outside=Join-Path $root 'outside';New-Item -ItemType Directory -Path $root,$source,$outside|Out-Null try{ - Refuses {Assert-NewBackupPassword 'short'} 'short password is refused' - Refuses {Assert-NewBackupPassword "long-enough-but`nmultiline"} 'multiline password is refused' - Refuses {Assert-NewBackupPassword ' leading-space-password'} 'leading whitespace is refused' + Refuses {Assert-NewBackupPassword 'short'} 'Use 12*characters*' 'short password is refused' + Refuses {Assert-NewBackupPassword "long-enough-but`nmultiline"} 'Use 12*characters*' 'multiline password is refused' + Refuses {Assert-NewBackupPassword ' leading-space-password'} 'Use 12*characters*' 'leading whitespace is refused' $key=Join-Path $outside 'key.txt';New-PrivateTextFile $key 'nonsecret test value' $acl=Get-Acl -LiteralPath $key Assert ($acl.AreAccessRulesProtected -and $acl.Access.Count -eq 2) 'file starts with a protected two-principal ACL' - Refuses {New-PrivateTextFile $key 'must not replace'} 'private file creation never overwrites existing data' + Refuses {New-PrivateTextFile $key 'must not replace'} '*already exists*' 'private file creation never overwrites existing data' Assert ([IO.File]::ReadAllText($key) -ceq 'nonsecret test value') 'failed overwrite leaves existing data intact' Set-PrivateTextFile $key 'replacement nonsecret value' Assert ([IO.File]::ReadAllText($key) -ceq 'replacement nonsecret value' -and (Get-Acl -LiteralPath $key).AreAccessRulesProtected) 'atomic private-file replacement preserves restricted permissions' - Refuses {Assert-PrivateExportPath (Join-Path $source 'key.txt') @($source) @()} 'source directory cannot hold the portable key' - Refuses {Assert-PrivateExportPath $key @($source) @($key)} 'credential and configuration paths cannot be overwritten by export' - Refuses {Assert-PrivateExportPath 'relative-key.txt' @($source) @()} 'relative export paths are refused' - Refuses {Assert-PrivateExportPath ($key+':hidden.txt') @($source) @()} 'alternate data-stream export is refused' + Refuses {Assert-PrivateExportPath (Join-Path $source 'key.txt') @($source) @()} '*outside the source*' 'source directory cannot hold the portable key' + Refuses {Assert-PrivateExportPath $key @($source) @($key)} '*must differ*' 'credential and configuration paths cannot be overwritten by export' + Refuses {Assert-PrivateExportPath 'relative-key.txt' @($source) @()} '*full recovery-key file path*' 'relative export paths are refused' + Refuses {Assert-PrivateExportPath ($key+':hidden.txt') @($source) @()} '*alternate streams*' 'alternate data-stream export is refused' $link=Join-Path $root 'linked';New-Item -ItemType Junction -Path $link -Target $outside|Out-Null - Refuses {Assert-PrivateExportPath (Join-Path $link 'key.txt') @($source) @()} 'linked export ancestors are refused' + Refuses {Assert-PrivateExportPath (Join-Path $link 'key.txt') @($source) @()} '*Linked recovery-key paths*' 'linked export ancestors are refused' # Compare full forms: a temp path with a short (8.3) name, as on CI runners, comes back expanded. Assert ((Assert-PrivateExportPath (Join-Path $outside 'new-key.txt') @($source) @()) -eq [IO.Path]::GetFullPath((Join-Path $outside 'new-key.txt'))) 'new outside recovery path is accepted' }finally{ diff --git a/Test-BackupSystem.ps1 b/Test-BackupSystem.ps1 index 01ede22..57115c7 100644 --- a/Test-BackupSystem.ps1 +++ b/Test-BackupSystem.ps1 @@ -24,7 +24,8 @@ function Run($action,$extra=@{}){ $state } $first=Run 'Backup';Assert ($first.Snapshot.Length -gt 7) 'encrypted first snapshot created' -Assert ($first.PeakMemoryMiB -lt 512) '512 MiB memory budget observed' +# Peak resident memory as reported by the worker; the hard cap itself is tested by Test-BackupResourceControls.ps1. +Assert ($first.PeakMemoryMiB -gt 0 -and $first.PeakMemoryMiB -lt 512) 'peak memory of the engine is reported and stays below the 512 MiB budget' $restored=Join-Path $private 'restored-first' [void](Run 'Restore' @{RestoreTarget=$restored;Snapshot=$first.Snapshot}) $file=Get-ChildItem -LiteralPath $restored -Recurse -File|Where-Object {$_.Name -eq 'עברית [sample].txt'} @@ -90,7 +91,7 @@ Assert ($remaining.Count -eq 1 -and $remaining[0].id -eq $forced.Snapshot) 'expl Assert ([IO.File]::ReadAllText($changed) -eq 'forced version') 'retained snapshot restores after history pruning without source changes' [IO.File]::WriteAllText($changed,'version with clean-up') $kept=Run 'Backup' @{KeepDays=7;KeepWeeks=4;KeepMonths=3} -Assert ($kept.Retention -and -not $kept.Retention.PSObject.Properties['Error'] -and $kept.Retention.Removed -eq 0 -and $kept.Retention.Kept -eq 2) 'a backup with Keep versions plans the clean-up and keeps both versions of today' +Assert ($kept.Retention -and -not $kept.Retention.PSObject.Properties['Error'] -and $kept.Retention.Removed -eq 0 -and $kept.Retention.Kept -eq 2) 'a backup with Keep versions plans the clean-up; with only two versions nothing is deleted (the oldest one inside the period is kept)' Assert (Test-Path -LiteralPath (Join-Path $kept.RunDirectory 'clean-up-plan.json')) 'the clean-up plan is saved with the run' Write-Output ('Artifacts: '+$private) Write-Output ('Disposable source: '+$root) diff --git a/Test-BackupVersions.ps1 b/Test-BackupVersions.ps1 new file mode 100644 index 0000000..c701402 --- /dev/null +++ b/Test-BackupVersions.ps1 @@ -0,0 +1,222 @@ +#Requires -Version 7.2 +# The versioned worker against a disposable repository, with real restic and no second drive: everything +# (source, repository, password files, run folders, engine cache) lives in one new folder under the +# temporary directory, which Backup-Worker.ps1 -TestRoot uses in place of the backup drive. +# Covers: a partial version (a file that cannot be read), the clean-up of old versions beside partial +# versions and another computer's versions, deleting versions, a restore below protected folders, saving +# the recovery key, a pending password change after the repository moved, a lock left in the repository +# (by a run that was killed: removed; by a backup that is still running: left alone), and plaintext +# passwords in files. +# Skips when restic is not installed (Install-BackupEngine.ps1). +$ErrorActionPreference='Stop' +Import-Module (Join-Path $PSScriptRoot 'Backup.Status.psm1') -Force +Import-Module (Join-Path $PSScriptRoot 'Backup.History.psm1') -Force +Import-Module (Join-Path $PSScriptRoot 'Backup.Security.psm1') -Force +function Assert($ok,[string]$message){if(-not $ok){throw "FAIL $message"};Write-Output "PASS $message"} +$engine=Join-Path $PSScriptRoot 'bin\restic_0.19.1_windows_amd64.exe' +if(-not(Test-Path -LiteralPath $engine)){Write-Output 'SKIP: bin\restic_0.19.1_windows_amd64.exe is not installed (Install-BackupEngine.ps1)';return} +$base=Join-Path ([IO.Path]::GetFullPath([IO.Path]::GetTempPath())) ('backup-versions-test-'+[guid]::NewGuid().ToString('N').Substring(0,10)) +$drive=Join-Path $base 'drive';$moved=Join-Path $base 'drive-moved';$work=Join-Path $base 'work';$source=Join-Path $work 'source';$private=Join-Path $base 'private' +New-Item -ItemType Directory -Path $drive,$moved,$source,$private|Out-Null +$config=Get-Content (Join-Path $PSScriptRoot 'config.example.json') -Raw|ConvertFrom-Json +$config.Source=$work+'\';$config.SourceVolumeId='test-source-volume';$config.DestinationVolumeId='test-backup-volume';$config.RepositoryFolder='Test-Versions' +$config.PasswordDpapiPath=Join-Path $private 'key.dpapi';$config.RecoveryKeyPath=Join-Path $private 'recovery-first.txt';$config.Include=@('source');$config.Choices=@([pscustomobject]@{Path='source';Mode='Versioned';Reason='Disposable test source'});$config.DefaultMode='Skip';$config.RateMBps=0;$config.MemoryMiB=512;$config.CpuThreads=1;$config.ReadConcurrency=1;$config.PackMiB=8 +$configPath=Join-Path $private 'settings.json';$config|ConvertTo-Json -Depth 7|Set-Content -LiteralPath $configPath +$repo=Join-Path $drive $config.RepositoryFolder;$journalPath=$config.PasswordDpapiPath+'.rotation.local.json' +$passwords=[Collections.Generic.List[string]]::new() +function Run([string]$action,[hashtable]$extra=@{},$request=$null,[string]$testRoot=$drive){ + $run=Join-Path $private ('runs\'+$action+'-'+[guid]::NewGuid().ToString('N').Substring(0,8)) + if($null -ne $request){$requestPath=Join-Path $private ('request-'+[guid]::NewGuid().ToString('N').Substring(0,8)+'.json');$request|ConvertTo-Json -Depth 6|Set-Content -LiteralPath $requestPath;$extra.RequestPath=$requestPath} + & (Join-Path $PSScriptRoot 'Backup-Worker.ps1') -ConfigPath $configPath -Action $action -RunDirectory $run -TestRoot $testRoot @extra + Read-BackupStatus (Join-Path $run 'result.json') +} +function Read-Password {[Net.NetworkCredential]::new('',(((Get-Content -LiteralPath $config.PasswordDpapiPath -Raw).Trim())|ConvertTo-SecureString)).Password} +function Restic([string[]]$arguments,[string]$password=(Read-Password),[string]$repository=$repo){ + # The engine on its own, to prepare dated versions and to check results independently of the worker. + $psi=[Diagnostics.ProcessStartInfo]::new($engine);$psi.UseShellExecute=$false;$psi.CreateNoWindow=$true;$psi.RedirectStandardOutput=$true;$psi.RedirectStandardError=$true;$psi.Environment['RESTIC_PASSWORD']=$password + foreach($a in @('-r',$repository,'--cache-dir',(Join-Path $base 'direct-cache'))+$arguments){[void]$psi.ArgumentList.Add($a)} + $p=[Diagnostics.Process]::Start($psi);$out=$p.StandardOutput.ReadToEndAsync();$err=$p.StandardError.ReadToEndAsync();$p.WaitForExit() + $result=[pscustomobject]@{Code=$p.ExitCode;Out=$out.GetAwaiter().GetResult();Err=$err.GetAwaiter().GetResult()};$p.Dispose();$result +} +function Snapshots {@((Restic @('--json','snapshots')).Out|ConvertFrom-Json)} +function Lock-File([string]$path,[scriptblock]$while){$handle=[IO.File]::Open($path,[IO.FileMode]::Open,[IO.FileAccess]::ReadWrite,[IO.FileShare]::None);try{& $while}finally{$handle.Dispose()}} +$inside='/'+$source.Replace(':','').Replace('\','/') +try{ + # --- The test folder cannot be pointed at real settings or outside the temporary directory ---------- + $outside=$null;try{& (Join-Path $PSScriptRoot 'Backup-Worker.ps1') -ConfigPath $configPath -Action Snapshots -RunDirectory (Join-Path $private 'runs\outside') -TestRoot $PSScriptRoot}catch{$outside=$_.Exception.Message} + Assert ($outside -like '*test suite only*' -and -not(Test-Path -LiteralPath (Join-Path $private 'runs\outside'))) 'a test folder outside the temporary directory is refused before anything is written' + $realSettings=Join-Path $private 'runs\real-settings';& (Join-Path $PSScriptRoot 'Backup-Worker.ps1') -ConfigPath (Join-Path $PSScriptRoot 'config.example.json') -Action Snapshots -RunDirectory $realSettings -TestRoot $drive + Assert ((Read-BackupStatus (Join-Path $realSettings 'result.json')).Error -like '*test suite only*') 'the test folder is refused together with settings outside the temporary directory' + + # --- A file that cannot be read: the version is saved, marked partial, and the run fails ----------- + [IO.File]::WriteAllText((Join-Path $source 'a.txt'),'a');$important=Join-Path $source 'important.txt';[IO.File]::WriteAllText($important,'important v1') + $partialRun=Lock-File $important {Run 'Backup'} + $passwords.Add((Read-Password)) + $all=@(Snapshots) + Assert ($partialRun.State -eq 'Failed' -and $partialRun.Partial -eq $true -and $partialRun.Error -like '*marked as partial*') 'a backup that could not read a file fails and reports a partial version' + Assert ($all.Count -eq 1 -and (Test-PartialSnapshot $all[0]) -and @($all[0].tags) -notcontains 'devdrive' -and $all[0].id -eq $partialRun.Snapshot -and $partialRun.BackupSummary.snapshot_id -eq $partialRun.Snapshot) 'the partial version is marked in the repository and the run names its final ID' + $completeRun=Run 'Backup' + Assert ($completeRun.State -eq 'Completed' -and $completeRun.Partial -eq $false) 'the next backup, with the file readable, completes' + Assert ($completeRun.BackupSummary.snapshot_id -eq $completeRun.Snapshot -and $completeRun.BackupSummary.total_files_processed -eq 2 -and $completeRun.BackupSummary.files_new -ge 1 -and $completeRun.BackupSummary.data_added -gt 0 -and $null -ne $completeRun.BackupSummary.files_changed -and $completeRun.BackupSummary.total_bytes_processed -gt 0) 'the result carries the backup''s own summary (files, bytes, version)' + Assert (-not $completeRun.Progress.PSObject.Properties['files_new']) 'Progress still ends with the summary of the check, as before' + + # --- Clean-up of old versions beside partial versions and another computer's versions --------------- + $computer=(Snapshots|Where-Object {$_.id -eq $completeRun.Snapshot}).hostname;$today=(Get-Date).Date;$ids=@{} + function Seed([string]$name,[datetime]$time,[string]$computerName,[string]$content,[switch]$Unreadable){ + [IO.File]::WriteAllText($important,$content) + $arguments=@('--json','backup','--host',$computerName,'--tag','devdrive','--time',$time.ToString('yyyy-MM-dd HH:mm:ss'),$source) + $o=if($Unreadable){Lock-File $important {Restic $arguments}}else{Restic $arguments} + if($o.Code -ne $(if($Unreadable){3}else{0})){throw "Preparing version '$name' ended with code $($o.Code): $($o.Err)"} + $id=@($o.Out -split "`r?`n"|ForEach-Object {try{$_|ConvertFrom-Json}catch{}}|Where-Object {$_ -and $_.message_type -eq 'summary'})[0].snapshot_id + if($Unreadable){$m=Restic (@('--json')+(Get-PartialMarkArguments $id));if($m.Code -ne 0){throw "Marking '$name' failed: $($m.Err)"};$id=@($m.Out -split "`r?`n"|ForEach-Object {try{$_|ConvertFrom-Json}catch{}}|Where-Object {$_ -and $_.message_type -eq 'changed'})[0].new_snapshot_id} + $ids[$name]=$id + } + Seed 'old-400' $today.AddDays(-400).AddHours(10) $computer 'important, 400 days ago' + Seed 'old-300' $today.AddDays(-300).AddHours(10) $computer 'important, 300 days ago' + Seed 'other-200' $today.AddDays(-200).AddHours(10) 'other-computer' 'another computer, 200 days ago' + Seed 'other-50' $today.AddDays(-50).AddHours(10) 'other-computer' 'another computer, 50 days ago' + Seed 'day-2' $today.AddDays(-2).AddHours(10) $computer 'important, two days ago' + Seed 'day-1-complete' $today.AddDays(-1).AddHours(10) $computer 'important, yesterday morning: the only copy of this text' + Seed 'day-1-partial' $today.AddDays(-1).AddHours(18) $computer 'important, being edited' -Unreadable + [IO.File]::WriteAllText($important,'important, today') + $kept=Run 'Backup' @{KeepDays=7;KeepWeeks=4;KeepMonths=3} + $left=@(Snapshots|ForEach-Object {$_.id}) + Assert ($kept.State -eq 'Completed' -and -not $kept.Retention.PSObject.Properties['Error']) 'a backup with Keep versions completes and cleans up without an error' + Assert ($ids['day-1-complete'] -in $left -and (Restic @('dump',$ids['day-1-complete'],($inside+'/important.txt'))).Out -eq 'important, yesterday morning: the only copy of this text') 'the complete version of a day survives a later partial version of that day' + Assert ($ids['day-1-partial'] -in $left -and $partialRun.Snapshot -in $left) 'partial versions are left alone by the clean-up' + Assert ($ids['other-200'] -in $left -and $ids['other-50'] -in $left) 'another computer''s versions are left alone, also those outside this computer''s rule' + Assert ($ids['old-400'] -notin $left -and $ids['old-300'] -notin $left -and $completeRun.Snapshot -notin $left -and $ids['day-2'] -in $left -and $kept.Snapshot -in $left) 'this computer''s versions outside the rule are deleted: the two old ones and today''s earlier one' + Assert ($kept.Retention.Removed -eq 3 -and $kept.Retention.Kept -eq 3 -and $kept.Retention.SpaceFreed -eq $true) 'the result counts this computer''s complete versions only: 3 deleted, 3 kept' + Assert ($kept.BackupSummary.snapshot_id -eq $kept.Snapshot) 'the backup''s summary is kept through the clean-up' + + # --- Deleting versions by hand ---------------------------------------------------------------------- + $refused=Run 'Forget' @{} @{SnapshotIds=@($kept.Snapshot);Confirmation='DELETE SELECTED HISTORY';ReclaimSpace=$false} + Assert ($refused.State -eq 'Failed' -and $refused.Error -like '*protected*') 'the newest version cannot be deleted' + $deleted=Run 'Forget' @{} @{SnapshotIds=@($ids['other-200'],$partialRun.Snapshot);Confirmation='DELETE SELECTED HISTORY';ReclaimSpace=$true} + $left=@(Snapshots|ForEach-Object {$_.id}) + Assert ($deleted.State -eq 'Completed' -and $ids['other-200'] -notin $left -and $partialRun.Snapshot -notin $left -and $left.Count -eq 5) 'chosen older versions, a partial one among them, are deleted and their space freed' + + # --- Restore: verified, or honestly reported as not verified ---------------------------------------- + $target=Join-Path $base 'restored' + $restored=Run 'Restore' @{Snapshot=$kept.Snapshot;RestoreTarget=$target} + Assert ($restored.State -eq 'Completed') 'a restore whose only trouble is the date of protected parent folders completes' + $same=$true;foreach($file in Get-ChildItem -LiteralPath $source -File){$copy=Join-Path (Join-Path $target $source.Replace(':','')) $file.Name;if(-not(Test-Path -LiteralPath $copy) -or (Get-FileHash -LiteralPath $copy).Hash -ne (Get-FileHash -LiteralPath $file.FullName).Hash){$same=$false}} + Assert $same 'every restored file has the same SHA-256 as its source' + # Below a user folder an ordinary account cannot set the date of the recreated C:\Users, and restic + # then skips its content check. An administrator (as on a build server) can, and the check runs. + if($restored.PSObject.Properties['Warnings'] -and @($restored.Warnings).Count){Assert ($restored.ContentVerified -eq $false -and @($restored.Warnings|Where-Object {$_ -notlike 'failed to restore timestamp*' -and $_ -notlike 'failed to restore attributes*'}).Count -eq 0) 'with such warnings the restore is reported as not verified, never as verified'} + else{Assert ($restored.ContentVerified -eq $true) 'without warnings the restore is reported as verified'} + + # --- Saving the recovery key ------------------------------------------------------------------------ + $export=Join-Path $private 'recovery-saved.txt' + $saved=Run 'ExportKey' @{} @{RecoveryKeyPath=$export} + Assert ($saved.State -eq 'Completed' -and [IO.File]::ReadAllText($export).Contains('Password: '+(Read-Password))) 'a recovery key is saved when the stored password opens the repository' + $goodKey=[IO.File]::ReadAllText($config.PasswordDpapiPath);$wrong='not-the-password-'+[guid]::NewGuid().ToString('N');$passwords.Add($wrong) + Set-PrivateTextFile $config.PasswordDpapiPath ($wrong|ConvertTo-SecureString -AsPlainText -Force|ConvertFrom-SecureString) + try{$bad=Join-Path $private 'recovery-wrong.txt';$refused=Run 'ExportKey' @{} @{RecoveryKeyPath=$bad}}finally{Set-PrivateTextFile $config.PasswordDpapiPath $goodKey} + Assert ($refused.State -eq 'Failed' -and $refused.Error -like '*does not open*' -and -not(Test-Path -LiteralPath $bad)) 'no recovery key is saved when the stored password does not open the repository' + + # --- Password change, and resuming it after the repository came back somewhere else ------------------ + function New-Secret([string]$name){$p='test-'+$name+'-'+[Convert]::ToBase64String([Security.Cryptography.RandomNumberGenerator]::GetBytes(24));$passwords.Add($p);$f=Join-Path $private ($name+'.dpapi');New-PrivateTextFile $f ($p|ConvertTo-SecureString -AsPlainText -Force|ConvertFrom-SecureString);[pscustomobject]@{Password=$p;File=$f}} + function Current-KeyId([string]$repository=$repo){@((Restic @('--json','key','list') (Read-Password) $repository).Out|ConvertFrom-Json|Where-Object {$_.current})[0].id} + function New-Journal([string]$name,[string]$repository,[hashtable]$more){ + # What an interrupted change leaves behind right after it was prepared. + $next=New-Secret $name + $journal=[ordered]@{Version=1;Repository=$repository;ConfigPath=$configPath;RecoveryKeyPath=(Join-Path $private ('recovery-'+$name+'.txt'));OldKeyId=(Current-KeyId $repository);NewKeyId=$null;OldPasswordDpapi=([IO.File]::ReadAllText($config.PasswordDpapiPath)).Trim();NewPasswordDpapi=([IO.File]::ReadAllText($next.File)).Trim();Phase='Prepared';CreatedUtc=[DateTime]::UtcNow.ToString('o')} + foreach($k in $more.Keys){$journal[$k]=$more[$k]} + New-PrivateTextFile $journalPath ($journal|ConvertTo-Json -Depth 6);$next + } + $first=New-Secret 'changed';$before=Read-Password + $change=Run 'ChangePassword' @{} @{RecoveryKeyPath=(Join-Path $private 'recovery-changed.txt');NewPasswordDpapiPath=$first.File} + $completedJournal=Get-Content -LiteralPath (Get-ChildItem -LiteralPath $private -Filter '*.completed-*'|Select-Object -First 1).FullName -Raw|ConvertFrom-Json + Assert ($change.State -eq 'Completed' -and (Read-Password) -ceq $first.Password -and (Restic @('--json','key','list') $before).Code -eq 12) 'a password change completes and retires the old password' + Assert ($completedJournal.DestinationVolumeId -ceq $config.DestinationVolumeId -and $completedJournal.RepositoryFolder -ceq $config.RepositoryFolder) 'the journal of a password change records the volume and folder of its repository' + $earlier=New-Journal 'earlier-format' $repo @{} + $resumed=Run 'ResumePasswordChange' + Assert ($resumed.State -eq 'Completed' -and (Read-Password) -ceq $earlier.Password -and -not(Test-Path -LiteralPath $journalPath)) 'a journal written by an earlier version (path only) still resumes' + $foreign=New-Journal 'other-volume' $repo @{DestinationVolumeId='some-other-volume';RepositoryFolder=$config.RepositoryFolder} + $refused=Run 'ResumePasswordChange' + Assert ($refused.State -eq 'Failed' -and $refused.Error -like '*another*' -and (Read-Password) -ceq $earlier.Password) 'a journal of another volume is refused and changes nothing' + Remove-Item -LiteralPath $journalPath + $pending=New-Journal 'after-move' $repo @{DestinationVolumeId=$config.DestinationVolumeId;RepositoryFolder=$config.RepositoryFolder} + Move-Item -LiteralPath $repo -Destination (Join-Path $moved $config.RepositoryFolder);$repo=Join-Path $moved $config.RepositoryFolder + $blocked=Run 'Snapshots' @{} $null $moved + Assert ($blocked.State -eq 'Failed' -and $blocked.Error -like '*resumed*') 'a pending password change still blocks other work' + $resumed=Run 'ResumePasswordChange' @{} $null $moved + Assert ($resumed.State -eq 'Completed' -and (Read-Password) -ceq $pending.Password -and (Restic @('--json','key','list') $earlier.Password).Code -eq 12 -and -not(Test-Path -LiteralPath $journalPath)) 'a pending password change resumes after the repository came back at another path (new drive letter)' + $after=Run 'Backup' @{} $null $moved + Assert ($after.State -eq 'Completed') 'backups continue after the resumed password change' + + # --- A lock left by a run that was killed: the next run removes it by itself and carries on --------- + function Lock-Names {@(Get-ChildItem -LiteralPath (Join-Path $repo 'locks') -File|ForEach-Object {$_.Name})} + function Wait-Lock {for($i=0;$i -lt 300 -and -not @(Lock-Names).Count;$i++){Start-Sleep -Milliseconds 200}} + function Start-Hidden([string]$file,[string[]]$arguments,[hashtable]$environment=@{}){ + $psi=[Diagnostics.ProcessStartInfo]::new($file);$psi.UseShellExecute=$false;$psi.CreateNoWindow=$true + foreach($k in $environment.Keys){$psi.Environment[$k]=$environment[$k]};foreach($a in $arguments){[void]$psi.ArgumentList.Add($a)} + [Diagnostics.Process]::Start($psi) + } + function Start-Engine([string[]]$arguments){Start-Hidden $engine (@('-r',$repo,'--cache-dir',(Join-Path $base 'direct-cache'),'--quiet')+$arguments) @{RESTIC_PASSWORD=(Read-Password)}} + Assert ($after.PSObject.Properties['LockReleased'] -and $null -eq $after.LockReleased -and -not(Test-Path -LiteralPath (Join-Path $after.RunDirectory 'ReadingLocks.jsonl'))) 'a run that meets no lock looks for none and reports none' + [IO.File]::WriteAllBytes((Join-Path $source 'large.bin'),[Security.Cryptography.RandomNumberGenerator]::GetBytes(6MB)) + # The worker itself, slowed down so that it is still backing up when it is killed with its whole tree. + $slowPath=Join-Path $private 'settings-slow.json';$config.RateMBps=0.05;$config|ConvertTo-Json -Depth 7|Set-Content -LiteralPath $slowPath;$config.RateMBps=0 + $killedRun=Join-Path $private 'runs\Backup-killed' + $worker=Start-Hidden ([Environment]::ProcessPath) @('-NoProfile','-NonInteractive','-File',(Join-Path $PSScriptRoot 'Backup-Worker.ps1'),'-ConfigPath',$slowPath,'-Action','Backup','-RunDirectory',$killedRun,'-TestRoot',$moved) + try{Wait-Lock}finally{$worker.Kill($true);$worker.WaitForExit();$worker.Dispose()} + $left=@(Lock-Names) + Assert ($left.Count -eq 1 -and (Read-BackupStatus (Join-Path $killedRun 'status.json')).State -eq 'BackingUp' -and -not(Test-Path -LiteralPath (Join-Path $killedRun 'result.json'))) 'a backup whose worker is killed part way leaves its lock in the repository' + $healed=Run 'Backup' @{} $null $moved + Assert ($healed.State -eq 'Completed' -and $healed.LockReleased.Stage -eq 'CheckingStructure' -and @($healed.LockReleased.Locks).Count -eq 1 -and $healed.LockReleased.Locks[0].Id -ceq $left[0] -and $healed.LockReleased.Locks[0].Pid -gt 0 -and $healed.LockReleased.Locks[0].Exclusive -eq $false -and $healed.LockReleased.Locks[0].CreatedUtc) 'the next backup removes the lock of the killed run by itself, completes, and records which lock it removed' + Assert (@(Lock-Names).Count -eq 0 -and [IO.File]::ReadAllText((Join-Path $healed.RunDirectory 'CheckingStructure.locked.errors.log')) -like '*already locked*' -and [IO.File]::ReadAllText((Join-Path $healed.RunDirectory 'CheckingStructure.errors.log')) -notlike '*already locked*') 'the step that met the lock was tried once more and passed; what its first try reported is kept beside it' + # A check that was killed leaves a lock of the other kind (exclusive), which stops even listing the versions. + $check=Start-Engine @('--limit-download','20','check','--read-data') + try{Wait-Lock}finally{$check.Kill($true);$check.WaitForExit();$check.Dispose()} + $left=@(Lock-Names) + $listed=Run 'Snapshots' @{} $null $moved + Assert ($left.Count -eq 1 -and $listed.State -eq 'Completed' -and $listed.LockReleased.Stage -eq 'ListingSnapshots' -and $listed.LockReleased.Locks[0].Id -ceq $left[0] -and $listed.LockReleased.Locks[0].Exclusive -eq $true -and @(Lock-Names).Count -eq 0) 'the lock of a killed check is removed the same way when the versions are listed' + + # --- A lock of a backup that is still running is never removed; a partial version that cannot be marked meanwhile --- + [IO.File]::WriteAllBytes((Join-Path $source 'large2.bin'),[Security.Cryptography.RandomNumberGenerator]::GetBytes(8MB)) + $running=Start-Engine @('--limit-upload','30','backup',$source) + try{ + Wait-Lock;$held=@(Lock-Names) + $lockedRun=Run 'Backup' @{} $null $moved + Assert ($held.Count -eq 1 -and $lockedRun.State -eq 'Failed' -and $lockedRun.Error -like "The new version was saved, but it could not be checked.*locked: a backup is still running on this computer (process $($running.Id))*Nothing is damaged*" -and $lockedRun.Error -notlike '*memory*') 'beside a backup that is still running, the next backup says in plain words that the version was saved, who holds the lock and that nothing is damaged' + Assert ($null -eq $lockedRun.LockReleased -and ((Lock-Names) -join ',') -ceq ($held -join ',') -and -not(Test-Path -LiteralPath (Join-Path $lockedRun.RunDirectory 'ReleasingLock.jsonl'))) 'the lock of the running backup is left alone' + [IO.File]::WriteAllText($important,'important, while the repository is locked') + $partialsBefore=@(Snapshots|Where-Object {Test-PartialSnapshot $_}).Count + $unmarked=Lock-File $important {Run 'Backup' @{} $null $moved} + $pendingPath=$config.PasswordDpapiPath+'.partial-pending.local.json' + Assert ($unmarked.State -eq 'Failed' -and $unmarked.Partial -eq $true -and $unmarked.Error -like '*could not be marked as partial yet*' -and (Test-Path -LiteralPath $pendingPath)) 'a partial version that cannot be marked on a locked repository is noted for later' + $refused=Run 'Forget' @{} @{SnapshotIds=@($ids['day-2']);Confirmation='DELETE SELECTED HISTORY';ReclaimSpace=$false} $moved + Assert ($refused.State -eq 'Failed' -and $refused.Error -like '*not marked yet*' -and $ids['day-2'] -in @(Snapshots|ForEach-Object {$_.id})) 'until it is marked, nothing is deleted' + Assert (-not $running.HasExited -and ((Lock-Names) -join ',') -ceq ($held -join ',')) 'the other backup was running, with its lock, all the while' + # The backup has ended, but restic itself still sees its process (here because this test keeps it + # open; a program that took over its process number has the same effect). restic then keeps the lock + # until 30 minutes after it was made, and so does the worker. + $running.Kill($true);$running.WaitForExit() + $keptLock=Run 'Check' @{} $null $moved + Assert ($keptLock.State -eq 'Failed' -and $keptLock.Error -like '*locked: the run that made the lock has ended*30 minutes*Nothing is damaged*' -and $null -eq $keptLock.LockReleased -and ((Lock-Names) -join ',') -ceq ($held -join ',')) 'a lock that restic itself still keeps is not removed behind its back, and the run says when to try again' + }finally{if(-not $running.HasExited){$running.Kill($true);$running.WaitForExit()};$ended=$running.Id;$running.Dispose()} + [IO.File]::WriteAllText($important,'important, after the lock') + $final=Run 'Backup' @{KeepDays=7;KeepWeeks=4;KeepMonths=3} $null $moved + $now=@(Snapshots) + Assert ($final.State -eq 'Completed' -and @($final.LockReleased.Locks).Count -eq 1 -and $final.LockReleased.Locks[0].Id -ceq $held[0] -and $final.LockReleased.Locks[0].Pid -eq $ended -and @(Lock-Names).Count -eq 0) 'once that backup has really ended, the next run removes its lock and completes' + Assert (-not $final.Retention.PSObject.Properties['Error'] -and -not(Test-Path -LiteralPath $pendingPath) -and $unmarked.Snapshot -notin @($now|ForEach-Object {$_.id}) -and @($now|Where-Object {Test-PartialSnapshot $_}).Count -eq $partialsBefore+1) 'the noted partial version is marked before the next clean-up, which then leaves it alone' + + # --- No plaintext password in anything written ------------------------------------------------------- + $leaks=@() + foreach($file in Get-ChildItem -LiteralPath $base -Recurse -File -Force|Where-Object {$_.Name -notlike 'recovery-*.txt'}){ + $bytes=[Text.Encoding]::Latin1.GetString([IO.File]::ReadAllBytes($file.FullName)) + foreach($p in $passwords){foreach($encoding in @([Text.Encoding]::UTF8,[Text.Encoding]::Unicode)){if($bytes.Contains([Text.Encoding]::Latin1.GetString($encoding.GetBytes($p)))){$leaks+=$file.FullName}}} + } + Assert ($passwords.Count -ge 6 -and $leaks.Count -eq 0) 'no file of any kind holds a password in plain text (UTF-8 or UTF-16), recovery keys aside' + Assert ([IO.File]::ReadAllText((Join-Path $private 'recovery-after-move.txt')).Contains('Password: '+$pending.Password)) 'the recovery key of the last change holds the password that now opens the repository' +}finally{ + if(Test-Path -LiteralPath $base){ + # A restore recreates protected folders with their original permissions: reset them so the folder can go. + if(Test-Path -LiteralPath (Join-Path $base 'restored')){& icacls (Join-Path $base 'restored') /reset /T /C /Q 2>&1|Out-Null} + Get-ChildItem -LiteralPath $base -Recurse -Force -File|ForEach-Object {$_.IsReadOnly=$false};Remove-Item -LiteralPath $base -Recurse -Force + } +} +Write-Output 'All versioned worker tests passed.' diff --git a/Test-BackupVersionsRules.ps1 b/Test-BackupVersionsRules.ps1 new file mode 100644 index 0000000..442613e --- /dev/null +++ b/Test-BackupVersionsRules.ps1 @@ -0,0 +1,133 @@ +#Requires -Version 7.2 +# Rules of the versioned engine that need no repository, no restic and no second drive: partial versions, +# the clean-up of old versions on a shared repository, deleting many versions at once, restore warnings, +# a pending password change after the drive letter changed, the wording of a locked repository, and which +# repository locks may be removed without asking (only one left by a run on this computer that has ended). +$ErrorActionPreference='Stop' +Import-Module (Join-Path $PSScriptRoot 'Backup.Status.psm1') -Force +Import-Module (Join-Path $PSScriptRoot 'Backup.History.psm1') -Force +Import-Module (Join-Path $PSScriptRoot 'Backup.Security.psm1') -Force +function Assert($ok,[string]$message){if(-not $ok){throw "FAIL $message"};Write-Output "PASS $message"} +function Assert-Throws([scriptblock]$Block,[string]$Like,[string]$message){ + try{& $Block;throw "FAIL $message (no error)"}catch{if($_.Exception.Message -like 'FAIL*'){throw};if($_.Exception.Message -notlike $Like){throw "FAIL $message (got: $($_.Exception.Message))"}} + Write-Output "PASS $message" +} +function Id([int]$n){('{0:x2}' -f $n)*32} +function Snap([int]$n,[string]$time,[string]$computer='pc',[string[]]$tags=@('devdrive')){[pscustomobject]@{id=(Id $n);time=$time;hostname=$computer;paths=@('E:\');tags=$tags}} + +# --- Partial versions (a backup that could not read every file) ---------------------------------- +$complete=Snap 1 '2026-06-29T10:00:00Z';$partial=Snap 2 '2026-06-29T18:00:00Z' 'pc' @('devdrive-partial');$older=Snap 3 '2026-06-28T10:00:00Z' +Assert ((Test-PartialSnapshot $partial) -and -not (Test-PartialSnapshot $complete) -and -not (Test-PartialSnapshot ([pscustomobject]@{id=(Id 9);time='2026-01-01T00:00:00Z';hostname='pc';paths=@('E:\')}))) 'a partial version is recognised by its mark; unmarked and untagged versions are not partial' +$mark=Get-PartialMarkArguments (Id 2) +Assert ($mark[0] -eq 'tag' -and ($mark -join ' ') -like '*--remove devdrive*' -and ($mark -join ' ') -like '*--add devdrive-partial*' -and $mark[-1] -eq (Id 2)) 'marking a version partial takes it out of the versions the clean-up rule sees' +Assert-Throws {Get-PartialMarkArguments 'latest'} '*exact snapshot ID*' 'only an exact version can be marked partial' +$set=@($complete,$partial,$older) +Assert-Throws {Get-HistoryDeletionPlan $set @($complete.id)} '*protected*' 'the newest complete version is protected even when a newer partial version exists' +Assert-Throws {Get-HistoryDeletionPlan $set @($partial.id)} '*protected*' 'the newest version stays protected as before, also when it is partial' +Assert (@(Get-HistoryDeletionPlan $set @($older.id)).Count -eq 1) 'an older version can still be deleted' +$onlyPartial=@((Snap 4 '2026-06-29T18:00:00Z' 'pc' @('devdrive-partial')),(Snap 5 '2026-06-28T18:00:00Z' 'pc' @('devdrive-partial'))) +Assert (@(Get-HistoryDeletionPlan $onlyPartial @((Id 5))).Count -eq 1) 'a set with only partial versions still lets an older one go' +# A plan that would delete a partial version (restic never plans one, because the mark replaces the tag). +Assert-Throws {Get-RetentionRemovals @([pscustomobject]@{keep=@($complete);remove=@($partial,$older)}) $complete.id @($complete,$partial,$older) 'pc'} '*partial*' 'the clean-up never deletes a partial version' +Assert-Throws {Get-RetentionRemovals @([pscustomobject]@{keep=@($partial,$complete);remove=@($older)}) $partial.id @($complete,$partial,$older) 'pc'} '*partial*' 'the clean-up never runs on behalf of a partial version' + +# --- The version just made -------------------------------------------------------------------------- +$future=Snap 6 '2027-01-01T00:00:00Z';$new=Snap 7 '2026-11-12T10:00:00Z' +$wrongClock=@([pscustomobject]@{keep=@($future);remove=@($new)}) +Assert-Throws {Get-RetentionRemovals $wrongClock '' @($future,$new) 'pc'} '*not known*' 'without the ID of the version just made nothing is deleted' +Assert-Throws {Get-RetentionRemovals $wrongClock $null @($future,$new) 'pc'} '*not known*' 'a missing ID of the version just made is refused too' + +# --- A repository shared by two computers ------------------------------------------------------------- +$a=Get-RetentionArguments 7 4 3 'pc' +Assert (($a -join ' ') -like '*--host pc*' -and '--dry-run' -in $a) 'the clean-up rule is limited to this computer''s versions' +Assert (-not ((Get-RetentionArguments 7 4 3) -contains '--host')) 'without a computer name the rule is unchanged (older callers)' +$mine=@((Snap 10 '2026-06-30T12:00:00Z'),(Snap 11 '2026-01-01T12:00:00Z'));$theirs=@((Snap 12 '2026-06-01T12:00:00Z' 'laptop'),(Snap 13 '2025-06-01T12:00:00Z' 'laptop')) +Assert-Throws {Get-RetentionRemovals @([pscustomobject]@{keep=@($mine[0]);remove=@($mine[1])},[pscustomobject]@{keep=@($theirs[0]);remove=@($theirs[1])}) $mine[0].id @($mine+$theirs) 'pc'} '*another computer*' 'a plan that would delete another computer''s version is refused' +$r=@(Get-RetentionRemovals @([pscustomobject]@{keep=@($mine[0]);remove=@($mine[1])}) $mine[0].id @($mine+$theirs) 'pc') +Assert ($r.Count -eq 1 -and $r[0].id -eq $mine[1].id) 'this computer''s old version is still returned for deletion' + +# --- Deleting many versions at once ----------------------------------------------------------------- +$ids=@(1..520|ForEach-Object {[guid]::NewGuid().ToString('N')*2}) +$batches=@(Split-SnapshotIdBatches $ids) +Assert ($batches.Count -eq 3 -and @($batches|Where-Object {@($_).Count -gt 200}).Count -eq 0) '520 versions are deleted in batches of at most 200' +Assert ((@($batches|ForEach-Object {$_}) -join ',') -ceq ($ids -join ',')) 'every version is in exactly one batch, in the same order' +# Windows refuses a command line over 32,767 characters; the worker's fixed arguments take a few hundred. +Assert (@($batches|Where-Object {((@($_) -join ' ').Length+2000) -ge 32767}).Count -eq 0) 'each batch fits a Windows command line with room for the other arguments' +$one=@(Split-SnapshotIdBatches @($ids[0])) +Assert ($one.Count -eq 1 -and @($one[0]).Count -eq 1 -and @($one[0])[0] -ceq $ids[0]) 'a single version is one batch' +Assert (@(Split-SnapshotIdBatches @()).Count -eq 0) 'nothing to delete gives no batch' + +# --- Restore warnings --------------------------------------------------------------------------------- +$src='C:\Users\YOUR_USER\Documents\work\' +$stamp='{"message_type":"error","error":{"message":"failed to restore timestamp of \"D:\\\\out\\\\C\\\\Users\": Access is denied."},"during":"restore","item":"\\C\\Users"}' +$exit='{"message_type":"exit_error","code":1,"message":"Fatal: There were 1 errors"}' +$w=@(Get-RestoreParentWarnings ($stamp+"`n"+$exit) $src ([pscustomobject]@{total_files=14;files_restored=13})) +Assert ($w.Count -eq 1 -and $w[0] -like 'failed to restore timestamp*') 'a date that could not be set on a folder above the source is a warning' +$inside='{"message_type":"error","error":{"message":"failed to restore timestamp of \"D:\\\\out\\\\C\\\\Users\\\\someone\\\\Documents\\\\work\\\\sub\": Access is denied."},"during":"restore","item":"\\C\\Users\\someone\\Documents\\work\\sub"}' +Assert (@(Get-RestoreParentWarnings ($inside+"`n"+$exit) $src $null).Count -eq 0) 'the same error on a folder inside the source is a failure' +$content='{"message_type":"error","error":{"message":"open D:\\out\\C\\Users\\someone\\Documents\\work\\a.txt: Access is denied."},"during":"restore","item":"\\C\\Users\\someone\\Documents\\work\\a.txt"}' +Assert (@(Get-RestoreParentWarnings ($stamp+"`n"+$content+"`n"+$exit) $src $null).Count -eq 0) 'a file that could not be written is a failure, also beside a harmless warning' +Assert (@(Get-RestoreParentWarnings ("ignoring error for \C\Users: something`n"+$exit) $src $null).Count -eq 0) 'output that cannot be understood is a failure' +Assert (@(Get-RestoreParentWarnings $exit $src $null).Count -eq 0) 'a failure without any item error is a failure' +Assert (@(Get-RestoreParentWarnings ($stamp+"`n"+$exit) $src ([pscustomobject]@{total_files=14;files_restored=10})).Count -eq 0) 'more items missing than warnings is a failure' +Assert (@(Get-RestoreParentWarnings ($stamp+"`n"+$exit) 'C:\Users\' $null).Count -eq 0) 'the source folder itself is not a folder above the source' +# The folders above are judged by what the restored version itself backed up, not by today's settings. +Assert (@(Get-RestoreParentWarnings ($stamp+"`n"+$exit) @($src,'C:\Users') $null).Count -eq 0) 'a folder that the version backed up itself (an older, wider source) is not a folder above the source' +Assert (@(Get-RestoreParentWarnings ($stamp+"`n"+$exit) @('C:\Users\YOUR_USER\Documents\work','C:\Users\YOUR_USER\Pictures') $null).Count -eq 1) 'a version with several source folders accepts the folders above them' +Assert (@(Get-RestoreParentWarnings ($stamp+"`n"+$exit) @() $null).Count -eq 0) 'without the version''s source paths nothing is accepted' + +# --- A pending password change after the drive letter changed ------------------------------------- +$journal=[pscustomobject]@{Version=1;Repository='Y:\Work-Versions';ConfigPath='C:\tool\jobs\work.local.json';DestinationVolumeId='volume-1';RepositoryFolder='Work-Versions'} +Assert-RotationJournalTarget $journal 'volume-1' 'Work-Versions' 'Z:\Work-Versions' +Assert $true 'a pending password change is recognised by volume and folder after the drive letter changed' +Assert-Throws {Assert-RotationJournalTarget $journal 'volume-2' 'Work-Versions' 'Y:\Work-Versions'} '*another*' 'the same path on another volume is refused' +Assert-Throws {Assert-RotationJournalTarget $journal 'volume-1' 'Other-Versions' 'Y:\Other-Versions'} '*another*' 'another repository folder on the same volume is refused' +$oldJournal=[pscustomobject]@{Version=1;Repository='Y:\Work-Versions';ConfigPath='C:\tool\jobs\work.local.json'} +Assert-RotationJournalTarget $oldJournal 'volume-1' 'Work-Versions' 'Y:\Work-Versions' +Assert-RotationJournalTarget $oldJournal 'volume-1' 'Work-Versions' 'Z:\Work-Versions' +Assert $true 'a journal written by an earlier version resumes on the same path and after a drive-letter change' +Assert-Throws {Assert-RotationJournalTarget $oldJournal 'volume-1' 'Other-Versions' 'Y:\Other-Versions'} '*another*' 'an earlier journal for another folder is refused' + +# --- A locked repository ------------------------------------------------------------------------------ +$text=Get-EngineFailureText 'CheckingStructure' 11 +Assert ($text -like '*exit code 11*' -and $text -like '*locked*' -and $text -like '*Nothing is damaged*' -and $text -like '*CheckingStructure.errors.log*' -and $text -notlike '*memory*') 'a locked repository is named as such, says nothing is damaged and does not blame memory' +Assert ((Get-EngineFailureText 'Verifying' 1) -like '*exit code 1*memory allocation*') 'other failures keep the general wording' +# --- Removing a lock left by a run that ended -------------------------------------------------------- +# A lock as restic records it (cat lock), with what now runs on this computer under its process number: +# nothing ($null), or a process with its name and start time (no start time: it cannot be examined). +function Left([int]$n,$process=$null,[string]$computer='Work-PC',[string]$time='2026-11-14T10:00:00.1234567+02:00',$number=4242,$exclusive=$false){ + [pscustomobject]@{Id=(Id $n);Lock=[pscustomobject]@{time=$time;exclusive=$exclusive;hostname=$computer;username='Work-PC\me';pid=$number};Process=$process} +} +function Proc([string]$name,$started){[pscustomobject]@{Name=$name;StartedUtc=$(if($started){[DateTimeOffset]::Parse($started,[Globalization.CultureInfo]::InvariantCulture).UtcDateTime}else{$null})}} +$d=Get-LockReleaseDecision @((Left 1)) 'WORK-pc' +Assert ($d.Release -eq $true -and $d.Reason -eq '' -and @($d.Locks).Count -eq 1 -and $d.Locks[0].State -eq 'Ended') 'a lock of this computer (the name compared without regard to case) whose process is gone may be removed' +Assert ($d.Locks[0].Id -ceq (Id 1) -and $d.Locks[0].Pid -eq 4242 -and $d.Locks[0].CreatedUtc -ceq '2026-11-14T08:00:00.1234567Z' -and $d.Locks[0].Exclusive -eq $false -and $d.Locks[0].Computer -ceq 'Work-PC') 'the decision names the lock: its ID, process number, creation time (UTC) and kind' +$d=Get-LockReleaseDecision @((Left 2 $null 'Laptop')) 'Work-PC' +Assert ($d.Release -eq $false -and $d.Locks[0].State -eq 'OtherComputer' -and $d.Reason -like '*another computer (Laptop)*left in place*') 'a lock of another computer is never removed, and the reason names that computer' +$d=Get-LockReleaseDecision @((Left 3 (Proc 'restic_0.19.1_windows_amd64' '2026-11-14T07:59:58Z'))) 'Work-PC' +Assert ($d.Release -eq $false -and $d.Locks[0].State -eq 'Running' -and $d.Reason -like '*still running on this computer (process 4242)*') 'a lock whose restic process is still running is never removed' +$d=Get-LockReleaseDecision @((Left 4 (Proc 'notepad' '2026-11-14T08:20:00Z'))) 'Work-PC' +Assert ($d.Release -eq $true -and $d.Locks[0].State -eq 'Ended') 'a process number reused by another program that started after the lock was made: the run that made the lock has ended' +$d=Get-LockReleaseDecision @((Left 5 (Proc 'restic' '2026-11-14T08:20:00Z'))) 'Work-PC' +Assert ($d.Release -eq $false -and $d.Locks[0].State -eq 'Unclear' -and $d.Reason -like '*process number*4242*') 'a restic process under that number that seems to have started after the lock is unclear (the clock may have changed): not removed' +$d=Get-LockReleaseDecision @((Left 6 (Proc 'svchost' '2026-11-01T08:00:00Z'))) 'Work-PC' +Assert ($d.Release -eq $false -and $d.Locks[0].State -eq 'Unclear') 'another program that was already running under that number when the lock was made is unclear: not removed' +$d=Get-LockReleaseDecision @((Left 7 (Proc '' $null))) 'Work-PC' +Assert ($d.Release -eq $false -and $d.Locks[0].State -eq 'Unclear' -and $d.Reason -like '*process 4242*could not be examined*') 'a process that cannot be examined is unclear: not removed' +$unreadable=[pscustomobject]@{Id=(Id 8);Lock=$null;Process=$null} +$d=Get-LockReleaseDecision @($unreadable) 'Work-PC' +Assert ($d.Release -eq $false -and $d.Locks[0].State -eq 'Unclear' -and $d.Reason -like '*could not be read*') 'a lock that cannot be read is not removed' +foreach($broken in @((Left 9 $null 'Work-PC' 'yesterday'),(Left 9 $null 'Work-PC' '2026-11-14T10:00:00+02:00' 0),(Left 9 $null 'Work-PC' '2026-11-14T10:00:00+02:00' '4242x'),(Left 9 $null ''),[pscustomobject]@{Id='latest';Lock=(Left 9).Lock;Process=$null},[pscustomobject]@{Id=(Id 9);Lock=(Left 9).Lock},[pscustomobject]@{Id=(Id 9);Lock=[pscustomobject]@{hostname='Work-PC';pid=4242};Process=$null},[pscustomobject]@{Id=(Id 9);Lock=[pscustomobject]@{time='2026-11-14T10:00:00+02:00';hostname='Work-PC';pid=4242};Process=$null},(Left 9 $null 'Work-PC' '2026-11-14T10:00:00+02:00' 4242 'no'))){ + if((Get-LockReleaseDecision @($broken) 'Work-PC').Release){throw 'FAIL an incomplete lock record was accepted'} +} +Assert $true 'a lock without a usable time, process number, computer name, kind or ID, or one whose process was not looked up, is not removed' +Assert ((Get-LockReleaseDecision @((Left 1)) '').Release -eq $false) 'without this computer''s name nothing is removed' +$d=Get-LockReleaseDecision @((Left 10),(Left 11 $null 'Laptop'),(Left 12 $null 'Work-PC' '2026-11-14T10:05:00+02:00' 5151 $true)) 'Work-PC' +Assert ($d.Release -eq $false -and @($d.Locks).Count -eq 3 -and @($d.Locks|Where-Object {$_.State -eq 'Ended'}).Count -eq 2 -and $d.Reason -like '*Laptop*') 'one lock that may not be removed keeps every lock in place' +$d=Get-LockReleaseDecision @((Left 10),(Left 12 $null 'Work-PC' '2026-11-14T10:05:00+02:00' 5151 $true)) 'Work-PC' +Assert ($d.Release -eq $true -and @($d.Locks).Count -eq 2 -and $d.Locks[1].Exclusive -eq $true -and $d.Locks[1].Pid -eq 5151) 'several locks that all belong to ended runs of this computer may be removed together' +$d=Get-LockReleaseDecision @() 'Work-PC' +Assert ($d.Release -eq $false -and $d.Reason -eq '' -and @($d.Locks).Count -eq 0) 'no lock: nothing to remove and nothing to report' +$text=Get-EngineFailureText 'CheckingStructure' 11 'a backup is still running on this computer (process 4242). Wait until it has finished, then try again.' +Assert ($text -like '*exit code 11*locked: a backup is still running on this computer (process 4242).*Nothing is damaged*CheckingStructure.errors.log*' -and $text -notlike '*memory*' -and $text -notlike '*An earlier run was stopped*') 'a lock that was examined and left in place is reported with its own reason instead of the general guess' +Write-Output 'All versioned-engine rule tests passed.' From bd75621061051f0d7618d01e9b8109bc52f2615b Mon Sep 17 00:00:00 2001 From: Raz Ben Aharon <181751257+razbenaharon@users.noreply.github.com> Date: Sat, 14 Nov 2026 20:00:19 +0200 Subject: [PATCH 03/23] Window: one window, settings recovery, run warnings, honest run results Starting twice brings the open window forward. A damaged settings file can be restored from the last good copy and is always kept. The overview warns when the latest run failed. Runs are recorded as failed, stopped or unknown instead of a wrong success. The window refreshes itself, explains disabled buttons and offers Repair. Tests open the window off-screen (-Quiet). Co-Authored-By: Claude Fable 5.2 --- Backup.Taskbar.cs | 6 +- DevDriveBackup.Data.psm1 | 554 ++++++++++++++++++++++++++++++++---- DevDriveBackup.Theme.psm1 | 21 +- DevDriveBackup.ps1 | 581 +++++++++++++++++++++++++++++++------- Test-BackupActivity.ps1 | 21 ++ Test-BackupData.ps1 | 343 ++++++++++++++++++++++ Test-BackupTheme.ps1 | 23 ++ Test-BackupWindow.ps1 | 174 ++++++++++++ Test-DevDriveBackup.ps1 | 2 +- 9 files changed, 1553 insertions(+), 172 deletions(-) create mode 100644 Test-BackupData.ps1 create mode 100644 Test-BackupWindow.ps1 diff --git a/Backup.Taskbar.cs b/Backup.Taskbar.cs index 02c9e9d..cab9c75 100644 --- a/Backup.Taskbar.cs +++ b/Backup.Taskbar.cs @@ -25,8 +25,10 @@ private struct PropertyKey public uint PropertyId; } - // Only the two kinds used here: empty (clears a value) and a wide string. - [StructLayout(LayoutKind.Explicit, Size = 16)] + // Only the two kinds used here: empty (clears a value) and a wide string. The size is the full + // PROPVARIANT of 64-bit Windows (24 bytes; 16 on 32-bit): Windows reads the whole value, so a + // shorter one would have it read past the end. + [StructLayout(LayoutKind.Explicit, Size = 24)] private struct PropVariant { [FieldOffset(0)] public ushort Type; diff --git a/DevDriveBackup.Data.psm1 b/DevDriveBackup.Data.psm1 index 45fb034..4245449 100644 --- a/DevDriveBackup.Data.psm1 +++ b/DevDriveBackup.Data.psm1 @@ -32,13 +32,16 @@ function Get-CenterLoadPresets($Hardware) { # Four levels scaled to this computer. Threads = processor threads used for compression and # encryption (versions); CopyThreads = files copied at once (plain copy); MemoryMiB = hard cap # for the versions engine; Priority decides who wins when your own programs need the computer. - $t=[int]$Hardware.Threads;$mem=[double]$Hardware.MemoryGB*1024 + $t=[math]::Max(1,[int]$Hardware.Threads);$mem=[double]$Hardware.MemoryGB*1024 + # Maximum leaves one thread for the rest of the computer, but never works with fewer than Strong + # (on 2 and 3 threads "all but one" would be below it). + $strong=[int][math]::Min($t,[math]::Max(2,[math]::Floor($t/2))) $m={param([double]$Share,[int]$Min,[int]$Max)[int][math]::Max($Min,[math]::Min($Max,[math]::Floor($mem*$Share/256)*256))} [ordered]@{ Light=[pscustomobject]@{Name='Light';CpuThreads=[int][math]::Max(1,[math]::Floor($t/6));MemoryMiB=(& $m 0.03 512 1024);CopyThreads=4;ReadConcurrency=1;Priority='BelowNormal'} Balanced=[pscustomobject]@{Name='Balanced';CpuThreads=[int][math]::Min($t,[math]::Max(2,[math]::Floor($t/3)));MemoryMiB=(& $m 0.06 512 2048);CopyThreads=8;ReadConcurrency=2;Priority='BelowNormal'} - Strong=[pscustomobject]@{Name='Strong';CpuThreads=[int][math]::Min($t,[math]::Max(2,[math]::Floor($t/2)));MemoryMiB=(& $m 0.12 1024 4096);CopyThreads=16;ReadConcurrency=4;Priority='Normal'} - Maximum=[pscustomobject]@{Name='Maximum';CpuThreads=[int][math]::Max(1,$t-1);MemoryMiB=(& $m 0.25 1024 8192);CopyThreads=32;ReadConcurrency=8;Priority='Normal'} + Strong=[pscustomobject]@{Name='Strong';CpuThreads=$strong;MemoryMiB=(& $m 0.12 1024 4096);CopyThreads=16;ReadConcurrency=4;Priority='Normal'} + Maximum=[pscustomobject]@{Name='Maximum';CpuThreads=[int][math]::Max($strong,$t-1);MemoryMiB=(& $m 0.25 1024 8192);CopyThreads=32;ReadConcurrency=8;Priority='Normal'} } } function Get-CenterLoad($Profile,$Hardware=(Get-CenterHardware)) { @@ -64,19 +67,111 @@ function Set-CenterRunPriority($Process,[string]$Priority) { function Set-CenterPaths([string]$Root) { # Root '' = the real locations; tests pass a disposable folder. if($Root){ - $script:Paths=[pscustomobject]@{Profiles=(Join-Path $Root 'profiles.local.json');Jobs=(Join-Path $Root 'jobs');History=(Join-Path $Root 'history.jsonl');Keys=(Join-Path $Root 'keys');Documents=(Join-Path $Root 'documents');DriveCheck=(Join-Path $Root 'drive-check');Appearance=(Join-Path $Root 'appearance.json')} + $script:Paths=[pscustomobject]@{Profiles=(Join-Path $Root 'profiles.local.json');Jobs=(Join-Path $Root 'jobs');History=(Join-Path $Root 'history.jsonl');Keys=(Join-Path $Root 'keys');Documents=(Join-Path $Root 'documents');DriveCheck=(Join-Path $Root 'drive-check');Appearance=(Join-Path $Root 'appearance.json');Logs=(Join-Path $Root 'logs')} }else{ $app=Join-Path $env:LOCALAPPDATA 'DevDriveMirror' - $script:Paths=[pscustomobject]@{Profiles=(Join-Path $script:ToolRoot 'profiles.local.json');Jobs=(Join-Path $script:ToolRoot 'jobs');History=(Join-Path $app 'center\history.jsonl');Keys=$app;Documents=(Join-Path $env:USERPROFILE $(if(Test-Path -LiteralPath (Join-Path $env:USERPROFILE 'Backup Center recovery keys')){'Backup Center recovery keys'}else{'DevDriveBackup recovery keys'}));DriveCheck=(Join-Path $app 'drive-check');Appearance=(Join-Path $app 'center\appearance.json')} # local, not a synced Documents folder + $script:Paths=[pscustomobject]@{Profiles=(Join-Path $script:ToolRoot 'profiles.local.json');Jobs=(Join-Path $script:ToolRoot 'jobs');History=(Join-Path $app 'center\history.jsonl');Keys=$app;Documents=(Join-Path $env:USERPROFILE $(if(Test-Path -LiteralPath (Join-Path $env:USERPROFILE 'Backup Center recovery keys')){'Backup Center recovery keys'}else{'DevDriveBackup recovery keys'}));DriveCheck=(Join-Path $app 'drive-check');Appearance=(Join-Path $app 'center\appearance.json');Logs=(Join-Path $app 'center')} # local, not a synced Documents folder } $script:Paths } function Get-CenterPaths {if(-not $script:Paths){[void](Set-CenterPaths '')};$script:Paths} +function Get-CenterProfilesProblem([string]$Text,[ref]$Data) { + # Why this text is not a settings file ('' = it is one). -Data receives what was read. + $d=$null + try{$d=$Text|ConvertFrom-Json}catch{return $_.Exception.Message} + if($d -isnot [pscustomobject] -or -not $d.PSObject.Properties['Profiles'] -or $null -eq $d.Profiles){return 'it has no list of profiles'} + if(@($d.Profiles|Where-Object {$q=$_;$q -isnot [pscustomobject] -or @('Id','Name','DestinationVolumeId','Sources'|Where-Object {-not $q.PSObject.Properties[$_]}).Count}).Count){return 'a profile in it is incomplete'} + # Checked profile by profile and without a pipeline: a profile with no sources (nothing, or an empty + # list) is fine, but an empty place inside the list is damage, and a pipeline would not tell the two apart. + foreach($profile in @($d.Profiles)){ + if($null -eq $profile.Sources){continue} + foreach($q in $profile.Sources){if($null -eq $q -or $q -isnot [pscustomobject] -or @('Id','Path','ConfigPath'|Where-Object {-not $q.PSObject.Properties[$_]}).Count){return 'a folder listed in it is incomplete'}} + } + if($Data){$Data.Value=$d} + '' +} +# --- The copy of the last good settings ------------------------------------------------------ +# Beside the settings file, under a name the same ignore rule covers (*.local.json). It is only ever +# written from a settings file that was read back from disk and is valid, so a damaged file can be +# replaced by it. Nothing here deletes or overwrites a damaged file. +function Get-CenterLastGoodPath {$p=(Get-CenterPaths).Profiles;if($p -match '\.local\.json$'){$p -replace '\.local\.json$','.lastgood.local.json'}else{$p+'.lastgood'}} +function Read-CenterFileText([string]$Path,[ref]$Bytes) { + # The text of a file as the settings are read (a byte order mark is not part of it), and its exact bytes. + $b=[IO.File]::ReadAllBytes($Path);if($Bytes){$Bytes.Value=$b} + $r=[IO.StreamReader]::new([IO.MemoryStream]::new($b),[Text.UTF8Encoding]::new($false),$true);try{$r.ReadToEnd()}finally{$r.Dispose()} +} +function Save-CenterLastGood { + # Copies the settings file as it is on disk right now to the last good copy, when it is valid. + # Returns whether a good copy of exactly these settings exists afterwards. Never an error: a copy + # that could not be written must not fail the save that was just made. + try{ + $p=(Get-CenterPaths).Profiles;$g=Get-CenterLastGoodPath + if(-not(Test-Path -LiteralPath $p -PathType Leaf)){return $false} + $bytes=$null;if(Get-CenterProfilesProblem (Read-CenterFileText $p ([ref]$bytes))){return $false} + if((Test-Path -LiteralPath $g -PathType Leaf) -and [Linq.Enumerable]::SequenceEqual([byte[]][IO.File]::ReadAllBytes($g),[byte[]]$bytes)){return $true} + $tmp=$g+'.'+[guid]::NewGuid().ToString('N')+'.tmp' + try{ + [IO.File]::WriteAllBytes($tmp,$bytes) + # Its date is the date of the settings it holds, which is what is shown when it is offered. + [IO.File]::SetLastWriteTime($tmp,[IO.File]::GetLastWriteTime($p)) + if([IO.File]::Exists($g)){[IO.File]::Replace($tmp,$g,[System.Management.Automation.Language.NullString]::Value)}else{[IO.File]::Move($tmp,$g)} + }finally{if([IO.File]::Exists($tmp)){try{[IO.File]::Delete($tmp)}catch{}}} + $true + }catch{$false} +} +function Get-CenterSettingsRecovery { + # What can be done about the settings file: Damaged (and Why), whether the last good copy can be put + # back (CanRestore, its date and what it holds), and the Message to show. It changes nothing. + $p=(Get-CenterPaths).Profiles;$g=Get-CenterLastGoodPath + $r=[pscustomobject]@{Path=$p;Damaged=$false;Why='';LastGood=$g;CanRestore=$false;LastGoodWhen=$null;LastGoodProfiles=0;LastGoodFolders=0;Message=''} + if(-not(Test-Path -LiteralPath $p -PathType Leaf)){return $r} + $r.Why=try{Get-CenterProfilesProblem (Read-CenterFileText $p)}catch{$_.Exception.Message} + if(-not $r.Why){return $r} + $r.Damaged=$true + $head="DevDriveBackup cannot open, because its settings file cannot be read.`n`nThe settings file $p is damaged ($($r.Why)).`n`nYour backups on the drives, the settings of each folder and the keys are not affected, and this file was not changed." + $byHand="put back an earlier copy of the file yourself. Or move the file somewhere else and start DevDriveBackup again: it opens without profiles, and when you add the drive and its folders again, the earlier settings of each folder are picked up." + if(-not(Test-Path -LiteralPath $g -PathType Leaf)){$r.Message="$head`n`nThere is no earlier good copy of the settings to put back.`n`nWhat you can do: $byHand";return $r} + $old=$null;$bad=try{Get-CenterProfilesProblem (Read-CenterFileText $g) ([ref]$old)}catch{$_.Exception.Message} + if($bad){$r.Message="$head`n`nThe copy of the last good settings ($g) cannot be read either, so it is not offered.`n`nWhat you can do: $byHand";return $r} + $r.CanRestore=$true;$r.LastGoodWhen=[IO.File]::GetLastWriteTime($g);$r.LastGoodProfiles=@($old.Profiles).Count;$r.LastGoodFolders=@($old.Profiles|ForEach-Object {$_.Sources}|Where-Object {$_}).Count + # Only the last part is a format: the path and the reason above may hold braces of their own. + $r.Message="$head`n`n"+("What you can do: restore the last good copy of the settings, from {0:d MMM yyyy, HH:mm} ({1} and {2}). Settings changed after that time are not in it. The damaged file is kept beside it under a new name; nothing is deleted.") -f $r.LastGoodWhen,$(if($r.LastGoodProfiles -eq 1){'1 backup drive'}else{"$($r.LastGoodProfiles) backup drives"}),$(if($r.LastGoodFolders -eq 1){'1 folder'}else{"$($r.LastGoodFolders) folders"}) + $r +} +function Restore-CenterLastGood { + # Puts the last good copy in the place of the settings file. The file that is there now is renamed + # aside with the time in its name and never deleted or written over. The new file is made ready and + # checked first, so the settings are never left missing: if putting it in place fails, the old file + # goes back where it was. + $p=(Get-CenterPaths).Profiles;$g=Get-CenterLastGoodPath + if(-not(Test-Path -LiteralPath $g -PathType Leaf)){throw 'There is no good copy of the settings to restore.'} + $bytes=$null;$bad=try{Get-CenterProfilesProblem (Read-CenterFileText $g ([ref]$bytes))}catch{$_.Exception.Message} + if($bad){throw "The good copy of the settings ($g) cannot be read ($bad). Nothing was changed."} + $when=[IO.File]::GetLastWriteTime($g);$dir=[IO.Path]::GetDirectoryName($p) + $tmp=$p+'.'+[guid]::NewGuid().ToString('N')+'.tmp' + try{ + [IO.File]::WriteAllBytes($tmp,$bytes) + if(Get-CenterProfilesProblem (Read-CenterFileText $tmp)){throw 'The copy could not be written correctly. Nothing was changed.'} + $kept=$null + if(Test-Path -LiteralPath $p -PathType Leaf){ + $base=[IO.Path]::GetFileName($p) -replace '\.local\.json$','';$tail=if($p -match '\.local\.json$'){'.local.json'}else{''};$stamp=Get-Date -Format 'yyyyMMdd-HHmmss' + for($i=1;;$i++){$kept=Join-Path $dir ($base+'.damaged-'+$stamp+$(if($i -gt 1){"-$i"})+$tail);if(-not(Test-Path -LiteralPath $kept)){break}} + [IO.File]::Move($p,$kept) # fails rather than overwrites when the name is taken after all + } + try{[IO.File]::Move($tmp,$p)}catch{if($kept -and -not [IO.File]::Exists($p)){[IO.File]::Move($kept,$p)};throw} + }finally{if([IO.File]::Exists($tmp)){try{[IO.File]::Delete($tmp)}catch{}}} + [pscustomobject]@{KeptAs=$kept;RestoredFrom=$g;When=$when} +} + function Read-CenterProfiles { $p=(Get-CenterPaths).Profiles if(-not(Test-Path -LiteralPath $p)){return [pscustomobject]@{Version=1;Profiles=@()}} - $data=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json + # A file that is empty, cut off or not shaped like the settings is reported by name and left exactly as + # it is: guessing (for example "no profiles") would let the next save write over what is left of it. + $data=$null + $why=try{Get-CenterProfilesProblem (Get-Content -LiteralPath $p -Raw) ([ref]$data)}catch{$_.Exception.Message} + if($why){throw "The settings file $p is damaged and cannot be read ($why). Nothing was changed."} $data.Profiles=@($data.Profiles|ForEach-Object {$_.Sources=@($_.Sources);$_}) # A source's settings file is stored by full path. When the tool folder is moved or renamed, that path # goes stale; a file of the same name in the jobs folder or beside the profiles file is the same file, moved. @@ -100,6 +195,8 @@ function Save-CenterProfiles($Data) { $p=(Get-CenterPaths).Profiles New-Item -ItemType Directory -Path ([IO.Path]::GetDirectoryName($p)) -Force|Out-Null [void](Write-BackupStatus $Data $p) + # What was just saved, read back from disk and found valid, becomes the copy to fall back on. + [void](Save-CenterLastGood) } function Get-CenterProfile($Data,[string]$Id) { $hit=@($Data.Profiles|Where-Object {$_.Id -eq $Id}) @@ -116,20 +213,50 @@ function Get-SafeName([string]$Text) { } function New-ShortId {[guid]::NewGuid().ToString('N').Substring(0,8)} -$script:VolumeCache=$null -function Get-CenterVolumes { - # Listing the volumes takes about a second; one refresh of the window asks several times, so the list - # is reused for 3 seconds. Plugging a drive in or out shows up on the next refresh. - if(-not $script:VolumeCache -or ((Get-Date)-$script:VolumeCache.At).TotalSeconds -gt 3){ - $script:VolumeCache=[pscustomobject]@{At=Get-Date;List=@(Get-Volume -ErrorAction SilentlyContinue)} +$script:VolumeCache=$null;$script:VolumeQueries=0 +function Get-CenterDriveKey { + # The lettered drives as Windows shows them right now, read in a few milliseconds: letter, size, label + # and file system of each. It changes when a drive is plugged in, unplugged or swapped. + try{@([IO.DriveInfo]::GetDrives()|ForEach-Object {try{if($_.IsReady){$_.Name+'|'+$_.TotalSize+'|'+$_.VolumeLabel+'|'+$_.DriveFormat}else{$_.Name+'|not ready'}}catch{$_.Name+'|?'}}) -join "`n"}catch{[guid]::NewGuid().ToString()} +} +function Get-CenterVolumes([switch]$Fresh) { + # Listing the volumes takes a second or more, on the window's own thread. For showing (the refresh of + # the window, the status line of a running backup) the list is reused while the same drives are + # plugged in, for at most a minute. -Fresh, used by everything that starts work on a drive, always + # lists again: the identity of the drive must be the one that is plugged in at that moment. + $key=if($Fresh){''}else{Get-CenterDriveKey} + if($Fresh -or -not $script:VolumeCache -or $script:VolumeCache.Key -ne $key -or ((Get-Date)-$script:VolumeCache.At).TotalSeconds -gt 60){ + $script:VolumeQueries++ + $list=@(Get-Volume -ErrorAction SilentlyContinue) + $script:VolumeCache=[pscustomobject]@{At=Get-Date;Key=(Get-CenterDriveKey);List=$list} } $script:VolumeCache.List } -function Get-CenterDrive([string]$VolumeId) { +function Get-CenterVolumeQueryCount {$script:VolumeQueries} # for tests: how often the volumes were listed +function Get-CenterFreeBytes([string]$Letter) { + # Free space of a lettered drive without listing the volumes (asked every few seconds during a backup). + try{$d=[IO.DriveInfo]::new($Letter.Substring(0,1)+':');if($d.IsReady){[long]$d.AvailableFreeSpace}else{$null}}catch{$null} +} +function Get-CenterDrive([string]$VolumeId,[switch]$Fresh) { # The drive of a profile, or Connected=$false when it is not plugged in. - $v=@(Get-CenterVolumes|Where-Object {$_.UniqueId -eq $VolumeId -and $_.DriveLetter}) + $v=@(Get-CenterVolumes -Fresh:$Fresh|Where-Object {$_.UniqueId -eq $VolumeId -and $_.DriveLetter}) if($v.Count -ne 1){return [pscustomobject]@{Connected=$false;Letter=$null;Label=$null;FileSystem=$null;SizeGB=$null;FreeGB=$null}} - [pscustomobject]@{Connected=$true;Letter=[string]$v[0].DriveLetter;Label=$v[0].FileSystemLabel;FileSystem=$v[0].FileSystem;SizeGB=[math]::Round($v[0].Size/1GB,1);FreeGB=[math]::Round($v[0].SizeRemaining/1GB,1)} + # Free space changes while the list is reused, so it is read now. + $free=Get-CenterFreeBytes ([string]$v[0].DriveLetter);if($null -eq $free){$free=$v[0].SizeRemaining} + [pscustomobject]@{Connected=$true;Letter=[string]$v[0].DriveLetter;Label=$v[0].FileSystemLabel;FileSystem=$v[0].FileSystem;SizeGB=[math]::Round($v[0].Size/1GB,1);FreeGB=[math]::Round($free/1GB,1)} +} +function Get-CenterBusyReason { + # One job per drive: a backup, a drive check and a speed test of the same drive would compete for it + # (and a speed test would measure the backup). Returns why -For cannot start on this profile's drive + # now, or '' when it can. The window uses it for the buttons and again at the moment a job starts, + # because a backup that waited for AI sessions starts by itself, long after the button was pressed. + param([string]$ProfileId,[ValidateSet('Backup','Check','Speed')][string]$For,[string]$RunProfileId,[string]$WaitProfileId,[string]$CheckProfileId,[string]$SpeedProfileId) + if(-not $ProfileId){return ''} + if($For -ne 'Backup' -and $RunProfileId -eq $ProfileId){return 'A backup of this drive is running. Wait until it has finished.'} + if($For -ne 'Backup' -and $WaitProfileId -eq $ProfileId){return 'A backup of this drive is waiting for AI sessions to finish and starts by itself. Stop the wait first, or wait for the backup.'} + if($For -ne 'Check' -and $CheckProfileId -eq $ProfileId){return 'A drive check of this drive is running. Wait until it has finished, or stop it on the Drive check page.'} + if($For -ne 'Speed' -and $SpeedProfileId -eq $ProfileId){return 'A speed test of this drive is running. It takes about half a minute; try again then.'} + '' } function Get-CenterCandidateDrives { # Volumes that can hold a backup: lettered, with a file system, not the system drive. @@ -159,19 +286,24 @@ function Test-PathOverlap([string]$A,[string]$B) { } function Add-CenterSource { - param($Data,[string]$ProfileId,[string]$Path,[string]$Label,[ValidateSet('Mirror','Versioned','Both','Skip')][string]$DefaultMode='Mirror',[string]$ExistingConfigPath,[string]$MirrorFolder) + # -Restored ([ref]): set to the earlier settings (copy folder, versions folder, mode of unlisted folders) + # when this folder was a source of this drive before and its kept settings were taken up again, so the + # caller can say that -Label and -DefaultMode were not applied. Left alone otherwise. + param($Data,[string]$ProfileId,[string]$Path,[string]$Label,[ValidateSet('Mirror','Versioned','Both','Skip')][string]$DefaultMode='Mirror',[string]$ExistingConfigPath,[string]$MirrorFolder,[ref]$Restored) $profile=Get-CenterProfile $Data $ProfileId $root=[IO.Path]::GetFullPath($Path).TrimEnd('\')+'\' + if($root -notmatch '^[A-Za-z]:\\'){throw "Choose a folder on a drive of this computer (one with a drive letter). A network folder such as $($root.TrimEnd('\')) cannot be backed up from here."} if(-not(Test-Path -LiteralPath $root -PathType Container)){throw "Folder not found: $root"} if((Get-Item -LiteralPath $root -Force).Attributes -band [IO.FileAttributes]::ReparsePoint){throw 'Choose the real folder, not a shortcut (junction or link) to it'} - $volume=Get-Volume -DriveLetter $root.Substring(0,1) + $volume=try{Get-Volume -DriveLetter $root.Substring(0,1) -ErrorAction Stop}catch{$null} + if(-not $volume){throw "Drive $($root.Substring(0,1)): is not a disk of this computer that Windows can identify (a network or substituted drive letter?). Choose the folder on its real drive."} if($volume.UniqueId -eq $profile.DestinationVolumeId){throw 'This folder is on the backup drive itself'} foreach($s in $profile.Sources){if(Test-PathOverlap $s.Path $root){throw "Overlaps with the existing source '$($s.Label)' ($($s.Path))"}} if(-not $Label.Trim()){$Label=if($root.Length -eq 3){'Drive '+$root.Substring(0,1)}else{Split-Path -Leaf $root.TrimEnd('\')}} $safe=Get-SafeName $Label # Names that are not Latin letters all become "Source"; keep them apart. if($safe -eq 'Source'){$safe='Source-'+(New-ShortId).Substring(0,4)} - $drive=Get-CenterDrive $profile.DestinationVolumeId + $drive=Get-CenterDrive $profile.DestinationVolumeId -Fresh # The folder names this drive's job files record, also those of sources removed earlier: their plain # copy and versions stay on the drive, so a new source must never get one, even while it is unplugged. $jobFiles=@(Get-ChildItem -LiteralPath (Get-CenterPaths).Jobs -Filter '*.local.json' -File -ErrorAction SilentlyContinue|ForEach-Object { @@ -201,7 +333,10 @@ function Add-CenterSource { $old=@($jobFiles|Where-Object { try{(([IO.Path]::GetFullPath([string]$_.Job.Source).TrimEnd('\')+'\') -ieq $root)}catch{$false} }|Where-Object {$current -inotcontains $_.Path -and -not ($_.Job.PSObject.Properties['RepositoryFolder'] -and $inUse -icontains [string]$_.Job.RepositoryFolder)}|Sort-Object {(Get-Item -LiteralPath $_.Path).LastWriteTime} -Descending) - if($old.Count){$ExistingConfigPath=$old[0].Path} + if($old.Count){ + $ExistingConfigPath=$old[0].Path + if($Restored){$j=$old[0].Job;$Restored.Value=[pscustomobject]@{ConfigPath=$ExistingConfigPath;MirrorFolder=$(if($j.PSObject.Properties['MirrorFolder']){[string]$j.MirrorFolder}else{''});RepositoryFolder=$(if($j.PSObject.Properties['RepositoryFolder']){[string]$j.RepositoryFolder}else{''});DefaultMode=$(if($j.PSObject.Properties['DefaultMode']){[string]$j.DefaultMode}else{''})}} + } } if(-not $ExistingConfigPath){while(& $taken $safe){$safe=(Get-SafeName $Label)+'-'+(New-ShortId).Substring(0,4)}} if($ExistingConfigPath){ @@ -238,16 +373,18 @@ function Remove-CenterSource($Data,[string]$ProfileId,[string]$SourceId) { } function Read-CenterJob([string]$ConfigPath) {Get-Content -LiteralPath $ConfigPath -Raw|ConvertFrom-Json} -function Save-CenterJob($Job,[string]$ConfigPath,[scriptblock]$Update) { +function Save-CenterJob($Job,[string]$ConfigPath,[scriptblock]$Update,[int]$WaitMilliseconds=3000) { # Writes under the job's run lock (as Backup-Worker.ps1 and Set-BackupDestination.ps1 take it) after # reading the file again, so a password change running elsewhere keeps its new recovery-key location. # With -Update the change is made to that fresh copy instead of writing $Job. + # -WaitMilliseconds: how long to try for the lock. A backup or an open restore window holds it for + # minutes, so waiting longer does not help; the window passes a short wait to stay responsive. $key={param($j)if($j -and $j.PSObject.Properties['PasswordDpapiPath'] -and $j.PasswordDpapiPath){[IO.Path]::GetFullPath([string]$j.PasswordDpapiPath)}else{''}} $keyPath=if(Test-Path -LiteralPath $ConfigPath){& $key (Read-CenterJob $ConfigPath)}else{& $key $Job} $lock=$null if($keyPath){ New-Item -ItemType Directory -Path ([IO.Path]::GetDirectoryName($keyPath)) -Force|Out-Null - for($i=0;-not $lock;$i++){try{$lock=[IO.File]::Open($keyPath+'.run.lock',[IO.FileMode]::OpenOrCreate,[IO.FileAccess]::ReadWrite,[IO.FileShare]::None)}catch [IO.IOException]{if($i -ge 30){throw 'This source is busy: a backup, check, restore or password change is using it. Try again when it has finished.'};Start-Sleep -Milliseconds 100}} + for($i=0;-not $lock;$i++){try{$lock=[IO.File]::Open($keyPath+'.run.lock',[IO.FileMode]::OpenOrCreate,[IO.FileAccess]::ReadWrite,[IO.FileShare]::None)}catch [IO.IOException]{if($i*100 -ge $WaitMilliseconds){throw 'This source is busy: a backup, check, restore or password change is using it. Try again when it has finished.'};Start-Sleep -Milliseconds 100}} } try{ $fresh=if(Test-Path -LiteralPath $ConfigPath){Read-CenterJob $ConfigPath}else{$null} @@ -270,7 +407,7 @@ function Get-CenterSourceKinds($Source) { } } -function Sync-CenterJob($Profile,$Source,$Load=(Get-CenterLoad $Profile)) { +function Sync-CenterJob($Profile,$Source,$Load=(Get-CenterLoad $Profile),[int]$WaitMilliseconds=3000) { # Keeps a job file pointed at its profile's drive and the chosen load, and records its plain copy # folder (so adding the folder again later finds the same copy). $light=$Load.Name -eq 'Light' @@ -285,7 +422,7 @@ function Sync-CenterJob($Profile,$Source,$Load=(Get-CenterLoad $Profile)) { $changed}.GetNewClosure() # Checked first without the lock: most calls change nothing and must not wait for a running backup. # A change is made to a fresh copy under the job's lock. - if(& $apply (Read-CenterJob $Source.ConfigPath)){Save-CenterJob $null $Source.ConfigPath -Update $apply} + if(& $apply (Read-CenterJob $Source.ConfigPath)){Save-CenterJob $null $Source.ConfigPath -Update $apply -WaitMilliseconds $WaitMilliseconds} } # --- History and freshness ---------------------------------------------------------------- @@ -300,26 +437,93 @@ function Get-CenterHistory { # empty list (a returned empty array would arrive as $null). if(-not(Test-Path -LiteralPath $p)){return} # Keep only well-formed entries; a damaged line must not break the window. + # A line whose finish time cannot be read is skipped as well: every list and age is ordered by it. + # The time is handed on as a local time, whatever form it was written in. Get-Content -LiteralPath $p -Encoding utf8|Where-Object {$_.Trim()}|ForEach-Object {try{$_|ConvertFrom-Json}catch{}}| - Where-Object {$_ -is [pscustomobject] -and $_.PSObject.Properties['SourceId'] -and $_.PSObject.Properties['Kind'] -and $_.PSObject.Properties['Finished'] -and $_.PSObject.Properties['Success']} + Where-Object {$_ -is [pscustomobject] -and $_.PSObject.Properties['SourceId'] -and $_.PSObject.Properties['Kind'] -and $_.PSObject.Properties['Finished'] -and $_.PSObject.Properties['Success']}| + ForEach-Object {$t=ConvertTo-CenterTime $_.Finished;if($t){$_.Finished=$t;$_}} +} +function ConvertTo-CenterTime($Value) { + # A time from a history or result file as a local time, or nothing when it cannot be read. Reading + # JSON already turns "2026-01-02T10:00:00+02:00" into a local time, but leaves "...Z" in UTC and a + # damaged value as text; used unchanged, a UTC time is off by the time zone and text stops the window. + if($null -eq $Value){return $null} + $t=$null + if($Value -is [datetime]){$t=$Value} + elseif($Value -is [DateTimeOffset]){return $Value.LocalDateTime} + else{ + $o=[DateTimeOffset]::MinValue;$s=([string]$Value).Trim() + if(-not $s -or -not [DateTimeOffset]::TryParse($s,[Globalization.CultureInfo]::InvariantCulture,[Globalization.DateTimeStyles]::AssumeLocal,[ref]$o)){return $null} + return $o.LocalDateTime + } + if($t.Kind -eq [DateTimeKind]::Utc){$t.ToLocalTime()}else{$t} +} +function Get-CenterElapsed($When,[datetime]$Now) { + # Time passed since $When, or nothing when it cannot be read. Both ends are compared as moments (UTC), + # so a clock change for summer time in between does not add or lose an hour. + $t=ConvertTo-CenterTime $When;if(-not $t){return $null} + $Now.ToUniversalTime()-$t.ToUniversalTime() } function Get-LastSuccess($History,[string]$SourceId,[string]$Kind) { - $hit=@(@($History)|Where-Object {$_ -and $_.SourceId -eq $SourceId -and $_.Kind -eq $Kind -and $_.Success -eq $true}|Sort-Object {[datetime]$_.Finished} -Descending) - if($hit.Count){[datetime]$hit[0].Finished}else{$null} + $hit=@(@($History)|Where-Object {$_ -and $_.SourceId -eq $SourceId -and $_.Kind -eq $Kind -and $_.Success -eq $true}|ForEach-Object {ConvertTo-CenterTime $_.Finished}|Where-Object {$_}|Sort-Object {$_.ToUniversalTime()} -Descending) + if($hit.Count){$hit[0]}else{$null} } function Get-FreshnessState($When,[int]$ReminderDays,[datetime]$Now) { - if(-not $When){return 'Never'} - $age=$Now-[datetime]$When + $age=Get-CenterElapsed $When $Now + if($null -eq $age){return 'Never'} if($age.TotalHours -lt 24){'Fresh'}elseif($age.TotalDays -lt $ReminderDays){'OK'}else{'Stale'} } function Format-CenterAge($When,[datetime]$Now=(Get-Date)) { if(-not $When){return 'never'} - $age=$Now-[datetime]$When + $age=Get-CenterElapsed $When $Now + if($null -eq $age){return 'unknown'} if($age.TotalMinutes -lt 2){'just now'} - elseif($age.TotalHours -lt 1){'{0} minutes ago' -f [int]$age.TotalMinutes} + elseif($age.TotalHours -lt 1){'{0} minutes ago' -f [int][math]::Floor($age.TotalMinutes)} elseif($age.TotalHours -lt 48){$h=[int][math]::Floor($age.TotalHours);if($h -eq 1){'1 hour ago'}else{"$h hours ago"}} else{'{0} days ago' -f [int][math]::Floor($age.TotalDays)} } +function Format-CenterElapsed([timespan]$Span) { + # "1 min 40 s" for a running or finished step. Whole minutes are rounded down: rounding to the nearest + # showed 100 seconds as "2 min 40 s". + '{0} min {1} s' -f [int][math]::Floor($Span.TotalMinutes),$Span.Seconds +} +function Format-CenterRunDuration($Entry) { + # History entries carry exact Seconds; older ones only Minutes, rounded to a tenth. + if($Entry.PSObject.Properties['Seconds'] -and $null -ne $Entry.Seconds){$s=[double]$Entry.Seconds;return $(if($s -lt 60){'{0} s' -f [int][math]::Floor($s)}else{Format-CenterElapsed ([timespan]::FromSeconds($s))})} + if($Entry.PSObject.Properties['Minutes'] -and $null -ne $Entry.Minutes){return '{0:N1} min' -f [double]$Entry.Minutes} + '' +} +function Get-CenterRunSpeed($Entry) { + # Average speed of a finished run in MB/s (millions of bytes, as drives are sold and as the speed test + # reports), or '' when it cannot be said honestly: a content check copies nothing; a run under 100 MB + # is mostly waiting; an old entry knows its time only to a tenth of a minute, too rough below a minute; + # and versions store only what is new after compression, so their speed is what was read + # (ProcessedBytes), when the worker reported it. + $get={param($n)if($Entry.PSObject.Properties[$n] -and $null -ne $Entry.$n){[double]$Entry.$n}else{$null}} + if([string]$Entry.Kind -like '*Check'){return ''} + $bytes=if([string]$Entry.Kind -eq 'Versioned'){& $get 'ProcessedBytes'}else{& $get 'CopiedBytes'} + $seconds=& $get 'Seconds' + if($null -eq $seconds){$m=& $get 'Minutes';if($null -ne $m -and $m -ge 1){$seconds=$m*60}} + if($null -eq $bytes -or $bytes -lt 100e6 -or $null -eq $seconds -or $seconds -lt 1){return ''} + '{0:N0} MB/s' -f ($bytes/1e6/$seconds) +} +function ConvertFrom-CenterToolJson([string]$Text,[string]$Require) { + # The result a tool printed as JSON, found among its other output: warnings before it, error text + # after it, lines that merely contain a brace. A result starts with "{" at the start of a line; of + # several, the last one that reads as JSON (and has the property -Require names) is taken. When there + # is none, the error is the tool's own words rather than a parsing error. + $starts=@([regex]::Matches($Text,'(?m)^\{')|ForEach-Object {$_.Index});[array]::Reverse($starts) + foreach($s in $starts){ + $ends=@($Text.Length)+@([regex]::Matches($Text.Substring($s),'(?m)^\}')|ForEach-Object {$s+$_.Index+1}|Sort-Object -Descending) + $line=$Text.IndexOf("`n",$s);if($line -gt $s){$ends+=$line} + foreach($e in $ends|Select-Object -Unique){ + $o=try{$Text.Substring($s,$e-$s)|ConvertFrom-Json -ErrorAction Stop}catch{$null} + if($o -is [pscustomobject] -and (-not $Require -or $o.PSObject.Properties[$Require])){return $o} + } + } + $said=($Text.Trim() -replace '\s+',' ');if($said.Length -gt 600){$said=$said.Substring(0,600)+'...'} + throw $(if($said){$said}else{'The tool gave no result.'}) +} function Get-CenterFreshness($Profile,$History,[datetime]$Now=(Get-Date)) { # A source is as fresh as the older of its two kinds; a profile as fresh as its oldest source. $rows=foreach($s in $Profile.Sources){ @@ -327,7 +531,7 @@ function Get-CenterFreshness($Profile,$History,[datetime]$Now=(Get-Date)) { $lm=if($kinds.Mirror){Get-LastSuccess $History $s.Id 'Mirror'}else{$null} $lv=if($kinds.Versioned){Get-LastSuccess $History $s.Id 'Versioned'}else{$null} $needed=@();if($kinds.Mirror){$needed+=,$lm};if($kinds.Versioned){$needed+=,$lv} - $complete=if($needed.Count -eq 0 -or @($needed|Where-Object {$null -eq $_}).Count){$null}else{($needed|Sort-Object)[0]} + $complete=if($needed.Count -eq 0 -or @($needed|Where-Object {$null -eq $_}).Count){$null}else{@($needed|Sort-Object {$_.ToUniversalTime()})[0]} $problem=if($kinds.PSObject.Properties['Error']){[string]$kinds.Error}else{$null} [pscustomobject]@{Source=$s;Mirror=$kinds.Mirror;Versioned=$kinds.Versioned;LastMirror=$lm;LastVersioned=$lv;LastComplete=$complete;Problem=$problem State=if($problem){'Error'}elseif($needed.Count -eq 0){'Nothing selected'}else{Get-FreshnessState $complete $Profile.ReminderDays $Now}} @@ -335,11 +539,180 @@ function Get-CenterFreshness($Profile,$History,[datetime]$Now=(Get-Date)) { $rows=@($rows) # A source whose settings cannot be read is not backed up, so it keeps the drive from looking fresh. $active=@($rows|Where-Object {$_.State -ne 'Nothing selected'}) - $profileComplete=if(-not $active.Count -or @($active|Where-Object {$null -eq $_.LastComplete}).Count){$null}else{($active.LastComplete|Sort-Object)[0]} + $profileComplete=if(-not $active.Count -or @($active|Where-Object {$null -eq $_.LastComplete}).Count){$null}else{@($active.LastComplete|Sort-Object {$_.ToUniversalTime()})[0]} $state=if(-not $active.Count){'Nothing selected'}elseif(@($active|Where-Object {$_.State -eq 'Error'}).Count){'Error'}else{Get-FreshnessState $profileComplete $Profile.ReminderDays $Now} [pscustomobject]@{Rows=$rows;LastComplete=$profileComplete;State=$state} } +# --- The latest run failed; files a content check found different ------------------------------ +function Get-CenterOpenCheck($Entries,[string]$Kind) { + # The newest content check of one source and kind ('Mirror' or 'Versioned') when it failed and nothing + # after it put that right, or nothing. $Entries: that source's history, oldest first. A check that was + # stopped says nothing about the backup, so the one before it counts. Put right means: a later repair + # that succeeded, or a later plain copy that compared every file (it records CheckedFiles) and succeeded. + # An ordinary backup does not: it copies by size and time and would leave a file that only differs inside. + $checks=@($Entries|Where-Object {$_.Kind -eq $Kind+'Check' -and -not ($_.PSObject.Properties['Stopped'] -and $_.Stopped)}) + if(-not $checks.Count -or $checks[-1].Success){return $null} + $c=$checks[-1];$at=$c.Finished.ToUniversalTime() + if($Kind -eq 'Mirror' -and @($Entries|Where-Object {$_.Success -and $_.Finished.ToUniversalTime() -gt $at -and ($_.Kind -eq 'MirrorRepair' -or ($_.Kind -eq 'Mirror' -and $_.PSObject.Properties['CheckedFiles'] -and $null -ne $_.CheckedFiles))}).Count){return $null} + $c +} +function Get-CenterSourceRuns($History,[string]$SourceId) { + # One source's runs, oldest first, each with its finish time as a time (entries handed in by a caller + # may still carry it as text); the entries themselves are not changed. + @(@($History)|Where-Object {$_ -and $_.SourceId -eq $SourceId}|ForEach-Object {$t=ConvertTo-CenterTime $_.Finished;if($t){$c=$_.PSObject.Copy();$c.Finished=$t;$c}}|Sort-Object {$_.Finished.ToUniversalTime()}) +} +function Get-CenterEntryText($Entry,[string]$Name) { + # A text field of a history entry, '' when the entry has none (older entries, and successes, carry fewer fields). + if($Entry.PSObject.Properties[$Name] -and $null -ne $Entry.$Name){[string]$Entry.$Name}else{''} +} +function Get-CenterRepairState($History,[string]$SourceId) { + # Whether "Repair differing files" has something to do for this source: its newest content check of + # the plain copy found files that differ, and no later repair or comparing backup put them right. + # The number is the entry's Differences; entries written before that field say it in their error text. + $c=Get-CenterOpenCheck (Get-CenterSourceRuns $History $SourceId) 'Mirror' + $n=0;$said=if($c){Get-CenterEntryText $c 'Error'}else{''} + if($c){ + if($c.PSObject.Properties['Differences'] -and $null -ne $c.Differences){try{$n=[int]$c.Differences}catch{}} + elseif($said -match '^(\d+) files differ'){$n=[int]$Matches[1]} + } + [pscustomobject]@{Needed=($n -gt 0);Differences=$n;When=$(if($c){$c.Finished});Details=$(if($c){$d=Get-CenterEntryText $c 'Details';if(-not $d -and $said -match ' in (.+)$'){$d=$Matches[1]};$d})} +} +function Get-CenterRunWarnings($Profile,$History,[datetime]$Now=(Get-Date)) { + # What the overview warns about beside the age: for every source of this profile and each kind it + # uses, the newest backup when it failed (Level Failed) or was stopped (Stopped); and, milder (Check), + # a content check that failed and was not put right since. Age and colour are not changed by this: + # they keep saying how old the last complete backup is. Runs of removed sources, of a kind a source no + # longer uses, and repairs are not backups and are left out. Failed backups first, newest first. + $short={param($t)$s=(([string]$t -split ';\s|(?<=[.!?])\s',2)[0]).Trim().TrimEnd('.');if($s.Length -gt 110){$s=$s.Substring(0,110).TrimEnd()+'...'};$s} + $list=foreach($s in @($Profile.Sources)){ + $kinds=try{Get-CenterSourceKinds $s}catch{[pscustomobject]@{Mirror=$true;Versioned=$true}} + $runs=Get-CenterSourceRuns $History $s.Id + foreach($k in @(@('Mirror','plain copy'),@('Versioned','versions'))){ + if(-not $kinds.($k[0])){continue} + $last=@($runs|Where-Object {$_.Kind -eq $k[0]})|Select-Object -Last 1 + if($last -and -not $last.Success){ + $stopped=[bool]($last.PSObject.Properties['Stopped'] -and $last.Stopped);$why=& $short (Get-CenterEntryText $last 'Error') + [pscustomobject]@{Level=$(if($stopped){'Stopped'}else{'Failed'});SourceId=$s.Id;Label=$s.Label;Kind=$k[0];When=$last.Finished;Reason=$why + Text=if($stopped){"The latest backup of $($s.Label) ($($k[1])) was stopped $(Format-CenterAge $last.Finished $Now), before it finished"}else{"The latest backup of $($s.Label) ($($k[1])) failed $(Format-CenterAge $last.Finished $Now)$(if($why){': '+$why})"}} + } + $c=Get-CenterOpenCheck $runs $k[0] + if($c){$why=& $short (Get-CenterEntryText $c 'Error');[pscustomobject]@{Level='Check';SourceId=$s.Id;Label=$s.Label;Kind=$k[0];When=$c.Finished;Reason=$why;Text="The last content check of $($s.Label) ($($k[1])) found a problem $(Format-CenterAge $c.Finished $Now)$(if($why){': '+$why})"}} + } + } + @($list|Sort-Object @{e={if($_.Level -eq 'Check'){1}else{0}}},@{e={$_.When.ToUniversalTime()};Descending=$true}) +} +function Format-CenterRunWarnings($Warnings) { + # One line for the overview (the first warning, how many more, where the details are), every warning + # for the tooltip, and the Level of the worst one ('' = nothing to warn about). + $w=@($Warnings|Where-Object {$_}) + if(-not $w.Count){return [pscustomobject]@{Level='';Text='';Full='';Count=0}} + $where='Details: Recent runs on the Speed and load page.' + [pscustomobject]@{Level=$w[0].Level;Count=$w.Count + Text=$w[0].Text+$(if($w.Count -gt 1){" (and $($w.Count-1) more)"})+'. '+$where + Full=(@($w|ForEach-Object {$_.Text+'.'})+$where) -join "`n"} +} +function Remove-CenterOldStepLogs([int]$Days=30,[datetime]$Now=(Get-Date)) { + # The detailed output of each tool run (step--