From 2927631f54a936c8524f4f14c63e93d15df149cb Mon Sep 17 00:00:00 2001 From: Raz Ben Aharon <181751257+razbenaharon@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:10:55 +0200 Subject: [PATCH 1/2] Window: speed and time left of a running backup come from the tools, not from free space The status line guessed the copy speed from how fast the backup drive's free space shrank. That jumps when robocopy sets a file's size before writing it, stays at zero while unchanged files are compared, and says nothing true for versions, which write only compressed new data. The time left came only from restic's own guess, rounded up to whole minutes: on a real run it said one or two minutes when 10 to 25 seconds were left. - Plain copy: files and bytes copied are read from robocopy's own log records, only the new part every 2 seconds; the speed is over the last 10 seconds. - Versions: "Counting files" until restic has counted them all, then "N of M files checked". - Time left: the usual length of the step (the middle of its last three runs that went through), blended with the tool's own guess by how far it is, plus the usual length of the steps still waiting. "Taking longer than usual" when there is nothing left to go by. Co-Authored-By: Claude Opus 5.5 --- DevDriveBackup.Data.psm1 | 104 +++++++++++++++++++++++++++++++++++++++ DevDriveBackup.ps1 | 62 +++++++++++++++++------ Test-BackupData.ps1 | 41 +++++++++++++++ 3 files changed, 191 insertions(+), 16 deletions(-) diff --git a/DevDriveBackup.Data.psm1 b/DevDriveBackup.Data.psm1 index 3470ad3..37e3335 100644 --- a/DevDriveBackup.Data.psm1 +++ b/DevDriveBackup.Data.psm1 @@ -616,6 +616,109 @@ function Get-CenterRunSpeed($Entry) { if($null -eq $bytes -or $bytes -lt 100e6 -or $null -eq $seconds -or $seconds -lt 1){return ''} '{0:N0} MB/s' -f ($bytes/1e6/$seconds) } +# --- Progress of a running step --------------------------------------------------------------- +# The status line of a running backup says how fast files are being copied and how long is left. Both +# come from what the tools themselves report, never from the free space of the backup drive: that +# shrinks when robocopy sets a file's size (before the data is written), stays still while unchanged +# files are compared or deleted files are moved aside, and a versioned backup writes only compressed +# new data while it reads far more. +function Measure-CenterCopyLog([string]$Text) { + # Files and bytes robocopy names as copied in a piece of its log (/FP /BYTES /NP, without /V): one + # record " New File 1234E:\path" per copied file. Extras, skipped files and the + # totals at the end do not count. With /MT two records can share a line, so records are not anchored + # to the start of a line. + $files=0;$bytes=[long]0 + foreach($m in [regex]::Matches($Text,'\t[ ]*(?:New File|Newer|Older|Changed|Modified|Tweaked)[ ]*\t+[ ]*(\d+)\t')){$files++;$bytes+=[long]$m.Groups[1].Value} + [pscustomobject]@{Files=$files;Bytes=$bytes} +} +function New-CenterCopyProgress([datetime]$Started,[string]$Root=(Join-Path $env:LOCALAPPDATA 'DevDriveMirror\fast-mirror')) { + # What the window keeps about one running plain copy: where its run folder is (found once it exists), + # how far each copy log was read, and what was copied so far. + @{Started=$Started;Root=$Root;Folder=$null;Logs=@{};Files=0;Bytes=[long]0;Samples=[Collections.Generic.List[object]]::new()} +} +function Update-CenterCopyProgress($State) { + # Reads what robocopy added to the copy logs of the running plain copy since the last call. Only the + # new part is read (every few seconds, on the window's thread), and a line still being written waits + # for the next call. The run folder is the newest one the tool made after the step started (the tool + # is started first, so its folder is never older than the step; the folder of the step before is). + if(-not $State.Folder){ + if(-not (Test-Path -LiteralPath $State.Root)){return $State} + $d=Get-ChildItem -LiteralPath $State.Root -Directory -ErrorAction SilentlyContinue|Where-Object {$_.CreationTime -ge $State.Started}|Sort-Object CreationTime -Descending|Select-Object -First 1 + if(-not $d){return $State};$State.Folder=$d.FullName + } + foreach($f in @(try{[IO.Directory]::GetFiles($State.Folder,'copy-*.log')}catch{})){ + $k=$State.Logs[$f];if(-not $k){$k=@{Offset=[long]0;Rest=''};$State.Logs[$f]=$k} + try{ + $fs=[IO.FileStream]::new($f,[IO.FileMode]::Open,[IO.FileAccess]::Read,[IO.FileShare]::ReadWrite -bor [IO.FileShare]::Delete) + try{ + $n=[long][math]::Min($fs.Length-$k.Offset,16MB);$n-=$n%2 # UTF-16: whole characters only + if($n -le 0){continue} + $buf=[byte[]]::new($n);$fs.Position=$k.Offset;$got=0 + while($got -lt $n){$r=$fs.Read($buf,$got,$n-$got);if($r -le 0){break};$got+=$r} + $got-=$got%2;$k.Offset+=$got + $text=$k.Rest+[Text.Encoding]::Unicode.GetString($buf,0,$got) + }finally{$fs.Dispose()} + }catch{continue} + $cut=$text.LastIndexOf("`n") + if($cut -lt 0){$k.Rest=$text;continue} + $k.Rest=$text.Substring($cut+1) + $m=Measure-CenterCopyLog $text.Substring(0,$cut+1);$State.Files+=$m.Files;$State.Bytes+=$m.Bytes + } + $State +} +function Add-CenterRateSample($State,[datetime]$Now,[long]$Bytes,[double]$WindowSeconds=10) { + # Bytes a second over about the last $WindowSeconds, or nothing until two readings at least two + # seconds apart exist. A file is counted when robocopy names it, which is when its copy starts, so a + # short window would jump; the average over the whole step would hide that copying stopped. + $s=$State.Samples;$s.Add([pscustomobject]@{T=$Now;Bytes=$Bytes}) + while($s.Count -gt 2 -and ($Now-$s[1].T).TotalSeconds -ge $WindowSeconds){$s.RemoveAt(0)} + $dt=($Now-$s[0].T).TotalSeconds + if($dt -lt 2){return $null} + [math]::Max([double]0,($Bytes-$s[0].Bytes)/$dt) +} +function Get-CenterUsualSeconds($History,$Step) { + # How long this step usually takes: of the last three runs of the same source, kind and purpose that + # went through (not failed, not stopped), the middle one; nothing when there is none. A plain copy + # spends most of its time comparing files that did not change, so its length repeats from one run to + # the next much better than the amount of new data does. + # A content check or repair is recorded as MirrorCheck, VersionedRepair... (Complete-CenterStep). + $purpose=if($Step.PSObject.Properties['Purpose'] -and $Step.Purpose){[string]$Step.Purpose}else{'Backup'} + $kind=[string]$Step.Kind+$(switch($purpose){'Verify'{'Check'}'Repair'{'Repair'}default{''}}) + $runs=@(@($History)|Where-Object { + $_ -and [string]$_.SourceId -eq [string]$Step.SourceId -and [string]$_.Kind -eq $kind -and $_.Success -eq $true -and + -not ($_.PSObject.Properties['Stopped'] -and $_.Stopped) -and + $(if($_.PSObject.Properties['Purpose'] -and $_.Purpose){[string]$_.Purpose}else{'Backup'}) -eq $purpose -and (ConvertTo-CenterTime $_.Finished) + }|Sort-Object {(ConvertTo-CenterTime $_.Finished).ToUniversalTime()} -Descending|ForEach-Object { + if($_.PSObject.Properties['Seconds'] -and $null -ne $_.Seconds){[double]$_.Seconds}elseif($_.PSObject.Properties['Minutes'] -and $null -ne $_.Minutes){[double]$_.Minutes*60} + }|Where-Object {$_ -gt 0}|Select-Object -First 3) + if(-not $runs.Count){return $null} + $sorted=@($runs|Sort-Object);$sorted[[int][math]::Floor($sorted.Count/2)] +} +function Get-CenterStepLeft([double]$Elapsed,$Usual,$Fraction,$ToolLeft) { + # Seconds left in the running step, or nothing when it cannot be said. $Usual: how long the step + # usually takes (Get-CenterUsualSeconds); $Fraction (0..1) and $ToolLeft: how far the tool says it is + # and what it expects, when it says so. A tool's own guess assumes the rest goes as fast as the start, + # which is far off when the new files come first and the unchanged ones go fast (restic said a minute + # when 20 seconds were left); so it is weighted by how far the tool is, the usual length by the rest. + # A step that already took longer than usual has no usual length left to go by. + $byUsual=if($null -ne $Usual -and $Elapsed -lt [double]$Usual){[double]$Usual-$Elapsed}else{$null} + $byTool=if($null -ne $ToolLeft -and [double]$ToolLeft -ge 0){[double]$ToolLeft}else{$null} + # [double] 0 and 1: with whole numbers [math]::Max would round the fraction to a whole number too. + if($null -ne $byTool -and $null -ne $byUsual){$w=if($null -ne $Fraction){[math]::Min([double]1,[math]::Max([double]0,[double]$Fraction))}else{0.5};return $w*$byTool+(1-$w)*$byUsual} + if($null -ne $byTool){return $byTool} + $byUsual +} +function Format-CenterTimeLeft([double]$Seconds) { + # "About 3 min left." Under a minute is not counted down in seconds: no estimate is that exact. + if($Seconds -lt 50){return 'Less than a minute left.'} + $m=[int][math]::Round($Seconds/60,[MidpointRounding]::AwayFromZero) + if($m -lt 60){return "About $m min left."} + 'About {0} h {1} min left.' -f [int][math]::Floor($m/60),($m%60) +} +function Format-CenterBytes([double]$Bytes) { + # Millions of bytes, as the speeds are given and drives are sold. + if($Bytes -ge 1e9){'{0:N1} GB' -f ($Bytes/1e9)}else{'{0:N0} MB' -f ($Bytes/1e6)} +} function ConvertFrom-CenterToolJson([string]$Text,[string]$Require) { # The result a tool printed as JSON, found among its other output: warnings before it, error text # after it, lines that merely contain a brace. A result starts with "{" at the start of a line; of @@ -1453,6 +1556,7 @@ function Get-CenterStaleProfiles([datetime]$Now=(Get-Date)) { Export-ModuleMember -Function Get-CenterHardware,Get-CenterLoadPresets,Get-CenterLoad,Set-CenterRunPriority,Set-CenterPaths,Get-CenterPaths,Read-CenterProfiles,Save-CenterProfiles,Get-CenterProfile,Get-CenterKeepDeleted,Get-CenterVersionRetention,Format-CenterVersionRetention,Set-CenterVersionRetention,Get-CenterRetentionPoints,Test-CenterRetentionConfirmed,Get-CenterRetentionState,Confirm-CenterRetention,Get-CenterRetentionPreview,Get-CenterRetentionExpected,Test-CenterRetentionPreviewWhole,Format-CenterRetentionPreview,Format-CenterRetentionList,Get-CenterDrive,Get-CenterCandidateDrives, New-CenterProfile,Remove-CenterProfile,Add-CenterSource,Remove-CenterSource,Read-CenterJob,Save-CenterJob,Get-CenterSourceKinds,Sync-CenterJob, Add-CenterHistory,Get-CenterHistory,Get-LastSuccess,Get-FreshnessState,Format-CenterAge,Get-CenterFreshness,ConvertTo-CenterTime,Format-CenterElapsed,Format-CenterRunDuration,Get-CenterRunSpeed,ConvertFrom-CenterToolJson, + Measure-CenterCopyLog,New-CenterCopyProgress,Update-CenterCopyProgress,Add-CenterRateSample,Get-CenterUsualSeconds,Get-CenterStepLeft,Format-CenterTimeLeft,Format-CenterBytes, Get-CenterBusyReason,Get-CenterFreeBytes,Get-CenterVolumeQueryCount, Get-CenterSaveReport,Find-CenterEarlierSource,Get-CenterWindowFit,Get-CenterRetentionApproved,Set-CenterRetentionApproved,Remove-CenterRetentionApproved,Set-CenterKeepDeleted,Complete-CenterRetentionRun, Get-CenterRecoveryAdvice,Get-CenterLockedOffer,Format-CenterLockedOffer,Set-CenterLockedAsked,Add-CenterLockedExclude,Get-CenterPersonalExcludes,Get-CenterMissingRepositories, diff --git a/DevDriveBackup.ps1 b/DevDriveBackup.ps1 index 1e13357..44a97d8 100644 --- a/DevDriveBackup.ps1 +++ b/DevDriveBackup.ps1 @@ -619,12 +619,12 @@ function Save-Settings {Save-CenterProfiles $script:data;Show-SaveNote} # The detailed output of tool runs older than 30 days is deleted; the run history is kept. try{[void](Remove-CenterOldStepLogs)}catch{} $script:queue=[Collections.Generic.Queue[object]]::new();$script:current=$null;$script:runResults=@();$script:runProfile=$null;$script:totalSteps=0;$script:waiting=$null -$script:lastFree=$null;$script:lastTick=$null;$script:loadingModes=$false;$script:meterRatio=0;$script:shown=$false;$script:zoomedFor=$null +$script:copyProgress=$null;$script:copyTick=$null;$script:copyText='';$script:stepUsual=$null;$script:queuedUsual=$null;$script:loadingModes=$false;$script:meterRatio=0;$script:shown=$false;$script:zoomedFor=$null $script:check=$null;$script:install=$null;$script:adminRiskAccepted=$false;$script:speedRun=$null;$script:speedProfile=$null # What the pages show now, so a refresh can keep the user's place: the profile, the source whose folder # choices are in the grid, and whether those choices were changed and not saved yet. $script:detailsProfileId=$null;$script:modesSourceId=$null;$script:modesDirty=$false;$script:suppressModes=$false -$script:autoRefresh=$false;$script:refreshing=$false;$script:lastRefresh=[datetime]::MinValue;$script:rateText='' +$script:autoRefresh=$false;$script:refreshing=$false;$script:lastRefresh=[datetime]::MinValue # The reason last written to the status line for buttons that are off, the history the overview was # filled from, and whether a repair could start now (see Update-Details). $script:shownReason=$null;$script:history=@();$script:canRepair=$false;$script:hardware=$null;$script:testLog=[ordered]@{};$script:saveNote='' @@ -994,7 +994,15 @@ function Start-NextStep { $script:runResults+=[pscustomobject]@{SourceId=$step.SourceId;Label=$step.Label;Kind=$step.Kind;Started=(Get-Date).ToString('o');Success=$false;Error='Could not start: '+$_.Exception.Message} Start-NextStep;return } - $script:lastFree=$null;$script:lastTick=$null;$script:rateText='' + # What the status line goes by for this step: the copy logs of a plain copy, and how long this step + # and those still waiting took in recent runs. The history is read once per step, not every second. + $script:copyProgress=if($step.Kind -eq 'Mirror'){New-CenterCopyProgress $script:current.Started}else{$null};$script:copyTick=$null;$script:copyText='' + $script:stepUsual=$null;$script:queuedUsual=$null + try{ + $h=@(Get-CenterHistory);$script:stepUsual=Get-CenterUsualSeconds $h $step + $rest=@(foreach($q in $script:queue.ToArray()){Get-CenterUsualSeconds $h $q}) + if($rest.Count -eq $script:queue.Count -and $null -notin $rest){$script:queuedUsual=[double](($rest|Measure-Object -Sum).Sum)} + }catch{} $timer.Start();Update-Details } function Update-RunStatus { @@ -1002,23 +1010,45 @@ function Update-RunStatus { $n=$script:totalSteps-$script:queue.Count $what=if($c.Step.Kind -eq 'Mirror'){'plain copy'}else{'versions'} if($c.Step.Purpose -eq 'Verify'){$what='checking '+$what}elseif($c.Step.Purpose -eq 'Repair'){$what='repairing files in the '+$what} - $line="Step $n of $($script:totalSteps): $($c.Step.Label) - $what. Running for $(Format-CenterElapsed ((Get-Date)-$c.Started))." - # This runs every second on the window's own thread, so it must not ask Windows for the list of - # volumes (a second or more each time). The drive is looked up from the kept list and its free space - # is read directly, every 5 seconds: the writing speed is how fast the free space shrinks. - $d=Get-CenterDrive $script:runProfile.DestinationVolumeId;$now=Get-Date - if($d.Connected -and (-not $script:lastTick -or ($now-$script:lastTick).TotalSeconds -ge 5)){ - $free=Get-CenterFreeBytes $d.Letter - if($null -ne $free){ - if($null -ne $script:lastFree -and $script:lastTick){$rate=($script:lastFree-$free)/1e6/($now-$script:lastTick).TotalSeconds;$script:rateText=if($rate -ge 0){" Writing about {0:N0} MB/s." -f $rate}else{''}} - $script:lastFree=$free;$script:lastTick=$now - } + $now=Get-Date;$elapsed=($now-$c.Started).TotalSeconds + $line="Step $n of $($script:totalSteps): $($c.Step.Label) - $what. Running for $(Format-CenterElapsed ($now-$c.Started))." + # This runs every second on the window's own thread: only what the tools added since the last look is + # read. Speed and time left come from what the tools report (see Measure-CenterCopyLog), not from the + # free space of the backup drive. + $fraction=$null;$toolLeft=$null + if($c.Step.Kind -eq 'Mirror' -and $c.Step.Purpose -eq 'Backup' -and $script:copyProgress -and (-not $script:copyTick -or ($now-$script:copyTick).TotalSeconds -ge 2)){ + $script:copyTick=$now + $cp=Update-CenterCopyProgress $script:copyProgress;$rate=Add-CenterRateSample $cp $now $cp.Bytes + $script:copyText=if($cp.Files -eq 0){if($null -ne $rate){' Looking for new and changed files.'}else{''}} + else{ + $t=" Copied $(Format-CenterBytes $cp.Bytes) ($('{0:N0}' -f $cp.Files) $(if($cp.Files -eq 1){'file'}else{'files'})) so far" + if($null -eq $rate){"$t."}elseif($rate -ge 1e6){"$t, about {0:N0} MB/s now." -f ($rate/1e6)}else{"$t; checking the files that did not change."} + } } - if($d.Connected){$line+=$script:rateText} + if($c.Step.Kind -eq 'Mirror'){$line+=$script:copyText} if($c.Step.Kind -eq 'Versioned' -and $c.Step.RunDirectory){ $st=Join-Path $c.Step.RunDirectory 'status.json' - if(Test-Path -LiteralPath $st){try{$s=Get-Content -LiteralPath $st -Raw|ConvertFrom-Json;if($s.Progress -and $s.Progress.PSObject.Properties['percent_done']){$line+=" {0:P0} done." -f [double]$s.Progress.percent_done;if($s.Progress.PSObject.Properties['seconds_remaining']){$line+=' About {0} min left.' -f [math]::Ceiling([double]$s.Progress.seconds_remaining/60)}}}catch{}} + if(Test-Path -LiteralPath $st){try{ + $s=Get-Content -LiteralPath $st -Raw|ConvertFrom-Json;$p=$s.Progress + if($p -and $p.PSObject.Properties['percent_done'] -and [string]$p.message_type -eq 'status'){ + $has={param($k)$p.PSObject.Properties[$k] -and $null -ne $p.$k} + $pd=[double]$p.percent_done + if([string]$s.State -eq 'BackingUp' -and (& $has 'total_files')){ + # Until restic has counted every file, its total grows and its share done jumps (29% then 11%). + # It gives a time left only once the count is complete. + if((& $has 'seconds_remaining') -or $pd -ge 1){$line+=" {0:N0} of {1:N0} files checked ({2:P0})." -f [double]$p.files_done,[double]$p.total_files,$pd;$fraction=$pd;if(& $has 'seconds_remaining'){$toolLeft=[double]$p.seconds_remaining}} + else{$line+=" Counting files: {0:N0} so far." -f [double]$p.total_files} + }else{$line+=" {0:P0} done." -f $pd;$fraction=$pd;if(& $has 'seconds_remaining'){$toolLeft=[double]$p.seconds_remaining}} + } + }catch{}} } + # Time left for the whole run: this step, then the usual length of each step still waiting. When one + # of them has never run before, only this step's time is given. + $left=Get-CenterStepLeft $elapsed $script:stepUsual $fraction $toolLeft + if($null -ne $left){ + if($null -ne $script:queuedUsual){$line+=' '+(Format-CenterTimeLeft ($left+$script:queuedUsual))} + else{$line+=' '+((Format-CenterTimeLeft $left) -replace ' left\.$',' left in this step.')} + }elseif($null -ne $script:stepUsual -and $elapsed -ge $script:stepUsual){$line+=' Taking longer than usual.'} $runStatus.Text=$line } function Finish-Run { diff --git a/Test-BackupData.ps1 b/Test-BackupData.ps1 index a330db3..ca4be5e 100644 --- a/Test-BackupData.ps1 +++ b/Test-BackupData.ps1 @@ -57,6 +57,47 @@ try{ Check 'an old entry of a long run still shows its speed' {(Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=[long]50e9;Minutes=60})) -eq '14 MB/s'} Check 'no speed for small amounts, checks, or versions without the amount read' {(Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=5e6;Seconds=60})) -eq '' -and (Get-CenterRunSpeed ([pscustomobject]@{Kind='MirrorCheck';CopiedBytes=[long]9e9;Seconds=60})) -eq '' -and (Get-CenterRunSpeed ([pscustomobject]@{Kind='Versioned';CopiedBytes=[long]9e9;Seconds=60})) -eq '' -and (Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=$null;Seconds=60})) -eq ''} Check 'versions show the speed of what was read' {(Get-CenterRunSpeed ([pscustomobject]@{Kind='Versioned';CopiedBytes=1000;ProcessedBytes=[long]6e9;Seconds=60})) -eq '100 MB/s'} + + # --- Progress of a running step --- + $log="`t *EXTRA File `t`t 360`tG:\M\a.txt`r`n`t New File `t`t 1000`tE:\a.txt`r`n`t Newer `t`t 500`tE:\b.txt`t Changed `t`t 20`tE:\c.txt`r`n Total Copied`r`n Bytes : 99999 1520`r`n" + Check 'copied files are counted from robocopy records, two on one line included; extras and totals are not' {$m=Measure-CenterCopyLog $log;$m.Files -eq 3 -and $m.Bytes -eq 1520} + Check 'a running copy log is read in parts: a line still being written waits for the next look' { + $fm=Join-Path $root 'fast-mirror';$run=Join-Path $fm '20261118-083353';New-Item -ItemType Directory -Path $run -Force|Out-Null + $f=Join-Path $run 'copy-main.log';$enc=[Text.Encoding]::Unicode + $first="`t New File `t`t 1000`tE:\a.txt`r`n`t Newer `t`t 5" + [IO.File]::WriteAllBytes($f,$enc.GetPreamble()+$enc.GetBytes($first)) + $st=New-CenterCopyProgress (Get-Date).AddSeconds(-2) $fm;[void](Update-CenterCopyProgress $st);$a=$st.Bytes + $fs=[IO.File]::Open($f,'Append');$b=$enc.GetBytes("00`tE:\b.txt`r`n");$fs.Write($b,0,$b.Length);$fs.Dispose() + [void](Update-CenterCopyProgress $st);[void](Update-CenterCopyProgress $st) + $a -eq 1000 -and $st.Bytes -eq 1500 -and $st.Files -eq 2 -and $st.Folder -eq $run + } + Check 'no run folder made since the step started: nothing is read' {$st=New-CenterCopyProgress (Get-Date).AddMinutes(5) (Join-Path $root 'fast-mirror');[void](Update-CenterCopyProgress $st);$null -eq $st.Folder -and $st.Bytes -eq 0} + Check 'speed is over the last ten seconds, and nothing until two seconds have passed' { + $st=New-CenterCopyProgress $now;$t=$now + $r0=Add-CenterRateSample $st $t 0;$r1=Add-CenterRateSample $st $t.AddSeconds(1) 100e6;$r2=Add-CenterRateSample $st $t.AddSeconds(2) 200e6 + for($i=3;$i -le 20;$i++){$last=Add-CenterRateSample $st $t.AddSeconds($i) 200e6} # copying stopped at 2 s + $null -eq $r0 -and $null -eq $r1 -and $r2 -eq 100e6 -and $last -eq 0 + } + $hist=@( + [pscustomobject]@{SourceId='s';Kind='Mirror';Purpose='Backup';Finished=$now.AddDays(-9);Success=$true;Minutes=25.2} + [pscustomobject]@{SourceId='s';Kind='Mirror';Finished=$now.AddDays(-6);Success=$true;Minutes=2.5} + [pscustomobject]@{SourceId='s';Kind='Mirror';Purpose='Backup';Finished=$now.AddDays(-5);Success=$true;Seconds=192;Minutes=3.2} + [pscustomobject]@{SourceId='s';Kind='Mirror';Purpose='Backup';Finished=$now.AddDays(-4);Success=$false;Seconds=5} + [pscustomobject]@{SourceId='s';Kind='Mirror';Purpose='Backup';Finished=$now.AddDays(-3);Success=$true;Stopped=$true;Seconds=7} + [pscustomobject]@{SourceId='s';Kind='MirrorCheck';Purpose='Verify';Finished=$now.AddDays(-2);Success=$true;Seconds=900} + [pscustomobject]@{SourceId='s';Kind='Mirror';Purpose='Backup';Finished=$now.AddDays(-1);Success=$true;Seconds=204} + ) + Check 'usual length: the middle of the last three runs that went through, of the same kind and purpose' {(Get-CenterUsualSeconds $hist ([pscustomobject]@{SourceId='s';Kind='Mirror';Purpose='Backup'})) -eq 192} + Check 'a content check goes by earlier content checks, which the history names MirrorCheck' {(Get-CenterUsualSeconds $hist ([pscustomobject]@{SourceId='s';Kind='Mirror';Purpose='Verify'})) -eq 900} + Check 'the run folder of the step before is not taken for this one' { + $fm=Join-Path $root 'fast-mirror-before';$old=Join-Path $fm '20261118-090000';New-Item -ItemType Directory -Path $old -Force|Out-Null + $st=New-CenterCopyProgress ((Get-Item -LiteralPath $old).CreationTime.AddSeconds(1)) $fm;[void](Update-CenterCopyProgress $st);$null -eq $st.Folder + } + Check 'no usual length for a step that never went through' {$null -eq (Get-CenterUsualSeconds $hist ([pscustomobject]@{SourceId='s';Kind='Versioned';Purpose='Backup'})) -and $null -eq (Get-CenterUsualSeconds @() ([pscustomobject]@{SourceId='s';Kind='Mirror';Purpose='Backup'}))} + Check 'time left: the usual length minus what passed, weighted with the tool''s own guess by how far the tool is' {(Get-CenterStepLeft 60 200 $null $null) -eq 140 -and (Get-CenterStepLeft 10 30 0.25 40) -eq 25 -and (Get-CenterStepLeft 10 $null 0.25 40) -eq 40} + Check 'time left: nothing to go by once a step takes longer than usual and the tool says nothing' {$null -eq (Get-CenterStepLeft 250 200 $null $null) -and $null -eq (Get-CenterStepLeft 10 $null $null $null) -and (Get-CenterStepLeft 250 200 0.9 12) -eq 12} + Check 'time left reads in whole minutes, and under a minute is not counted in seconds' {(Format-CenterTimeLeft 20) -eq 'Less than a minute left.' -and (Format-CenterTimeLeft 61) -eq 'About 1 min left.' -and (Format-CenterTimeLeft 222) -eq 'About 4 min left.' -and (Format-CenterTimeLeft 7380) -eq 'About 2 h 3 min left.'} + Check 'amounts copied are in MB and GB of a million and a billion bytes' {(Format-CenterBytes 1387789076) -eq '1.4 GB' -and (Format-CenterBytes 52e6) -eq '52 MB'} Check 'duration: exact for new entries, minutes for old ones' {(Format-CenterRunDuration ([pscustomobject]@{Seconds=100.4;Minutes=1.7})) -eq '1 min 40 s' -and (Format-CenterRunDuration ([pscustomobject]@{Minutes=1.7})) -eq '1.7 min' -and (Format-CenterRunDuration ([pscustomobject]@{Seconds=24.9;Minutes=0.4})) -eq '24 s'} # --- Load levels --- From 6afed804f6337def0ccf04a448660da16bf31f22 Mon Sep 17 00:00:00 2001 From: Raz Ben Aharon <181751257+razbenaharon@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:26:27 +0200 Subject: [PATCH 2/2] Recent runs: speed of a plain copy is over the time robocopy copied; versions show none The Speed column divided what was copied by the length of the whole run. Most of a plain copy is comparing files that did not change and moving deleted ones aside, so a copy made at about 140 MB/s showed 7 MB/s. Versions divided what restic went through by the run's length, but restic passes most files by size and date without reading them, so 397 MB/s was no speed either. - Invoke-FastMirror times each robocopy run (Seconds in summary.Runs); robocopy's own Times add up every thread under /MT. - Complete-CenterStep records CopySeconds: the time of the runs that copied something, unknown when one of them has no time or no readable bytes. - Get-CenterRunSpeed: a plain copy's speed is CopiedBytes / CopySeconds; versions, checks, repairs and entries from before this show none. - The Help page says what the speed means. Co-Authored-By: Claude Opus 5.5 --- DevDriveBackup.Data.psm1 | 31 +++++++++++++++++++------------ DevDriveBackup.ps1 | 2 ++ Invoke-FastMirror.ps1 | 7 +++++-- Test-BackupData.ps1 | 19 ++++++++++++------- 4 files changed, 38 insertions(+), 21 deletions(-) diff --git a/DevDriveBackup.Data.psm1 b/DevDriveBackup.Data.psm1 index 37e3335..80a23e1 100644 --- a/DevDriveBackup.Data.psm1 +++ b/DevDriveBackup.Data.psm1 @@ -603,16 +603,17 @@ function Format-CenterRunDuration($Entry) { '' } function Get-CenterRunSpeed($Entry) { - # Average speed of a finished run in MB/s (millions of bytes, as drives are sold and as the speed test - # reports), or '' when it cannot be said honestly: a content check copies nothing; a run under 100 MB - # is mostly waiting; an old entry knows its time only to a tenth of a minute, too rough below a minute; - # and versions store only what is new after compression, so their speed is what was read - # (ProcessedBytes), when the worker reported it. + # Copy speed of a finished plain copy in MB/s (millions of bytes, as drives are sold and as the speed + # test reports): what was copied, divided by how long robocopy ran in the parts that copied something + # (CopySeconds). The length of the whole run is not used: most of it is comparing files that did not + # change and moving deleted ones aside, which showed 7 MB/s for a copy made at 140 MB/s. + # '' when it cannot be said honestly: an entry from before CopySeconds was recorded; a run under 100 MB + # or under a second; content checks and repairs; and versions, because restic passes most files by + # their size and date without reading them, so neither what it went through (shown as 397 MB/s) nor + # what it stored is a speed. $get={param($n)if($Entry.PSObject.Properties[$n] -and $null -ne $Entry.$n){[double]$Entry.$n}else{$null}} - if([string]$Entry.Kind -like '*Check'){return ''} - $bytes=if([string]$Entry.Kind -eq 'Versioned'){& $get 'ProcessedBytes'}else{& $get 'CopiedBytes'} - $seconds=& $get 'Seconds' - if($null -eq $seconds){$m=& $get 'Minutes';if($null -ne $m -and $m -ge 1){$seconds=$m*60}} + if([string]$Entry.Kind -ne 'Mirror'){return ''} + $bytes=& $get 'CopiedBytes';$seconds=& $get 'CopySeconds' if($null -eq $bytes -or $bytes -lt 100e6 -or $null -eq $seconds -or $seconds -lt 1){return ''} '{0:N0} MB/s' -f ($bytes/1e6/$seconds) } @@ -1386,7 +1387,7 @@ function Complete-CenterStep($Running,[string]$ProfileId,[switch]$Stopped) { # Seconds is exact; Minutes (a tenth of a minute) stays for older readers. ProcessedBytes: what a # versioned backup read, where CopiedBytes is only what it had to store. $e=[ordered]@{ProfileId=$ProfileId;SourceId=$s.SourceId;Label=$s.Label;Kind=$s.Kind;Purpose=$s.Purpose;Started=$Running.Started.ToString('o');Finished=(Get-Date).ToString('o') - Minutes=[math]::Round(((Get-Date)-$Running.Started).TotalMinutes,1);Seconds=[math]::Round(((Get-Date)-$Running.Started).TotalSeconds,1);Success=$false;Stopped=$false;Error=$null;CopiedFiles=$null;CopiedBytes=$null;ProcessedBytes=$null;CheckedFiles=$null;Differences=$null;Details=$null;Note=$null;RetentionPreview=$null;RetentionPlanFailed=$false + Minutes=[math]::Round(((Get-Date)-$Running.Started).TotalMinutes,1);Seconds=[math]::Round(((Get-Date)-$Running.Started).TotalSeconds,1);Success=$false;Stopped=$false;Error=$null;CopiedFiles=$null;CopiedBytes=$null;CopySeconds=$null;ProcessedBytes=$null;CheckedFiles=$null;Differences=$null;Details=$null;Note=$null;RetentionPreview=$null;RetentionPlanFailed=$false RetentionPhase=$null;RecoveryFolderCount=$null;LockedFiles=$null;RepositoryMissing=$false} # What Get-CenterSteps noted on the step beside its arguments; a step built elsewhere has none of it. $noted={param($n)if($s.PSObject.Properties[$n]){$s.$n}else{$null}} @@ -1399,12 +1400,18 @@ function Complete-CenterStep($Running,[string]$ProfileId,[switch]$Stopped) { $e.Details=$r.RunFolder # Files and bytes copied are the second number of each robocopy total. A total that was not read # (the tool could not find it in the log) leaves both unknown: 0 would claim that nothing was copied. + # CopySeconds: how long robocopy ran in the runs that copied something, the time the speed of the + # copy is measured over; unknown when a run's bytes or time were not reported (an older tool). $files=[long]0;$bytes=[long]0;$known=[bool]@($r.Runs).Count # no run listed: nothing was read either + $copySeconds=[double]0;$timeKnown=$true foreach($run in @($r.Runs)){ if([string]$run.Files -match '^Files : \d+ (\d+)'){$files+=[long]$Matches[1]}else{$known=$false} - if([string]$run.Bytes -match '^Bytes : \d+ (\d+)'){$bytes+=[long]$Matches[1]}else{$known=$false} + if([string]$run.Bytes -match '^Bytes : \d+ (\d+)'){ + $b=[long]$Matches[1];$bytes+=$b + if($b -gt 0){if($run.PSObject.Properties['Seconds'] -and $null -ne $run.Seconds){$copySeconds+=[double]$run.Seconds}else{$timeKnown=$false}} + }else{$known=$false} } - if($known){$e.CopiedFiles=$files;$e.CopiedBytes=$bytes} + if($known){$e.CopiedFiles=$files;$e.CopiedBytes=$bytes;if($timeKnown -and $bytes -gt 0){$e.CopySeconds=[math]::Round($copySeconds,1)}} $verifyOk=$true # Differences is kept as a number of its own: "Repair differing files" is offered from it, also when # the error text below names something else. diff --git a/DevDriveBackup.ps1 b/DevDriveBackup.ps1 index 44a97d8..2469062 100644 --- a/DevDriveBackup.ps1 +++ b/DevDriveBackup.ps1 @@ -115,6 +115,8 @@ You can change the load while a backup runs. The priority changes at once; threa Speed test writes one temporary file (2 GB) to the backup drive, reads it back and deletes it. It also shows how the drive is connected: USB 2 is slow (about 40 MB/s), 5 Gbit/s USB reaches about 450 MB/s, 10 Gbit/s USB and Thunderbolt more. Many small files are limited by the drive they are read from, not by the backup drive. A low MB/s during a backup of a code folder is normal. + +Speed in "Recent runs" is the average while a plain copy was copying: what was copied, divided by the time the copy tool ran in the parts of the drive where it copied something. The time spent only comparing unchanged files is left out. Versions show no speed: most files are passed by their size and date without being read, so there is no honest number. '@ DriveCheck=@' Quick check (a few minutes) asks the drive for its own health report and runs its two-minute self-test, reads samples at the start, middle and end of the drive, looks in the Windows log for disk and connection errors of the last 30 days, checks the connection speed and scans the file system. diff --git a/Invoke-FastMirror.ps1 b/Invoke-FastMirror.ps1 index 0dd4262..b34d4c8 100644 --- a/Invoke-FastMirror.ps1 +++ b/Invoke-FastMirror.ps1 @@ -254,8 +254,11 @@ function Invoke-CopyRun([string[]]$RoboArgs,[string]$Part,[string]$Log){ # One robocopy run, added to the summary. The totals are read as soon as the run ends, from the log # of this run alone. Repeated marks a run whose folder was copied again after a file-or-folder swap # in it was healed: the later run tells what is left uncopied there. - $code=Invoke-Robo $RoboArgs $Part;$result=Get-RoboRunResult $Log $code;$t=$result.Totals - $entry=[ordered]@{Part=$Part;ExitCode=$code;Log=$Log;Files=$(if($t){'Files : '+($t.Files -join ' ')}else{''});Bytes=$(if($t){'Bytes : '+($t.Bytes -join ' ')}else{''});NotCopied=$result.NotCopied;Complete=$result.Complete;Repeated=$false} + # Seconds: how long robocopy ran, the time the window divides the copied bytes by for the speed of + # the copy (robocopy's own Times add up the time of every thread under /MT). + $clock=[Diagnostics.Stopwatch]::StartNew();$code=Invoke-Robo $RoboArgs $Part;$clock.Stop() + $result=Get-RoboRunResult $Log $code;$t=$result.Totals + $entry=[ordered]@{Part=$Part;ExitCode=$code;Log=$Log;Files=$(if($t){'Files : '+($t.Files -join ' ')}else{''});Bytes=$(if($t){'Bytes : '+($t.Bytes -join ' ')}else{''});NotCopied=$result.NotCopied;Complete=$result.Complete;Repeated=$false;Seconds=[math]::Round($clock.Elapsed.TotalSeconds,2)} $summary.Runs+=$entry [pscustomobject]@{Entry=$entry;Mismatch=([bool]($code -band 4) -or ($t -and ($t.Files[3]+$t.Dirs[3]) -gt 0))} } diff --git a/Test-BackupData.ps1 b/Test-BackupData.ps1 index ca4be5e..0ad0316 100644 --- a/Test-BackupData.ps1 +++ b/Test-BackupData.ps1 @@ -52,11 +52,10 @@ try{ Remove-Item -LiteralPath $hp # --- Run numbers shown in "Recent runs" --- - Check 'speed uses the exact seconds: 600 MiB in 3.1 s is 203 MB/s' {(Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=629145600;Seconds=3.1;Minutes=0.1})) -eq '203 MB/s'} - Check 'an old entry with only rounded minutes shows no speed for a short run' {(Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=629145600;Minutes=0.1})) -eq ''} - Check 'an old entry of a long run still shows its speed' {(Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=[long]50e9;Minutes=60})) -eq '14 MB/s'} - Check 'no speed for small amounts, checks, or versions without the amount read' {(Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=5e6;Seconds=60})) -eq '' -and (Get-CenterRunSpeed ([pscustomobject]@{Kind='MirrorCheck';CopiedBytes=[long]9e9;Seconds=60})) -eq '' -and (Get-CenterRunSpeed ([pscustomobject]@{Kind='Versioned';CopiedBytes=[long]9e9;Seconds=60})) -eq '' -and (Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=$null;Seconds=60})) -eq ''} - Check 'versions show the speed of what was read' {(Get-CenterRunSpeed ([pscustomobject]@{Kind='Versioned';CopiedBytes=1000;ProcessedBytes=[long]6e9;Seconds=60})) -eq '100 MB/s'} + Check 'speed is over the time the copy ran, not the whole run: 1.39 GB copied in 10 s of a 204 s run is 139 MB/s' {(Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=1387789076;CopySeconds=10.0;Seconds=203.6})) -eq '139 MB/s'} + Check 'an entry from before the copy time was recorded shows no speed' {(Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=[long]50e9;Seconds=3600;Minutes=60})) -eq '' -and (Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=629145600;Minutes=0.1})) -eq ''} + Check 'no speed for small amounts, under a second, checks or repairs' {(Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=5e6;CopySeconds=60})) -eq '' -and (Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=[long]9e9;CopySeconds=0.5})) -eq '' -and (Get-CenterRunSpeed ([pscustomobject]@{Kind='MirrorCheck';CopiedBytes=[long]9e9;CopySeconds=60})) -eq '' -and (Get-CenterRunSpeed ([pscustomobject]@{Kind='MirrorRepair';CopiedBytes=[long]9e9;CopySeconds=60})) -eq '' -and (Get-CenterRunSpeed ([pscustomobject]@{Kind='Mirror';CopiedBytes=$null;CopySeconds=60})) -eq ''} + Check 'versions show no speed: restic passes most files without reading them' {(Get-CenterRunSpeed ([pscustomobject]@{Kind='Versioned';CopiedBytes=871093093;ProcessedBytes=12547883974;Seconds=31.6})) -eq ''} # --- Progress of a running step --- $log="`t *EXTRA File `t`t 360`tG:\M\a.txt`r`n`t New File `t`t 1000`tE:\a.txt`r`n`t Newer `t`t 500`tE:\b.txt`t Changed `t`t 20`tE:\c.txt`r`n Total Copied`r`n Bytes : 99999 1520`r`n" @@ -361,8 +360,14 @@ try{ Check 'a plain copy: files and bytes copied, files checked, exact seconds' {$e.Success -and $e.Kind -eq 'Mirror' -and $e.CopiedFiles -eq 3 -and $e.CopiedBytes -eq 629145600 -and $e.CheckedFiles -eq 10 -and $e.Seconds -gt 0 -and $e.Seconds -lt 60 -and $null -ne $e.Minutes} $e=Invoke-Fake "Write-Output 'WARNING: skipped {temp} folder';Write-Output '$json'" Check 'a line with a brace before the result does not fail the plain copy' {$e.Success -and $e.CopiedFiles -eq 3} + Check 'a result from a tool that did not time its runs records no copy time' {$null -eq $e.CopySeconds} + $timed='{"RunFolder":"X","CopyErrors":0,"Runs":[{"Files":"Files : 9 2 7 0 0 0","Bytes":"Bytes : 9 600000000 0 0 0 0","Seconds":8.4},{"Files":"Files : 99 0 99 0 0 0","Bytes":"Bytes : 99 0 99 0 0 0","Seconds":70},{"Files":"Files : 5 1 4 0 0 0","Bytes":"Bytes : 5 1000 4 0 0 0","Seconds":1.6}]}' + $e=Invoke-Fake "Write-Output '$timed'" + Check 'copy time is the time of the runs that copied something; a run that only compared files is left out' {$e.CopiedBytes -eq 600001000 -and $e.CopySeconds -eq 10 -and (Get-CenterRunSpeed $e) -eq '60 MB/s'} + $e=Invoke-Fake ("Write-Output '"+$timed.Replace(',"Seconds":1.6','')+"'") + Check 'a run that copied something but has no time leaves the copy time unknown' {$e.CopiedBytes -eq 600001000 -and $null -eq $e.CopySeconds -and (Get-CenterRunSpeed $e) -eq ''} $e=Invoke-Fake ("Write-Output '"+$json.Replace('Files : 10 3 7 0 0 0','').Replace('Bytes : 5000 629145600 0 0 0 0','')+"'") - Check 'totals that could not be read are recorded as unknown, not as 0 files' {$e.Success -and $null -eq $e.CopiedFiles -and $null -eq $e.CopiedBytes} + Check 'totals that could not be read are recorded as unknown, not as 0 files' {$e.Success -and $null -eq $e.CopiedFiles -and $null -eq $e.CopiedBytes -and $null -eq $e.CopySeconds} $e=Invoke-Fake ("Write-Output '"+$json.Replace('Files :','Dateien:')+"'") Check 'totals in an unexpected form are recorded as unknown too' {$e.Success -and $null -eq $e.CopiedFiles -and $null -eq $e.CopiedBytes} $e=Invoke-Fake ("Write-Output '"+'{"RunFolder":"X","CopyErrors":0,"Runs":[]}'+"'") @@ -494,7 +499,7 @@ try{ Check 'the full list of a short preview holds what the question shows' {$t=Format-CenterRetentionList $pv;$t -like '*1 Feb 2026, 09:30, partial*E:\Projects, E:\Docs*' -and $t -like '*2026-01-02_101500*2026-01-09_101500*' -and -not (Format-CenterRetentionList $none)} Check 'a long version ID is shown by its first 8 characters, as the history shows it' {(Format-CenterRetentionList (Get-CenterRetentionPreview @([pscustomobject]@{Label='L';SourceId='s';RetentionPreview=[pscustomobject]@{Versions=@([pscustomobject]@{Id=('ab12'*16);Time='2026-01-05T10:00:00';Partial=$false;Host='PC';Paths=@('E:\P')});RecoveryFolders=@()}}))) -like '*(ab12ab12)*'} $h=@(Get-CenterHistory) - Check 'every run above is in the history, readable, stopped ones marked' {$h.Count -eq 43 -and @($h|Where-Object {$_.PSObject.Properties['Stopped'] -and $_.Stopped}).Count -eq 3 -and @($h|Where-Object {$_.Finished -isnot [datetime]}).Count -eq 0} + Check 'every run above is in the history, readable, stopped ones marked' {$h.Count -eq 45 -and @($h|Where-Object {$_.PSObject.Properties['Stopped'] -and $_.Stopped}).Count -eq 3 -and @($h|Where-Object {$_.Finished -isnot [datetime]}).Count -eq 0} # --- The settings file was damaged, removed or changed on disk after this window read it --- # A save never writes over what it does not know: the file on disk is kept aside under a dated name,