diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 795e270..79a2fb9 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -7,9 +7,30 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 + # Every commit being pushed is checked, not only the newest: a detail added and removed again + # would otherwise stay in the history. - name: Check for private backup details shell: pwsh - run: ./Test-PrivacyGuard.ps1 + env: + BEFORE: ${{ github.event.before }} + BASE: ${{ github.event.pull_request.base.sha }} + DEFAULT: ${{ github.event.repository.default_branch }} + BRANCH: ${{ github.ref_name }} + run: | + # A new branch is compared with the default branch. The first push of the default branch itself + # (a new repository or fork) has nothing to compare with, so its whole history is checked. + $range = if ($env:BASE) { "$env:BASE..HEAD" } elseif ($env:BEFORE -and $env:BEFORE -notmatch '^0+$') { "$env:BEFORE..HEAD" } elseif ($env:BRANCH -eq $env:DEFAULT) { 'HEAD' } else { "origin/$env:DEFAULT..HEAD" } + ./Test-PrivacyGuard.ps1 -Range $range + - name: Verify the privacy check itself + shell: pwsh + run: pwsh -NoProfile -File ./Test-PrivacyGuardRules.ps1 + - name: Check that no built-in variable is assigned to + shell: pwsh + run: | + Install-Module PSScriptAnalyzer -RequiredVersion 1.25.0 -Scope CurrentUser -Force + pwsh -NoProfile -File ./Test-BackupStyle.ps1 - name: Parse all PowerShell sources shell: pwsh run: | @@ -36,28 +57,62 @@ jobs: if ($config.MemoryMiB -lt 512 -or $config.Include.Count -eq 0) { throw 'Invalid example' } - name: Verify concurrent status access shell: pwsh - run: ./Test-BackupStatus.ps1 + run: pwsh -NoProfile -File ./Test-BackupStatus.ps1 - name: Verify drive check rules shell: pwsh - run: ./Test-BackupDriveCheck.ps1 + run: pwsh -NoProfile -File ./Test-BackupDriveCheck.ps1 - name: Verify the AI session activity rule shell: pwsh - run: ./Test-BackupActivity.ps1 + run: pwsh -NoProfile -File ./Test-BackupActivity.ps1 - name: Verify colour themes and contrast shell: pwsh - run: ./Test-BackupTheme.ps1 - - name: Verify the clean-up of old versions - shell: pwsh - run: ./Test-BackupRetention.ps1 + run: pwsh -NoProfile -File ./Test-BackupTheme.ps1 - name: Verify history selection safeguards shell: pwsh - run: ./Test-BackupHistory.ps1 + run: pwsh -NoProfile -File ./Test-BackupHistory.ps1 - name: Verify private credential file safeguards shell: pwsh - run: ./Test-BackupSecuritySafety.ps1 + run: pwsh -NoProfile -File ./Test-BackupSecuritySafety.ps1 - name: Verify plain copy rules shell: pwsh - run: ./Test-BackupCopyRules.ps1 + run: pwsh -NoProfile -File ./Test-BackupCopyRules.ps1 + - name: Verify plain copy runs, failures and recovery folders + shell: pwsh + run: pwsh -NoProfile -File ./Test-BackupMirror.ps1 + - name: Verify the rules for versions + shell: pwsh + run: pwsh -NoProfile -File ./Test-BackupVersionsRules.ps1 + - name: Verify versions and the clean-up of old versions against a disposable repository + shell: pwsh + run: | + ./Install-BackupEngine.ps1 + # Without the engine the test would skip its real part and still pass. + if (-not (Test-Path -LiteralPath ./bin/restic_0.19.1_windows_amd64.exe)) { throw 'The versions engine was not installed, so the test would not run.' } + pwsh -NoProfile -File ./Test-BackupVersions.ps1 + if ($LASTEXITCODE) { exit $LASTEXITCODE } + pwsh -NoProfile -File ./Test-BackupRetention.ps1 + - name: Verify the window's data and run results + shell: pwsh + run: pwsh -NoProfile -File ./Test-BackupData.ps1 + # Windows cuts a window down to the screen it is on, and the runner's screen (1024 x 768) is smaller + # than the window at full size, so the test would measure the runner instead of the window. + - name: Give the window tests a screen with room for the window + shell: powershell + run: | + Set-DisplayResolution -Width 1920 -Height 1080 -Force -ErrorAction Stop + Add-Type -AssemblyName System.Windows.Forms + $largest = [Windows.Forms.SystemInformation]::MaxWindowTrackSize + "Largest window Windows allows now: $($largest.Width) x $($largest.Height)" + if ($largest.Width -lt 1216 -or $largest.Height -lt 839) { throw 'The screen is still too small for the window at full size.' } + - name: Verify the window opens on every page + shell: pwsh + run: pwsh -NoProfile -File ./Test-BackupWindow.ps1 + - name: Verify drive tools and the report generator + shell: pwsh + run: | + pwsh -NoProfile -File ./Test-BackupTools.ps1 + if ($LASTEXITCODE) { exit $LASTEXITCODE } # a failing first test must not be hidden by the next command + python ./Test-BackupToolsReport.py - name: Verify the hard memory limit shell: pwsh - run: ./Test-BackupResourceControls.ps1 + run: pwsh -NoProfile -File ./Test-BackupResourceControls.ps1 diff --git a/BACKUP-SYSTEM.md b/BACKUP-SYSTEM.md index 34e3677..8e8e58f 100644 --- a/BACKUP-SYSTEM.md +++ b/BACKUP-SYSTEM.md @@ -8,7 +8,7 @@ All application text, reports and recommendations are English-only and use left- ## Requirements and installation -- Windows 11, PowerShell 7.2+ (64-bit), and a separate destination volume. +- Windows 10 or 11, PowerShell 7.2+ (64-bit), and a separate destination volume. A Dev Drive as the source needs Windows 11. - Restic 0.19.1. `./Install-BackupEngine.ps1` downloads the official Windows x64 archive and checks its pinned SHA-256 before extraction. No PATH changes or administrator installation. - Python 3.8+ only for generating the optional HTML report. The backup itself has no Python dependency. - Source may be a physical Dev Drive or a mounted ReFS VHD/VHDX. Back up the files in the mounted drive. Do not copy a live VHDX and assume it is consistent. @@ -29,13 +29,13 @@ Start `DevDriveBackup-Advanced.cmd` (or use the main DevDriveBackup window) or: pwsh -NoProfile -File ./Backup-Manager.ps1 ``` -Profiles are starting points, not a hardware-independent optimum. Review exclusions and deselect anything you do not want backed up. Backup runs create snapshots; source deletions do not automatically delete older snapshots. No scheduled task is installed. Old versions are deleted automatically only when **Keep versions** is set on the main window's overview (`Backup-Worker.ps1 -KeepDays -KeepWeeks -KeepMonths`; off by default). Manual history deletion and optional pruning are available through **History and restore**. +Profiles are starting points, not a hardware-independent optimum. Review exclusions and deselect anything you do not want backed up. Backup runs create snapshots; source deletions do not automatically delete older snapshots. No scheduled task is installed. Old versions are deleted automatically only when **Keep versions** is set on the main window's overview (`Backup-Worker.ps1 -KeepLast N -KeepMonths M`; off by default). For each group of computer and source paths the worker keeps the N newest complete versions and the newest complete version of each calendar month in the month of the group's newest complete version and the M-1 months before it, and also in the M months that end with the month of the version just made (so one version dated far ahead by a wrong clock does not move every month out of the rule); a version that meets either rule stays, and the newest complete version of a group and the version just made are never deleted. The plan is computed by `Get-VersionRetentionPlan` in `Backup.History.psm1` (restic only lists and deletes by exact ID) and checked again by `Get-RetentionRemovals` before anything is deleted. `-RetentionPreview` plans only: nothing is forgotten or pruned, and `result.json` lists what would go (`Retention.Candidates`). `-RetentionApprovedFile ` (one full version ID per line) limits one clean-up to the listed versions that the plan still wants deleted; `Retention.Deferred` counts what was left for the next clean-up, and a missing, empty or unreadable list deletes nothing. If the versions folder holds no repository although this source was backed up before (its password file exists), every action fails with `RepositoryMissing` in `result.json`; only `Backup -CreateNewRepository` starts a new one. `-KeepDays` and `-KeepWeeks` of earlier releases are refused with a plain message. Manual history deletion and optional pruning are available through **History and restore**. ## History browser -`Backup-History.ps1` lists backup versions by date, computer, source folders and exact snapshot ID. Optional From/To dates include the entire selected local calendar day. Browsing loads one directory at a time and displays at most 500 rows per page, avoiding expansion of an entire development drive. The restore selection can contain files and folders from several directories in the same version; opening another version clears it. Selected names are treated literally, including Unicode and bracket characters, rather than as restore wildcard patterns. Restore targets must be new folders outside the configured source and repository. Restic verifies restored content and uses `--overwrite never`. +`Backup-History.ps1` lists backup versions by date, computer, source folders and exact snapshot ID. Optional From/To dates include the entire selected local calendar day. Browsing loads one directory at a time and displays at most 500 rows per page, avoiding expansion of an entire development drive. The restore selection can contain files and folders from several directories in the same version; opening another version clears it. Selected names are treated literally, including Unicode and bracket characters, rather than as restore wildcard patterns. Restore targets must be new folders outside the configured source and repository. Restic verifies restored content and uses `--overwrite never`. When the dates or attributes of folders above the restored paths cannot be set (for example below `C:\Users`), the restore completes with a warning, restic skips its verification, and the result says the content was not verified. Versions from a backup that could not read every file are listed as "(partial)". -History deletion always takes explicit full snapshot IDs. The GUI displays the exact deletion list and requires typing `DELETE`; the worker independently validates confirmation, current repository membership and newest-version protection for each computer/source set. Empty, abbreviated, stale and protected selections fail. Equal-time newest versions are all protected. **Select older versions** respects the displayed date range; it is not an automatic retention policy. Removing history makes those versions unavailable, even if some data packs remain until prune. Optional prune removes unreferenced data across the repository, including abandoned data from interrupted backups, and may need temporary free space. Source files are not deleted. All operations use the existing repository worker lock; a concurrent backup or verification causes the new request to fail with a busy/lock error rather than interrupting the existing operation. +History deletion always takes explicit full snapshot IDs. The GUI displays the exact deletion list and requires typing `DELETE`; the worker independently validates confirmation, current repository membership and newest-version protection for each computer/source set. Empty, abbreviated, stale and protected selections fail. Equal-time newest versions are all protected. Protection is checked two ways and both must pass: the newest version and the newest complete version of each computer/source set counting every version in the repository, sets compared without regard to case (as Windows names folders); and the newest complete version this tool made of each set as the clean-up groups them (`Get-SnapshotGroupKey`: this tool's versions only, case kept apart). So neither a newer version saved by another program nor the same folder written with other capital letters can leave the tool's newest complete version unprotected. **Select older versions** respects the displayed date range; it is not an automatic retention policy. Removing history makes those versions unavailable, even if some data packs remain until prune. Optional prune removes unreferenced data across the repository, including abandoned data from interrupted backups, and may need temporary free space. Source files are not deleted. When restic reports a locked repository (exit code 11), the worker reads every lock. If all were made on this computer by processes that have ended, it runs `restic unlock` (never `--remove-all`), retries the step once and records `LockReleased` in `result.json`. Otherwise nothing is removed and the error names the holder: another computer, a backup still running here, or a lock that cannot be judged. One release per run. When Windows has already handed the ended process's number to another program, the worker judges the lock ended but `restic unlock` keeps it until it is 30 minutes old; the run fails with that time in its message, and a run after it removes the lock. `Test-BackupVersionsRules.ps1` checks the judgement against real running programs every time; the kill test in `Test-BackupVersions.ps1` checks, when the number happens to be taken, that the outcome is a kept lock with that message or a removal once the number is free. All operations use the existing repository worker lock; a concurrent backup or verification causes the new request to fail with a busy/lock error rather than interrupting the existing operation. ## First backup and keys @@ -43,11 +43,11 @@ On the first **Backup**, the worker generates a random 256-bit repository passwo **Copy the portable recovery key into a password manager or separate offline location.** A DPAPI file depends on the Windows account and machine. If that environment is lost, the portable password is needed to recover the repository. Never commit either file or put the only recovery key inside the encrypted repository. The recovery file is plaintext, protected by its local ACL, so anyone receiving a copy can decrypt the backup. -The UI's preview is a Restic dry run and requires an initialized repository. The first Backup initializes it. Exit code 3 is treated as failure/incomplete backup, even when Restic saved a partial snapshot. Error details remain in that run's logs. Correct the source/exclusions and run again; do not label a partial snapshot as complete. +The UI's preview is a Restic dry run and requires an initialized repository. The first Backup initializes it. Exit code 3 is treated as failure/incomplete backup, even when Restic saved a partial snapshot. That snapshot is retagged `devdrive-partial` (its ID changes), so the automatic clean-up never counts it and never keeps it in place of a complete version; it deletes it only once three complete versions of the same computer and paths are newer, which can remove content that exists only in that partial snapshot; partial snapshots made by earlier releases carry no such mark. Error details remain in that run's logs. Correct the source/exclusions and run again; do not label a partial snapshot as complete. ## Password changes and portable key export -Choose **Password and recovery key** in DevDriveBackup or **Password and recovery** in the advanced window. The window does not contact the repository until an action is chosen. Saving a recovery key exports the current repository password to a new file and records the location in the local configuration. Existing files, linked paths, reserved credential/configuration paths, and paths inside the source, repository or project are refused. Recovery files are created with a protected current-user/SYSTEM ACL before any content is written. Exported files are plaintext and must be kept separately and privately. +Choose **Password and recovery key** in DevDriveBackup or **Password and recovery** in the advanced window. The window does not contact the repository until an action is chosen. Saving a recovery key first checks that the stored password opens the repository, then exports the current repository password to a new file and records the location in the local configuration. Existing files, linked paths, reserved credential/configuration paths, and paths inside the source, repository or project are refused. Recovery files are created with a protected current-user/SYSTEM ACL before any content is written. Exported files are plaintext and must be kept separately and privately. Password changes replace the access key currently used by this tool; independently added access keys remain intact. The repository master key, data packs and snapshots are not re-encrypted or recreated. Changing a password does not revoke access to an earlier independently copied repository. This feature does not repair an unhealthy backup drive. @@ -109,9 +109,9 @@ pwsh -NoProfile -File ./Invoke-DriveCheck.ps1 -Drive Y -OutputDirectory ./audit/ pwsh -NoProfile -File ./Invoke-DriveCheck.ps1 -Drive Y -OutputDirectory ./audit/check -Mode LongSelfTest ``` -Before it asks Windows for administrator rights, the window checks who could change what the check runs: `Invoke-DriveCheck.ps1`, the modules and script it loads, `smartctl` and its folder, and every folder above the tool folder (whoever can rename one of those can swap the tool folder itself). When an account other than you, SYSTEM or Administrators can, it names those accounts and asks before it starts; keep the tool under your user folder to avoid that. The check runs the bundled `smartctl.exe` only when its SHA-256 matches the 64-bit build in the official 7.5 installer, and keeps the file open, readable only, until it ends. The window runs these through a Windows administrator prompt and keeps the results per profile in `%LOCALAPPDATA%\DevDriveMirror\drive-check\`: `report.json` (last quick check), `surface.json` (full scan position and findings), `run.json` (last run), `progress.json` (live). An elevated process cannot be ended by the window, so *Stop* creates `stop.request` and the check stops itself after the current step; a full scan saves its position every 512 MiB and continues from there. +Before it asks Windows for administrator rights, the window checks who could change what the check runs: `Invoke-DriveCheck.ps1`, the modules and script it loads, `smartctl` and its folder, and every folder above the tool folder (whoever can rename one of those can swap the tool folder itself). When an account other than you, SYSTEM or Administrators can, it names those accounts and asks before it starts; keep the tool under your user folder to avoid that. The check runs the bundled `smartctl.exe` only when its SHA-256 matches the 64-bit build in the official 7.5 installer, and keeps the file open, readable only, until it ends. The window runs these through a Windows administrator prompt and keeps the results per profile in `%LOCALAPPDATA%\DevDriveMirror\drive-check\`: `report.json` (last quick check), `surface.json` (full scan position and findings), `run.json` (last run), `progress.json` (live). An elevated process cannot be ended by the window, so *Stop* creates `stop.request` and the check stops itself within one 4 MiB read; a full scan saves its position every 512 MiB and continues from there. -Reads use `\\.\PhysicalDriveN` with unbuffered aligned 4 MiB reads. A failing chunk is read again in 64 KiB pieces to locate the damaged place. A chunk over one second counts as very slow (the drive retried); one over 250 ms and over ten times the median of its 512 MiB stretch counts as slow; many slow places (more than 20, and more than 0.2% of what was read) are a warning. Errors that mean the drive disappeared (not ready, no such device, disconnected) end the run instead of being recorded as bad areas. The check itself writes nothing to the drive; Windows' online file system scan may note what it finds in the file system's own records. The health report tries smartctl's own device-type detection, then standard SAT pass-through; vendor-specific USB bridge commands are not tried blindly. Windows log entries (warnings and errors of the last 30 days) name a disk by number, and numbers move between drives as they are plugged in. Each entry is matched to the drive that held that number at the time, using Windows' own record of drive arrivals (`Microsoft-Windows-Partition/Diagnostic`, event 1006: disk number, serial number, model, size), so another drive's errors are not held against this one. The verdict rules live in `Backup.DriveCheck.psm1` and are tested by `Test-BackupDriveCheck.ps1` without hardware. +Reads use `\\.\PhysicalDriveN` with unbuffered aligned 4 MiB reads. A failing chunk is read again in 64 KiB pieces to locate the damaged place. A chunk over one second counts as very slow (the drive retried); one over 250 ms and over ten times the median of its 512 MiB stretch counts as slow; many slow places (more than 20, and more than 0.2% of what was read) are a warning. Errors that mean the drive disappeared (not ready, no such device, disconnected) end the run instead of being recorded as bad areas. A chunk that fails with a general device error is treated the same way only when the drive answers nowhere else either (the last good place, the start of the disk, the next chunk): the scan then ends as not completed, the drive may have been disconnected, and continues from that place later. If the drive still answers elsewhere the chunk is read once more and what fails again is recorded as bad areas. The check itself writes nothing to the drive; Windows' online file system scan may note what it finds in the file system's own records. The health report tries smartctl's own device-type detection, then standard SAT pass-through; vendor-specific USB bridge commands are not tried blindly. Windows log entries (warnings and errors of the last 30 days) name a disk by number, and numbers move between drives as they are plugged in. Each entry is matched to the drive that held that number at the time, using Windows' own record of drive arrivals (`Microsoft-Windows-Partition/Diagnostic`, event 1006: disk number, serial number, model, size), so another drive's errors are not held against this one. The verdict rules live in `Backup.DriveCheck.psm1` and are tested by `Test-BackupDriveCheck.ps1` without hardware. ## What to preserve diff --git a/Backup-History.ps1 b/Backup-History.ps1 index aa7141d..85aef19 100644 --- a/Backup-History.ps1 +++ b/Backup-History.ps1 @@ -36,7 +36,7 @@ $add=C Button 520 648 230 32 'Add selected files / folders' $clear=C Button 765 648 165 32 'Clear selection' $restore=C Button 945 648 185 32 'Restore selected...' $selectionLabel=C Label 20 691 1110 24 'Restore selection: 0 items. Folders include their contents. Double-click a folder to open it.' -$status=C Label 20 725 1110 48 'Ready. Refresh history to load saved versions. The newest version of each computer/source set is protected from deletion.' +$status=C Label 20 725 1110 48 'Ready. Refresh history to load saved versions. The newest version and the newest complete version of each computer/source set are protected from deletion.' $logs=C LinkLabel 20 782 1110 25 '' $script:history=@();$script:entries=@();$script:page=0;$script:pageSize=500;$script:currentPath='/';$script:snapshot=$null;$script:worker=$null;$script:run=$null;$script:operation=$null;$script:target=$null $script:basket=[Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) @@ -49,7 +49,8 @@ function Show-Versions { foreach($snapshot in @($script:history|Sort-Object {(ConvertTo-HistoryDate $_.time).UtcTicks} -Descending)){ $date=(ConvertTo-HistoryDate $snapshot.time).LocalDateTime if($from.Checked -and $date.Date -lt $from.Value.Date){continue};if($to.Checked -and $date.Date -gt $to.Value.Date){continue} - $i=$versions.Rows.Add($false,$date.ToString('yyyy-MM-dd HH:mm:ss'),$snapshot.hostname,(@($snapshot.paths)-join '; '),$snapshot.id.Substring(0,12));$versions.Rows[$i].Tag=$snapshot + # A partial version (a backup that could not read every file) is labelled, so it is not taken for a complete one. + $i=$versions.Rows.Add($false,$date.ToString('yyyy-MM-dd HH:mm:ss')+$(if(Test-PartialSnapshot $snapshot){' (partial)'}else{''}),$snapshot.hostname,(@($snapshot.paths)-join '; '),$snapshot.id.Substring(0,12));$versions.Rows[$i].Tag=$snapshot } } function Show-Files { @@ -73,13 +74,13 @@ function Open-Folder([string]$path) {if(-not $script:snapshot){throw 'Browse a b function Confirm-Deletion($plan) { $dialog=[Windows.Forms.Form]::new();$dialog.Text='Delete backup history';$dialog.ClientSize=[Drawing.Size]::new(740,470);$dialog.StartPosition='CenterParent';$dialog.Font=$form.Font;$dialog.RightToLeft='No';$dialog.FormBorderStyle='FixedDialog';$dialog.MaximizeBox=$false $label=[Windows.Forms.Label]::new();$label.SetBounds(20,15,700,58);$label.Text='This permanently removes the listed backup versions. Their unique file versions will no longer be recoverable. Your source files are not deleted.';$dialog.Controls.Add($label) - $list=[Windows.Forms.TextBox]::new();$list.SetBounds(20,80,700,180);$list.Multiline=$true;$list.ReadOnly=$true;$list.ScrollBars='Both';$list.WordWrap=$false;$list.Text=(@($plan|ForEach-Object {(ConvertTo-HistoryDate $_.time).LocalDateTime.ToString('yyyy-MM-dd HH:mm:ss')+' '+$_.id})-join [Environment]::NewLine);$dialog.Controls.Add($list) + $list=[Windows.Forms.TextBox]::new();$list.SetBounds(20,80,700,180);$list.Multiline=$true;$list.ReadOnly=$true;$list.ScrollBars='Both';$list.WordWrap=$false;$list.Text=(@($plan|ForEach-Object {(ConvertTo-HistoryDate $_.time).LocalDateTime.ToString('yyyy-MM-dd HH:mm:ss')+' '+$_.id+$(if(Test-PartialSnapshot $_){' (partial)'}else{''})})-join [Environment]::NewLine);$dialog.Controls.Add($list) $reclaim=[Windows.Forms.CheckBox]::new();$reclaim.SetBounds(20,274,700,28);$reclaim.Text='Also reclaim unused storage (prune; may take time and temporarily need free space)';$dialog.Controls.Add($reclaim) $hint=[Windows.Forms.Label]::new();$hint.SetBounds(20,311,700,25);$hint.Text='Type DELETE to confirm deletion of '+$plan.Count+' selected version(s):';$dialog.Controls.Add($hint) - $input=[Windows.Forms.TextBox]::new();$input.SetBounds(20,341,700,28);$dialog.Controls.Add($input) + $typed=[Windows.Forms.TextBox]::new();$typed.SetBounds(20,341,700,28);$dialog.Controls.Add($typed) $ok=[Windows.Forms.Button]::new();$ok.SetBounds(470,402,120,35);$ok.Text='Delete history';$ok.Enabled=$false;$ok.DialogResult='OK';$dialog.Controls.Add($ok) $cancel=[Windows.Forms.Button]::new();$cancel.SetBounds(600,402,120,35);$cancel.Text='Cancel';$cancel.DialogResult='Cancel';$dialog.Controls.Add($cancel);$dialog.CancelButton=$cancel - $input.Add_TextChanged({$ok.Enabled=$input.Text -ceq 'DELETE'}) + $typed.Add_TextChanged({$ok.Enabled=$typed.Text -ceq 'DELETE'}) try{if($dialog.ShowDialog($form) -eq 'OK'){return @{SnapshotIds=@($plan.id);Confirmation='DELETE SELECTED HISTORY';ReclaimSpace=$reclaim.Checked}};return $null}finally{$dialog.Dispose()} } $refresh.Add_Click({try{Start-HistoryAction 'Snapshots'}catch{Error-Message $_.Exception.Message}}) @@ -108,7 +109,8 @@ $timer.Add_Tick({try{ switch($script:operation){ 'Snapshots' {$script:history=@(Get-Content -LiteralPath (Join-Path $script:run 'ListingSnapshots.jsonl') -Raw|ConvertFrom-Json);Show-Versions;$status.Text='Loaded '+$script:history.Count+' backup version(s). Select a row and choose Browse selected version.'} 'Files' {$items=[Collections.Generic.List[object]]::new();foreach($line in [IO.File]::ReadLines((Join-Path $script:run 'ListingFiles.jsonl'))){$item=$line|ConvertFrom-Json;if($item.type -and $item.path -ne $script:currentPath){$items.Add($item)}};$script:entries=@($items|Sort-Object @{Expression={if($_.type -eq 'dir'){0}else{1}}},name);$script:page=0;Show-Files;$status.Text='Browsing snapshot '+$script:snapshot.Substring(0,12)+'. Files are loaded one folder at a time.'} - 'Restore' {$status.Text='Restore completed and contents verified. Restored to: '+$script:target;[void][Windows.Forms.MessageBox]::Show($form,$status.Text,'Restore completed')} + # restic skips its content check when it could not set the date of a protected parent folder: say so. + 'Restore' {$status.Text=if($s.PSObject.Properties['ContentVerified'] -and $s.ContentVerified -eq $false){'Restore completed, but the content of the restored files was NOT verified: the dates of '+@($s.Warnings).Count+' protected parent folder(s) could not be set, and the check is skipped then. The files are in: '+$script:target}else{'Restore completed and contents verified. Restored to: '+$script:target};[void][Windows.Forms.MessageBox]::Show($form,$status.Text,'Restore completed')} 'Forget' {$script:entries=@();$script:snapshot=$null;Reset-Selection;Show-Files;Start-HistoryAction 'Snapshots'} } }catch{$timer.Stop();foreach($control in $busyControls){$control.Enabled=$true};Error-Message $_.Exception.Message}}) diff --git a/Backup-Worker.ps1 b/Backup-Worker.ps1 index 82bf6e1..60fb2d7 100644 --- a/Backup-Worker.ps1 +++ b/Backup-Worker.ps1 @@ -9,11 +9,33 @@ param( [switch]$ReadAllData, [switch]$ForceRead, [string]$RequestPath, - # Backup only: afterwards keep one version per day / week / month for this long and delete the rest - # (all 0: keep every version). - [ValidateRange(0,3650)][int]$KeepDays=0, - [ValidateRange(0,520)][int]$KeepWeeks=0, - [ValidateRange(0,240)][int]$KeepMonths=0 + # Backup only, Keep versions: afterwards keep the KeepLast newest complete versions and the newest + # complete version of each month for KeepMonths months, and delete the rest (both 0: keep every + # version). The rule is Get-VersionRetentionPlan in Backup.History.psm1. + [ValidateRange(0,10000)][int]$KeepLast=0, + [ValidateRange(0,240)][int]$KeepMonths=0, + # Backup only: work the clean-up out as usual and report it, but delete nothing and free no space. + [switch]$RetentionPreview, + # Backup only, the first clean-up after the user approved a preview: a UTF-8 text file with one full + # version ID a line (blank lines ignored; spaces around an ID and capital letters make no difference). + # The clean-up is planned as usual and then deletes only the versions that are both in its plan and in + # this file; whatever else the plan would delete stays for the next clean-up (Retention.Deferred). The + # file only narrows: a listed version that the rule keeps, or that no longer exists, is not deleted. A + # file that is missing, unreadable, empty or holds anything but full IDs deletes nothing + # (Retention.Error), never the whole plan. With -RetentionPreview the preview wins and the file is not used. + [string]$RetentionApprovedFile, + # Backup only: the versions folder is gone although this source was backed up before (see + # RepositoryMissing below), and the user chose to start a new, empty one. Without it such a backup + # stops and changes nothing. Not needed for a first-ever backup; ignored while the repository exists. + [switch]$CreateNewRepository, + # The earlier rule (one version a day / a week). Still accepted only so that a run started by an older + # window fails with a plain message instead of applying another rule than the one that was chosen. + [int]$KeepDays=0, + [int]$KeepWeeks=0, + # Test suite only: an existing folder under the temporary directory that stands in for the backup + # drive (repository, engine cache, run folders), so the tests need no second drive and never touch + # the real state. A real run never passes it and keeps every volume check. + [string]$TestRoot ) $ErrorActionPreference='Stop' Set-StrictMode -Version Latest @@ -21,9 +43,22 @@ Import-Module (Join-Path $PSScriptRoot 'Backup.Status.psm1') -Force Import-Module (Join-Path $PSScriptRoot 'Backup.History.psm1') -Force Import-Module (Join-Path $PSScriptRoot 'Backup.Security.psm1') -Force Import-Module (Join-Path $PSScriptRoot 'Backup.Selection.psm1') -Force -if(-not $RunDirectory){$RunDirectory=Join-Path $env:LOCALAPPDATA ('DevDriveMirror\runs\'+(Get-Date -Format 'yyyyMMdd-HHmmss')+'-'+[guid]::NewGuid().ToString('N'))} +if($TestRoot){ + $TestRoot=[IO.Path]::GetFullPath($TestRoot).TrimEnd('\') + if(-not ($TestRoot+'\').StartsWith([IO.Path]::GetFullPath([IO.Path]::GetTempPath()).TrimEnd('\')+'\',[StringComparison]::OrdinalIgnoreCase) -or -not(Test-Path -LiteralPath $TestRoot -PathType Container)){throw 'TestRoot is for the test suite only: an existing folder under the temporary directory'} +} +if(-not $RunDirectory){$RunDirectory=Join-Path $(if($TestRoot){$TestRoot}else{Join-Path $env:LOCALAPPDATA 'DevDriveMirror'}) ('runs\'+(Get-Date -Format 'yyyyMMdd-HHmmss')+'-'+[guid]::NewGuid().ToString('N'))} New-Item -ItemType Directory -Path $RunDirectory -Force | Out-Null -$script:status=[ordered]@{State='Starting';Action=$Action;WorkerPid=$PID;StartedUtc=[DateTime]::UtcNow.ToString('o');UpdatedUtc='';Progress=$null;MemoryMiB=0;PeakMemoryMiB=0;CpuPercent=0;Error=$null;Snapshot=$null;RunDirectory=$RunDirectory} +# BackupSummary: restic's own summary of the backup step (Progress ends up holding the summary of the +# check that follows). Partial: the backup saved a version but could not read every file. LockReleased: +# nothing, or the locks of an earlier run that this run removed before trying a step again (Run-Engine). +# RepositoryMissing: the run stopped because the versions folder holds no repository although this source +# was backed up before (its protected password file exists): the folder was renamed, moved, deleted or +# emptied. Nothing is created then; only a Backup with -CreateNewRepository starts a new one. +# RepositoryCreated: this run created the repository (a first-ever backup, or -CreateNewRepository; Note +# then says that the earlier versions are not in it). +$script:lockExamined=$false +$script:status=[ordered]@{State='Starting';Action=$Action;WorkerPid=$PID;StartedUtc=[DateTime]::UtcNow.ToString('o');UpdatedUtc='';Progress=$null;MemoryMiB=0;PeakMemoryMiB=0;CpuPercent=0;Error=$null;Snapshot=$null;RunDirectory=$RunDirectory;BackupSummary=$null;Partial=$false;LockReleased=$null;RepositoryMissing=$false;RepositoryCreated=$false;Note=$null} function Save-Status { $script:status.UpdatedUtc=[DateTime]::UtcNow.ToString('o') [void](Write-BackupStatus $script:status (Join-Path $RunDirectory 'status.json') -BestEffort) @@ -32,7 +67,7 @@ function Assert-PlainAncestors([string]$Path) { $p=[IO.Path]::GetFullPath($Path) while($p){if(Test-Path -LiteralPath $p){if((Get-Item -LiteralPath $p -Force).Attributes -band [IO.FileAttributes]::ReparsePoint){throw "Linked repository/root/restore path refused: $p"}};$parent=[IO.Directory]::GetParent($p);if(-not $parent){break};$p=$parent.FullName} } -function Run-Engine([string[]]$EngineArguments,[string]$Stage,[string]$InputSecret) { +function Invoke-EngineStep([string[]]$EngineArguments,[string]$Stage,[string]$InputSecret) { $script:status.State=$Stage;Save-Status $psi=[Diagnostics.ProcessStartInfo]::new($script:engine);$psi.UseShellExecute=$false;$psi.CreateNoWindow=$true;$psi.RedirectStandardOutput=$true;$psi.RedirectStandardError=$true # Pruning needs at least two connections, whether it runs with forget or on its own. @@ -60,7 +95,8 @@ function Run-Engine([string[]]$EngineArguments,[string]$Stage,[string]$InputSecr $line=$lineTask.Result if($null -eq $line){break} $output.WriteLine($line);$output.Flush() - try{$event=$line|ConvertFrom-Json;if($event.message_type -eq 'status'){$script:status.Progress=$event};if($event.message_type -eq 'summary'){$script:status.Progress=$event;if($event.PSObject.Properties['snapshot_id']){$script:status.Snapshot=$event.snapshot_id}}}catch{} + try{$event=$line|ConvertFrom-Json;if($event.message_type -eq 'status'){$script:status.Progress=$event};if($event.message_type -eq 'summary'){$script:status.Progress=$event;if($event.PSObject.Properties['snapshot_id']){$script:status.Snapshot=$event.snapshot_id} + if($Stage -eq 'BackingUp'){$own=[ordered]@{};foreach($name in @('files_new','files_changed','data_added','total_files_processed','total_bytes_processed','snapshot_id')){$own[$name]=if($event.PSObject.Properties[$name]){$event.$name}else{$null}};$script:status.BackupSummary=$own}}}catch{} $lineTask=$process.StandardOutput.ReadLineAsync() } if($watch.Elapsed.TotalSeconds-$lastSave -ge 2){ @@ -71,23 +107,93 @@ function Run-Engine([string[]]$EngineArguments,[string]$Stage,[string]$InputSecr } } $process.WaitForExit();$errorText=$errors.GetAwaiter().GetResult();[IO.File]::WriteAllText((Join-Path $RunDirectory ($Stage+'.errors.log')),$errorText) - # A restore below a protected folder (for example C:\Users) can restore every file but fail to set - # the timestamps of the recreated parent folders. That is a warning, not a failed restore. - $ignored=@($errorText -split "`r?`n"|Where-Object {$_ -match '^ignoring error for '}) - if($Stage -eq 'Restoring' -and $process.ExitCode -eq 1 -and $ignored.Count -and -not @($ignored|Where-Object {$_ -notmatch 'failed to restore (timestamp|attributes|security descriptor)'}).Count){ - $script:status|Add-Member -NotePropertyName Warnings -NotePropertyValue @($ignored) -Force - }elseif($process.ExitCode -ne 0){ - $hint=if($Stage -eq 'DeletingHistory'){'History deletion may be partial. Refresh history before retrying.'}elseif($Stage -eq 'BackingUp'){'Code 3 means an incomplete snapshot.'}else{'The operation did not complete successfully.'} - $engineError=[InvalidOperationException]::new("Restic $Stage returned exit code $($process.ExitCode). $hint See $Stage.errors.log. A memory allocation failure requires a higher memory budget.") + if($process.ExitCode -ne 0){ + $engineError=[InvalidOperationException]::new((Get-EngineFailureText $Stage $process.ExitCode)) $engineError.Data['ExitCode']=$process.ExitCode;throw $engineError } }finally{if($started -and -not $process.HasExited){$process.Kill($true)};$output.Dispose();$job.Dispose();$process.Dispose()} } +function Read-RepositoryLockIds([string]$Stage) { + Invoke-EngineStep @('list','locks','--no-lock') $Stage + @([IO.File]::ReadAllLines((Join-Path $RunDirectory ($Stage+'.jsonl')))|ForEach-Object {$_.Trim()}|Where-Object {$_}) +} +function Clear-EndedLocks { + # Reads every lock in the repository and removes them only when all of them were left by runs on this + # computer that have ended (Get-LockReleaseDecision). Removing is left to restic's own "unlock", never + # with --remove-all: it lets go of a lock of this computer whose process is gone, or of one that is + # over 30 minutes old, and keeps every other one, so a lock goes only when both tests agree. Returns + # the locks that are gone afterwards and, when none is, why. The worker's own run lock is held here. + $found=@(foreach($id in @(Read-RepositoryLockIds 'ReadingLocks')){ + $record=$null;$process=$null + if($id -cmatch '^[0-9a-f]{64}$'){try{Invoke-EngineStep @('cat','lock',$id,'--no-lock') 'ReadingLock';$record=Get-Content -LiteralPath (Join-Path $RunDirectory 'ReadingLock.jsonl') -Raw|ConvertFrom-Json}catch{$record=$null}} + if($record -is [pscustomobject] -and $record.PSObject.Properties['pid']){$process=Get-LockProcess $record.pid} + [pscustomobject]@{Id=$id;Lock=$record;Process=$process} + }) + # restic records the computer's DNS host name, as this returns it. + $decision=Get-LockReleaseDecision $found ([Net.Dns]::GetHostName()) + if(-not $decision.Release){return [pscustomobject]@{Released=@();Reason=$decision.Reason}} + Invoke-EngineStep @('unlock') 'ReleasingLock' + $left=@(Read-RepositoryLockIds 'ReadingLocksAgain') + $gone=@($decision.Locks|Where-Object {$_.Id -cnotin $left}) + $kept=@($decision.Locks|Where-Object {$_.Id -cin $left}) + [pscustomobject]@{Released=$gone;Reason=$(if($kept.Count){"the run that made the lock has ended, but restic still sees a process under its number ($($kept[0].Pid)) and keeps the lock until $(([DateTimeOffset]::Parse($kept[0].CreatedUtc,[Globalization.CultureInfo]::InvariantCulture)).ToLocalTime().AddMinutes(30).ToString('HH:mm')), 30 minutes after it was made. Try again after that."}else{''})} +} +function Run-Engine([string[]]$EngineArguments,[string]$Stage,[string]$InputSecret) { + # One restic step. A run that was stopped, crashed or was ended by the memory cap leaves its lock in the + # repository, and every later step that needs the repository then ends with exit code 11. The first + # time that happens in a run the locks are examined; when they were all left by ended runs of this + # computer they are removed, the result records it (LockReleased) and the step is tried once more. + # Otherwise nothing is removed and the failure says whose lock it is. A step that is locked out later + # in the same run fails as it is: one removal and one second try per run. + try{Invoke-EngineStep @PSBoundParameters;return}catch{if($_.Exception.Data['ExitCode'] -ne 11 -or $script:lockExamined){throw};$locked=$_} + $script:lockExamined=$true;$outcome=$null + try{$outcome=Clear-EndedLocks}catch{} + if(-not $outcome -or (-not @($outcome.Released).Count -and -not $outcome.Reason)){throw $locked} + if(-not @($outcome.Released).Count){ + $stillLocked=[InvalidOperationException]::new((Get-EngineFailureText $Stage 11 $outcome.Reason));$stillLocked.Data['ExitCode']=11;throw $stillLocked + } + $script:status.LockReleased=[ordered]@{Stage=$Stage;Locks=@($outcome.Released|ForEach-Object {[ordered]@{Id=$_.Id;Pid=$_.Pid;CreatedUtc=$_.CreatedUtc;Exclusive=$_.Exclusive}})} + # What the first try reported stays beside the second try's files. + try{[IO.File]::Move((Join-Path $RunDirectory ($Stage+'.errors.log')),(Join-Path $RunDirectory ($Stage+'.locked.errors.log')),$true)}catch{} + Invoke-EngineStep @PSBoundParameters +} function Save-RecoveryLocation([string]$Path) { $fresh=Get-Content -LiteralPath $ConfigPath -Raw|ConvertFrom-Json if($fresh.PasswordDpapiPath -ine $script:config.PasswordDpapiPath -or $fresh.DestinationVolumeId -cne $script:config.DestinationVolumeId -or $fresh.RepositoryFolder -cne $script:config.RepositoryFolder){throw 'Configuration changed during key management. Resume with the original repository configuration.'} $fresh.RecoveryKeyPath=$Path;[void](Write-BackupStatus $fresh $ConfigPath);$script:config=$fresh } +function Set-PartialMark([string]$Id) { + # Marks a version partial and returns its new ID (changing the tag changes the ID). + Run-Engine (Get-PartialMarkArguments $Id) 'MarkingPartial' + $changed=@([IO.File]::ReadAllLines((Join-Path $RunDirectory 'MarkingPartial.jsonl'))|ForEach-Object {try{$_|ConvertFrom-Json}catch{}}|Where-Object {$_ -and $_ -is [pscustomobject] -and $_.PSObject.Properties['message_type'] -and $_.message_type -eq 'changed'}) + if($changed.Count -ne 1 -or $changed[0].new_snapshot_id -cnotmatch '^[0-9a-f]{64}$'){throw 'the engine did not report the marked version'} + $changed[0].new_snapshot_id +} +# A partial version that could not be marked (for example while the repository is locked) still looks +# like a complete one. It is noted beside the password file and marked on a later run, before anything +# is deleted: until then neither the clean-up nor a deletion by hand goes ahead. +function Save-PendingPartialMark([string]$Id) { + $path=$keyPath+'.partial-pending.local.json' + $ids=@(if(Test-Path -LiteralPath $path){Get-Content -LiteralPath $path -Raw|ConvertFrom-Json})+$Id + [void](Write-BackupStatus @($ids|Select-Object -Unique) $path) +} +function Get-PendingPartialMarks { + # The IDs of the partial versions that are still waiting for their mark. + $path=$keyPath+'.partial-pending.local.json' + if(Test-Path -LiteralPath $path){@(Get-Content -LiteralPath $path -Raw|ConvertFrom-Json|ForEach-Object {[string]$_})} +} +function Complete-PendingPartialMarks { + $path=$keyPath+'.partial-pending.local.json' + if(-not(Test-Path -LiteralPath $path)){return} + $pending=@(Get-Content -LiteralPath $path -Raw|ConvertFrom-Json) + Run-Engine @('snapshots') 'ReadingPartialMarks' + $saved=@(Get-Content -LiteralPath (Join-Path $RunDirectory 'ReadingPartialMarks.jsonl') -Raw|ConvertFrom-Json) + foreach($id in $pending){ + $version=@($saved|Where-Object {$_.id -ceq $id}) + if($version.Count -and -not (Test-PartialSnapshot $version[0])){[void](Set-PartialMark $id)} + } + [IO.File]::Delete($path) +} function Read-RepositoryKeys([string]$Stage) { Run-Engine @('key','list') $Stage @(Get-Content -LiteralPath (Join-Path $RunDirectory ($Stage+'.jsonl')) -Raw|ConvertFrom-Json) @@ -98,7 +204,7 @@ function Invoke-KeyManagement { if(-not(Test-Path -LiteralPath $journalPath)){throw 'There is no interrupted password change to resume'} $journal=Get-Content -LiteralPath $journalPath -Raw|ConvertFrom-Json if($journal.Version -ne 1 -or $journal.OldKeyId -cnotmatch '^[0-9a-f]{64}$'){throw 'Password-change journal is invalid; keep the existing protected credential and recovery files'} - if($journal.Repository -ine $script:repository -or $journal.ConfigPath -ine [IO.Path]::GetFullPath($ConfigPath)){throw 'Password-change journal belongs to another configuration or repository'} + Assert-RotationJournalTarget $journal ([string]$script:config.DestinationVolumeId) ([string]$script:config.RepositoryFolder) $script:repository if($RequestPath){ $request=Get-Content -LiteralPath $RequestPath -Raw|ConvertFrom-Json if($request.RecoveryKeyPath -ine $journal.RecoveryKeyPath){ @@ -113,6 +219,8 @@ function Invoke-KeyManagement { $export=Assert-PrivateExportPath $request.RecoveryKeyPath @($source,$script:repository,$PSScriptRoot) @($keyPath,$ConfigPath,$journalPath) if(Test-Path -LiteralPath $export){throw 'The recovery-key destination already exists. Choose a new file; existing files are never overwritten.'} if($Action -eq 'ExportKey'){ + # A recovery key that does not open the repository is worse than none: check the stored password first. + try{[void](Read-RepositoryKeys 'CheckingPassword')}catch{if($_.Exception.Data['ExitCode'] -eq 12){throw 'The password stored on this computer does not open this backup repository, so no recovery key was saved. Use an earlier recovery key file to regain access.'};throw} New-PrivateTextFile $export (Get-RecoveryKeyText $script:repository $script:password) Save-RecoveryLocation $export $script:status|Add-Member -NotePropertyName RecoveryKeyPath -NotePropertyValue $export -Force @@ -124,7 +232,7 @@ function Invoke-KeyManagement { if($newPassword -ceq $script:password){throw 'The new password must differ from the current password'} $keys=@(Read-RepositoryKeys 'ReadingAccessKeys');$current=@($keys|Where-Object {$_.current -eq $true}) if($current.Count -ne 1 -or $current[0].id -cnotmatch '^[0-9a-f]{64}$'){throw 'Cannot identify the currently used repository access key'} - $journal=[pscustomobject]@{Version=1;Repository=$script:repository;ConfigPath=[IO.Path]::GetFullPath($ConfigPath);RecoveryKeyPath=$export;OldKeyId=$current[0].id;NewKeyId=$null;OldPasswordDpapi=([IO.File]::ReadAllText($keyPath)).Trim();NewPasswordDpapi=$protectedNew.Trim();Phase='Prepared';CreatedUtc=[DateTime]::UtcNow.ToString('o')} + $journal=[pscustomobject]@{Version=1;Repository=$script:repository;ConfigPath=[IO.Path]::GetFullPath($ConfigPath);DestinationVolumeId=[string]$script:config.DestinationVolumeId;RepositoryFolder=[string]$script:config.RepositoryFolder;RecoveryKeyPath=$export;OldKeyId=$current[0].id;NewKeyId=$null;OldPasswordDpapi=([IO.File]::ReadAllText($keyPath)).Trim();NewPasswordDpapi=$protectedNew.Trim();Phase='Prepared';CreatedUtc=[DateTime]::UtcNow.ToString('o')} New-PrivateTextFile $journalPath ($journal|ConvertTo-Json -Depth 6) } $oldPassword=[Net.NetworkCredential]::new('',($journal.OldPasswordDpapi|ConvertTo-SecureString)).Password @@ -157,7 +265,10 @@ function Invoke-KeyManagement { } try { Save-Status + if($PSBoundParameters.ContainsKey('KeepDays') -or $PSBoundParameters.ContainsKey('KeepWeeks')){throw 'This run was started with the earlier Keep versions settings (days and weeks), which no longer exist: the rule is now the last N versions and one version a month. Nothing was backed up and nothing was deleted. Update every file of DevDriveBackup, set Keep versions again and run the backup again.'} $script:config=Get-Content -LiteralPath $ConfigPath -Raw|ConvertFrom-Json + # The test folder never goes together with real settings or key files. + if($TestRoot){$temporary=[IO.Path]::GetFullPath([IO.Path]::GetTempPath()).TrimEnd('\')+'\';foreach($testPath in @($ConfigPath,$script:config.PasswordDpapiPath,$script:config.RecoveryKeyPath)){if(-not [IO.Path]::GetFullPath([string]$testPath).StartsWith($temporary,[StringComparison]::OrdinalIgnoreCase)){throw 'TestRoot is for the test suite only: the settings and key files must be under the temporary directory too'}}} $script:engine=Join-Path $PSScriptRoot 'bin\restic_0.19.1_windows_amd64.exe' if(-not(Test-Path -LiteralPath $script:engine)){throw 'Run Install-BackupEngine.ps1 first'} if($script:config.MemoryMiB -lt 512 -or $script:config.MemoryMiB -gt 8192){throw 'Memory budget must be 512..8192 MiB'} @@ -169,14 +280,18 @@ try { $script:backendConnections=if($script:config.PSObject.Properties['BackendConnections']){[int]$script:config.BackendConnections}else{1} if($script:backendConnections -lt 1 -or $script:backendConnections -gt 8){throw 'BackendConnections must be 1..8'} $source=[IO.Path]::GetFullPath($script:config.Source).TrimEnd('\')+'\' - if($Action -in @('Backup','Preview')){Assert-PlainAncestors $source;if((Get-Volume -DriveLetter $source.Substring(0,1)).UniqueId -ne $script:config.SourceVolumeId){throw 'Source volume identity changed'}} - $volume=@(Get-Volume -UniqueId $script:config.DestinationVolumeId|Where-Object {$_.DriveLetter}) - if($volume.Count -ne 1){throw 'Expected destination volume is missing or ambiguous'} - if($volume[0].UniqueId -eq $script:config.SourceVolumeId){throw 'Backup must use a separate volume'} + if($Action -in @('Backup','Preview')){Assert-PlainAncestors $source;if(-not $TestRoot -and (Get-Volume -DriveLetter $source.Substring(0,1)).UniqueId -ne $script:config.SourceVolumeId){throw 'Source volume identity changed'}} + if($TestRoot){$destinationRoot=$TestRoot;$free=[IO.DriveInfo]::new([IO.Path]::GetPathRoot($TestRoot)).AvailableFreeSpace} + else{ + $volume=@(Get-Volume -UniqueId $script:config.DestinationVolumeId|Where-Object {$_.DriveLetter}) + if($volume.Count -ne 1){throw 'Expected destination volume is missing or ambiguous'} + if($volume[0].UniqueId -eq $script:config.SourceVolumeId){throw 'Backup must use a separate volume'} + $destinationRoot=$volume[0].DriveLetter+':\';$free=$volume[0].SizeRemaining + } $folder=[string]$script:config.RepositoryFolder if($folder -notmatch '^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$' -or $folder.EndsWith('.')){throw 'RepositoryFolder must be a single plain folder name'} - $script:repository=Join-Path ($volume[0].DriveLetter+':\') $folder;Assert-PlainAncestors $script:repository - if($Action -in @('Backup','Preview') -and $volume[0].SizeRemaining -lt 5GB){throw 'Less than 5 GiB free on the backup volume'} + $script:repository=Join-Path $destinationRoot $folder;Assert-PlainAncestors $script:repository + if($Action -in @('Backup','Preview') -and $free -lt 5GB){throw 'Less than 5 GiB free on the backup volume'} # What to back up comes from the per-folder Mode shared with the plain mirror (Versioned or Both). $sources=@();$selectionExcludes=@();$privateExcludes=@() if($Action -in @('Backup','Preview')){ @@ -213,7 +328,7 @@ try { }) $script:status.Sources=$sources;$script:status.SelectionExcludes=$selectionExcludes } - $script:cache=Join-Path $env:LOCALAPPDATA 'DevDriveMirror\restic-cache';New-Item -ItemType Directory -Path $script:cache -Force|Out-Null + $script:cache=if($TestRoot){Join-Path $TestRoot 'engine-cache'}else{Join-Path $env:LOCALAPPDATA 'DevDriveMirror\restic-cache'} $keyPath=[IO.Path]::GetFullPath($script:config.PasswordDpapiPath);$exists=Test-Path -LiteralPath (Join-Path $script:repository 'config') $recoveryPath=[IO.Path]::GetFullPath($script:config.RecoveryKeyPath) if($keyPath -ieq $recoveryPath){throw 'Protected password and portable recovery key must be different files'} @@ -221,7 +336,22 @@ try { New-Item -ItemType Directory -Path ([IO.Path]::GetDirectoryName($keyPath)) -Force|Out-Null $script:runLock=[IO.File]::Open($keyPath+'.run.lock',[IO.FileMode]::OpenOrCreate,[IO.FileAccess]::ReadWrite,[IO.FileShare]::None) if((Test-Path -LiteralPath ($keyPath+'.rotation.local.json')) -and $Action -ne 'ResumePasswordChange'){throw 'An interrupted password change must be resumed from Password and recovery before using this repository.'} - if(Test-Path -LiteralPath $keyPath){$secure=(Get-Content -LiteralPath $keyPath -Raw).Trim()|ConvertTo-SecureString;$script:password=[Net.NetworkCredential]::new('',$secure).Password} + $missing=$false + if(Test-Path -LiteralPath $keyPath){ + $secure=(Get-Content -LiteralPath $keyPath -Raw).Trim()|ConvertTo-SecureString;$script:password=[Net.NetworkCredential]::new('',$secure).Password + # The protected password file says that this source was backed up before. When the versions folder + # then holds no repository, it was renamed, moved, deleted or emptied. Starting a new, empty one here + # would look like a healthy backup with every earlier version gone, so the run stops before anything + # is created (no folder, no recovery-key file, no engine cache) and the user decides: bring the folder + # back, or run a Backup with -CreateNewRepository. + if(-not $exists){ + $missing=$true;$script:status.RepositoryMissing=$true + if($Action -ne 'Backup' -or -not $CreateNewRepository){ + if(Test-Path -LiteralPath $script:repository){throw "The versions folder $folder on the backup drive is empty or incomplete: the saved versions are not in it. They may have been moved or deleted. Nothing was changed."} + throw "The versions folder $folder was not found on the backup drive. It may have been renamed, moved or deleted. Nothing was changed." + } + } + } elseif($exists){throw 'Repository already exists but its protected password file is missing'} else { if($Action -ne 'Backup'){throw 'Repository is not initialized; the first Backup creates it'} @@ -233,6 +363,7 @@ try { New-PrivateTextFile $keyPath ($script:password|ConvertTo-SecureString -AsPlainText -Force|ConvertFrom-SecureString) New-PrivateTextFile $recovery (Get-RecoveryKeyText $script:repository $script:password) } + New-Item -ItemType Directory -Path $script:cache -Force|Out-Null if(-not $exists -and -not(Test-Path -LiteralPath $recoveryPath)){ New-Item -ItemType Directory -Path ([IO.Path]::GetDirectoryName($recoveryPath)) -Force|Out-Null New-PrivateTextFile $recoveryPath (Get-RecoveryKeyText $script:repository $script:password) @@ -244,6 +375,9 @@ try { # empty one (a wiped or renamed backup folder would otherwise look like a healthy empty history). if($Action -ne 'Backup'){throw "No backup repository was found at $($script:repository). Check that the right drive is connected and that the folder was not renamed or deleted."} Run-Engine @('init') 'Initializing' + $script:status.RepositoryCreated=$true + # Created on the user's choice in place of one that is gone: it opens with the password already stored. + if($missing){$script:status.RepositoryMissing=$false;$script:status.Note="A new, empty versions folder $folder was created, as chosen, because the earlier one was not found. The versions saved before are not in it."} } $base=@('backup','--read-concurrency',[string]$script:config.ReadConcurrency,'--tag','devdrive') if($ForceRead){$base+='--force'} @@ -251,42 +385,115 @@ try { switch($Action){ {$_ -in @('ExportKey','ChangePassword','ResumePasswordChange')} {Invoke-KeyManagement} 'Backup' { - Run-Engine ($base+@('--')+$sources) 'BackingUp';Run-Engine @('check') 'CheckingStructure' - if($KeepDays+$KeepWeeks+$KeepMonths -gt 0){ - # The backup itself has succeeded and been checked. A clean-up that fails is reported, not a failed - # backup. Old versions are deleted first and their space freed in a second step, so the result says - # how far it got. Once versions were deleted the repository is checked again, outside that report: - # a repository that fails this check fails the run, since it may be damaged. + # The first clean-up after an approved preview is limited to the list of approved versions. The list + # is read now, as it was when the run started; one that cannot be used is reported with the clean-up + # (Retention.Error) and deletes nothing. A preview wins: the list is then not used at all. + $approving=$PSBoundParameters.ContainsKey('RetentionApprovedFile') -and -not $RetentionPreview + $approvedIds=@();$approvedProblem=$null + if($approving){ + try{ + if(-not ([string]$RetentionApprovedFile).Trim()){throw 'No list of approved versions was named, so the clean-up deleted nothing.'} + $approvedPath=$ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($RetentionApprovedFile) + if(-not(Test-Path -LiteralPath $approvedPath -PathType Leaf)){throw "The list of approved versions was not found ($approvedPath), so the clean-up deleted nothing."} + try{$approvedText=[IO.File]::ReadAllText($approvedPath)}catch{throw "The list of approved versions could not be read ($approvedPath), so the clean-up deleted nothing. $($_.Exception.GetBaseException().Message)"} + $approvedIds=@(Get-RetentionApprovedIds $approvedText) + }catch{$approvedProblem=$_.Exception.Message;$approvedIds=@()} + } + try{Run-Engine ($base+@('--')+$sources) 'BackingUp'} + catch{ + # Exit code 3: restic saved a version although some files could not be read. Mark it (the tag + # changes, and with it the version's ID) so the clean-up rule and the newest-complete protection + # never treat it as a complete version, then fail the run as before. + if($_.Exception.Data['ExitCode'] -ne 3 -or -not $script:status.Snapshot){throw} + $script:status.Partial=$true;$markError=$null + try{ + $marked=Set-PartialMark $script:status.Snapshot + $script:status.Snapshot=$marked;if($script:status.BackupSummary){$script:status.BackupSummary.snapshot_id=$marked} + }catch{$markError=$_.Exception.Message;try{Save-PendingPartialMark $script:status.Snapshot}catch{$markError+=' It could not be noted for later either: '+$_.Exception.Message}} + $partialText='Some files could not be read, so this version is incomplete (restic exit code 3). ' + if($markError){throw ($partialText+"It was saved, but could not be marked as partial yet, so for now the history shows it like a complete version; it is marked on a later run, before any version is deleted. See BackingUp.errors.log for the files that were skipped, then back up again. ($markError)")} + throw ($partialText+'It was saved and marked as partial: the history shows it as partial, it never counts as one of the versions to keep, and the automatic clean-up of old versions deletes it only after three complete versions were made after it. See BackingUp.errors.log for the files that were skipped (for example files another program keeps open), then back up again.') + } + try{Run-Engine @('check') 'CheckingStructure'} + catch{if($_.Exception.Data['ExitCode'] -eq 11){throw ('The new version was saved, but it could not be checked. '+$_.Exception.Message)};throw} + if($KeepLast -gt 0 -or $KeepMonths -gt 0 -or $RetentionPreview -or $approving){ + # Keep versions. The backup itself has succeeded and been checked: a clean-up that fails is reported + # (Retention.Error), not a failed backup. The plan is worked out from the list of versions + # (Get-VersionRetentionPlan), checked (Get-RetentionRemovals) and saved with the run before anything + # is deleted. A preview stops there. Otherwise the versions are deleted by exact ID and their space + # is freed in a second step, so the result says how far it got. Once versions were deleted the + # repository is checked again, outside that report: a repository that fails this check fails the + # run, since it may be damaged. + # Retention in the result: Kept = complete versions of this computer that stay; Candidates = the + # versions a preview would delete now, or the ones this run really deleted (Removed of them, + # PartialRemoved of those partial). No size is given: deleting a version frees only the data that no + # remaining version needs, and restic 0.19 does not report that amount in a form that can be read. + # With a list of approved versions (-RetentionApprovedFile, the first clean-up after a preview) the + # plan is narrowed to that list before anything is deleted: Approved = the list was read and the plan + # was narrowed to it; Deferred = the versions the rule would delete now that were not on the list, + # which stay for the next clean-up (they count in Kept while they are complete and still there). $new=$script:status.Snapshot - $retention=[ordered]@{Days=$KeepDays;Weeks=$KeepWeeks;Months=$KeepMonths;Removed=0;Kept=$null;SpaceFreed=$false} - $remove=@();$deleting=$false + $retention=[ordered]@{Preview=[bool]$RetentionPreview;Approved=$false;Last=$KeepLast;Months=$KeepMonths;Kept=$null;Removed=0;PartialRemoved=0;Deferred=0;Candidates=@();Error=$null} + $remove=@();$gone=@();$deleting=$false try{ - Run-Engine (Get-RetentionArguments $KeepDays $KeepWeeks $KeepMonths) 'PlanningCleanUp' - $plan=@(Get-Content -LiteralPath (Join-Path $RunDirectory 'PlanningCleanUp.jsonl') -Raw|ConvertFrom-Json) - Run-Engine @('snapshots','--tag','devdrive') 'ReadingHistory' + # Before anything else: without a usable list nothing is marked, planned or deleted. + if($approving -and ($approvedProblem -or -not $approvedIds.Count)){throw $(if($approvedProblem){$approvedProblem}else{'The list of approved versions is empty, so the clean-up deleted nothing.'})} + # A partial version that is still waiting for its mark would pass for a complete one. It is marked + # first (the mark changes its ID). A preview changes no version: it only plans such a version as + # the partial one it is, under the ID it still has. + $waiting=@() + if($RetentionPreview){try{$waiting=@(Get-PendingPartialMarks)}catch{throw "The note of an earlier partial version that is not marked yet could not be read, so nothing was planned. $($_.Exception.Message)"}} + else{try{Complete-PendingPartialMarks}catch{throw "An earlier partial version is not marked yet, so nothing was cleaned up. $($_.Exception.Message)"}} + # Every version in the repository: the rule needs the partial ones too, and the checks need to see + # what belongs to another computer or another program. The computer name is read from the version + # just made, as restic recorded it. + Run-Engine @('snapshots') 'ReadingHistory' $available=@(Get-Content -LiteralPath (Join-Path $RunDirectory 'ReadingHistory.jsonl') -Raw|ConvertFrom-Json) - $remove=@(Get-RetentionRemovals $plan $new $available) - [IO.File]::WriteAllText((Join-Path $RunDirectory 'clean-up-plan.json'),(ConvertTo-Json @($remove) -Depth 8)) - $retention.Kept=$available.Count-$remove.Count - if($remove.Count){ + if($waiting.Count){$available=@($available|ForEach-Object { + if($_.id -cin $waiting -and $_.PSObject.Properties['tags'] -and -not (Test-PartialSnapshot $_)){$as=$_.PSObject.Copy();$as.tags=@(Get-PartialTags $_.tags);$as}else{$_} + })} + $made=@($available|Where-Object {$new -and $_.id -ceq $new}) + if($made.Count -ne 1 -or -not $made[0].hostname){throw 'The version this backup just made was not found among the saved versions, so the clean-up cannot protect it. Nothing was deleted.'} + $computer=[string]$made[0].hostname + $plan=@(Get-VersionRetentionPlan $available $KeepLast $KeepMonths $computer $new) + $remove=@(Get-RetentionRemovals $plan $new $available $computer) + if($approving){ + # Only what was shown and approved: the plan is narrowed to the list, and the checks run again on + # what is left. A version goes only when the rule, the checks and the list all agree. + $wanted=@($remove|ForEach-Object {[string]$_.id}) + $plan=@(Get-ApprovedRetentionPlan $plan $approvedIds) + $remove=@(Get-RetentionRemovals $plan $new $available $computer) + if(@($remove|Where-Object {[string]$_.id -cnotin $approvedIds -or [string]$_.id -cnotin $wanted}).Count){throw 'The clean-up would delete a version that was not approved. Nothing was deleted.'} + $retention.Approved=$true;$retention.Deferred=$wanted.Count-$remove.Count + } + [IO.File]::WriteAllText((Join-Path $RunDirectory 'clean-up-plan.json'),(ConvertTo-Json @($plan|ForEach-Object {[ordered]@{Id=$_.Id;Time=$_.Time.UtcDateTime.ToString('o');Month=$_.Month;Host=$_.Host;Paths=$_.Paths;Partial=$_.Partial;Keep=$_.Keep;Reasons=$_.Reasons}}) -Depth 6)) + $retention.Kept=@($plan|Where-Object {$_.Keep -and -not $_.Partial}).Count + if($RetentionPreview){$gone=$remove} + elseif($remove.Count){ $deleting=$true - Run-Engine (@('forget')+@($remove|ForEach-Object {$_.id})) 'DeletingOldVersions' - $retention.Removed=$remove.Count + # In batches: one command line cannot hold more than about 500 IDs. + foreach($batch in @(Split-SnapshotIdBatches @($remove|ForEach-Object {$_.id}))){Run-Engine (@('forget')+@($batch)) 'DeletingOldVersions';$gone+=@($remove|Where-Object {$_.id -cin @($batch)})} Run-Engine @('prune') 'FreeingSpace' - $retention.SpaceFreed=$true } }catch{ $retention.Error=$_.Exception.Message - # Deleting may have stopped part way: count the versions that are really gone. - if($remove.Count -and -not $retention.Removed){ + # Deleting may have stopped part way through a batch: see which versions are really gone. + if($deleting -and $gone.Count -lt $remove.Count){ try{ - Run-Engine @('snapshots','--tag','devdrive') 'ReadingHistoryAgain' + Run-Engine @('snapshots') 'ReadingHistoryAgain' $still=@(Get-Content -LiteralPath (Join-Path $RunDirectory 'ReadingHistoryAgain.jsonl') -Raw|ConvertFrom-Json|ForEach-Object {$_.id}) - $retention.Removed=@($remove|Where-Object {$_.id -notin $still}).Count - $retention.Kept=@($still).Count - }catch{} + # A list without the version just made is not the list of this repository. + if($new -cnotin $still){throw 'The saved versions could not be listed.'} + $gone=@($remove|Where-Object {$_.id -cnotin $still}) + }catch{$retention.Error+=' How many versions were deleted could not be confirmed: the count given is the least, and more may be gone. Refresh the history to see what is left.'} } } + $retention.Candidates=@($gone|ForEach-Object {[ordered]@{Id=[string]$_.id;Time=(ConvertTo-HistoryDate $_.time).UtcDateTime.ToString('o');Partial=(Test-PartialSnapshot $_);Host=[string]$_.hostname;Paths=[string[]]@($_.paths)}}) + if(-not $RetentionPreview){ + $retention.Removed=$gone.Count;$retention.PartialRemoved=@($gone|Where-Object {Test-PartialSnapshot $_}).Count + # A complete version that was to go and is still there is still kept. + if($null -ne $retention.Kept){$retention.Kept+=@($remove|Where-Object {$_.id -cnotin @($gone|ForEach-Object {$_.id}) -and -not (Test-PartialSnapshot $_)}).Count} + } $script:status.Retention=$retention # Checked whenever deleting started, also when how far it got is unknown. if($deleting){ @@ -296,7 +503,7 @@ try { } } 'Preview' {Run-Engine ($base+@('--dry-run','--')+$sources) 'Previewing'} - 'Check' {$args=@('check');if($ReadAllData){$args+='--read-data'};Run-Engine $args 'Verifying'} + 'Check' {$checkArgs=@('check');if($ReadAllData){$checkArgs+='--read-data'};Run-Engine $checkArgs 'Verifying'} 'Snapshots' {Run-Engine @('snapshots') 'ListingSnapshots'} 'Files' { if(-not $RequestPath){throw 'A history request is required'} @@ -308,13 +515,18 @@ try { if(-not $RequestPath){throw 'An explicit history deletion request is required'} $request=Get-Content -LiteralPath $RequestPath -Raw|ConvertFrom-Json if($request.Confirmation -cne 'DELETE SELECTED HISTORY'){throw 'History deletion was not confirmed'} + try{Complete-PendingPartialMarks}catch{throw "An earlier partial version is not marked yet, so nothing was deleted. $($_.Exception.Message)"} Run-Engine @('snapshots') 'ReadingHistory' $available=Get-Content -LiteralPath (Join-Path $RunDirectory 'ReadingHistory.jsonl') -Raw|ConvertFrom-Json $plan=@(Get-HistoryDeletionPlan $available @($request.SnapshotIds)) [IO.File]::WriteAllText((Join-Path $RunDirectory 'deletion-plan.json'),($plan|ConvertTo-Json -Depth 8)) - $forgetArgs=@('forget')+@($plan.id) - if($request.ReclaimSpace -eq $true){$forgetArgs+='--prune'} - Run-Engine $forgetArgs 'DeletingHistory' + # In batches: one command line cannot hold more than about 500 IDs. Space is reclaimed with the last. + $batches=@(Split-SnapshotIdBatches @($plan|ForEach-Object {$_.id})) + for($i=0;$i -lt $batches.Count;$i++){ + $forgetArgs=@('forget')+@($batches[$i]) + if($request.ReclaimSpace -eq $true -and $i -eq $batches.Count-1){$forgetArgs+='--prune'} + Run-Engine $forgetArgs 'DeletingHistory' + } Run-Engine @('check') 'CheckingStructure' } 'Restore' { @@ -338,7 +550,23 @@ try { $restoreArguments+=@('--include-file',$includeFile) } foreach($path in $RestoreInclude){$restoreArguments+=@('--include',$path)} - Run-Engine $restoreArguments 'Restoring' + try{Run-Engine $restoreArguments 'Restoring'} + catch{ + # A restore below a protected folder (for example C:\Users) can restore every file but fail to set + # the dates of the recreated parent folders; restic then ends with exit code 1 and skips its content + # check. That is a warning, not a failed restore, but the restore is reported as not verified. Which + # folders are parents is read from the restored version itself. + if($_.Exception.Data['ExitCode'] -ne 1){throw} + $failure=$_;$summary=$script:status.Progress;$warnings=@() + try{ + Run-Engine @('snapshots',$Snapshot) 'ReadingVersion' + $version=@(Get-Content -LiteralPath (Join-Path $RunDirectory 'ReadingVersion.jsonl') -Raw|ConvertFrom-Json) + if($version.Count -eq 1){$warnings=@(Get-RestoreParentWarnings ([IO.File]::ReadAllText((Join-Path $RunDirectory 'Restoring.errors.log'))) @($version[0].paths) $summary)} + }catch{} + if(-not $warnings.Count){throw $failure} + $script:status.Warnings=$warnings + } + $script:status.ContentVerified=-not $script:status.Contains('Warnings') } } $script:status.State='Completed';$script:status.UpdatedUtc=[DateTime]::UtcNow.ToString('o');[void](Write-BackupStatus $script:status (Join-Path $RunDirectory 'result.json'));Save-Status diff --git a/Backup.DriveCheck.psm1 b/Backup.DriveCheck.psm1 index 76abb91..4ed3b7b 100644 --- a/Backup.DriveCheck.psm1 +++ b/Backup.DriveCheck.psm1 @@ -8,8 +8,9 @@ # FileSystem the file system's own structures (Windows online scan) # # Pure functions over data that Invoke-DriveCheck.ps1 collects as administrator, so the rules -# are tested without hardware. Statuses: OK, Warning, Problem, Unavailable, NotRun. The one -# exception, Read-DriveCheckPathAccess, reads who may change the files the check runs. +# are tested without hardware. Statuses: OK, Warning, Problem, Unavailable, NotRun. The exceptions +# are named where they stand: the full scan's saved position, stop request and reader, the health +# tool's checksum, and Read-DriveCheckPathAccess, which reads who may change the files the check runs. Set-StrictMode -Version Latest $ErrorActionPreference='Stop' @@ -29,13 +30,17 @@ function Format-DriveHours($Hours) { if($h -ge 24*365){$text+=' (about {0:N1} years of running)' -f ($h/24/365)}elseif($h -ge 48){$text+=' (about {0:N0} days of running)' -f ($h/24)} $text } -function Format-DriveOffset([long]$Offset){if($Offset -ge 1GB){'{0:N1} GB' -f ($Offset/1GB)}else{'{0:N0} MB' -f ($Offset/1MB)}} +# Places on the disk are counted in binary units, so they are labelled that way (1 GiB = 1024 MiB). +function Format-DriveOffset([long]$Offset){if($Offset -ge 1GB){'{0:N1} GiB' -f ($Offset/1GB)}else{'{0:N0} MiB' -f ($Offset/1MB)}} function Test-DriveSmartAnswer($Smart) { - # True when the drive really answered, not only the USB box in front of it. + # True when the drive's own counters came through, not only the USB box in front of it. A box that + # passes on just the overall "passed" flag gives nothing to judge, so that is no answer; a drive + # that says it is failing always is. if(-not $Smart){return $false} - ($null -ne (Get-Field $Smart 'smart_status','passed')) -or [bool](Get-Field $Smart 'ata_smart_attributes','table') -or [bool](Get-Field $Smart 'nvme_smart_health_information_log') + ((Get-Field $Smart 'smart_status','passed') -eq $false) -or [bool](Get-Field $Smart 'ata_smart_attributes','table') -or [bool](Get-Field $Smart 'nvme_smart_health_information_log') } +function Test-DriveSmartFlagOnly($Smart){[bool]$Smart -and -not (Test-DriveSmartAnswer $Smart) -and ((Get-Field $Smart 'smart_status','passed') -eq $true)} function Get-DriveSelfTest($Smart) { # The self-test running now, if any, and from the drive's own log (newest first) the newest entry # (Last) and the newest finished short and long test (LastShort, LastLong). Each kind is judged on @@ -51,13 +56,14 @@ function Get-DriveSelfTest($Smart) { foreach($e in $ata){ $code=[int](Get-Field $e 'status','value') -shr 4 $outcome=switch($code){0{'Passed'}{$_ -in 1,2}{'Interrupted'}15{'Running'}default{'Failed'}} - [pscustomobject]@{Kind=if([string](Get-Field $e 'type','string') -match 'Extended|Long'){'Long'}else{'Short'};Outcome=$outcome;Text=[string](Get-Field $e 'status','string');Hours=Get-Field $e 'lifetime_hours'} + $kind=switch -Regex ([string](Get-Field $e 'type','string')){'Extended|Long'{'Long';break}'Short'{'Short';break}'Selective'{'Selective';break}'Conveyance'{'Conveyance';break}default{'Other'}} + [pscustomobject]@{Kind=$kind;Outcome=$outcome;Text=[string](Get-Field $e 'status','string');Hours=Get-Field $e 'lifetime_hours'} } }else{ foreach($e in $nvme){ $code=[int](Get-Field $e 'self_test_result','value') $outcome=switch($code){0{'Passed'}{$_ -in 5,6,7}{'Failed'}default{'Interrupted'}} - [pscustomobject]@{Kind=if([int](Get-Field $e 'self_test_code','value') -eq 2){'Long'}else{'Short'};Outcome=$outcome;Text=[string](Get-Field $e 'self_test_result','string');Hours=Get-Field $e 'power_on_hours'} + [pscustomobject]@{Kind=switch([int](Get-Field $e 'self_test_code','value')){2{'Long'}1{'Short'}default{'Other'}};Outcome=$outcome;Text=[string](Get-Field $e 'self_test_result','string');Hours=Get-Field $e 'power_on_hours'} } }) $last=if($entries.Count){$entries[0]}else{$null} @@ -69,14 +75,17 @@ function Get-DriveSelfTest($Smart) { $newestLong=@($entries|Where-Object {$_.Kind -eq 'Long' -and $_.Outcome -ne 'Running'})|Select-Object -First 1 $shortInterrupted=if($newestShort -and $newestShort.Outcome -eq 'Interrupted'){$newestShort}else{$null} $longInterrupted=if($newestLong -and $newestLong.Outcome -eq 'Interrupted'){$newestLong}else{$null} + # Selective, conveyance and maker-specific tests: the newest finished one of each kind, so one that + # failed is not hidden by a short test that passed after it. + $others=@(foreach($k in @($entries|ForEach-Object {$_.Kind}|Where-Object {$_ -notin 'Short','Long'}|Select-Object -Unique)){@($entries|Where-Object {$_.Kind -eq $k -and $_.Outcome -in 'Passed','Failed'})|Select-Object -First 1}) $supported=[bool](Get-Field $Smart 'ata_smart_data','capabilities','self_tests_supported') -or [bool](Get-Field $Smart 'nvme_optional_admin_commands','self_test') - [pscustomobject]@{Supported=$supported;Running=$now;Last=$last;LastShort=$lastShort;LastLong=$lastLong;ShortInterrupted=$shortInterrupted;LongInterrupted=$longInterrupted;ShortMinutes=Get-Field $Smart 'ata_smart_data','self_test','polling_minutes','short';LongMinutes=Get-Field $Smart 'ata_smart_data','self_test','polling_minutes','extended'} + [pscustomobject]@{Supported=$supported;Running=$now;Last=$last;LastShort=$lastShort;LastLong=$lastLong;Others=$others;ShortInterrupted=$shortInterrupted;LongInterrupted=$longInterrupted;ShortMinutes=Get-Field $Smart 'ata_smart_data','self_test','polling_minutes','short';LongMinutes=Get-Field $Smart 'ata_smart_data','self_test','polling_minutes','extended'} } function Get-DriveHealthFindings($Smart,[string]$Unavailable='') { # Unavailable: why there is no report (no tool installed, or the box does not pass the request on). if(-not (Test-DriveSmartAnswer $Smart)){ - $why=if($Unavailable){$Unavailable}else{'The drive did not answer. Its USB box or adapter probably does not pass the request on.'} + $why=if(Test-DriveSmartFlagOnly $Smart){'The USB box passed on only the drive''s overall "passed" flag, not its counters or self-tests, so the drive''s health cannot be judged through this box.'}elseif($Unavailable){$Unavailable}else{'The drive did not answer. Its USB box or adapter probably does not pass the request on.'} return New-DriveGroup 'Unavailable' @($why) } $bad=[Collections.Generic.List[string]]::new();$warn=[Collections.Generic.List[string]]::new();$info=[Collections.Generic.List[string]]::new() @@ -117,13 +126,13 @@ function Get-DriveHealthFindings($Smart,[string]$Unavailable='') { if($t.Running){$info.Add("A self-test is running now, about $($t.Running.Remaining)% left.")} # The newest finished test of each kind counts, so a quick check's short test cannot hide a long # test that failed: that stays a problem until a newer long test passes. - $tests=@(@($t.LongInterrupted,$t.LastLong,$t.ShortInterrupted,$t.LastShort)|Where-Object {$_}) + $tests=@(@($t.LongInterrupted,$t.LastLong,$t.ShortInterrupted,$t.LastShort)+@($t.Others)|Where-Object {$_}) foreach($test in $tests){ - $kind=if($test.Kind -eq 'Long'){'Long self-test'}else{'Short self-test'} + $kind=switch($test.Kind){'Long'{'Long self-test'}'Short'{'Short self-test'}'Other'{'Self-test of another kind'}default{"$($test.Kind) self-test"}} $when=if($null -ne $test.Hours -and $null -ne $hours){$ago=[double]$hours-[double]$test.Hours;if($ago -lt 1){', just now'}else{' {0:N0} running hours ago' -f $ago}}else{''} switch($test.Outcome){ 'Passed'{$info.Add("${kind}: passed$when.")} - 'Failed'{$bad.Add("${kind}: failed$when ($($test.Text)).$(if($test.Kind -eq 'Long'){' This counts until a newer long self-test passes.'})")} + 'Failed'{$bad.Add("${kind}: failed$when ($($test.Text)).$(if($test.Kind -eq 'Long'){' This counts until a newer long self-test passes.'}elseif($test.Kind -ne 'Short'){' This counts until a newer self-test of the same kind passes.'})")} 'Interrupted'{$info.Add("${kind}: did not finish$when ($($test.Text)). Run it again when the drive can stay on.")} } } @@ -135,27 +144,62 @@ function Get-DriveHealthFindings($Smart,[string]$Unavailable='') { function Get-DriveSurfaceFindings($Sample,$Scan) { # Sample: the start/middle/end reads of the last quick check. Scan: the full scan record. $bad=[Collections.Generic.List[string]]::new();$warn=[Collections.Generic.List[string]]::new();$info=[Collections.Generic.List[string]]::new() - if($Sample){ - $regions=@($Sample.Regions) + # A quick check stopped before its first read leaves a sample without regions: that is no sample. + $regions=@(@(Get-Field $Sample 'Regions')|Where-Object {$_}) + if($regions.Count){ $speeds=($regions|ForEach-Object {'{0:N0}' -f $_.MBps}) -join ', ' - $errors=($regions|Measure-Object -Property BadCount -Sum).Sum + $errors=[long]0;$again=[long]0;foreach($r in $regions){$errors+=[long](Get-Field $r 'BadCount');$again+=[long](Get-Field $r 'RecoveredCount')} if($errors){$bad.Add("Sample reads at the start, middle and end: $errors unreadable areas.")} else{$info.Add("Sample reads at the start, middle and end: no errors ($speeds MB/s).")} + if($again){$warn.Add("Sample reads: $again places could be read only at the second try. That is often an early sign of damage, or of a loose cable.")} } - if($Scan){ + $known=Test-DriveScanRecord $Scan + $cut=$known -and -not $Scan.Complete -and [bool](Get-Field $Scan 'Disconnected') + if($known){ $pct=if($Scan.Size){[math]::Floor(100*[double]$Scan.NextOffset/[double]$Scan.Size)}else{0} - $first=@($Scan.Bad|Select-Object -First 3|ForEach-Object {Format-DriveOffset $_.Offset}) -join ', ' - if($Scan.BadCount){$bad.Add("Full scan: $($Scan.BadCount) areas could not be read (first at $first). Do not trust this drive with your only copy.")} - if($Scan.VerySlowCount){$warn.Add("Full scan: $($Scan.VerySlowCount) places took over a second to read. The drive had to try hard there; that is often an early sign of damage.")} - # Some slow places come from Windows or other programs using the drive at the same moment; many do not. - elseif($Scan.SlowCount -gt [math]::Max(20,[double]$Scan.NextOffset/4MB/500)){$warn.Add("Full scan: $($Scan.SlowCount) places were much slower than the rest.")} + $found=Get-DriveScanLines $Scan 'Full scan' '' + foreach($l in $found.Bad){$bad.Add($l)};foreach($l in $found.Warn){$warn.Add($l)} if($Scan.Complete){ - if(-not $Scan.BadCount){$info.Add("Full scan: finished $(Format-DriveAge $Scan.Finished), every part could be read.")} - }else{$info.Add("Full scan: stopped at $pct%. Continue it any time; it goes on from there.")} + if(-not $found.Bad.Count){$info.Add("Full scan: finished $(Format-DriveAge (Get-Field $Scan 'Finished')), every part could be read.")} + }else{ + # A scan that ended because the drive stopped answering says neither that the drive is fine nor + # that it is damaged: nothing was recorded at that place, and reading the disk is not checked (below). + if($cut){$info.Add("Full scan: not completed. The drive stopped answering at $pct% ($(Format-DriveOffset ([long](Get-Field $Scan 'DisconnectOffset')))); it may have been disconnected (cable, USB box, power). Nothing was recorded as damaged there. Check the connection, then continue the scan; it goes on from that place.")} + else{$info.Add("Full scan: stopped at $pct%. Continue it any time; it goes on from there.")} + # A scan that has not finished says nothing about the parts it has not reached, so what the last + # finished scan found still counts until this one is through. + $before=Get-Field $Scan 'Previous' + if(Test-DriveScanRecord $before){ + $found=Get-DriveScanLines $before 'Earlier full scan' ' This counts until a new full scan finishes.' + foreach($l in $found.Bad){$bad.Add($l)};foreach($l in $found.Warn){$warn.Add($l)} + } + } }else{$info.Add('Full scan: not run yet. It reads the whole drive and takes hours.')} - $status=if($bad.Count){'Problem'}elseif($warn.Count){'Warning'}elseif($Sample -or ($Scan -and $Scan.Complete)){'OK'}else{'NotRun'} + # What was really found (here, in the samples, or by the last finished scan) counts as always. Without + # such a finding, a scan the drive dropped out of is not a verdict, whatever the samples read. + $status=if($bad.Count){'Problem'}elseif($warn.Count){'Warning'}elseif($cut){'NotRun'}elseif($regions.Count -or ($known -and $Scan.Complete)){'OK'}else{'NotRun'} New-DriveGroup $status @(@($bad)+@($warn)+@($info)) } +function Test-DriveScanRecord($Scan) { + # A scan record with the fields every version wrote. Newer fields (RecoveredCount, Previous) are + # read with Get-Field, so a record saved by an older version keeps working. + $Scan -is [pscustomobject] -and $Scan -isnot [Array] -and $null -ne (Get-Field $Scan 'Size') -and $null -ne (Get-Field $Scan 'NextOffset') -and $null -ne (Get-Field $Scan 'Complete') +} +function Get-DriveScanLines($Scan,[string]$Name,[string]$Until) { + # What one scan record found, as problem lines (Bad) and warning lines (Warn). + $bad=@();$warn=@() + $n=[long](Get-Field $Scan 'BadCount');$very=[long](Get-Field $Scan 'VerySlowCount');$slow=[long](Get-Field $Scan 'SlowCount');$again=[long](Get-Field $Scan 'RecoveredCount') + if($n){ + $first=@(@(Get-Field $Scan 'Bad')|Where-Object {$_}|Select-Object -First 3|ForEach-Object {Format-DriveOffset $_.Offset}) -join ', ' + $bad+="${Name}: $n areas could not be read (first at $first). Do not trust this drive with your only copy.$Until" + } + # A read that failed and then worked in smaller pieces: the drive, or the way to it, needed a second try. + if($again){$warn+="${Name}: $again places could be read only at the second try. That is often an early sign of damage, or of a loose cable.$Until"} + if($very){$warn+="${Name}: $very places took over a second to read. The drive had to try hard there; that is often an early sign of damage.$Until"} + # Some slow places come from Windows or other programs using the drive at the same moment; many do not. + elseif($slow -gt [math]::Max(20,[double]$Scan.NextOffset/4MB/500)){$warn+="${Name}: $slow places were much slower than the rest.$Until"} + [pscustomobject]@{Bad=@($bad);Warn=@($warn)} +} # Windows event log entries about disks (warnings and errors only). Kind decides the group; # Problem/Warning the verdict. @@ -172,24 +216,27 @@ $script:DiskEvents=@( @{Provider='Ntfs';Id=55;Group='FileSystem';Status='Problem';Text='Windows found damage in the file system'} @{Provider='Ntfs';Id=140;Group='FileSystem';Status='Warning';Text='a write to the file system failed'} ) -$script:ResetProviders=@('uaspstor','USBSTOR','stornvme','storahci') function Get-DriveConnectionKey([string]$Serial,[string]$Model,$Capacity) { # The same drive under any disk number: its serial number, or model and size when it reports none. $s=$Serial.Trim().TrimEnd('.').Trim() if($s -and $s -ne 'NULL'){'S:'+$s}else{'M:'+$Model.Trim()+'|'+$Capacity} } -function Get-DriveEventFindings($Events,[int]$DiskNumber,[string]$Letter,[bool]$Usb,$Connections=@()) { +function Get-DriveEventFindings($Events,[int]$DiskNumber,[string]$Letter,[bool]$Usb,$Connections=@(),[string]$BusType='') { # Events name the disk as \Device\Harddisk\ (in their data, in every Windows language) and # volumes by letter. Disk numbers move between drives as they are plugged in, so an entry counts # only when this drive held that number at the time. Connections: Windows' record of every drive # arrival (Time, DiskNumber, Key); the newest one for this number is this drive's own arrival. # Without that record only the number is compared. Letters and controller resets (129) name no - # disk, so they count only after this drive arrived. + # disk, so they count only after this drive arrived, and a reset only when it was written by the + # driver of this drive's own kind of connection (BusType): a reset of the internal drive says + # nothing about a USB drive. Without BusType, Usb alone decides as far as it can. + $isUsb=if($BusType){$BusType -eq 'USB'}else{$Usb} + $resetFrom=@(switch -Regex ($BusType){'^USB$'{'uaspstor','USBSTOR';break}'^NVMe$'{'stornvme';break}'^(SATA|ATA|ATAPI|RAID)$'{'storahci';break}'^$'{if($Usb){'uaspstor','USBSTOR'}else{'stornvme','storahci'};break}}) $byNumber=@{};foreach($c in @($Connections|Where-Object {$_}|Sort-Object {[datetime]$_.Time})){$n=[int]$c.DiskNumber;if(-not $byNumber.ContainsKey($n)){$byNumber[$n]=[Collections.Generic.List[object]]::new()};$byNumber[$n].Add($c)} $own=if($byNumber.ContainsKey($DiskNumber)){$byNumber[$DiskNumber][$byNumber[$DiskNumber].Count-1]}else{$null} $since=if($own){[datetime]$own.Time}else{[datetime]::MinValue} - $found=@{} + $found=@{};$unmatched=0 foreach($e in @($Events)){ if(-not $e){continue} $text=[string]$e.Message;$when=[datetime]$e.TimeCreated @@ -200,12 +247,15 @@ function Get-DriveEventFindings($Events,[int]$DiskNumber,[string]$Letter,[bool]$ if($n -ne $DiskNumber){continue} if($own -and $when -lt $since){ $then=$null;foreach($c in $byNumber[$n]){if([datetime]$c.Time -le $when){$then=$c}else{break}} - if(-not $then -or $then.Key -ne $own.Key){continue} + # Older than every arrival Windows still records for this number: it cannot be said which drive + # the entry was about. It is left out of every verdict, and counted for a note of its own. + if(-not $then){$unmatched++;continue} + if($then.Key -ne $own.Key){continue} } }elseif($Letter -and $when -ge $since -and $text -match "(^|[^A-Za-z])$([regex]::Escape($Letter.ToUpperInvariant())):"){} else{continue} - }elseif([int]$e.Id -eq 129 -and $e.ProviderName -in $script:ResetProviders -and $when -ge $since -and ($Usb -or $e.ProviderName -notin 'uaspstor','USBSTOR')){ - $kind=@{Group='Connection';Status='Warning';Text=if($Usb){'a USB drive stopped answering and Windows reset the connection'}else{'the connection to a drive was reset'}} + }elseif([int]$e.Id -eq 129 -and $e.ProviderName -in $resetFrom -and $when -ge $since){ + $kind=@{Group='Connection';Status='Warning';Text=if($isUsb){'a USB drive stopped answering and Windows reset the connection'}else{'the connection to a drive was reset'}} }else{continue} $key=$kind.Text if(-not $found.ContainsKey($key)){$found[$key]=[pscustomobject]@{Group=$kind.Group;Status=$kind.Status;Text=$kind.Text;Count=0;Last=$null}} @@ -215,25 +265,35 @@ function Get-DriveEventFindings($Events,[int]$DiskNumber,[string]$Letter,[bool]$ $times=if($f.Count -eq 1){'once'}else{"$($f.Count) times"} [pscustomobject]@{Group=$f.Group;Status=$f.Status;Line=[string]::Format([Globalization.CultureInfo]::GetCultureInfo('en-GB'),'Windows log: {0}, {1} in the last 30 days (last on {2:d MMM HH:mm}).',($f.Text.Substring(0,1).ToUpper()+$f.Text.Substring(1)),$times,$f.Last)} } + # A note, not a finding (Info): Merge-DriveEvents shows it after the group's own lines and never + # lets it change a verdict. + if($unmatched){ + $what=if($unmatched -eq 1){'1 older disk entry could not be matched to a drive (Windows no longer has a record of which drive had that disk number then); it is not counted for or against this drive.'}else{"$unmatched older disk entries could not be matched to a drive (Windows no longer has a record of which drive had that disk number then); they are not counted for or against this drive."} + [pscustomobject]@{Group='Connection';Status='NotRun';Info=$true;Line='Windows log: '+$what} + } } function Get-DriveConnectionFindings($Link,[string]$BusType,$Smart,$Counters) { $warn=[Collections.Generic.List[string]]::new();$info=[Collections.Generic.List[string]]::new() + # checked: something about the connection was really read. A line saying that the link speed + # could not be read is no check, so alone it gives "not available", never "no problems". + $checked=$false if($BusType -eq 'USB'){ if($Link -and $Link.PSObject.Properties['Speed']){ + $checked=$true if(-not $Link.SuperSpeedActive){$warn.Add("Connected at USB 2 speed ($($Link.Speed)). Use a USB 3 port and the drive's own cable, without a hub.")} elseif([string]$Link.Speed -match 'SuperSpeedPlus'){$info.Add('Connected at full speed (10 Gbit/s or more).')} else{$info.Add('Connected at 5 Gbit/s. That is fine for most drives.')} }else{$info.Add('The USB link speed could not be read.')} - }elseif($BusType){$info.Add("Connected directly ($BusType), not through USB.")} + }elseif($BusType){$checked=$true;$info.Add("Connected directly ($BusType), not through USB.")} $crc=$null;foreach($a in @(Get-Field $Smart 'ata_smart_attributes','table')){if($a -and [int]$a.id -eq 199){$crc=[long](Get-Field $a 'raw','value')}} - if($null -ne $crc){if($crc -gt 0){$warn.Add("The drive counted $crc damaged transfers over its life. They usually come from a cable or port, not the drive; if this number grows, change the cable.")}else{$info.Add('Damaged transfers counted by the drive: none.')}} + if($null -ne $crc){$checked=$true;if($crc -gt 0){$warn.Add("The drive counted $crc damaged transfers over its life. They usually come from a cable or port, not the drive; if this number grows, change the cable.")}else{$info.Add('Damaged transfers counted by the drive: none.')}} if($Counters){ $failed=[long]$Counters.ReadErrorsUncorrected+[long]$Counters.WriteErrorsUncorrected if($failed -gt 0){$warn.Add("Windows counted $failed reads or writes that failed on this drive.")} - elseif($null -ne $Counters.ReadErrorsTotal){$info.Add('Failed reads and writes counted by Windows: none.')} + elseif($null -ne $Counters.ReadErrorsTotal){$checked=$true;$info.Add('Failed reads and writes counted by Windows: none.')} } - $status=if($warn.Count){'Warning'}elseif($info.Count){'OK'}else{'Unavailable'} + $status=if($warn.Count){'Warning'}elseif($checked){'OK'}else{'Unavailable'} New-DriveGroup $status @(@($warn)+@($info)) } @@ -253,10 +313,14 @@ function Get-DriveFileSystemFindings([string]$Letter,[string]$FileSystem,[string function Merge-DriveEvents($Group,$Events,[string]$Name) { # Adds the log lines that belong to this group; a log entry can only make the verdict worse. + # A note (Info, see Get-DriveEventFindings) is no entry about this drive: it stands after the + # group's own lines and has no say in the verdict. $mine=@($Events|Where-Object {$_ -and $_.Group -eq $Name}) if(-not $mine.Count){return $Group} - $status=Get-WorstStatus (@($Group.Status)+@($mine.Status)) - New-DriveGroup $status (@($mine.Line)+@($Group.Lines)) + $notes=@($mine|Where-Object {$_.PSObject.Properties['Info'] -and $_.Info}) + $entries=@($mine|Where-Object {-not($_.PSObject.Properties['Info'] -and $_.Info)}) + $status=Get-WorstStatus (@($Group.Status)+@($entries|ForEach-Object {$_.Status})) + New-DriveGroup $status (@($entries|ForEach-Object {$_.Line})+@($Group.Lines)+@($notes|ForEach-Object {$_.Line})) } function Get-DriveCheckView($Report,$Scan) { @@ -268,7 +332,207 @@ function Get-DriveCheckView($Report,$Scan) { $connection=if($Report){$Report.Connection}else{New-DriveGroup 'NotRun' @('Not checked yet.')} $fs=if($Report){$Report.FileSystem}else{New-DriveGroup 'NotRun' @('Not checked yet.')} $groups=[ordered]@{Health=$health;Surface=$surface;Connection=$connection;FileSystem=$fs} - [pscustomobject]@{Groups=$groups;Overall=Get-WorstStatus @($groups.Values.Status);When=if($Report){$Report.Finished}else{$null}} + $overall=Get-WorstStatus @($groups.Values.Status) + # Unchecked: the parts with no verdict. Summary: the one line for the overview, which must not read + # "no problems" as if the whole drive had been checked when a part could not be. + $plain=@{Health='the drive''s own health report';Surface='reading the disk';Connection='the connection';FileSystem='the file system'} + $unchecked=@($groups.Keys|Where-Object {$groups[$_].Status -in 'Unavailable','NotRun'}|ForEach-Object {$plain[$_]}) + $summary=switch($overall){'Problem'{'problem found'}'Warning'{'worth a look'}'OK'{if($unchecked.Count){'no problems in what could be checked; not checked: '+($unchecked -join ', ')}else{'no problems'}}default{'not checked yet'}} + [pscustomobject]@{Groups=$groups;Overall=$overall;Unchecked=$unchecked;Summary=$summary;When=if($Report){$Report.Finished}else{$null}} +} + +# --- The full scan: its saved position, stopping, and the reader ---------------------------------- +# These read or move files (and the reader reads a disk), unlike the rules above. They live here so +# that they are tested without a drive; Invoke-DriveCheck.ps1 only puts them together. + +function New-DriveScanState([string]$Identity,[long]$Size,$Previous=$null) { + # Previous: what the last scan of this drive found (Read-DriveScanState), kept until this one finishes. + # Disconnected: the last stretch ended because the drive stopped answering (see the reader), at DisconnectOffset. + [pscustomobject]@{Identity=$Identity;Size=$Size;NextOffset=[long]0;Complete=$false;Started=(Get-Date).ToString('o');Finished=$null;Seconds=0.0;BadCount=0;VerySlowCount=0;SlowCount=0;RecoveredCount=0;Bad=@();Slow=@();Recovered=@();Previous=$Previous;Disconnected=$false;DisconnectOffset=$null;DisconnectCode=$null} +} +function Update-DriveScanState($State,$Result,[long]$Position) { + # Adds what the reader found in one stretch of the full scan, read from Position, to the scan record. + # A stretch cut short (a stop request, or a drive that stopped answering) ends at its last whole + # chunk, and the scan goes on from there. How the stretch ended is kept: after a drive that stopped + # answering the record says so, with the place, until a later stretch reads again. + foreach($p in 'Disconnected','DisconnectOffset','DisconnectCode'){if(-not $State.PSObject.Properties[$p]){$State|Add-Member -NotePropertyName $p -NotePropertyValue $null}} + $State.BadCount+=$Result.BadCount;$State.SlowCount+=$Result.SlowCount;$State.VerySlowCount+=$Result.VerySlowCount;$State.RecoveredCount+=$Result.RecoveredCount;$State.Seconds=[double]$State.Seconds+[double]$Result.Seconds + $State.Bad=@(@($State.Bad)+@($Result.Bad)|Where-Object {$_}|Select-Object -First 200) + $State.Slow=@(@($State.Slow)+@($Result.Slow)|Where-Object {$_}|Select-Object -First 200) + $State.Recovered=@(@($State.Recovered)+@($Result.Recovered)|Where-Object {$_}|Select-Object -First 200) + $State.NextOffset=$Position+[long]$Result.Bytes + $cut=[bool](Get-Field $Result 'Disconnected') + $State.Disconnected=$cut;$State.DisconnectOffset=if($cut){[long]$State.NextOffset}else{$null};$State.DisconnectCode=if($cut){Get-Field $Result 'DisconnectCode'}else{$null} +} +function Read-DriveScanState([string]$Path,[string]$Identity) { + # Reads surface.json before a scan starts or continues. State: the unfinished scan of this very + # drive, to continue; $null starts a new one. Previous: the findings of this drive's finished scan, + # for the new record, so that starting again does not erase known bad areas. SetAside: where a file + # that cannot be read was moved. It is never deleted, and never stops the scan from starting. + $r=[pscustomobject]@{State=$null;Previous=$null;SetAside=$null} + if(-not (Test-Path -LiteralPath $Path)){return $r} + $old=$null;try{$old=Get-Content -LiteralPath $Path -Raw|ConvertFrom-Json}catch{} + $ok=(Test-DriveScanRecord $old) -and (Get-Field $old 'Identity') -is [string] + if($ok){try{$ok=[double]$old.Size -ge 0 -and [double]$old.NextOffset -ge 0 -and [double]$old.NextOffset -le [double]$old.Size}catch{$ok=$false}} + if(-not $ok){ + $aside=Join-Path (Split-Path -Parent $Path) ('surface.damaged-'+(Get-Date -Format 'yyyyMMdd-HHmmss')+'.json') + for($i=2;Test-Path -LiteralPath $aside;$i++){$aside=Join-Path (Split-Path -Parent $Path) ('surface.damaged-'+(Get-Date -Format 'yyyyMMdd-HHmmss')+"-$i.json")} + Move-Item -LiteralPath $Path -Destination $aside + $r.SetAside=$aside;return $r + } + if($old.Identity -ne $Identity){return $r} + if($old.Complete){ + $r.Previous=[pscustomobject]@{Size=$old.Size;NextOffset=$old.NextOffset;Complete=$true;Finished=Get-Field $old 'Finished';BadCount=[long](Get-Field $old 'BadCount');VerySlowCount=[long](Get-Field $old 'VerySlowCount');SlowCount=[long](Get-Field $old 'SlowCount');RecoveredCount=[long](Get-Field $old 'RecoveredCount');Bad=@(@(Get-Field $old 'Bad')|Where-Object {$_}|Select-Object -First 200)} + return $r + } + # An unfinished scan saved by an older version lacks the newer fields. + $new=New-DriveScanState $Identity ([long]$old.Size) + foreach($p in $new.PSObject.Properties){if(-not $old.PSObject.Properties[$p.Name]){$old|Add-Member -NotePropertyName $p.Name -NotePropertyValue $p.Value}} + $r.State=$old;$r +} +function Test-DriveStopRequest([string]$Path,[datetime]$Since) { + # Since: when Windows started the check's process. A stop.request left by an earlier run is older + # and does not count; one made while this check was still starting does. + try{(Test-Path -LiteralPath $Path) -and (Get-Item -LiteralPath $Path).LastWriteTime -ge $Since}catch{$false} +} + +$script:ScannerSource=@' +using System; using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; using System.Globalization; using System.IO; using System.Runtime.InteropServices; using System.Text; using Microsoft.Win32.SafeHandles; +public static class BackupDriveScanner { + [DllImport("kernel32.dll",CharSet=CharSet.Unicode,SetLastError=true)] static extern SafeFileHandle CreateFile(string n,uint a,uint s,IntPtr sa,uint d,uint f,IntPtr t); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool ReadFile(SafeFileHandle h,IntPtr b,uint n,out uint done,IntPtr o); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool SetFilePointerEx(SafeFileHandle h,long d,out long np,uint m); + [DllImport("kernel32.dll",SetLastError=true)] static extern IntPtr VirtualAlloc(IntPtr a,UIntPtr n,uint t,uint p); + [DllImport("kernel32.dll")] static extern bool VirtualFree(IntPtr a,UIntPtr n,uint t); + [DllImport("kernel32.dll")] static extern uint SetThreadExecutionState(uint flags); + // These mean the drive is gone (unplugged, asleep, box reset), not that an area is bad. + static readonly int[] Gone={2,3,6,21,55,433,1167}; + // These two come from a place the drive cannot read and from a drive that stopped answering (a loose + // cable, a USB box that reset, power) alike: 31, a device is not functioning; 1117, I/O device error. + static readonly int[] Unclear={31,1117}; + // For tests only: the error one read (position, length) should end with instead of being read; 0 reads it. + public static Func Fault; + // Whether the drive still answers: one piece is read at each of these places (below 0: none) until one reads. + static bool Answers(SafeFileHandle h,IntPtr buf,int piece,long[] places){ + foreach(long p in places){if(p>=0&&ReadAt(h,buf,p,piece)==0)return true;} + return false; + } + static string N(double v){return v.ToString("0.###",CultureInfo.InvariantCulture);} + public static void KeepAwake(bool on){SetThreadExecutionState(on?0x80000001u:0x80000000u);} + static int ReadAt(SafeFileHandle h,IntPtr buf,long pos,int n){ + if(Fault!=null){int f=Fault(pos,n);if(f!=0)return f;} + long np;if(!SetFilePointerEx(h,pos,out np,0))return Marshal.GetLastWin32Error(); + uint got;if(!ReadFile(h,buf,(uint)n,out got,IntPtr.Zero))return Marshal.GetLastWin32Error(); + return got==(uint)n?0:38; + } + // A stop.request written since the check's process started (stopSince, local ticks). + static bool StopAsked(string file,long since){ + try{return !string.IsNullOrEmpty(file)&&File.Exists(file)&&File.GetLastWriteTime(file).Ticks>=since;}catch(Exception){return false;} + } + // Reads [start, start+length) of a physical disk without the Windows cache, 4 MiB at a time. + // A chunk that fails is read again in 64 KiB pieces so the report names the damaged place; one + // whose pieces then all read is counted as Recovered (the drive needed a second try). + // VerySlow: a chunk over verySlowMs (the drive retried hard). Slow: over 10x the median and 250 ms. + // A stop request is looked for before every chunk and every piece, so a stretch on a failing drive, + // where one read can take many seconds, does not have to finish first. Bytes then ends at the last + // whole chunk, and nothing of the chunk that was cut short is counted. + // Disconnected: a chunk in which pieces failed with one of the two unclear errors (31, 1117), on a + // drive that then answers nowhere any more: not at the last place that read, not at its start, not + // further on. That is a drive that went away, not thousands of bad areas. The scan ends there like + // a stop: Bytes ends before that chunk, nothing of it is recorded as bad (it is read again when the + // scan continues), and DisconnectCode holds the error. The same when, after nothing but such errors + // in the chunk, Windows says outright that the drive is gone. A drive that still answers elsewhere + // has a damaged place there, or had a short break. Where only some pieces failed, they are recorded + // as bad areas, whatever the error. Where the whole chunk failed with nothing but unclear errors, + // the pieces are read once more, and what fails again while the drive again answers elsewhere is + // recorded as bad areas, so that a scan can get past a damaged stretch instead of ending at it every + // time it is continued. + public static string Read(string device,long start,long length,int sector,double verySlowMs,string stopFile,long stopSince){ + const int chunk=4*1024*1024;int piece=Math.Max(sector,64*1024); + IntPtr buf=VirtualAlloc(IntPtr.Zero,(UIntPtr)chunk,0x3000,4);if(buf==IntPtr.Zero)throw new Win32Exception(); + var times=new List();var where=new List();var bad=new StringBuilder();var again=new StringBuilder();int badCount=0,againCount=0,cutCode=0;long done=0;bool stopped=false,cut=false;var total=Stopwatch.StartNew(); + try{ + using(var h=CreateFile(device,0x80000000,3,IntPtr.Zero,3,0x20000000,IntPtr.Zero)){ + if(h.IsInvalid)throw new Win32Exception(Marshal.GetLastWin32Error()); + long end=start+length; + for(long pos=start;pos=0)throw new Win32Exception(err,"The drive stopped answering ("+err+"). Check that it is still connected."); + var found=new List(); + for(int look=0;look<2;look++){ + found.Clear();int pieces=0,unclear=0,first=0;long good=-1; + for(long p=pos;p=0){ + // Gone after nothing but unclear errors in this chunk: the drive went away while it was read. + if(found.Count==pieces-1&&unclear==found.Count&&(unclear>0||Array.IndexOf(Unclear,err)>=0)){cut=true;cutCode=e;break;} + throw new Win32Exception(e,"The drive stopped answering ("+e+"). Check that it is still connected."); + } + if(Array.IndexOf(Unclear,e)>=0){unclear++;if(first==0)first=e;} + found.Add("{\"Offset\":"+p+",\"Length\":"+m+",\"Code\":"+e+"}"); + } + if(stopped||cut||unclear==0)break; + // A piece failed with an unclear error. Does the drive still answer anywhere: at the last + // piece of this chunk that read, at the last chunk that read, at its start, further on? + long[] places={good,where.Count>0?where[where.Count-1]:-1,pos>=chunk?0:-1,pos+n+piece<=end?pos+n:-1}; + if(!Answers(h,buf,piece,places)){cut=true;cutCode=first;break;} + // It does. Only a chunk that failed whole, with nothing but such errors, is read once more. + if(found.Count0)again.Append(',');again.Append("{\"Offset\":"+pos+",\"Code\":"+err+"}");}againCount++;} + foreach(string f in found){if(badCount<200){if(bad.Length>0)bad.Append(',');bad.Append(f);}badCount++;} + } + done+=n; + } + } + }finally{VirtualFree(buf,UIntPtr.Zero,0x8000);} + double[] sorted=times.ToArray();Array.Sort(sorted);double median=sorted.Length>0?sorted[sorted.Length/2]:0;double max=sorted.Length>0?sorted[sorted.Length-1]:0; + var slow=new StringBuilder();int slowCount=0,verySlow=0;double slowMs=Math.Max(250,10*median); + for(int i=0;i=verySlowMs;if(very)verySlow++; + if(very||times[i]>=slowMs){if(slowCount<200){if(slow.Length>0)slow.Append(',');slow.Append("{\"Offset\":"+where[i]+",\"Ms\":"+N(times[i])+"}");}slowCount++;} + } + return "{\"Bytes\":"+done+",\"Stopped\":"+(stopped?"true":"false")+",\"Disconnected\":"+(cut?"true":"false")+",\"DisconnectCode\":"+cutCode+",\"Seconds\":"+N(total.Elapsed.TotalSeconds)+",\"MedianMs\":"+N(median)+",\"MaxMs\":"+N(max)+",\"BadCount\":"+badCount+",\"Bad\":["+bad+"],\"RecoveredCount\":"+againCount+",\"Recovered\":["+again+"],\"SlowCount\":"+slowCount+",\"VerySlowCount\":"+verySlow+",\"Slow\":["+slow+"]}"; + } +} +'@ +function Add-DriveScannerType{if(-not ('BackupDriveScanner' -as [type])){Add-Type -TypeDefinition $script:ScannerSource}} + +function Open-DriveCheckSmartTool([string]$Path,[string]$ToolHash=(Get-DriveCheckSmartToolHash),[string]$DatabaseHash=(Get-DriveCheckDatabaseHash)) { + # smartctl runs as administrator, so whichever copy is asked for, the bundled one or another path, + # must be the official 7.5 build. Hold: the file kept open (others may only read it) for as long as + # the check runs, so the file that was checked is the file that runs. Problem: why it will not run. + # (-ToolHash and -DatabaseHash are for the tests, which have no real smartctl.) + $no={param($why)[pscustomobject]@{Problem=$why;Hold=$null;DatabaseHold=$null}} + if(-not (Test-Path -LiteralPath $Path -PathType Leaf)){return & $no 'The health tool is not installed. Use "Install health tool" on this page, then check again.'} + $hold=[IO.File]::Open($Path,'Open','Read','Read') + $official=try{(Get-FileHash -InputStream $hold -Algorithm SHA256).Hash -eq $ToolHash}catch{$false} + if(-not $official){ + $hold.Dispose() + return & $no 'The health tool is not the official smartctl 7.5, so it was not run. Use "Install health tool" on this page again, then check again.' + } + # What lies beside it is loaded by it, as administrator too. A library there would be loaded in + # place of Windows' own, and the official installation puts none there, so any is refused. The + # drive database (drivedb.h) is read at every start: it must be the installer's, and is held open + # the same way. Without that file smartctl uses the database built into it. + $folder=[IO.Path]::GetDirectoryName([IO.Path]::GetFullPath($Path)) + $libraries=@(Get-ChildItem -LiteralPath $folder -Filter '*.dll' -File -Force -ErrorAction SilentlyContinue) + if($libraries.Count){$hold.Dispose();return & $no ("The health tool was not run, because its folder holds a file that does not belong to it: $(@($libraries|ForEach-Object Name) -join ', '). Remove it, or use `"Install health tool`" on this page again.")} + $dbHold=$null;$db=Join-Path $folder 'drivedb.h' + if(Test-Path -LiteralPath $db -PathType Leaf){ + # A database that cannot be opened or read is as unknown as a changed one; nothing stays held open. + $same=try{$dbHold=[IO.File]::Open($db,'Open','Read','Read');(Get-FileHash -InputStream $dbHold -Algorithm SHA256).Hash -eq $DatabaseHash}catch{$false} + if(-not $same){ + if($dbHold){$dbHold.Dispose()};$hold.Dispose() + return & $no 'The health tool was not run, because the drive database beside it (drivedb.h) could not be read or is not the one from the official 7.5 installer. Use "Install health tool" on this page again, then check again.' + } + } + [pscustomobject]@{Problem='';Hold=$hold;DatabaseHold=$dbHold} } # --- Running as administrator ----------------------------------------------------------------- @@ -278,6 +542,16 @@ function Get-DriveCheckView($Report,$Scan) { # SHA-256 of bin\smartctl.exe (64-bit) inside the official smartmontools 7.5 installer, whose own # hash Install-DriveTools.ps1 checks before it runs it. function Get-DriveCheckSmartToolHash {'B5DB94E5082C042BE44994B7A4FA8F7B5C8E713B2AB1C9A560D8F7A7995EA27D'} +# SHA-256 of bin\drivedb.h as the 7.5 Windows installer puts it beside smartctl.exe. The hash was +# taken from an installation made by that installer (the one whose own hash Install-DriveTools.ps1 +# checks); it was not derived again by unpacking the installer. The content of that file was compared +# with drivedb.h in the official source archive (smartmontools-7.5.tar.gz from the project's GitHub +# release, SHA-256 690B83CA331378DA9EA0D9D61008C4B22DDE391387B9BBAD7F29387F2595F76E): all 6,941 lines +# are the same except the version line, where the Windows file has an unexpanded "$Id$" and the source +# has the expanded revision text, and the Windows file has CRLF line ends. So this hash is not the +# hash of the source file. A database updated later (update-smart-drivedb) no longer matches, and the +# health report then asks for a reinstall. +function Get-DriveCheckDatabaseHash {'DD39C6A520D38895DA61923FE26FE7C9C5EB3F42325F2E4477A06ED7A61966D0'} function Get-DriveCheckPathRisks($Items,[string]$UserSid) { # Pure rule over what Read-DriveCheckPathAccess reads. Items: Path, Role, Owner (a SID) and Rules @@ -328,4 +602,4 @@ function Format-DriveAge($When) { if($span.TotalMinutes -lt 2){'just now'}elseif($span.TotalHours -lt 1){'{0:N0} minutes ago' -f $span.TotalMinutes}elseif($span.TotalHours -lt 36){'{0:N0} hours ago' -f $span.TotalHours}else{'{0:N0} days ago' -f $span.TotalDays} } -Export-ModuleMember -Function Get-Field,New-DriveGroup,Get-WorstStatus,Test-DriveSmartAnswer,Get-DriveSelfTest,Get-DriveHealthFindings,Get-DriveSurfaceFindings,Get-DriveConnectionKey,Get-DriveEventFindings,Get-DriveConnectionFindings,Get-DriveFileSystemFindings,Merge-DriveEvents,Get-DriveCheckView,Format-DriveOffset,Format-DriveAge,Get-DriveCheckSmartToolHash,Get-DriveCheckPathRisks,Read-DriveCheckPathAccess +Export-ModuleMember -Function Get-Field,New-DriveGroup,Get-WorstStatus,Test-DriveSmartAnswer,Get-DriveSelfTest,Get-DriveHealthFindings,Get-DriveSurfaceFindings,Get-DriveConnectionKey,Get-DriveEventFindings,Get-DriveConnectionFindings,Get-DriveFileSystemFindings,Merge-DriveEvents,Get-DriveCheckView,Format-DriveOffset,Format-DriveAge,Test-DriveSmartFlagOnly,Test-DriveScanRecord,New-DriveScanState,Update-DriveScanState,Read-DriveScanState,Test-DriveStopRequest,Add-DriveScannerType,Open-DriveCheckSmartTool,Get-DriveCheckSmartToolHash,Get-DriveCheckDatabaseHash,Get-DriveCheckPathRisks,Read-DriveCheckPathAccess diff --git a/Backup.History.psm1 b/Backup.History.psm1 index 4e4238e..e14775e 100644 --- a/Backup.History.psm1 +++ b/Backup.History.psm1 @@ -15,46 +15,320 @@ function ConvertTo-HistoryDate($Value) { if($Value -is [DateTime]){return [DateTimeOffset]::new($Value)} [DateTimeOffset]::Parse([string]$Value,[Globalization.CultureInfo]::InvariantCulture,[Globalization.DateTimeStyles]::RoundtripKind) } +# A backup that could not read every file (restic exit code 3) still saves a version. The worker marks +# it by replacing the tag devdrive with this one, so it never passes for a complete version (it is not +# one of the last N, not a month's version and not the newest complete one) and history can label it. +# The clean-up deletes it only once three complete versions of the same computer and folders are newer +# (Get-VersionRetentionPlan). Versions saved before this mark existed cannot be recognised. +$script:PartialTag='devdrive-partial' +function Test-PartialSnapshot($Snapshot) { + [bool]($Snapshot -and $Snapshot.PSObject.Properties['tags'] -and @($Snapshot.tags) -ccontains $script:PartialTag) +} +function Test-OwnSnapshot($Snapshot) { + # This tool's versions carry the tag devdrive (complete) or devdrive-partial (partial). Anything else in + # the repository was saved by another program and is never counted, kept or deleted by the clean-up. + [bool]($Snapshot -and $Snapshot.PSObject.Properties['tags'] -and (@($Snapshot.tags) -ccontains 'devdrive' -or @($Snapshot.tags) -ccontains $script:PartialTag)) +} +function Get-SnapshotGroupKey($Snapshot) { + # A group is one computer and one set of source folders, told apart exactly as restic does + # (--group-by host,paths): the paths in sorted order, capital and small letters kept apart, and a list + # that names one folder twice a set of its own (restic saves and groups it that way; this tool never + # passes a folder twice). Telling more sets apart only ever keeps more versions. The parts are joined + # with a character no path or computer name can hold, so two sets never share a key. + $paths=[string[]]@(@(if($Snapshot.PSObject.Properties['paths']){$Snapshot.paths})|Where-Object {$null -ne $_}|ForEach-Object {[string]$_}) + [Array]::Sort($paths,[StringComparer]::Ordinal) + ([string]$Snapshot.hostname)+"`0"+($paths -join "`0") +} +function Get-PartialMarkArguments([string]$Id) { + Assert-SnapshotId $Id + @('tag','--remove','devdrive','--add',$script:PartialTag,$Id) +} +function Get-PartialTags($Tags) { + # The tags a version carries once it is marked partial (what Get-PartialMarkArguments makes restic + # do). A preview of the clean-up uses them to plan a version that is still waiting for its mark. + @(@($Tags|Where-Object {$_ -and $_ -cne 'devdrive' -and $_ -cne $script:PartialTag})+$script:PartialTag) +} +function Split-SnapshotIdBatches([string[]]$Ids,[int]$Size=200) { + # Versions are deleted by exact ID on the command line, and Windows refuses a command line over 32,767 + # characters (about 500 IDs). 200 IDs are 13,000 characters. + if($Size -lt 1){throw 'Batch size must be at least 1'} + $all=@($Ids|Where-Object {$_}) + for($i=0;$i -lt $all.Count;$i+=$Size){,@($all[$i..([math]::Min($i+$Size,$all.Count)-1)])} +} +function Get-RestoreParentWarnings([string]$ErrorText,[string[]]$SourcePaths,$Summary=$null) { + # restic ends a restore with exit code 1 when any item reported an error, and then skips checking the + # restored content. The only errors accepted as warnings are dates or attributes that could not be set + # on the folders ABOVE what the version backed up ($SourcePaths: the version's own source paths, not + # today's settings): restic recreates them inside the restore folder with their original, protected + # permissions (for example C:\Users). Returns those messages; nothing when anything else went wrong, + # when the output cannot be understood, or when more items are missing than there are warnings. + $roots=@($SourcePaths|Where-Object {$_}|ForEach-Object {'/'+([string]$_).TrimEnd('\','/').Replace(':','').Replace('\','/')}) + if(-not $roots.Count){return} + $warnings=@() + foreach($line in @($ErrorText -split "`r?`n"|Where-Object {$_.Trim()})){ + $e=$null;try{$e=$line|ConvertFrom-Json}catch{return} + if(-not $e -or $e -isnot [pscustomobject] -or -not $e.PSObject.Properties['message_type']){return} + if($e.message_type -eq 'exit_error'){continue} + if($e.message_type -ne 'error' -or -not $e.PSObject.Properties['during'] -or $e.during -ne 'restore' -or -not $e.PSObject.Properties['item'] -or -not $e.PSObject.Properties['error']){return} + $item=([string]$e.item).Replace('\','/').TrimEnd('/');$message=[string]$e.error.message + if($message -notmatch '^failed to restore (timestamp|attributes) of ' -or -not $item){return} + # Above at least one backed-up path, and neither one of them nor inside one. + $above=@($roots|Where-Object {$_.StartsWith($item+'/',[StringComparison]::OrdinalIgnoreCase)}).Count + $content=@($roots|Where-Object {$item -ieq $_ -or $item.StartsWith($_+'/',[StringComparison]::OrdinalIgnoreCase)}).Count + if(-not $above -or $content){return} + $warnings+=$message + } + if($Summary -and $Summary.PSObject.Properties['total_files'] -and $Summary.PSObject.Properties['files_restored'] -and ([long]$Summary.total_files-[long]$Summary.files_restored) -gt $warnings.Count){return} + $warnings +} function Get-HistoryDeletionPlan($Snapshots,[string[]]$Ids) { $all=@($Snapshots);$selected=@($Ids|Select-Object -Unique) if($selected.Count -eq 0){throw 'Select at least one older snapshot'} foreach($id in $selected){Assert-SnapshotId $id;if($id -cnotin @($all.id)){throw 'A selected snapshot no longer exists. Refresh history.'}} - # Preserve the newest snapshot of each host/source set, including equal-time ties. + # Preserve the newest snapshot of each host/source set, including equal-time ties, and also the newest + # complete one: a newer partial version must never be what is left of a set. foreach($group in @($all|Group-Object {([string]$_.hostname)+'|'+((@($_.paths)|Sort-Object)-join '|')})){ $latest=$group.Group|ForEach-Object {(ConvertTo-HistoryDate $_.time).UtcTicks}|Sort-Object -Descending|Select-Object -First 1 - foreach($snapshot in $group.Group){if((ConvertTo-HistoryDate $snapshot.time).UtcTicks -eq $latest -and $snapshot.id -cin $selected){throw 'The newest snapshot for each computer/source set is protected. Select older versions.'}} + $latestComplete=$group.Group|Where-Object {-not (Test-PartialSnapshot $_)}|ForEach-Object {(ConvertTo-HistoryDate $_.time).UtcTicks}|Sort-Object -Descending|Select-Object -First 1 + foreach($snapshot in $group.Group){ + if($snapshot.id -cnotin $selected){continue} + $ticks=(ConvertTo-HistoryDate $snapshot.time).UtcTicks + if($ticks -eq $latest){throw 'The newest snapshot for each computer/source set is protected. Select older versions.'} + if($null -ne $latestComplete -and $ticks -eq $latestComplete -and -not (Test-PartialSnapshot $snapshot)){throw 'The newest complete snapshot for each computer/source set is protected: the newer ones are partial. Select older versions.'} + } + } + # Also the newest complete version this tool made of each computer and set of folders, told apart as + # the clean-up tells them apart (Get-SnapshotGroupKey: this tool's versions only, capital letters kept + # apart). The grouping above counts other programs' versions and ignores case, so a newer version of + # another program, or the same folder written with other capital letters, would leave it unprotected. + $newestOwn=[Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal) + foreach($s in $all){ + if(-not (Test-OwnSnapshot $s) -or (Test-PartialSnapshot $s)){continue} + $key=Get-SnapshotGroupKey $s;$ticks=(ConvertTo-HistoryDate $s.time).UtcTicks + if(-not $newestOwn.ContainsKey($key) -or $ticks -gt $newestOwn[$key]){$newestOwn[$key]=$ticks} + } + foreach($s in @($all|Where-Object {$_.id -cin $selected})){ + if(-not (Test-OwnSnapshot $s) -or (Test-PartialSnapshot $s)){continue} + if((ConvertTo-HistoryDate $s.time).UtcTicks -eq $newestOwn[(Get-SnapshotGroupKey $s)]){throw 'The newest complete snapshot for each computer/source set is protected. Select older versions.'} } @($all|Where-Object {$_.id -cin $selected}) } -function Get-RetentionArguments([int]$Days,[int]$Weeks,[int]$Months) { - # Automatic clean-up after a backup: one version per day for $Days days, per week for $Weeks weeks and - # per month for $Months months. restic counts these windows back from the newest version, not from - # today, so a long pause between backups never empties the history; --keep-last 1 keeps the newest - # version whatever the numbers. Only this tool's own versions (tag devdrive) are considered. Always a - # dry run: the versions to delete are checked first and then deleted by exact ID. - if($Days -lt 0 -or $Weeks -lt 0 -or $Months -lt 0 -or $Days -gt 3650 -or $Weeks -gt 520 -or $Months -gt 240){throw 'Retention numbers are out of range'} - if($Days+$Weeks+$Months -eq 0){throw 'Retention needs at least one of days, weeks or months'} - $a=@('forget','--dry-run','--tag','devdrive','--group-by','host,paths','--keep-last','1') - if($Days){$a+=@('--keep-within-daily',"$($Days)d")} - if($Weeks){$a+=@('--keep-within-weekly',"$($Weeks*7)d")} - if($Months){$a+=@('--keep-within-monthly',"$($Months)m")} - $a -} -function Get-RetentionRemovals($Plan,[string]$NewSnapshot,$Snapshots) { - # The versions a retention dry run would delete, after our own checks: the version this backup just - # made must be kept, every group keeps at least one version, and the newest version of each - # computer/source set stays protected (Get-HistoryDeletionPlan). Returns the snapshots to delete. - $groups=@($Plan|Where-Object {$_}) - $remove=@();$keep=@() - foreach($g in $groups){ - $k=@(if($g.PSObject.Properties['keep'] -and $g.keep){$g.keep});$r=@(if($g.PSObject.Properties['remove'] -and $g.remove){$g.remove}) - if($r.Count -and -not $k.Count){throw 'The clean-up would delete every version of a folder set. Nothing was deleted.'} - $keep+=@($k|ForEach-Object {[string]$_.id});$remove+=@($r|ForEach-Object {[string]$_.id}) - } - if($NewSnapshot -and $NewSnapshot -cnotin $keep){throw 'The clean-up would not keep the version this backup just made. Nothing was deleted.'} - $remove=@($remove|Where-Object {$_}|Select-Object -Unique) +function Get-VersionRetentionPlan($Snapshots,[int]$Last,[int]$Months,[string]$HostName,[string]$NewSnapshot,[TimeZoneInfo]$TimeZone=[TimeZoneInfo]::Local) { + # Keep versions, the automatic clean-up after a backup. Worked out here from the list of versions; + # restic only lists the versions and deletes the ones named, by exact ID. (restic's own + # --keep-within-monthly keeps a version in up to $Months+1 calendar months and knows nothing of + # partial versions, so it cannot plan this rule.) + # + # The rule, applied separately to each group = one computer and one set of source folders. Only the + # groups of $HostName are planned, and only this tool's versions (Test-OwnSnapshot): another computer + # that shares the repository cleans its own versions, with its own rule. + # - LastN: the $Last newest complete versions stay, whatever their dates. + # - Monthly: the newest complete version of each calendar month stays, for $Months months: the month + # of the group's newest complete version and the $Months-1 months before it. Counting back from that + # version and not from today, a long pause between backups never empties the history. The months + # ending with the month of the version just made count as well, so that one version saved while + # the clock was far ahead does not move every month out of the rule. + # - A version that any rule keeps stays. NewestComplete (the newest complete version of the group) + # and JustMade ($NewSnapshot, the version this backup made) stay whatever the numbers. + # - A partial version never counts for LastN, Monthly or NewestComplete. It stays (PartialKept) until + # at least three complete versions of its own group are newer than it; then it goes + # (PartialAfter3Complete). Its age alone is never a reason. + # - $Last = 0 and $Months = 0: the clean-up is off and every version stays (CleanUpOff). + # With both rules on, a group keeps at most $Last+$Months-1 complete versions by the two rules (the + # newest one is always both among the last N and its month's version: 15 with 10 and 6). After a + # clock that was wrong (versions dated later than the version just made) it can keep more: the version + # just made itself, and up to $Months more months' versions (the months before the version just made). + # Versions made at exactly the same moment count as one. Partial versions come on top. + # + # Month boundaries: a version belongs to the calendar month of the moment it was made as THIS computer's + # clock shows it ($TimeZone, by default the computer's own time zone), which is the date the history + # window and restic's own list show. The offset written in the version (the clock of the computer that + # made it, at that time) is not used: a version made at 23:30 on 30 June here belongs to June. + # Versions made at exactly the same moment are equally new: they are kept or dropped together. + # + # Returns one record for every version of this computer: Id, Time, Month, Host, Paths, Partial, Keep, + # Reasons and the version itself (Snapshot). Nothing is planned without the ID of the version just made, + # which must be a complete version of this computer. + if($Last -lt 0 -or $Months -lt 0 -or $Last -gt 10000 -or $Months -gt 240){throw 'Retention numbers are out of range'} + if(-not $NewSnapshot){throw 'The version this backup just made is not known, so the clean-up cannot protect it. Nothing was deleted.'} + if(-not $HostName){throw 'The name of this computer is not known, so the clean-up cannot tell its versions from another computer''s. Nothing was deleted.'} + $enabled=$Last -gt 0 -or $Months -gt 0 + $all=[Collections.Generic.List[object]]::new();$groups=[Collections.Generic.Dictionary[string,object]]::new([StringComparer]::Ordinal) + foreach($s in @($Snapshots|Where-Object {$_})){ + if([string]$s.hostname -cne $HostName -or -not (Test-OwnSnapshot $s)){continue} + $when=ConvertTo-HistoryDate $s.time;$local=[TimeZoneInfo]::ConvertTime($when,$TimeZone) + $d=[pscustomobject]@{Id=[string]$s.id;Time=$when;Month=('{0:d4}-{1:d2}' -f $local.Year,$local.Month);Host=[string]$s.hostname;Paths=[string[]]@($s.paths);Partial=(Test-PartialSnapshot $s);Keep=$true;Reasons=[string[]]@();Snapshot=$s;Ticks=$when.UtcTicks;MonthNumber=$local.Year*12+$local.Month} + $key=Get-SnapshotGroupKey $s + if(-not $groups.ContainsKey($key)){$groups[$key]=[Collections.Generic.List[object]]::new()} + $groups[$key].Add($d);$all.Add($d) + } + $made=@($all|Where-Object {$_.Id -ceq $NewSnapshot}) + if($made.Count -ne 1){throw 'The version this backup just made was not found among this computer''s saved versions, so the clean-up cannot protect it. Nothing was deleted.'} + if($made[0].Partial){throw 'The version this backup just made is partial, so nothing is cleaned up on its account. Nothing was deleted.'} + $madeMonth=$made[0].MonthNumber + foreach($group in $groups.Values){ + if(-not $enabled){foreach($d in $group){$d.Reasons=[string[]]@('CleanUpOff')};continue} + # Newest first. + $complete=@($group|Where-Object {-not $_.Partial}|Sort-Object @{Expression={$_.Ticks};Descending=$true},@{Expression={$_.Id}}) + if($complete.Count){ + $newest=$complete[0].Ticks + $lastFrom=if($Last -gt 0){$complete[[math]::Min($Last,$complete.Count)-1].Ticks}else{$null} + # The months inside the rule: the $Months months that end with the newest complete version's month, + # and the $Months months that end with the month of the version just made (today, by this computer's + # clock). While the clock is right both are the same or the second adds nothing; after a wrong clock + # one version dated far ahead or behind no longer moves every month out of the rule. + $windows=@(@($complete[0].MonthNumber,$madeMonth)|Select-Object -Unique) + $monthNewest=@{} + foreach($d in $complete){if(@($windows|Where-Object {$d.MonthNumber -gt $_-$Months -and $d.MonthNumber -le $_}).Count -and -not $monthNewest.ContainsKey($d.MonthNumber)){$monthNewest[$d.MonthNumber]=$d.Ticks}} + foreach($d in $complete){ + $why=@() + if($d.Id -ceq $NewSnapshot){$why+='JustMade'} + if($d.Ticks -eq $newest){$why+='NewestComplete'} + if($null -ne $lastFrom -and $d.Ticks -ge $lastFrom){$why+='LastN'} + if($monthNewest.ContainsKey($d.MonthNumber) -and $monthNewest[$d.MonthNumber] -eq $d.Ticks){$why+='Monthly'} + $d.Keep=[bool]$why.Count;$d.Reasons=[string[]]$(if($why.Count){$why}else{@('OutsideRules')}) + } + } + foreach($d in @($group|Where-Object {$_.Partial})){ + # Three complete versions are newer exactly when the third newest complete one is. + $followed=$complete.Count -ge 3 -and $complete[2].Ticks -gt $d.Ticks + $d.Keep=-not $followed;$d.Reasons=[string[]]@($(if($followed){'PartialAfter3Complete'}else{'PartialKept'})) + } + } + $all.ToArray()|Select-Object Id,Time,Month,Host,Paths,Partial,Keep,Reasons,Snapshot +} +function Get-RetentionRemovals($Plan,[string]$NewSnapshot,$Snapshots,[string]$HostName) { + # The versions a clean-up plan (Get-VersionRetentionPlan: records with Id and Keep) deletes, after + # checks that do not depend on how the plan was worked out. Refused, with nothing deleted: a plan made + # without knowing the version this backup just made, or one that does not keep it; a version just made + # that is partial; a plan that deletes a version of another computer or of another program, every + # version of a set of folders, the newest complete version of a set, or a partial version that fewer + # than three complete versions follow. The newest version of each computer/source set stays protected + # as for a deletion by hand (Get-HistoryDeletionPlan). Returns the saved versions to delete. + if(-not $NewSnapshot){throw 'The version this backup just made is not known, so the clean-up cannot protect it. Nothing was deleted.'} + if(-not $HostName){throw 'The name of this computer is not known, so the clean-up cannot tell its versions from another computer''s. Nothing was deleted.'} + $decisions=@($Plan|Where-Object {$_}) + $keep=@($decisions|Where-Object {$_.Keep}|ForEach-Object {[string]$_.Id}) + $remove=@($decisions|Where-Object {-not $_.Keep}|ForEach-Object {[string]$_.Id}|Where-Object {$_}|Select-Object -Unique) + if($NewSnapshot -cnotin $keep){throw 'The clean-up would not keep the version this backup just made. Nothing was deleted.'} + $known=[Collections.Generic.Dictionary[string,object]]::new([StringComparer]::Ordinal) + foreach($s in @($Snapshots|Where-Object {$_})){$known[[string]$s.id]=$s} + if(-not $known.ContainsKey($NewSnapshot)){throw 'The version this backup just made was not found among the saved versions, so the clean-up cannot protect it. Nothing was deleted.'} + if(Test-PartialSnapshot $known[$NewSnapshot]){throw 'The version this backup just made is partial, so nothing is cleaned up on its account. Nothing was deleted.'} if(-not $remove.Count){return @()} if(@($remove|Where-Object {$_ -cin $keep}).Count){throw 'The clean-up plan both keeps and deletes one version. Nothing was deleted.'} + $gone=[Collections.Generic.HashSet[string]]::new([string[]]$remove,[StringComparer]::Ordinal) + foreach($id in $remove){ + if(-not $known.ContainsKey($id)){throw 'The clean-up plan names a version that is not among the saved versions. Refresh and try again. Nothing was deleted.'} + if([string]$known[$id].hostname -cne $HostName){throw 'The clean-up plan would delete a version of another computer. Nothing was deleted.'} + if(-not (Test-OwnSnapshot $known[$id])){throw 'The clean-up plan would delete a version that this tool did not make. Nothing was deleted.'} + } + $groups=[Collections.Generic.Dictionary[string,object]]::new([StringComparer]::Ordinal) + foreach($s in $known.Values){ + if([string]$s.hostname -cne $HostName -or -not (Test-OwnSnapshot $s)){continue} + $key=Get-SnapshotGroupKey $s + if(-not $groups.ContainsKey($key)){$groups[$key]=[Collections.Generic.List[object]]::new()} + $groups[$key].Add([pscustomobject]@{Id=[string]$s.id;Ticks=(ConvertTo-HistoryDate $s.time).UtcTicks;Partial=(Test-PartialSnapshot $s)}) + } + foreach($group in $groups.Values){ + $leaving=@($group|Where-Object {$gone.Contains($_.Id)}) + if(-not $leaving.Count){continue} + if($leaving.Count -eq $group.Count){throw 'The clean-up would delete every version of a folder set. Nothing was deleted.'} + $complete=@($group|Where-Object {-not $_.Partial}|Sort-Object @{Expression={$_.Ticks};Descending=$true}) + if($complete.Count -and @($leaving|Where-Object {-not $_.Partial -and $_.Ticks -eq $complete[0].Ticks}).Count){throw 'The clean-up would delete the newest complete version of a folder set. Nothing was deleted.'} + foreach($partial in @($leaving|Where-Object {$_.Partial})){ + if($complete.Count -lt 3 -or $complete[2].Ticks -le $partial.Ticks){throw 'The clean-up would delete a partial version that fewer than three complete versions follow. Nothing was deleted.'} + } + } @(Get-HistoryDeletionPlan $Snapshots $remove) } -Export-ModuleMember -Function Assert-SnapshotId,Assert-SnapshotPath,ConvertTo-ResticLiteralPattern,ConvertTo-HistoryDate,Get-HistoryDeletionPlan,Get-RetentionArguments,Get-RetentionRemovals +# The first clean-up deletes only what the user saw. Until the user approves, backups only list what the +# clean-up would delete (a preview). The preview is itself a backup, so by the time of the first real +# clean-up the plan can hold more than was shown. That run is therefore given the list of the versions +# that were shown and approved, and deletes only the versions that are both in its own plan and on the +# list. The list only ever narrows the plan: the rule still decides what may go, and every check of +# Get-RetentionRemovals runs again on what is left. Whatever else the plan would delete stays for the +# next clean-up, which follows the rule alone. +function Get-RetentionApprovedIds([string]$Text) { + # Reads the list of approved versions: text with one full version ID a line. Blank lines, spaces + # around an ID, capital letters (an ID is hexadecimal) and a repeated ID make no difference. Anything + # else on a line (a shortened ID, a name such as latest, two IDs) makes the whole list unusable, and so + # does a list without any ID: nothing is guessed and nothing is deleted. Returns the IDs as restic + # writes them (small letters), each once. + $ids=[Collections.Generic.List[string]]::new() + foreach($line in @(([string]$Text) -split "`r`n|`r|`n")){ + $id=$line.Trim().Trim([char]0xFEFF).Trim().ToLowerInvariant() + if(-not $id){continue} + if($id -cnotmatch '^[0-9a-f]{64}$'){throw 'The list of approved versions holds a line that is not a full version ID, so the clean-up deleted nothing.'} + if(-not $ids.Contains($id)){$ids.Add($id)} + } + if(-not $ids.Count){throw 'The list of approved versions is empty, so the clean-up deleted nothing.'} + $ids.ToArray() +} +function Get-ApprovedRetentionPlan($Plan,[string[]]$ApprovedIds) { + # A clean-up plan (Get-VersionRetentionPlan) narrowed to the approved versions: a version the plan + # deletes stays to be deleted only when its exact full ID is on the list; any other one is kept for now + # (reason NotApproved, after the reason the rule gave). A version the plan keeps stays kept whatever the + # list says, and an ID on the list that is not in the plan (it no longer exists, it belongs to another + # computer, or it is the earlier ID of a partial version that was marked since) changes nothing. + # Returns new records; the plan handed in is left as it was. + $approved=[Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach($id in @($ApprovedIds|Where-Object {$_})){[void]$approved.Add([string]$id)} + foreach($decision in @($Plan|Where-Object {$_})){ + $copy=$decision.PSObject.Copy() + if(-not $copy.Keep -and -not $approved.Contains([string]$copy.Id)){ + $copy.Keep=$true + if($copy.PSObject.Properties['Reasons']){$copy.Reasons=[string[]](@($copy.Reasons)+'NotApproved')} + } + $copy + } +} +function Get-LockProcess($Id) { + # What runs on this computer under a lock's process number. Nothing: no process has that number. + # Without a start time: there is one, but it cannot be examined (for example another account's). + try{$p=[Diagnostics.Process]::GetProcessById([int]$Id)}catch [ArgumentException]{return}catch{return [pscustomobject]@{Name='';StartedUtc=$null}} + try{[pscustomobject]@{Name=$p.ProcessName;StartedUtc=$p.StartTime.ToUniversalTime()}} + catch{try{[void][Diagnostics.Process]::GetProcessById([int]$Id)}catch [ArgumentException]{return}catch{};[pscustomobject]@{Name='';StartedUtc=$null}} + finally{$p.Dispose()} +} +function Get-LockReleaseDecision($Locks,[string]$ComputerName) { + # Which locks in the repository may be removed without asking: only when EVERY lock present was made on + # this computer by a process that has ended. A run that was stopped, crashed or was ended by the memory + # cap leaves such a lock behind, and restic then refuses its next steps (exit code 11). + # $Locks: one record per lock file, with Id, Lock (what restic's "cat lock" printed: time, hostname, + # pid, exclusive; nothing when it could not be read, and a record that lacks one of them counts as + # unreadable) and Process (what runs on this computer under that process number now: nothing, or its + # Name and StartedUtc; no start time when it cannot be examined). + # The process has ended when nothing runs under its number, or when what does started after the lock was + # made and is not restic (Windows hands process numbers out again). A restic process that started before + # the lock is its maker and still running. Everything else is unclear and stays: another program that + # was already running under that number, a restic process that seems to have started after the lock + # (the clock may have been set back), a process or a lock that cannot be read. + $judged=@(foreach($l in @($Locks|Where-Object {$_})){ + $r=if($l.PSObject.Properties['Lock']){$l.Lock}else{$null};$p=$null;if($r -is [pscustomobject]){$p=$r.PSObject.Properties} + $state='Unclear';$why='the lock in the repository could not be read, so it was left in place.';$number=$null;$created=$null;$computer=$null;$exclusive=$null + if([string]$l.Id -cmatch '^[0-9a-f]{64}$' -and $l.PSObject.Properties['Process'] -and $ComputerName -and $p -and $p['hostname'] -and $p['pid'] -and $p['time'] -and $p['exclusive'] -and $r.exclusive -is [bool] -and ([string]$r.hostname).Trim() -and ($r.pid -is [int] -or $r.pid -is [long]) -and $r.pid -gt 0 -and $r.pid -le [int]::MaxValue){ + try{$created=(ConvertTo-HistoryDate $r.time).UtcDateTime}catch{$created=$null} + } + if($null -ne $created){ + $number=[int]$r.pid;$computer=[string]$r.hostname;$exclusive=$r.exclusive;$process=$l.Process + if($computer -ine $ComputerName){$state='OtherComputer';$why="another computer ($computer) is using this backup, or stopped while it was using it. This computer cannot tell which, so the lock was left in place."} + elseif($null -eq $process){$state='Ended';$why=''} + else{ + $started=$null;try{if($null -ne $process.StartedUtc){$started=(ConvertTo-HistoryDate $process.StartedUtc).UtcDateTime}}catch{$started=$null} + $engine=([string]$process.Name) -like 'restic*' + if($null -eq $started){$why="process $number on this computer may still be using this backup: it could not be examined, so the lock was left in place."} + elseif($started -le $created){ + if($engine){$state='Running';$why="a backup is still running on this computer (process $number). Wait until it has finished, then try again."} + else{$why="process $number on this computer was already running when the lock was made and may be using this backup, so the lock was left in place."} + } + elseif($engine){$why="another backup program now runs on this computer under the process number of the run that made the lock ($number), and the two cannot be told apart, so the lock was left in place."} + else{$state='Ended';$why=''} + } + } + [pscustomobject]@{Id=[string]$l.Id;Pid=$number;CreatedUtc=$(if($null -ne $created){$created.ToString('o')}else{$null});Exclusive=$exclusive;Computer=$computer;State=$state;Why=$why} + }) + $blocking=@($judged|Where-Object {$_.State -ne 'Ended'}) + [pscustomobject]@{Release=[bool]($judged.Count -and -not $blocking.Count);Reason=$(if($blocking.Count){$blocking[0].Why}else{''});Locks=$judged} +} +Export-ModuleMember -Function Assert-SnapshotId,Assert-SnapshotPath,ConvertTo-ResticLiteralPattern,ConvertTo-HistoryDate,Test-PartialSnapshot,Get-PartialMarkArguments,Get-PartialTags,Split-SnapshotIdBatches,Get-RestoreParentWarnings,Get-HistoryDeletionPlan,Get-VersionRetentionPlan,Get-RetentionRemovals,Get-RetentionApprovedIds,Get-ApprovedRetentionPlan,Get-LockProcess,Get-LockReleaseDecision diff --git a/Backup.MirrorVerifier.cs b/Backup.MirrorVerifier.cs index 8bf50fd..14a5283 100644 --- a/Backup.MirrorVerifier.cs +++ b/Backup.MirrorVerifier.cs @@ -1,50 +1,200 @@ // Content check of a plain copy (Invoke-FastMirror.ps1 -Verify and -VerifyOnly). Both sides are walked // with the rules the copy uses (per-folder mode, skipped names and paths, links never followed), so a // selected source file that is missing from the copy is a difference, not only a file whose content differs. -using System; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Security.Cryptography; using System.Collections.Concurrent; using System.Collections.Generic; using System.Text.RegularExpressions; +// The same rules serve the repair of single files (-Repair) and the look for names robocopy treats +// differently from what the lists say (Survey). +using System; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Security.Cryptography; using System.Collections.Concurrent; using System.Collections.Generic; using System.Text.RegularExpressions; using System.Runtime.InteropServices; public static class MirrorVerifier { static string Hash(string p){using(var s=new FileStream(p,FileMode.Open,FileAccess.Read,FileShare.ReadWrite|FileShare.Delete,1<<20))using(var h=SHA256.Create())return Convert.ToHexString(h.ComputeHash(s));} + static bool Same(string a,string b){return string.Equals(a,b,StringComparison.OrdinalIgnoreCase);} static bool Under(string path,string parent){return path.StartsWith(parent+"\\",StringComparison.OrdinalIgnoreCase);} + // Every path is used in its \\?\ form: Windows otherwise drops a dot or space at the end of a name + // (and then looks for another file), and stops at 260 characters. + static string Long(string p){p=Path.GetFullPath(p).TrimEnd('\\');return p.StartsWith("\\\\?\\")?p:p.StartsWith("\\\\")?"\\\\?\\UNC\\"+p.Substring(2):"\\\\?\\"+p;} + static string Plain(string text){return text.Replace("\\\\?\\UNC\\","\\\\").Replace("\\\\?\\","");} // Mirror mode of a relative path: the longest entry that equals or contains it, else the default. static bool IsOn(string rel,string[] paths,bool[] on,bool defaultOn){ int best=-1; - for(int i=0;ipaths[best].Length))best=i; + for(int i=0;ipaths[best].Length))best=i; return best<0?defaultOn:on[best]; } // robocopy wildcards in /XD and /XF names: * and ?, not case-sensitive. static Regex[] Globs(string[] names){return names.Select(n=>new Regex("^"+Regex.Escape(n).Replace("\\*",".*").Replace("\\?",".")+"$",RegexOptions.IgnoreCase|RegexOptions.CultureInvariant)).ToArray();} - sealed class Rules {public string[] Paths;public bool[] On;public bool DefaultOn;public Regex[] DirNames,FileNames;public string[] SkipPaths;} + sealed class Rules {public string[] Paths;public bool[] On;public bool DefaultOn;public Regex[] DirNames,FileNames;public string[] SkipPaths,Roots;} + static Rules Make(string[] paths,bool[] on,bool defaultOn,string[] skipDirNames,string[] skipFileNames,string[] skipPaths,string[] roots){ + return new Rules{Paths=paths,On=on,DefaultOn=defaultOn,DirNames=Globs(skipDirNames),FileNames=Globs(skipFileNames),SkipPaths=skipPaths.Select(s=>s.Trim('\\')).Where(s=>s.Length>0).ToArray(),Roots=roots.Select(s=>s.Trim('\\')).Where(s=>s.Length>0).ToArray()}; + } + // robocopy skips names and paths only below the folder it copies, never that folder itself. So a + // folder or file that was copied on its own (one of Roots) is checked whatever is skipped above it + // or by its own name, and a skipped folder is still entered, without checking its files, where + // such a root lies inside it. + // What the copy does with a folder. 0: left out. 1: read. 2: skipped, and entered only for a root inside it. + static int Folder(string rel,string name,bool skipped,Rules r){ + if(r.Roots.Any(x=>Same(x,rel)))return 1; + if(skipped||r.SkipPaths.Any(s=>Same(rel,s))||r.DirNames.Any(g=>g.IsMatch(name)))return r.Roots.Any(x=>Under(x,rel))?2:0; + // Unselected folders are entered only when a selected entry lies inside them. + return IsOn(rel,r.Paths,r.On,r.DefaultOn)||Enumerable.Range(0,r.Paths.Length).Any(i=>r.On[i]&&Under(r.Paths[i],rel))?1:0; + } + // Whether the copy takes a file. + static bool Taken(string rel,string name,bool skipped,Rules r){ + return (r.Roots.Any(x=>Same(x,rel))||(!skipped&&!r.SkipPaths.Any(s=>Same(rel,s))&&!r.FileNames.Any(g=>g.IsMatch(name))))&&IsOn(rel,r.Paths,r.On,r.DefaultOn); + } + // The same for one file by its path alone, folder by folder, without reading the disk. + static bool Selected(string rel,Rules r){ + var parts=rel.Split('\\');string d="";bool skipped=false; + for(int i=0;i Files(string root,Rules r,ConcurrentBag bad){ - var stack=new Stack();stack.Push(root); - while(stack.Count>0){var d=stack.Pop();FileSystemInfo[] items; - try{items=new DirectoryInfo(d).GetFileSystemInfos();}catch(Exception ex){bad.Add("ERROR\t"+d.Substring(root.Length).TrimStart('\\')+"\t"+ex.Message);continue;} + var stack=new Stack>();stack.Push(new KeyValuePair("",false)); + while(stack.Count>0){var top=stack.Pop();string d=top.Key;bool skipped=top.Value;FileSystemInfo[] items; + try{items=new DirectoryInfo(d.Length==0?root+"\\":root+"\\"+d+"\\").GetFileSystemInfos();}catch(Exception ex){bad.Add("ERROR\t"+d+"\t"+Plain(ex.Message));continue;} foreach(var e in items){ if((e.Attributes&FileAttributes.ReparsePoint)!=0)continue; - string rel=e.FullName.Substring(root.Length).TrimStart('\\'); - if(r.SkipPaths.Any(s=>string.Equals(rel,s,StringComparison.OrdinalIgnoreCase)||Under(rel,s)))continue; - if((e.Attributes&FileAttributes.Directory)!=0){ - if(r.DirNames.Any(g=>g.IsMatch(e.Name)))continue; - // Unselected folders are entered only when a selected entry lies inside them. - if(IsOn(rel,r.Paths,r.On,r.DefaultOn)||Enumerable.Range(0,r.Paths.Length).Any(i=>r.On[i]&&Under(r.Paths[i],rel)))stack.Push(e.FullName); - }else if(!r.FileNames.Any(g=>g.IsMatch(e.Name))&&IsOn(rel,r.Paths,r.On,r.DefaultOn))yield return rel;}} + string rel=d.Length==0?e.Name:d+"\\"+e.Name; + if((e.Attributes&FileAttributes.Directory)!=0){int k=Folder(rel,e.Name,skipped,r);if(k>0)stack.Push(new KeyValuePair(rel,k==2));} + else if(Taken(rel,e.Name,skipped,r))yield return rel;}} } public static string[] Compare(string mirror,string source,int threads,string[] paths,bool[] on,bool defaultOn,string[] skipDirNames,string[] skipFileNames,string[] skipPaths,out long files,out long bytes){ - var bad=new ConcurrentBag();long f=0,b=0; - var r=new Rules{Paths=paths,On=on,DefaultOn=defaultOn,DirNames=Globs(skipDirNames),FileNames=Globs(skipFileNames),SkipPaths=skipPaths.Select(s=>s.Trim('\\')).Where(s=>s.Length>0).ToArray()}; + return Compare(mirror,source,threads,paths,on,defaultOn,skipDirNames,skipFileNames,skipPaths,new string[0],out files,out bytes); + } + // roots: the selected folders and files that were copied on their own ("" is the source root). + public static string[] Compare(string mirror,string source,int threads,string[] paths,bool[] on,bool defaultOn,string[] skipDirNames,string[] skipFileNames,string[] skipPaths,string[] roots,out long files,out long bytes){ + var bad=new ConcurrentBag();long f=0,b=0;mirror=Long(mirror);source=Long(source); + var r=Make(paths,on,defaultOn,skipDirNames,skipFileNames,skipPaths,roots); // Every selected source file must be in the copy with the same content. Parallel.ForEach(Files(source,r,bad),new ParallelOptions{MaxDegreeOfParallelism=threads},rel=>{ - string s=Path.Combine(source,rel),m=Path.Combine(mirror,rel); + string s=source+"\\"+rel,m=mirror+"\\"+rel; try{ var si=new FileInfo(s);var mi=new FileInfo(m); if(!mi.Exists){bad.Add("MISSING-IN-MIRROR\t"+rel);return;} if(mi.Length!=si.Length){bad.Add("SIZE\t"+rel);return;} if(Hash(m)!=Hash(s)){bad.Add("CONTENT\t"+rel);return;} Interlocked.Increment(ref f);Interlocked.Add(ref b,mi.Length); - }catch(Exception ex){bad.Add("ERROR\t"+rel+"\t"+ex.Message);} + }catch(Exception ex){bad.Add("ERROR\t"+rel+"\t"+Plain(ex.Message));} }); // Files only in the copy (deleted from the source) are listed, not read: a copy that never deletes keeps them. - foreach(var rel in Files(mirror,r,bad))if(!File.Exists(Path.Combine(source,rel)))bad.Add("MISSING-IN-SOURCE\t"+rel); + foreach(var rel in Files(mirror,r,bad))if(!File.Exists(source+"\\"+rel))bad.Add("MISSING-IN-SOURCE\t"+rel); files=f;bytes=b;return bad.OrderBy(x=>x,StringComparer.Ordinal).ToArray(); } + + // --- Repair of single files (Invoke-FastMirror.ps1 -Repair) --------------------------------------- + // Each file named by an earlier content check is looked at again and, where the copy still differs + // from the source, copied again. Nothing is deleted or overwritten: the new copy is first written + // beside the old one and read back, and only when it matches the source by SHA-256 is the old version + // moved to the recovery folder and the new one given its name. So a file that cannot be read, copied + // or matched leaves the copy as it was. One line per file, with tabs between: what was done, the + // path, where the old version went (or nothing), and a note. + // REPAIRED copied again, and the copy now matches the source + // MATCHES the copy matches the source already; nothing was done + // SKIPPED not a file this copy takes any more (no longer selected, gone from the source) + // STILL-DIFFERENT the new copy did not match the source (the file is changing, or the drive fails) + // FAILED could not be read, copied or moved + public static string[] Repair(string mirror,string source,string recovery,int threads,string[] rels,string[] paths,bool[] on,bool defaultOn,string[] skipDirNames,string[] skipFileNames,string[] skipPaths,string[] roots){ + var done=new ConcurrentBag();mirror=Long(mirror);source=Long(source);recovery=Long(recovery); + var r=Make(paths,on,defaultOn,skipDirNames,skipFileNames,skipPaths,roots); + Parallel.ForEach(rels.Distinct(StringComparer.OrdinalIgnoreCase),new ParallelOptions{MaxDegreeOfParallelism=Math.Max(1,threads)},rel=>done.Add(RepairOne(mirror,source,recovery,rel,r))); + return done.OrderBy(x=>x,StringComparer.Ordinal).ToArray(); + } + // A link or junction on the way from root down to rel, rel itself included. The copy never follows one. + static bool Link(string root,string rel){ + string p=root; + foreach(var part in rel.Split('\\')){p+="\\"+part; + try{if((File.GetAttributes(p)&FileAttributes.ReparsePoint)!=0)return true;}catch(FileNotFoundException){return false;}catch(DirectoryNotFoundException){return false;}} + return false; + } + // A working file of the repair is the repair's own; nothing else is ever deleted. + static void Drop(string p){try{File.SetAttributes(p,FileAttributes.Normal);File.Delete(p);}catch{}} + static string RepairOne(string mirror,string source,string recovery,string rel,Rules r){ + string s=source+"\\"+rel,m=mirror+"\\"+rel,old=recovery+"\\"+rel,work=null,moved=""; + try{ + // The list comes from a file; a path that could lead out of the copy is never followed. + if(rel.Length==0||rel.IndexOfAny(new[]{':','/'})>=0||rel.Split('\\').Any(p=>p.Length==0||p=="."||p==".."))return "SKIPPED\t"+rel+"\t\tnot the path of a file in the copy"; + if(!Selected(rel,r))return "SKIPPED\t"+rel+"\t\tno longer selected for the plain copy"; + if(Link(source,rel))return "SKIPPED\t"+rel+"\t\tbehind a link or junction in the source, which is never copied"; + if(Directory.Exists(s))return "SKIPPED\t"+rel+"\t\ta folder in the source, not a file"; + if(!File.Exists(s))return "SKIPPED\t"+rel+"\t\tno longer in the source"; + if(Link(mirror,rel))return "FAILED\t"+rel+"\t\tthe copy has a link or junction on the way to this file"; + if(Directory.Exists(m))return "FAILED\t"+rel+"\t\tthe copy has a folder with this name; back up, which moves that folder aside"; + bool had=File.Exists(m); + // Looked at again now: the file may have been put right since the check. A copy that cannot be + // read is a damaged copy; a source that cannot be read fails the file (below). + if(had&&new FileInfo(m).Length==new FileInfo(s).Length){ + string hs=Hash(s),hm=null;try{hm=Hash(m);}catch(IOException){} + if(hm==hs)return "MATCHES\t"+rel+"\t\t"; + } + string dir=Path.GetDirectoryName(m);Directory.CreateDirectory(dir); + work=dir+"\\~repair-"+Guid.NewGuid().ToString("N").Substring(0,12)+".tmp"; + File.Copy(s,work,false); + try{File.SetCreationTimeUtc(work,File.GetCreationTimeUtc(s));}catch{} + if(Hash(work)!=Hash(s)){Drop(work);return "STILL-DIFFERENT\t"+rel+"\t\tthe new copy did not match the source: the file is changing, or the drive is failing";} + if(had){Directory.CreateDirectory(Path.GetDirectoryName(old));File.Move(m,old);moved=Plain(old);} + File.Move(work,m); + return "REPAIRED\t"+rel+"\t"+moved+"\t"; + }catch(Exception ex){ + if(work!=null)Drop(work); + // The old version had left its place and the new one could not take it: it is put back. + if(moved.Length>0&&!File.Exists(m)){try{File.Move(old,m);moved="";}catch{}} + return "FAILED\t"+rel+"\t"+moved+"\t"+Plain(ex.Message).Replace("\r"," ").Replace("\n"," ").Replace("\t"," "); + } + } + + // --- Names robocopy treats differently from what the lists say -------------------------------------- + [StructLayout(LayoutKind.Sequential,CharSet=CharSet.Unicode)] + struct Entry {public uint Attributes,C1,C2,A1,A2,W1,W2,SizeHigh,SizeLow,R0,R1;[MarshalAs(UnmanagedType.ByValTStr,SizeConst=260)]public string Name;[MarshalAs(UnmanagedType.ByValTStr,SizeConst=14)]public string Short;} + [DllImport("kernel32.dll",CharSet=CharSet.Unicode,SetLastError=true)]static extern IntPtr FindFirstFileW(string pattern,out Entry e); + [DllImport("kernel32.dll",CharSet=CharSet.Unicode,SetLastError=true)]static extern bool FindNextFileW(IntPtr h,out Entry e); + [DllImport("kernel32.dll")]static extern bool FindClose(IntPtr h); + // The entries of one folder with their short (8.3) names, which .NET does not give. A folder that + // cannot be read gives nothing here; the copy reports it. + static List List(string dir){ + var l=new List();Entry e;IntPtr h=FindFirstFileW(dir+"\\*",out e); + if(h==new IntPtr(-1))return l; + try{do{if(e.Name!="."&&e.Name!="..")l.Add(e);}while(FindNextFileW(h,out e));}finally{FindClose(h);} + return l; + } + // The short (8.3) name Windows gave a file or folder; empty when it has none. + public static string ShortName(string path){ + Entry e;IntPtr h=FindFirstFileW(Long(path),out e);if(h==new IntPtr(-1))return ""; + FindClose(h);return e.Short??""; + } + // Names in one folder that differ only by the case of their letters, each paired with the first of + // its kind (first, tab, other). A folder that keeps case apart can hold both; the copy holds one. + public static string[] SameNames(IEnumerable names){ + var first=new Dictionary(StringComparer.OrdinalIgnoreCase);var pairs=new List(); + foreach(var n in names){string f;if(!first.TryGetValue(n,out f))first[n]=n;else if(f!=n)pairs.Add(f+"\t"+n);} + return pairs.ToArray(); + } + // One pass over the names (no file is read) in the folders the copy takes, for two things robocopy + // does that the lists do not say: + // SHORT path, short name, name: a file or folder (ending in \) that robocopy skips because its + // short name fits one of shortNames (the wildcard names of the personal Exclude list), + // while its own name fits no skipped name. Nothing inside such a folder is copied. + // CASE path, path: two names in one folder that differ only by letter case. + public static string[] Survey(string source,string[] paths,bool[] on,bool defaultOn,string[] skipDirNames,string[] skipFileNames,string[] skipPaths,string[] roots,string[] shortNames){ + var found=new List();source=Long(source);var r=Make(paths,on,defaultOn,skipDirNames,skipFileNames,skipPaths,roots);var globs=Globs(shortNames); + Func fits=name=>{if(string.IsNullOrEmpty(name))return -1;for(int i=0;i>();stack.Push(new KeyValuePair("",false)); + while(stack.Count>0){var top=stack.Pop();string d=top.Key;bool skipped=top.Value;var names=new List(); + foreach(var e in List(d.Length==0?source:source+"\\"+d)){ + if((e.Attributes&(uint)FileAttributes.ReparsePoint)!=0)continue; + string rel=d.Length==0?e.Name:d+"\\"+e.Name;bool root=r.Roots.Any(x=>Same(x,rel));int i; + if((e.Attributes&(uint)FileAttributes.Directory)!=0){ + int k=Folder(rel,e.Name,skipped,r);if(k==0)continue; + // robocopy reads this folder as part of a copied one (a folder entered only to reach a + // selection inside it is not read by robocopy, and one copied on its own is never skipped). + if(k==1&&!root&&IsOn(rel,r.Paths,r.On,r.DefaultOn)){ + if((i=fits(e.Short))>=0){found.Add("SHORT\t"+rel+"\\\t"+e.Short+"\t"+shortNames[i]);k=r.Roots.Any(x=>Under(x,rel))?2:0;if(k==0)continue;} + else names.Add(e.Name); + } + stack.Push(new KeyValuePair(rel,k==2)); + }else if(Taken(rel,e.Name,skipped,r)){ + if(!root&&(i=fits(e.Short))>=0)found.Add("SHORT\t"+rel+"\t"+e.Short+"\t"+shortNames[i]);else names.Add(e.Name); + } + } + foreach(var pair in SameNames(names)){var p=pair.Split('\t');string pre=d.Length==0?"":d+"\\";found.Add("CASE\t"+pre+p[0]+"\t"+pre+p[1]);} + } + return found.OrderBy(x=>x,StringComparer.Ordinal).ToArray(); + } } diff --git a/Backup.Security.psm1 b/Backup.Security.psm1 index 755cdba..b27a9fe 100644 --- a/Backup.Security.psm1 +++ b/Backup.Security.psm1 @@ -28,4 +28,20 @@ function Assert-PrivateExportPath([string]$Path,[string[]]$ForbiddenRoots,[strin function Get-RecoveryKeyText([string]$Repository,[string]$Password) { "Restic repository recovery key`r`nRepository: $Repository`r`nPassword: $Password`r`nKeep this file in a password manager or separate offline location. Anyone with this password can decrypt the backup.`r`n" } -Export-ModuleMember -Function New-PrivateTextFile,Set-PrivateTextFile,Assert-NewBackupPassword,Assert-PrivateExportPath,Get-RecoveryKeyText +function Assert-RotationJournalTarget($Journal,[string]$VolumeId,[string]$Folder,[string]$Repository) { + # A pending password change belongs to one repository. The backup drive can come back under another + # letter between the interruption and the resume, so the repository is recognised by its volume and + # folder. A journal written before those were recorded holds only the path: the same path, or the + # same path under another drive letter, is accepted (the old password in the journal must still open + # the repository before anything changes). + $p=$Journal.PSObject.Properties + if($p['DestinationVolumeId'] -and $p['RepositoryFolder'] -and $Journal.DestinationVolumeId -and $Journal.RepositoryFolder){ + if([string]$Journal.DestinationVolumeId -cne $VolumeId -or [string]$Journal.RepositoryFolder -ine $Folder){throw 'Password-change journal belongs to another configuration or repository'} + return + } + $old=[string]$Journal.Repository + if($old -and $old -ieq $Repository){return} + if($old -match '^[A-Za-z]:\\.' -and $Repository -match '^[A-Za-z]:\\.' -and $old.Substring(1) -ieq $Repository.Substring(1)){return} + throw 'Password-change journal belongs to another configuration or repository' +} +Export-ModuleMember -Function New-PrivateTextFile,Set-PrivateTextFile,Assert-NewBackupPassword,Assert-PrivateExportPath,Assert-RotationJournalTarget,Get-RecoveryKeyText diff --git a/Backup.Selection.psm1 b/Backup.Selection.psm1 index 907b0c8..7fd409f 100644 --- a/Backup.Selection.psm1 +++ b/Backup.Selection.psm1 @@ -20,9 +20,17 @@ function Test-BackupModeIncludes([string]$Mode,[ValidateSet('Mirror','Versioned' $Mode -eq 'Both' -or $Mode -eq $Tool } +function Test-SameName([string]$A,[string]$B) { + # Names are compared the way Windows compares them: character by character, in any case. PowerShell's + # own -eq and -in compare the way a dictionary sorts and take an invisible character (a direction + # mark, a zero-width space) or another spelling of an accented letter as no difference, so two + # folders Windows keeps apart counted as one, and a name made only of such a character as empty. + [string]::Equals($A,$B,[StringComparison]::OrdinalIgnoreCase) +} + function ConvertTo-SelectionPath([string]$Path) { $p=$Path.Replace('/','\').Trim().Trim('\') - if(-not $p -or [IO.Path]::IsPathRooted($Path) -or $p.Contains(':') -or @($p.Split('\')|Where-Object {$_ -in @('','.','..')}).Count){throw "Choice path must be relative to the source root: '$Path'"} + if($p.Length -eq 0 -or [IO.Path]::IsPathRooted($Path) -or $p.Contains(':') -or @($p.Split('\')|Where-Object {$_.Length -eq 0 -or $_.Equals('.') -or $_.Equals('..')}).Count){throw "Choice path must be relative to the source root: '$Path'"} $p } @@ -33,11 +41,11 @@ function Get-BackupSelection($Config) { $entries=[Collections.Generic.List[object]]::new() if($hasModes){ $given=if($Config.PSObject.Properties['DefaultMode']){[string]$Config.DefaultMode}else{'Skip'} - $default=$script:Modes|Where-Object {$_ -ieq $given.Trim()} + $default=$script:Modes|Where-Object {Test-SameName $_ $given.Trim()} if(-not $default){$default=$given} foreach($c in $choices){ $given=if($c.PSObject.Properties['Mode']){[string]$c.Mode}else{''} - $mode=$script:Modes|Where-Object {$_ -ieq $given.Trim()} + $mode=$script:Modes|Where-Object {Test-SameName $_ $given.Trim()} if(-not $mode){throw "Choice '$($c.Path)' needs Mode Mirror, Versioned, Both or Skip (found '$given')"} $entries.Add([pscustomobject]@{Path=(ConvertTo-SelectionPath $c.Path);Mode=$mode}) } @@ -47,10 +55,10 @@ function Get-BackupSelection($Config) { foreach($p in @(if($Config.PSObject.Properties['Include']){$Config.Include})){if($p){$entries.Add([pscustomobject]@{Path=(ConvertTo-SelectionPath $p);Mode='Versioned'})}} } if($default -cnotin $script:Modes){throw "DefaultMode must be Mirror, Versioned, Both or Skip (found '$default')"} - $seen=@{} - foreach($e in $entries){if($seen.ContainsKey($e.Path.ToLowerInvariant())){throw "Choice path listed twice: $($e.Path)"};$seen[$e.Path.ToLowerInvariant()]=$true} + $seen=[Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + foreach($e in $entries){if(-not $seen.Add($e.Path)){throw "Choice path listed twice: $($e.Path)"}} foreach($never in @(if($Config.PSObject.Properties['NeverBackUp']){$Config.NeverBackUp|Where-Object {$_}|ForEach-Object {ConvertTo-SelectionPath $_}})){ - $hit=@($entries|Where-Object {$_.Path -ieq $never}) + $hit=@($entries|Where-Object {Test-SameName $_.Path $never}) if($hit.Count -and $hit[0].Mode -ne 'Skip'){throw "$never is listed in NeverBackUp; set its Mode to Skip"} if(-not $hit.Count){$entries.Add([pscustomobject]@{Path=$never;Mode='Skip'})} # A choice inside a NeverBackUp folder cannot bring part of it back: it is skipped too. @@ -61,7 +69,7 @@ function Get-BackupSelection($Config) { function Test-IsUnder([string]$Path,[string]$Parent) { # True when Path is strictly inside Parent; '' is the source root. - if($Parent -eq ''){return $Path -ne ''} + if($Parent.Length -eq 0){return $Path.Length -gt 0} $Path.StartsWith($Parent+'\',[StringComparison]::OrdinalIgnoreCase) } @@ -78,7 +86,7 @@ function Test-BackupPathInside([string]$Path,[string[]]$Roots) { function Get-BackupMode($Selection,[string]$Path) { $p=if($Path){ConvertTo-SelectionPath $Path}else{''} $best=$null - foreach($e in $Selection.Entries){if(($e.Path -ieq $p -or (Test-IsUnder $p $e.Path)) -and (-not $best -or $e.Path.Length -gt $best.Path.Length)){$best=$e}} + foreach($e in $Selection.Entries){if(((Test-SameName $e.Path $p) -or (Test-IsUnder $p $e.Path)) -and (-not $best -or $e.Path.Length -gt $best.Path.Length)){$best=$e}} if($best){$best.Mode}else{$Selection.DefaultMode} } @@ -103,18 +111,22 @@ function Get-BackupRegions($Selection,[ValidateSet('Mirror','Versioned')][string } } -function Get-RoboExtraPaths([string[]]$Lines,[string]$Root) { +function Get-RoboExtraPaths([string[]]$Lines,[string]$Root,[string[]]$OtherRoots=@()) { # The paths under Root that a robocopy /L /PURGE listing reports as extra. With /MT several threads # write into one log, so a record is not always one line: its path can land on the next line on its # own, and one line can hold the end of one record and the start of the next. Every tab-separated # field is therefore read, and a field is split again wherever a new path under Root starts. + # A path of the other side of the copy can follow at once as well (...\source\a\bX:\Copy\c\d), so + # the listing is read with that side's root in OtherRoots, and a field is split there too; a full + # path with its drive letter cannot stand inside a name. Only the paths under Root are returned. # Error lines are skipped: they can name a destination folder that is not extra. $prefix=$Root.TrimEnd('\')+'\' - $split='(?='+[regex]::Escape($prefix)+')' + $starts=@($prefix)+@($OtherRoots|Where-Object {$_}|ForEach-Object {$_.TrimEnd('\')+'\'}) + $split=[regex]::new('(?='+(@($starts|ForEach-Object {[regex]::Escape($_)}) -join '|')+')',[Text.RegularExpressions.RegexOptions]'IgnoreCase,CultureInvariant') foreach($line in $Lines){ if($line -match '\sERROR \d+ '){continue} foreach($field in ($line -split "`t")){ - foreach($p in ($field -split $split)){ + foreach($p in $split.Split($field)){ $p=$p.Trim() if($p.Length -gt $prefix.Length -and $p.StartsWith($prefix,[StringComparison]::OrdinalIgnoreCase)){$p} } @@ -122,4 +134,185 @@ function Get-RoboExtraPaths([string[]]$Lines,[string]$Root) { } } -Export-ModuleMember -Function Get-BackupModeNames,Test-BackupModeIncludes,ConvertTo-SelectionPath,Get-BackupSelection,Get-BackupMode,Get-BackupRegions,Get-RoboExtraPaths,Test-BackupPathInside +function Get-RoboLockedFiles([string[]]$Lines,[string]$SourceRoot,[int[]]$Codes=@(32,33,5)) { + # The source files a robocopy log names in an error record with one of Codes: 32 (another program + # is using the file), 33 (another program locked a part of it), 5 (access is denied). Each file + # once, as Path (below SourceRoot) and Code, in the order of the log; robocopy writes one record per + # try. A record reads "