Report repeated batch start failures once, with a count

ParallelKit.Harness.ps1

@@ -2519,6 +2519,104 @@ Test-Assert ($limitWorker.Count -eq 1 -and $limitWorker[0]['Seen'] -eq 99) "and
 Reset-Harness
 Write-Host ""
 
+Write-Host "=== AD: repeated start failures are reported once, with a count ===" -ForegroundColor Cyan
+
+function Get-StartMessageCount { param([string]$Like) return @($script:messages | Where-Object { $_.Message -like $Like }).Count }
+$missing = 'pk-no-such-program.exe'
+$missingCommands = @(1..5 | ForEach-Object { @{ FilePath = $missing; Arguments = "$_"; Tag = $_ } })
+
+Reset-Harness
+$completedCount = @{ N = 0 }
+$collapsed = Invoke-PkProcessBatch -Commands $missingCommands -MaxConcurrent 2 -OnComplete { param($c, $r) $completedCount.N++ }.GetNewClosure()
+Test-Assert (@($collapsed.Results | Where-Object { $_.StartError -like "*could not start '$missing'*" }).Count -eq 5) "every command that can't start still gets its own StartError"
+Test-Assert ($completedCount.N -eq 0) "-OnComplete still skips commands that couldn't start"
+Test-Assert ((Get-StartMessageCount "*$missing*") -eq 2) ("five failures of one program give two messages, the first and a summary (" + (Get-StartMessageCount "*$missing*") + ")")
+Test-Assert ((Get-StartMessageCount "*4 more commands couldn't start '$missing'*") -eq 1) "the summary counts the ones it left out"
+Test-Assert ((Get-StartMessageCount "*JobStarter*") -eq 0 -and (Get-StartMessageCount "*ProcessDispatch*") -eq 0) "batch messages don't name dispatcher internals"
+Test-Assert (@($script:messages | Where-Object { $_.Message -like "*more command*" -and $_.Message -like "*could not start*" }).Count -eq 0) "the summary gives Windows' reason, without the per-command text"
+
+Reset-Harness
+$null = Invoke-PkProcessBatch -Commands @(@{ FilePath = $missing }) -MaxConcurrent 2
+Test-Assert ((Get-StartMessageCount "*$missing*") -eq 1 -and (Get-StartMessageCount "*more command*") -eq 0) "one failure gets no summary"
+
+Reset-Harness
+$null = Invoke-PkProcessBatch -Commands @(@{ FilePath = $missing }, @{ FilePath = $missing }) -MaxConcurrent 2
+Test-Assert ((Get-StartMessageCount "*1 more command couldn't start '$missing'*") -eq 1) "the summary reads right for one more"
+
+Reset-Harness
+$null = Invoke-PkProcessBatch -MaxConcurrent 2 -Commands @(
+    @{ FilePath = 'pk-missing-a.exe' }, @{ FilePath = 'pk-missing-b.exe' }, @{ FilePath = 'pk-missing-a.exe' },
+    @{ FilePath = 'pk-missing-b.exe' }, @{ FilePath = 'pk-missing-a.exe' }, @{ FilePath = 'pk-missing-b.exe' })
+Test-Assert ((Get-StartMessageCount "*pk-missing-a.exe*") -eq 2 -and (Get-StartMessageCount "*pk-missing-b.exe*") -eq 2) "different programs are reported separately"
+
+Reset-Harness
+$null = Invoke-PkProcessBatch -Commands @(@{ FilePath = 'PK-MISSING-C.exe' }, @{ FilePath = 'pk-missing-c.EXE' }) -MaxConcurrent 2
+Test-Assert ((Get-StartMessageCount "*1 more command couldn't start*") -eq 1) "a program's name is matched as Windows matches it, ignoring case"
+
+# On 7 the text of this failure names the working directory, so it differs
+# between these: repeats have to be matched on Windows' error code.
+Reset-Harness
+$null = Invoke-PkProcessBatch -MaxConcurrent 2 -Commands @(
+    @{ FilePath = $missing; WorkingDirectory = $env:TEMP }, @{ FilePath = $missing; WorkingDirectory = $env:windir }, @{ FilePath = $missing; WorkingDirectory = $PSHOME })
+Test-Assert ((Get-StartMessageCount "*$missing*") -eq 2) ("the same failure from different folders counts as a repeat (" + (Get-StartMessageCount "*$missing*") + ")")
+
+# A different reason for the same program isn't a repeat.
+Reset-Harness
+$null = Invoke-PkProcessBatch -MaxConcurrent 2 -Commands @(
+    @{ FilePath = $cmdExe; Arguments = '/c exit 0'; WorkingDirectory = (Join-Path $env:TEMP 'pk-no-such-folder') },
+    @{ FilePath = $cmdExe; Arguments = ('/c rem ' + ('x' * 33000)) })
+Test-Assert ((Get-StartMessageCount "*couldn't start*") -eq 2 -and (Get-StartMessageCount "*more command*") -eq 0) "different reasons for one program are reported separately"
+
+Reset-Harness
+$mixed = Invoke-PkProcessBatch -MaxConcurrent 2 -Commands @(
+    @{ Name = 'bad'; FilePath = $cmdExe; Arguments = '/c exit 0'; WorkingDirectory = (Join-Path $env:TEMP 'pk-no-such-folder') },
+    @{ Name = 'good1'; FilePath = $cmdExe; Arguments = '/c exit 0' }, @{ Name = 'good2'; FilePath = $cmdExe; Arguments = '/c exit 0' })
+$mixedOk = @($mixed.Results | Where-Object { $_.ExitCode -eq 0 -and $_.Command['Name'] -like 'good*' }).Count
+Test-Assert ($mixedOk -eq 2 -and @($mixed.Results | Where-Object { $_.Command['Name'] -eq 'bad' -and $_.StartError }).Count -eq 1) "a command that can't start doesn't stop the good ones"
+
+Reset-Harness
+$badKeys = Invoke-PkProcessBatch -MaxConcurrent 2 -Commands @(
+    @{ FilePath = $cmdExe; Arguments = '/c exit 0'; TimeoutSeconds = 'soon' }, @{ FilePath = $cmdExe; Arguments = '/c exit 0'; TimeoutSeconds = 'soon' }, @{ FilePath = $cmdExe; Arguments = '/c exit 0'; TimeoutSeconds = 'soon' })
+Test-Assert ($badKeys.Results.Count -eq 3 -and (Get-StartMessageCount "*TimeoutSeconds must be*") -eq 2 -and (Get-StartMessageCount "*2 more commands*") -eq 1) "repeats of a bad TimeoutSeconds key are collapsed too"
+
+# The summary isn't lost when a callback stops the batch.
+Reset-Harness
+$thrown = $null
+try {
+    $null = Invoke-PkProcessBatch -MaxConcurrent 3 -OnComplete { throw 'pk-callback-stop' } -Commands @(
+        @{ FilePath = $missing }, @{ FilePath = $missing }, @{ FilePath = $cmdExe; Arguments = '/c exit 0' })
+} catch { $thrown = [string]$_ }
+Test-Assert ((Get-StartMessageCount "*1 more command couldn't start '$missing'*") -eq 1) ("the summary still prints when -OnComplete throws (threw: " + $thrown + ")")
+
+# Or when an abort ends the batch early.
+Reset-Harness
+$abortedBatch = Invoke-PkProcessBatch -MaxConcurrent 3 -AbortTimeoutSeconds 1 -OnComplete { Request-PkAbort } -Commands @(
+    @{ FilePath = $missing }, @{ FilePath = $missing }, @{ FilePath = $cmdExe; Arguments = '/c exit 0' }, @{ FilePath = $pingExe; Arguments = '-n 30 127.0.0.1' })
+Test-Assert ($abortedBatch.Aborted -and (Get-StartMessageCount "*1 more command couldn't start '$missing'*") -eq 1) ("the summary still prints after an abort (aborted: " + $abortedBatch.Aborted + ")")
+
+# The dispatcher's custom starters can throw anything, so it reports each one.
+Reset-Harness
+$dispatchMissing = Invoke-LimitGroup -Jobs @(
+    @{ Name = 'a'; FilePath = $missing }, @{ Name = 'b'; FilePath = $missing }, @{ Name = 'c'; FilePath = $missing })
+Test-Assert ($dispatchMissing.Result.JobStarterFailedCount -eq 3 -and (Get-StartMessageCount "*JobStarter failed to start a job*$missing*") -eq 3) "the dispatcher still reports every failed start"
+
+# A bad key is refused before the starter runs, so don't blame the starter.
+Reset-Harness
+$null = Invoke-LimitGroup -Jobs @(@{ Name = 'x'; FilePath = $cmdExe; Arguments = '/c exit 0'; TimeoutSeconds = 'soon' })
+Test-Assert ((Get-StartMessageCount "*ProcessDispatch: a job couldn't start:*TimeoutSeconds must be*") -eq 1 -and (Get-StartMessageCount "*JobStarter*") -eq 0) "a bad TimeoutSeconds key on a dispatcher job doesn't blame the JobStarter"
+
+Reset-Harness
+$collapseWorker = [System.Collections.Generic.List[object]]::new()
+$collapseOut = Invoke-PkRunspaceJobs -MaxParallel 1 -JobParams (New-JobList -Jobs @(@{ Missing = $missing })) -ResultHandler { param($r) $collapseWorker.Add($r) }.GetNewClosure() -WorkerScript {
+    param([string]$Missing)
+    $batch = Invoke-PkProcessBatch -Commands @(@{ FilePath = $Missing }, @{ FilePath = $Missing }, @{ FilePath = $Missing }) -MaxConcurrent 2
+    return @{ Failed = @($batch.Results | Where-Object { $_.StartError }).Count }
+}
+Test-Assert ($collapseOut.CrashedCount -eq 0 -and $collapseWorker.Count -eq 1 -and $collapseWorker[0]['Failed'] -eq 3) "start failures are handled inside a worker too"
+Test-Assert ((Get-StartMessageCount "*$missing*") -eq 2) ("and collapsed there (" + (Get-StartMessageCount "*$missing*") + " messages)")
+Reset-Harness
+Write-Host ""
+
 Set-PkMessageHandler -Handler $null
 Reset-PkAbortState
 

ParallelKit.ps1

@@ -885,7 +885,8 @@ function Start-PkProcessCore {
     } catch {
         $tried = ""
         if ($psi.FileName -ne $FilePath) { $tried = " (" + $psi.FileName + ")" }
-        throw ("Start-PkProcess: could not start '" + $FilePath + "'" + $tried + ": " + $_)
+        # The original goes along as the inner exception: the batch reads Windows' error code from it.
+        throw [System.Management.Automation.RuntimeException]::new("Start-PkProcess: could not start '" + $FilePath + "'" + $tried + ": " + $_, $_.Exception)
     }
     $null = [PkChildProcessJob]::Assign($proc)
     return @{
@@ -1505,7 +1506,9 @@ function Invoke-PkProcessDispatchCore {
                 }
                 $pkJob['SlotReleased'] = $false
                 # Before starting, so a bad key never leaves a program running.
+                $pkStartFailureText = "ProcessDispatch: a job couldn't start: "
                 $pkJob['TimeLimit'] = Get-PkJobTimeLimit -Job $pkJob -DefaultTimeoutSeconds $pkTimeoutSeconds
+                $pkStartFailureText = "ProcessDispatch: JobStarter failed to start a job: "
                 $pkStartedRaw = & $pkJobStarter $pkJob
                 foreach ($pkItem in @($pkStartedRaw)) {
                     if ($pkItem -isnot [hashtable] -or [object]::ReferenceEquals($pkItem, $pkJob)) { continue }
@@ -1524,7 +1527,8 @@ function Invoke-PkProcessDispatchCore {
                 # would otherwise leak (this job never reached $pkRunning, so
                 # the outer cleanup can't find it).
                 if ($pkIsSharedBudget) { $null = $pkMaxConcurrent.Release() }
-                Write-PkMessage ("ProcessDispatch: JobStarter failed to start a job: " + $_) -Level Error
+                # The batch sets this key on its own jobs: it reports start failures itself.
+                if (-not $pkJob['StartFailureReported']) { Write-PkMessage ($pkStartFailureText + $_) -Level Error }
                 $pkDispatchResult.JobStarterFailedCount++
                 $pkKey = $pkJob['GroupKey']
                 if ($pkGroupStates.ContainsKey($pkKey)) {
@@ -1796,6 +1800,19 @@ function New-PkBatchResult {
     return [pscustomobject]$r
 }
 
+function Get-PkStartFailureCause {
+    # Why a command couldn't start, for telling repeats apart: Windows' own text
+    # for the error code where there is one, because on 7 the exception's text
+    # for one failure differs by working directory.
+    param([Parameter(Mandatory)][System.Management.Automation.ErrorRecord]$StartError)
+    for ($e = $StartError.Exception; $null -ne $e; $e = $e.InnerException) {
+        if ($e -is [System.ComponentModel.Win32Exception]) {
+            return [System.ComponentModel.Win32Exception]::new($e.NativeErrorCode).Message.TrimEnd('.')
+        }
+    }
+    return [string]$StartError
+}
+
 function Invoke-PkProcessBatch {
     # .SYNOPSIS
     # Runs a list of external programs, a limited number at a time, and returns
@@ -1806,7 +1823,10 @@ function Invoke-PkProcessBatch {
     # TimeoutSeconds (overrides -TimeoutSeconds) and any keys of your own, which
     # come back on the result's Command. Inside
     # Invoke-PkRun, Ctrl+C lets running programs finish (up to -AbortTimeoutSeconds)
-    # and starts no new ones; a second Ctrl+C kills them straight away.
+    # and starts no new ones; a second Ctrl+C kills them straight away. Every
+    # command is tried, and one that can't start gets a StartError on its result.
+    # Repeats of the same failure for the same program are reported once, with a
+    # count at the end.
     # .PARAMETER Commands
     # The command hashtables. $null or empty returns no results.
     # .PARAMETER MaxConcurrent
@@ -1864,7 +1884,25 @@ function Invoke-PkProcessBatch {
     $pkNewBatchResult = ${function:New-PkBatchResult}
     $pkStartProcessCore = ${function:Start-PkProcessCore}
     $pkGetJobTimeLimit = ${function:Get-PkJobTimeLimit}
+    $pkGetStartFailureCause = ${function:Get-PkStartFailureCause}
     $pkCallerState = $PSCmdlet.SessionState
+    $pkStartFailures = [System.Collections.Specialized.OrderedDictionary]::new([System.StringComparer]::OrdinalIgnoreCase)
+
+    # The first failure for a program and cause goes out in full; the rest only
+    # add to the count that's reported after the dispatch.
+    $pkReportStartFailure = {
+        param($pkCommand, $pkError)
+        $pkResults.Add((& $pkNewBatchResult -Command $pkCommand -StartError ([string]$pkError)))
+        $pkFilePath = [string]$pkCommand['FilePath']
+        $pkCause = & $pkGetStartFailureCause -StartError $pkError
+        $pkFailureKey = $pkFilePath + '|' + $pkCause
+        if ($pkStartFailures.Contains($pkFailureKey)) {
+            $pkStartFailures[$pkFailureKey].More++
+            return
+        }
+        $pkStartFailures[$pkFailureKey] = @{ FilePath = $pkFilePath; Cause = $pkCause; More = 0 }
+        Write-PkMessage ("Invoke-PkProcessBatch: a command couldn't start: " + $pkError) -Level Error
+    }.GetNewClosure()
 
     $pkGroupProvider = {
         if ($pkState.Given) { return $null }
@@ -1884,8 +1922,7 @@ function Invoke-PkProcessBatch {
             try {
                 $null = & $pkGetJobTimeLimit -Job $pkC
             } catch {
-                $pkResults.Add((& $pkNewBatchResult -Command $pkC -StartError ([string]$_)))
-                Write-PkMessage ("Invoke-PkProcessBatch: a command couldn't start: " + $_) -Level Error
+                & $pkReportStartFailure $pkC $_
                 continue
             }
             $pkJobs.Add(@{ FilePath = $pkC['FilePath']; Arguments = [string]$pkC['Arguments']; WorkingDirectory = [string]$pkC['WorkingDirectory']; OutputEncoding = $pkEncoding; TimeoutSeconds = $pkC['TimeoutSeconds']; Command = $pkC })
@@ -1898,7 +1935,8 @@ function Invoke-PkProcessBatch {
         try {
             $pkStarted = & $pkStartProcessCore -CallerState $pkCallerState -FilePath $pkJob['FilePath'] -Arguments $pkJob['Arguments'] -WorkingDirectory $pkJob['WorkingDirectory'] -OutputEncoding $pkJob['OutputEncoding']
         } catch {
-            $pkResults.Add((& $pkNewBatchResult -Command $pkJob['Command'] -StartError ([string]$_)))
+            $pkJob['StartFailureReported'] = $true
+            & $pkReportStartFailure $pkJob['Command'] $_
             throw
         }
         $pkJob['Proc'] = $pkStarted.Proc; $pkJob['StdoutTask'] = $pkStarted.StdoutTask; $pkJob['StderrTask'] = $pkStarted.StderrTask
@@ -1924,7 +1962,15 @@ function Invoke-PkProcessBatch {
         JobFinalize = $pkJobFinalize; JobAbortCleanup = $pkJobAbortCleanup; MaxConcurrent = $pkMaxConcurrent
         AbortTimeoutSeconds = $pkAbortTimeoutSeconds; MaxQueueAhead = 1; TimeoutSeconds = $pkTimeoutSeconds
     }
-    $pkDispatch = Invoke-PkProcessDispatch @pkDispatchParams
+    try {
+        $pkDispatch = Invoke-PkProcessDispatch @pkDispatchParams
+    } finally {
+        foreach ($pkFailure in $pkStartFailures.Values) {
+            if ($pkFailure.More -eq 0) { continue }
+            $pkCommandWord = if ($pkFailure.More -eq 1) { 'command' } else { 'commands' }
+            Write-PkMessage ("Invoke-PkProcessBatch: " + $pkFailure.More + " more " + $pkCommandWord + " couldn't start '" + $pkFailure.FilePath + "' (" + $pkFailure.Cause + ").") -Level Error
+        }
+    }
 
     return [pscustomobject][ordered]@{ Results = $pkResults; Aborted = $pkDispatch.Aborted; ForceKilledCount = $pkDispatch.ForceKilledCount }
 }
@@ -1938,7 +1984,7 @@ function Get-PkWorkerSafeFunctionNames {
     return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool', 'Stop-PkProcessTree', 'Stop-PkJobProcesses', 'Resolve-PkEncoding',
              'Start-PkProcess', 'Wait-PkProcess', 'Wait-PkRunningJobs', 'Complete-PkProcess', 'Invoke-PkProcessWithTimeout',
              'Invoke-PkProcessDispatch', 'Invoke-PkProcessDispatchCore', 'Invoke-PkProcessBatch',
-             'Register-PkCancellationSource', 'Unregister-PkCancellationSource', 'New-PkBatchResult', 'ConvertTo-PkArgumentString', 'Get-PkForceStopReason', 'Resolve-PkFileSystemPath', 'Add-PkDispatchGroupJobs', 'Start-PkProcessCore', 'Get-PkJobTimeLimit', 'Stop-PkTimedOutJob')
+             'Register-PkCancellationSource', 'Unregister-PkCancellationSource', 'New-PkBatchResult', 'ConvertTo-PkArgumentString', 'Get-PkForceStopReason', 'Resolve-PkFileSystemPath', 'Add-PkDispatchGroupJobs', 'Start-PkProcessCore', 'Get-PkJobTimeLimit', 'Stop-PkTimedOutJob', 'Get-PkStartFailureCause')
 }
 
 function Get-PkHandlerParamCount {