From 866a1f19e205417f71bbae95f7ead68c8448f7f6 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:05:21 -0500 Subject: [PATCH 1/2] Report repeated batch start failures once, with a count A batch whose program isn't installed printed one identical red line per command, naming dispatcher internals the caller never touched. Now the first failure for a program and cause prints in full, and one line at the end counts the rest. Every command is still tried and still gets its own StartError. Repeats are matched on the file path and Windows' error code, not the message, because on PowerShell 7 the message for one failure differs by working directory. Start-PkProcess now keeps the original exception as the inner one. A bad TimeoutSeconds key on a dispatcher job no longer blames the JobStarter, which never ran. The dispatcher still reports each failed start, since custom starters can throw anything. Co-Authored-By: Claude Opus 5.5 --- ParallelKit.Harness.ps1 | 92 +++++++++++++++++++++++++++++++++++++++++ ParallelKit.ps1 | 62 +++++++++++++++++++++++---- README.md | 4 +- 3 files changed, 148 insertions(+), 10 deletions(-) diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index ed56e9c..1d1374b 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -2519,6 +2519,98 @@ Test-Assert ($limitWorker.Count -eq 1 -and $limitWorker[0]['Seen'] -eq 99) "and Reset-Harness Write-Host "" +Write-Host "=== AD: repeated start failures are reported once, with a count ===" -ForegroundColor Cyan + +function Get-StartMessageCount { param([string]$Like) return @($script:messages | Where-Object { $_.Message -like $Like }).Count } +$missing = 'pk-no-such-program.exe' +$missingCommands = @(1..5 | ForEach-Object { @{ FilePath = $missing; Arguments = "$_"; Tag = $_ } }) + +Reset-Harness +$completedCount = @{ N = 0 } +$collapsed = Invoke-PkProcessBatch -Commands $missingCommands -MaxConcurrent 2 -OnComplete { param($c, $r) $completedCount.N++ }.GetNewClosure() +Test-Assert (@($collapsed.Results | Where-Object { $_.StartError -like "*could not start '$missing'*" }).Count -eq 5) "every command that can't start still gets its own StartError" +Test-Assert ($completedCount.N -eq 0) "-OnComplete still skips commands that couldn't start" +Test-Assert ((Get-StartMessageCount "*$missing*") -eq 2) ("five failures of one program give two messages, the first and a summary (" + (Get-StartMessageCount "*$missing*") + ")") +Test-Assert ((Get-StartMessageCount "*4 more commands couldn't start '$missing'*") -eq 1) "the summary counts the ones it left out" +Test-Assert ((Get-StartMessageCount "*JobStarter*") -eq 0 -and (Get-StartMessageCount "*ProcessDispatch*") -eq 0) "batch messages don't name dispatcher internals" +Test-Assert (@($script:messages | Where-Object { $_.Message -like "*more command*" -and $_.Message -notlike "*cannot find the file*" }).Count -eq 0) "the summary gives Windows' reason, without the per-command text" + +Reset-Harness +$null = Invoke-PkProcessBatch -Commands @(@{ FilePath = $missing }) -MaxConcurrent 2 +Test-Assert ((Get-StartMessageCount "*$missing*") -eq 1 -and (Get-StartMessageCount "*more command*") -eq 0) "one failure gets no summary" + +Reset-Harness +$null = Invoke-PkProcessBatch -Commands @(@{ FilePath = $missing }, @{ FilePath = $missing }) -MaxConcurrent 2 +Test-Assert ((Get-StartMessageCount "*1 more command couldn't start '$missing'*") -eq 1) "the summary reads right for one more" + +Reset-Harness +$null = Invoke-PkProcessBatch -MaxConcurrent 2 -Commands @( + @{ FilePath = 'pk-missing-a.exe' }, @{ FilePath = 'pk-missing-b.exe' }, @{ FilePath = 'pk-missing-a.exe' }, + @{ FilePath = 'pk-missing-b.exe' }, @{ FilePath = 'pk-missing-a.exe' }, @{ FilePath = 'pk-missing-b.exe' }) +Test-Assert ((Get-StartMessageCount "*pk-missing-a.exe*") -eq 2 -and (Get-StartMessageCount "*pk-missing-b.exe*") -eq 2) "different programs are reported separately" + +Reset-Harness +$null = Invoke-PkProcessBatch -Commands @(@{ FilePath = 'PK-MISSING-C.exe' }, @{ FilePath = 'pk-missing-c.EXE' }) -MaxConcurrent 2 +Test-Assert ((Get-StartMessageCount "*1 more command couldn't start*") -eq 1) "a program's name is matched as Windows matches it, ignoring case" + +# On 7 the text of this failure names the working directory, so it differs +# between these: repeats have to be matched on Windows' error code. +Reset-Harness +$null = Invoke-PkProcessBatch -MaxConcurrent 2 -Commands @( + @{ FilePath = $missing; WorkingDirectory = $env:TEMP }, @{ FilePath = $missing; WorkingDirectory = $env:windir }, @{ FilePath = $missing; WorkingDirectory = $PSHOME }) +Test-Assert ((Get-StartMessageCount "*$missing*") -eq 2) ("the same failure from different folders counts as a repeat (" + (Get-StartMessageCount "*$missing*") + ")") + +# A different reason for the same program isn't a repeat. +Reset-Harness +$null = Invoke-PkProcessBatch -MaxConcurrent 2 -Commands @( + @{ FilePath = $cmdExe; Arguments = '/c exit 0'; WorkingDirectory = (Join-Path $env:TEMP 'pk-no-such-folder') }, + @{ FilePath = $cmdExe; Arguments = ('/c rem ' + ('x' * 33000)) }) +Test-Assert ((Get-StartMessageCount "*couldn't start*") -eq 2 -and (Get-StartMessageCount "*more command*") -eq 0) "different reasons for one program are reported separately" + +Reset-Harness +$mixed = Invoke-PkProcessBatch -MaxConcurrent 2 -Commands @( + @{ Name = 'bad'; FilePath = $cmdExe; Arguments = '/c exit 0'; WorkingDirectory = (Join-Path $env:TEMP 'pk-no-such-folder') }, + @{ Name = 'good1'; FilePath = $cmdExe; Arguments = '/c exit 0' }, @{ Name = 'good2'; FilePath = $cmdExe; Arguments = '/c exit 0' }) +$mixedOk = @($mixed.Results | Where-Object { $_.ExitCode -eq 0 -and $_.Command['Name'] -like 'good*' }).Count +Test-Assert ($mixedOk -eq 2 -and @($mixed.Results | Where-Object { $_.Command['Name'] -eq 'bad' -and $_.StartError }).Count -eq 1) "a command that can't start doesn't stop the good ones" + +Reset-Harness +$badKeys = Invoke-PkProcessBatch -MaxConcurrent 2 -Commands @( + @{ FilePath = $cmdExe; Arguments = '/c exit 0'; TimeoutSeconds = 'soon' }, @{ FilePath = $cmdExe; Arguments = '/c exit 0'; TimeoutSeconds = 'soon' }, @{ FilePath = $cmdExe; Arguments = '/c exit 0'; TimeoutSeconds = 'soon' }) +Test-Assert ($badKeys.Results.Count -eq 3 -and (Get-StartMessageCount "*TimeoutSeconds must be*") -eq 2 -and (Get-StartMessageCount "*2 more commands*") -eq 1) "repeats of a bad TimeoutSeconds key are collapsed too" + +# The summary isn't lost when a callback stops the batch. +Reset-Harness +$thrown = $null +try { + $null = Invoke-PkProcessBatch -MaxConcurrent 3 -OnComplete { throw 'pk-callback-stop' } -Commands @( + @{ FilePath = $missing }, @{ FilePath = $missing }, @{ FilePath = $cmdExe; Arguments = '/c exit 0' }) +} catch { $thrown = [string]$_ } +Test-Assert ((Get-StartMessageCount "*1 more command couldn't start '$missing'*") -eq 1) ("the summary still prints when -OnComplete throws (threw: " + $thrown + ")") + +# The dispatcher's custom starters can throw anything, so it reports each one. +Reset-Harness +$dispatchMissing = Invoke-LimitGroup -Jobs @( + @{ Name = 'a'; FilePath = $missing }, @{ Name = 'b'; FilePath = $missing }, @{ Name = 'c'; FilePath = $missing }) +Test-Assert ($dispatchMissing.Result.JobStarterFailedCount -eq 3 -and (Get-StartMessageCount "*JobStarter failed to start a job*$missing*") -eq 3) "the dispatcher still reports every failed start" + +# A bad key is refused before the starter runs, so don't blame the starter. +Reset-Harness +$null = Invoke-LimitGroup -Jobs @(@{ Name = 'x'; FilePath = $cmdExe; Arguments = '/c exit 0'; TimeoutSeconds = 'soon' }) +Test-Assert ((Get-StartMessageCount "*ProcessDispatch: a job couldn't start:*TimeoutSeconds must be*") -eq 1 -and (Get-StartMessageCount "*JobStarter*") -eq 0) "a bad TimeoutSeconds key on a dispatcher job doesn't blame the JobStarter" + +Reset-Harness +$collapseWorker = [System.Collections.Generic.List[object]]::new() +$collapseOut = Invoke-PkRunspaceJobs -MaxParallel 1 -JobParams (New-JobList -Jobs @(@{ Missing = $missing })) -ResultHandler { param($r) $collapseWorker.Add($r) }.GetNewClosure() -WorkerScript { + param([string]$Missing) + $batch = Invoke-PkProcessBatch -Commands @(@{ FilePath = $Missing }, @{ FilePath = $Missing }, @{ FilePath = $Missing }) -MaxConcurrent 2 + return @{ Failed = @($batch.Results | Where-Object { $_.StartError }).Count } +} +Test-Assert ($collapseOut.CrashedCount -eq 0 -and $collapseWorker.Count -eq 1 -and $collapseWorker[0]['Failed'] -eq 3) "start failures are handled inside a worker too" +Test-Assert ((Get-StartMessageCount "*$missing*") -eq 2) ("and collapsed there (" + (Get-StartMessageCount "*$missing*") + " messages)") +Reset-Harness +Write-Host "" + Set-PkMessageHandler -Handler $null Reset-PkAbortState diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index 48ef15a..86e9e77 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -885,7 +885,8 @@ function Start-PkProcessCore { } catch { $tried = "" if ($psi.FileName -ne $FilePath) { $tried = " (" + $psi.FileName + ")" } - throw ("Start-PkProcess: could not start '" + $FilePath + "'" + $tried + ": " + $_) + # The original goes along as the inner exception: the batch reads Windows' error code from it. + throw [System.Management.Automation.RuntimeException]::new("Start-PkProcess: could not start '" + $FilePath + "'" + $tried + ": " + $_, $_.Exception) } $null = [PkChildProcessJob]::Assign($proc) return @{ @@ -1505,7 +1506,9 @@ function Invoke-PkProcessDispatchCore { } $pkJob['SlotReleased'] = $false # Before starting, so a bad key never leaves a program running. + $pkStartFailureText = "ProcessDispatch: a job couldn't start: " $pkJob['TimeLimit'] = Get-PkJobTimeLimit -Job $pkJob -DefaultTimeoutSeconds $pkTimeoutSeconds + $pkStartFailureText = "ProcessDispatch: JobStarter failed to start a job: " $pkStartedRaw = & $pkJobStarter $pkJob foreach ($pkItem in @($pkStartedRaw)) { if ($pkItem -isnot [hashtable] -or [object]::ReferenceEquals($pkItem, $pkJob)) { continue } @@ -1524,7 +1527,8 @@ function Invoke-PkProcessDispatchCore { # would otherwise leak (this job never reached $pkRunning, so # the outer cleanup can't find it). if ($pkIsSharedBudget) { $null = $pkMaxConcurrent.Release() } - Write-PkMessage ("ProcessDispatch: JobStarter failed to start a job: " + $_) -Level Error + # The batch sets this key on its own jobs: it reports start failures itself. + if (-not $pkJob['StartFailureReported']) { Write-PkMessage ($pkStartFailureText + $_) -Level Error } $pkDispatchResult.JobStarterFailedCount++ $pkKey = $pkJob['GroupKey'] if ($pkGroupStates.ContainsKey($pkKey)) { @@ -1796,6 +1800,19 @@ function New-PkBatchResult { return [pscustomobject]$r } +function Get-PkStartFailureCause { + # Why a command couldn't start, for telling repeats apart: Windows' own text + # for the error code where there is one, because on 7 the exception's text + # for one failure differs by working directory. + param([Parameter(Mandatory)][System.Management.Automation.ErrorRecord]$StartError) + for ($e = $StartError.Exception; $null -ne $e; $e = $e.InnerException) { + if ($e -is [System.ComponentModel.Win32Exception]) { + return [System.ComponentModel.Win32Exception]::new($e.NativeErrorCode).Message.TrimEnd('.') + } + } + return [string]$StartError +} + function Invoke-PkProcessBatch { # .SYNOPSIS # Runs a list of external programs, a limited number at a time, and returns @@ -1806,7 +1823,10 @@ function Invoke-PkProcessBatch { # TimeoutSeconds (overrides -TimeoutSeconds) and any keys of your own, which # come back on the result's Command. Inside # Invoke-PkRun, Ctrl+C lets running programs finish (up to -AbortTimeoutSeconds) - # and starts no new ones; a second Ctrl+C kills them straight away. + # and starts no new ones; a second Ctrl+C kills them straight away. Every + # command is tried, and one that can't start gets a StartError on its result. + # Repeats of the same failure for the same program are reported once, with a + # count at the end. # .PARAMETER Commands # The command hashtables. $null or empty returns no results. # .PARAMETER MaxConcurrent @@ -1864,7 +1884,25 @@ function Invoke-PkProcessBatch { $pkNewBatchResult = ${function:New-PkBatchResult} $pkStartProcessCore = ${function:Start-PkProcessCore} $pkGetJobTimeLimit = ${function:Get-PkJobTimeLimit} + $pkGetStartFailureCause = ${function:Get-PkStartFailureCause} $pkCallerState = $PSCmdlet.SessionState + $pkStartFailures = [System.Collections.Specialized.OrderedDictionary]::new([System.StringComparer]::OrdinalIgnoreCase) + + # The first failure for a program and cause goes out in full; the rest only + # add to the count that's reported after the dispatch. + $pkReportStartFailure = { + param($pkCommand, $pkError) + $pkResults.Add((& $pkNewBatchResult -Command $pkCommand -StartError ([string]$pkError))) + $pkFilePath = [string]$pkCommand['FilePath'] + $pkCause = & $pkGetStartFailureCause -StartError $pkError + $pkFailureKey = $pkFilePath + '|' + $pkCause + if ($pkStartFailures.Contains($pkFailureKey)) { + $pkStartFailures[$pkFailureKey].More++ + return + } + $pkStartFailures[$pkFailureKey] = @{ FilePath = $pkFilePath; Cause = $pkCause; More = 0 } + Write-PkMessage ("Invoke-PkProcessBatch: a command couldn't start: " + $pkError) -Level Error + }.GetNewClosure() $pkGroupProvider = { if ($pkState.Given) { return $null } @@ -1884,8 +1922,7 @@ function Invoke-PkProcessBatch { try { $null = & $pkGetJobTimeLimit -Job $pkC } catch { - $pkResults.Add((& $pkNewBatchResult -Command $pkC -StartError ([string]$_))) - Write-PkMessage ("Invoke-PkProcessBatch: a command couldn't start: " + $_) -Level Error + & $pkReportStartFailure $pkC $_ continue } $pkJobs.Add(@{ FilePath = $pkC['FilePath']; Arguments = [string]$pkC['Arguments']; WorkingDirectory = [string]$pkC['WorkingDirectory']; OutputEncoding = $pkEncoding; TimeoutSeconds = $pkC['TimeoutSeconds']; Command = $pkC }) @@ -1898,7 +1935,8 @@ function Invoke-PkProcessBatch { try { $pkStarted = & $pkStartProcessCore -CallerState $pkCallerState -FilePath $pkJob['FilePath'] -Arguments $pkJob['Arguments'] -WorkingDirectory $pkJob['WorkingDirectory'] -OutputEncoding $pkJob['OutputEncoding'] } catch { - $pkResults.Add((& $pkNewBatchResult -Command $pkJob['Command'] -StartError ([string]$_))) + $pkJob['StartFailureReported'] = $true + & $pkReportStartFailure $pkJob['Command'] $_ throw } $pkJob['Proc'] = $pkStarted.Proc; $pkJob['StdoutTask'] = $pkStarted.StdoutTask; $pkJob['StderrTask'] = $pkStarted.StderrTask @@ -1924,7 +1962,15 @@ function Invoke-PkProcessBatch { JobFinalize = $pkJobFinalize; JobAbortCleanup = $pkJobAbortCleanup; MaxConcurrent = $pkMaxConcurrent AbortTimeoutSeconds = $pkAbortTimeoutSeconds; MaxQueueAhead = 1; TimeoutSeconds = $pkTimeoutSeconds } - $pkDispatch = Invoke-PkProcessDispatch @pkDispatchParams + try { + $pkDispatch = Invoke-PkProcessDispatch @pkDispatchParams + } finally { + foreach ($pkFailure in $pkStartFailures.Values) { + if ($pkFailure.More -eq 0) { continue } + $pkCommandWord = if ($pkFailure.More -eq 1) { 'command' } else { 'commands' } + Write-PkMessage ("Invoke-PkProcessBatch: " + $pkFailure.More + " more " + $pkCommandWord + " couldn't start '" + $pkFailure.FilePath + "' (" + $pkFailure.Cause + ").") -Level Error + } + } return [pscustomobject][ordered]@{ Results = $pkResults; Aborted = $pkDispatch.Aborted; ForceKilledCount = $pkDispatch.ForceKilledCount } } @@ -1938,7 +1984,7 @@ function Get-PkWorkerSafeFunctionNames { return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool', 'Stop-PkProcessTree', 'Stop-PkJobProcesses', 'Resolve-PkEncoding', 'Start-PkProcess', 'Wait-PkProcess', 'Wait-PkRunningJobs', 'Complete-PkProcess', 'Invoke-PkProcessWithTimeout', 'Invoke-PkProcessDispatch', 'Invoke-PkProcessDispatchCore', 'Invoke-PkProcessBatch', - 'Register-PkCancellationSource', 'Unregister-PkCancellationSource', 'New-PkBatchResult', 'ConvertTo-PkArgumentString', 'Get-PkForceStopReason', 'Resolve-PkFileSystemPath', 'Add-PkDispatchGroupJobs', 'Start-PkProcessCore', 'Get-PkJobTimeLimit', 'Stop-PkTimedOutJob') + 'Register-PkCancellationSource', 'Unregister-PkCancellationSource', 'New-PkBatchResult', 'ConvertTo-PkArgumentString', 'Get-PkForceStopReason', 'Resolve-PkFileSystemPath', 'Add-PkDispatchGroupJobs', 'Start-PkProcessCore', 'Get-PkJobTimeLimit', 'Stop-PkTimedOutJob', 'Get-PkStartFailureCause') } function Get-PkHandlerParamCount { diff --git a/README.md b/README.md index d120c5a..9f2b3d1 100644 --- a/README.md +++ b/README.md @@ -129,14 +129,14 @@ Programs run in your current location (`Get-Location`) unless a command says oth - `Command` is your original command hashtable, so extra keys like `Source` above come back with it. - `ExitCode` is whatever the program returned. The kit doesn't treat a non-zero code as a failure; that's for you to decide. -- `StartError` is set when the program couldn't be started (not found, say). +- `StartError` is set when the program couldn't be started (not found, say). Every command is still tried and gets its own result, but repeats of the same failure for the same program print once, with a count at the end. - `Aborted` is `$true` for a command an abort killed or kept from starting. - `TimedOut` is `$true` for a program `-TimeoutSeconds` killed. - `StreamTimedOut` / `StreamError` mean the output may be incomplete. The output read gives up 15 seconds after the program exits, which only happens when a program it started is still holding the output open. `Results` is in the order commands finished, not the order you gave them. Use `Command` to match them up. -`-OnComplete` runs as each command finishes, but only for commands that actually ran. Check `Results` for the ones that couldn't start or were aborted. Defaults: `-MaxConcurrent` 4, `-AbortTimeoutSeconds` 60 (how long Ctrl+C waits for running programs), `-TimeoutSeconds` 0 (no limit). +`-OnComplete` runs as each command finishes, but only for commands that actually ran. Check `Results` for the ones that couldn't start or were aborted. A progress counter driven from `-OnComplete` won't reach the total if any did, so count from `$batch.Results` afterwards for the final numbers. Defaults: `-MaxConcurrent` 4, `-AbortTimeoutSeconds` 60 (how long Ctrl+C waits for running programs), `-TimeoutSeconds` 0 (no limit). `-TimeoutSeconds 600` kills any program still running after 10 minutes. Its result has `TimedOut = $true`, still goes through `-OnComplete`, and the rest of the batch carries on. The default, 0, means no limit. Any programs it started are killed with it. It can be fractional. A command's own `TimeoutSeconds` key overrides this for that command; 0 opts it out of the limit. A bad value there gives that command a `StartError`. From cdd7a579d2b5f5a077ea29f8949dbb70da5ed604 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:10:03 -0500 Subject: [PATCH 2/2] Cover the start-failure summary after an abort; reserve the new job key Co-Authored-By: Claude Opus 5.5 --- ParallelKit.Harness.ps1 | 8 +++++++- README.md | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index 1d1374b..adf6f91 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -2533,7 +2533,7 @@ Test-Assert ($completedCount.N -eq 0) "-OnComplete still skips commands that cou Test-Assert ((Get-StartMessageCount "*$missing*") -eq 2) ("five failures of one program give two messages, the first and a summary (" + (Get-StartMessageCount "*$missing*") + ")") Test-Assert ((Get-StartMessageCount "*4 more commands couldn't start '$missing'*") -eq 1) "the summary counts the ones it left out" Test-Assert ((Get-StartMessageCount "*JobStarter*") -eq 0 -and (Get-StartMessageCount "*ProcessDispatch*") -eq 0) "batch messages don't name dispatcher internals" -Test-Assert (@($script:messages | Where-Object { $_.Message -like "*more command*" -and $_.Message -notlike "*cannot find the file*" }).Count -eq 0) "the summary gives Windows' reason, without the per-command text" +Test-Assert (@($script:messages | Where-Object { $_.Message -like "*more command*" -and $_.Message -like "*could not start*" }).Count -eq 0) "the summary gives Windows' reason, without the per-command text" Reset-Harness $null = Invoke-PkProcessBatch -Commands @(@{ FilePath = $missing }) -MaxConcurrent 2 @@ -2588,6 +2588,12 @@ try { } catch { $thrown = [string]$_ } Test-Assert ((Get-StartMessageCount "*1 more command couldn't start '$missing'*") -eq 1) ("the summary still prints when -OnComplete throws (threw: " + $thrown + ")") +# Or when an abort ends the batch early. +Reset-Harness +$abortedBatch = Invoke-PkProcessBatch -MaxConcurrent 3 -AbortTimeoutSeconds 1 -OnComplete { Request-PkAbort } -Commands @( + @{ FilePath = $missing }, @{ FilePath = $missing }, @{ FilePath = $cmdExe; Arguments = '/c exit 0' }, @{ FilePath = $pingExe; Arguments = '-n 30 127.0.0.1' }) +Test-Assert ($abortedBatch.Aborted -and (Get-StartMessageCount "*1 more command couldn't start '$missing'*") -eq 1) ("the summary still prints after an abort (aborted: " + $abortedBatch.Aborted + ")") + # The dispatcher's custom starters can throw anything, so it reports each one. Reset-Harness $dispatchMissing = Invoke-LimitGroup -Jobs @( diff --git a/README.md b/README.md index 9f2b3d1..dd637a3 100644 --- a/README.md +++ b/README.md @@ -240,7 +240,7 @@ All of these callbacks run on whichever thread called the dispatcher: the main t - `-JobFinalize` is called once per finished job. Collect its exit code and output there with `Complete-PkProcess -Job $job`; the dispatcher disposes the process right after. - Groups are pulled lazily, at most `-MaxQueueAhead` (default 3) ahead. - A group can be any value: a hashtable, a path, a name. Each job gets its group as `$job.Group`. -- The dispatcher keeps its state on the job hashtables. Don't use these names for your own job fields: `GroupKey`, `SlotReleased`, `Proc`, `StdoutTask`, `StderrTask`, `Stopwatch`, `TimeLimit`, `TimedOut`. (`Group` is yours if you set it yourself.) +- The dispatcher keeps its state on the job hashtables. Don't use these names for your own job fields: `GroupKey`, `SlotReleased`, `Proc`, `StdoutTask`, `StderrTask`, `Stopwatch`, `TimeLimit`, `TimedOut`, `StartFailureReported`. (`Group` is yours if you set it yourself.) - `-TimeoutSeconds 600` kills a job's program, and anything it started, if it's still running after 10 minutes. The default, 0, means no limit, and it can be fractional. A job's own `TimeoutSeconds` key overrides it; 0 opts the job out, and a bad value is a failed start. The killed job still goes through `-JobFinalize`, where `$job['TimedOut']` is `$true`, and other jobs carry on. The limit stays in force while an abort waits for running jobs. A retry gets a fresh clock. A job whose `.Proc` isn't a process isn't timed: `-JobPollCompleted` is the place for its limit. - To retry a job or add follow-up work, enqueue job hashtables onto `$pending` inside `-JobFinalize`. They join the same group, and the group isn't complete until they've all finished. A retry can start within milliseconds of the failure, so cap the number of retries, and don't count on the gap for something like a locked file to clear. - Supply `-JobStarter { param($job) ... }` to start jobs yourself. It must set `$job.Proc` and should return the job. A starter that is just `Start-PkProcess ...` works too: its result is folded into your job. Throw to report a job that couldn't start.