Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions ParallelKit.Harness.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -2715,6 +2715,62 @@ if ($compiled) {
Reset-Harness
Write-Host ""

Write-Host "=== AF: parameter handling: advanced functions, aliases, the safety switch ===" -ForegroundColor Cyan

# Every public function is advanced, so a wrong or extra argument is an error
# rather than silently ignored.
$simpleFunctions = @($publicFunctions | Where-Object { -not (Get-Command $_).CmdletBinding })
Test-Assert ($simpleFunctions.Count -eq 0) ("every public function rejects unknown arguments (not: " + ($simpleFunctions -join ', ') + ")")
$threw = $null
try { $null = Initialize-ParallelKit -MaxParallel 4 } catch { $threw = [string]$_ }
Test-Assert ($threw -like '*MaxParallel*') "Initialize-ParallelKit -MaxParallel (the engine's name) is an error, not ignored"
$threw = $null
try { $null = Test-PkAbortSignal 'extra' } catch { $threw = [string]$_ }
Test-Assert ($null -ne $threw) "Test-PkAbortSignal refuses an argument it doesn't take"
$threw = $null
try { Reset-PkAbortState -WhatIf } catch { $threw = [string]$_ }
Test-Assert ($null -ne $threw) "Reset-PkAbortState -WhatIf is an error, not a reset"
# Still called the usual ways.
Set-PkMessageHandler { param($Message, $Level) $messageSink.Add(@{ Message = $Message; Level = $Level }) }.GetNewClosure()
Write-PkMessage 'pk-positional-handler'
Test-Assert ((Get-StartMessageCount '*pk-positional-handler*') -eq 1) "Set-PkMessageHandler still takes the handler by position"

$threw = $null
try { $null = Invoke-PkProcessBatch -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) -MaxConcurrent 0 } catch { $threw = [string]$_ }
Test-Assert ($threw -like 'Invoke-PkProcessBatch:*') ("the batch's own -MaxConcurrent error names the batch (" + $threw + ")")

# The concurrency and abort-wait names work on every engine.
Reset-Harness
$aliasBatchParams = @{ Commands = @(@{ FilePath = $cmdExe; Arguments = '/c exit 3' }); MaxParallel = 2; DrainTimeoutSeconds = 5 }
$aliasBatch = $null
try { $aliasBatch = Invoke-PkProcessBatch @aliasBatchParams } catch { Write-Host " batch alias threw: $_" }
Test-Assert ($null -ne $aliasBatch -and $aliasBatch.Results[0].ExitCode -eq 3) "the batch takes -MaxParallel and -DrainTimeoutSeconds"
$aliasRunParams = @{ WorkerScript = { param($Id) $Id * 2 }; JobParams = (New-JobList -Jobs @(@{ Id = 21 })); MaxConcurrent = 2; AbortTimeoutSeconds = 5 }
$aliasRunResults = [System.Collections.Generic.List[object]]::new()
try { $null = Invoke-PkRunspaceJobs @aliasRunParams -ResultHandler { param($r) $aliasRunResults.Add($r) }.GetNewClosure() } catch { Write-Host " runspace alias threw: $_" }
Test-Assert ($aliasRunResults.Count -eq 1 -and $aliasRunResults[0] -eq 42) "Invoke-PkRunspaceJobs takes -MaxConcurrent and -AbortTimeoutSeconds"
$aliasFeed = @{ Given = $false }
$aliasDispatch = $null
try {
$aliasDispatch = Invoke-PkProcessDispatch -MaxParallel 1 -DrainTimeoutSeconds 5 `
-GroupProvider { if ($aliasFeed.Given) { return $null }; $aliasFeed.Given = $true; return 'g' }.GetNewClosure() `
-JobBuilder { param($g) return @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) }.GetNewClosure() `
-JobFinalize { param($job, $q) $null = Complete-PkProcess -Job $job }
} catch { Write-Host " dispatcher alias threw: $_" }
Test-Assert ($null -ne $aliasDispatch -and $aliasDispatch.GroupResults.Count -eq 1) "the dispatcher takes -MaxParallel and -DrainTimeoutSeconds"

# -CheckWorkerSafety: on unless turned off, however it's passed.
$unsafeWorker = { param($Id) $null = [Func[int]]{ 1 } }
$threw = $false
try { $null = Invoke-PkRunspaceJobs -WorkerScript $unsafeWorker -JobParams (New-JobList -Jobs @(@{ Id = 1 })) -MaxParallel 1 } catch { $threw = ($_.ToString() -like '*Func*') }
Test-Assert $threw "the worker-safety check is on when -CheckWorkerSafety isn't given"
$safetyOff = @{ CheckWorkerSafety = $false }
$out = $null
try { $out = Invoke-PkRunspaceJobs -WorkerScript $unsafeWorker -JobParams (New-JobList -Jobs @(@{ Id = 1 })) -MaxParallel 1 @safetyOff } catch { Write-Host " splatted opt-out threw: $_" }
Test-Assert ($null -ne $out -and $out.CrashedCount -eq 0) "a splatted CheckWorkerSafety = `$false turns it off"
Reset-Harness
Write-Host ""

Set-PkMessageHandler -Handler $null
Reset-PkAbortState

Expand Down
48 changes: 36 additions & 12 deletions ParallelKit.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,7 @@ function Set-PkMessageHandler {
# { param($Message, $Level) ... }, or $null to go back to the default (console).
# .EXAMPLE
# Set-PkMessageHandler { param($Message, $Level) Add-Content run.log "$Level $Message" }
[CmdletBinding()]
param([scriptblock]$Handler)
[PkState]::MessageHandler = $Handler
}
Expand Down Expand Up @@ -475,6 +476,7 @@ function Initialize-ParallelKit {
# .OUTPUTS
# An object with AbortHandler, ChildProcessJob and ThreadPool, each $true if that
# part worked.
[CmdletBinding()]
param([int]$MaxConcurrentProcesses = 0)
$handlerOk = Register-PkAbortHandler
$jobOk = Initialize-PkChildProcessJob
Expand Down Expand Up @@ -523,6 +525,8 @@ function Reset-PkAbortState {

# Deliberately leaves RunActive alone: that belongs to Invoke-PkRun, and
# clearing it here would disarm a run that calls this part-way through.
[CmdletBinding()]
param()
[PkState]::AbortLatched = $false
[PkState]::CloseAbort = $false
[PkConsoleCtrl]::AbortRequested = $false
Expand All @@ -534,6 +538,8 @@ function Reset-PkAbortState {
function Request-PkAbort {
# .SYNOPSIS
# Starts an abort from code, the same as the user pressing Ctrl+C.
[CmdletBinding()]
param()
[PkConsoleCtrl]::AbortRequested = $true
}

Expand All @@ -547,6 +553,8 @@ function Test-PkAbortSignal {
# the abort. Worker code should use Test-PkCancelRequested instead.
# .OUTPUTS
# [bool]
[CmdletBinding()]
param()
if ($null -ne $global:PkWorkerQueue) {
return [PkConsoleCtrl]::AbortRequested
}
Expand Down Expand Up @@ -582,6 +590,8 @@ function Test-PkCancelRequested {
# and can carry on once its current .NET call returns.
# .OUTPUTS
# [bool]
[CmdletBinding()]
param()
if ($null -ne $global:PkCancelToken -and $global:PkCancelToken.IsCancellationRequested) { return $true }
return [PkConsoleCtrl]::AbortRequested
}
Expand All @@ -592,6 +602,8 @@ function Get-PkAbortTriggerLabel {
# or "Request-PkAbort" for an abort started from code.
# .OUTPUTS
# [string]
[CmdletBinding()]
param()
if ([PkState]::CloseAbort) { return "window close (X button)" }
$pressedKey = [PkConsoleCtrl]::PressedKey
if ($null -ne $pressedKey) { return $pressedKey }
Expand Down Expand Up @@ -1237,7 +1249,7 @@ function Invoke-PkProcessDispatch {
# abort cut short, so check Test-PkCancelRequested before anything irreversible.
# .PARAMETER MaxConcurrent
# How many programs run at once: an int, or a SemaphoreSlim shared by several
# dispatchers that must stay under one combined limit.
# dispatchers that must stay under one combined limit. -MaxParallel also works.
# .PARAMETER AbortPollCallback
# { param($runningJobs) } run on every loop pass, with the jobs currently running.
# .PARAMETER CancelToken
Expand All @@ -1246,7 +1258,7 @@ function Invoke-PkProcessDispatch {
# .PARAMETER AbortTimeoutSeconds
# How long an abort waits for running jobs before force-stopping them
# (default 60). A force-stopped program is killed along with anything it
# started.
# started. -DrainTimeoutSeconds also works.
# .PARAMETER MaxQueueAhead
# How many groups to pull ahead of the ones running (default 3).
# .PARAMETER GroupReleaseCheck
Expand All @@ -1273,9 +1285,11 @@ function Invoke-PkProcessDispatch {
[scriptblock]$JobAbortCleanup = $null,
[scriptblock]$GroupCompleteCallback = $null,
[Parameter(Mandatory)]
[Alias('MaxParallel')]
$MaxConcurrent,
[scriptblock]$AbortPollCallback = $null,
[System.Threading.CancellationToken]$CancelToken,
[Alias('DrainTimeoutSeconds')]
[int]$AbortTimeoutSeconds = 60,
[int]$MaxQueueAhead = 3,
[scriptblock]$GroupReleaseCheck = $null,
Expand Down Expand Up @@ -1854,12 +1868,13 @@ function Invoke-PkProcessBatch {
# .PARAMETER Commands
# The command hashtables. $null or empty returns no results.
# .PARAMETER MaxConcurrent
# How many programs run at once (default 4).
# How many programs run at once (default 4). -MaxParallel also works.
# .PARAMETER OnComplete
# { param($command, $result) } run on the main thread as each program finishes.
# Not called for one that couldn't start or was stopped by an abort.
# .PARAMETER AbortTimeoutSeconds
# How long an abort waits for running programs before killing them (default 60).
# -DrainTimeoutSeconds also works.
# .PARAMETER TimeoutSeconds
# Kill a program still running after this long (default 0, no limit). Its
# result has TimedOut = $true.
Expand All @@ -1879,8 +1894,10 @@ function Invoke-PkProcessBatch {
[AllowNull()]
[AllowEmptyCollection()]
[hashtable[]]$Commands,
[Alias('MaxParallel')]
[int]$MaxConcurrent = 4,
[scriptblock]$OnComplete = $null,
[Alias('DrainTimeoutSeconds')]
[int]$AbortTimeoutSeconds = 60,
[double]$TimeoutSeconds = 0,
$OutputEncoding = $null
Expand All @@ -1899,6 +1916,8 @@ function Invoke-PkProcessBatch {
# Checked once here, so a bad name fails the call instead of every command.
$null = Resolve-PkEncoding -Encoding $pkOutputEncoding
# Invoke-PkProcessDispatch has the same check; this one names the batch.
if ($pkMaxConcurrent -lt 1) { throw ("Invoke-PkProcessBatch: -MaxConcurrent must be at least 1 (got " + $pkMaxConcurrent + ").") }
# Invoke-PkProcessDispatch has the same check; this one names the batch.
if ($pkTimeoutSeconds -lt 0) { throw "Invoke-PkProcessBatch: -TimeoutSeconds must be 0 (no limit) or more." }

$pkResults = [System.Collections.Generic.List[object]]::new()
Expand Down Expand Up @@ -2174,7 +2193,7 @@ function Invoke-PkRunspaceJobs {
# .PARAMETER SharedFunctionNames
# Your functions the worker calls.
# .PARAMETER MaxParallel
# How many workers run at once.
# How many workers run at once. -MaxConcurrent also works.
# .PARAMETER Cts
# Your own CancellationTokenSource for the run; an abort cancels it too.
# .PARAMETER ResultHandler
Expand All @@ -2189,12 +2208,12 @@ function Invoke-PkRunspaceJobs {
# as a message and the run carries on.
# .PARAMETER DrainTimeoutSeconds
# How long an abort waits for running workers before force-stopping them
# (default 60).
# (default 60). -AbortTimeoutSeconds also works.
# .PARAMETER AbortWaitMessage
# The message shown while an abort waits; '' for none.
# .PARAMETER CheckWorkerSafety
# Check the worker code and job values before starting (default on).
# -CheckWorkerSafety:$false turns it off.
# Check the worker code and job values before starting. On unless you pass
# -CheckWorkerSafety:$false, whatever Get-Help says the default is.
# .OUTPUTS
# An object with ForceStoppedCount, CrashedCount, Aborted and Crashed (one
# @{ Job; Error } entry per worker that threw).
Expand All @@ -2207,16 +2226,21 @@ function Invoke-PkRunspaceJobs {
[scriptblock]$NextJobProvider = $null,
[string[]]$SharedFunctionNames = @(),
[Parameter(Mandatory)]
[Alias('MaxConcurrent')]
[int]$MaxParallel,
[System.Threading.CancellationTokenSource]$Cts = $null,
[scriptblock]$ResultHandler = $null,
[scriptblock]$PollCallback = $null,
[scriptblock]$JobCrashedHandler = $null,
[Alias('AbortTimeoutSeconds')]
[int]$DrainTimeoutSeconds = 60,
[string]$AbortWaitMessage,
# On by default; -CheckWorkerSafety:$false turns it off.
[switch]$CheckWorkerSafety = $true
# On unless turned off; see $checkSafety below.
[switch]$CheckWorkerSafety
)
# A switch can't default to on without a lint warning, so "not passed"
# means on. A bare -CheckWorkerSafety (the v1 opt-in) still means on.
$checkSafety = (-not $PSBoundParameters.ContainsKey('CheckWorkerSafety')) -or $CheckWorkerSafety.IsPresent

if ($null -eq $JobParams -and $null -eq $NextJobProvider) {
throw "Invoke-PkRunspaceJobs: must supply either -JobParams or -NextJobProvider."
Expand Down Expand Up @@ -2247,7 +2271,7 @@ function Invoke-PkRunspaceJobs {
$outcome = [ordered]@{ ForceStoppedCount = 0; CrashedCount = 0; Aborted = $false; Crashed = [System.Collections.Generic.List[object]]::new() }
if ($null -ne $JobParams -and $JobParams.Count -eq 0) { return [pscustomobject]$outcome }

if ($CheckWorkerSafety) {
if ($checkSafety) {
$safetyFindings = Test-PkWorkerSafetyCore -WorkerScript $WorkerScript -SharedFunctionNames $SharedFunctionNames -SessionState $PSCmdlet.SessionState
if ($safetyFindings.Count -gt 0) {
$safetyLines = foreach ($f in $safetyFindings) { " " + $f.Text }
Expand All @@ -2273,7 +2297,7 @@ function Invoke-PkRunspaceJobs {
throw ("Invoke-PkRunspaceJobs: job " + $jobIndex + " supplies '" + $key + "', but -WorkerScript's param() block doesn't declare it. Nothing was started.")
}
}
if ($CheckWorkerSafety) {
if ($checkSafety) {
$sbKey = Get-PkScriptBlockJobKey -Job $JobParams[$jobIndex]
if ($null -ne $sbKey) { throw ((Get-PkScriptBlockJobMessage -Job ("job " + $jobIndex) -Key $sbKey) + " Nothing was started.") }
}
Expand Down Expand Up @@ -2373,7 +2397,7 @@ function Invoke-PkRunspaceJobs {
throw ("Invoke-PkRunspaceJobs: a job supplies '" + $key + "', but -WorkerScript's param() block doesn't declare it.")
}
}
if ($CheckWorkerSafety -and $null -eq $JobParams) {
if ($checkSafety -and $null -eq $JobParams) {
$sbKey = Get-PkScriptBlockJobKey -Job $nextJob
if ($null -ne $sbKey) { throw (Get-PkScriptBlockJobMessage -Job "a job from -NextJobProvider" -Key $sbKey) }
}
Expand Down
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ Programs run in your current location (`Get-Location`) unless a command says oth

`Results` is in the order commands finished, not the order you gave them. Use `Command` to match them up.

`-OnComplete` runs as each command finishes, but only for commands that actually ran. Check `Results` for the ones that couldn't start or were aborted. A progress counter driven from `-OnComplete` won't reach the total if any did, so count from `$batch.Results` afterwards for the final numbers. Defaults: `-MaxConcurrent` 4, `-AbortTimeoutSeconds` 60 (how long Ctrl+C waits for running programs), `-TimeoutSeconds` 0 (no limit).
`-OnComplete` runs as each command finishes, but only for commands that actually ran. Check `Results` for the ones that couldn't start or were aborted. A progress counter driven from `-OnComplete` won't reach the total if any did, so count from `$batch.Results` afterwards for the final numbers. Defaults: `-MaxConcurrent` 4, `-AbortTimeoutSeconds` 60 (how long Ctrl+C waits for running programs), `-TimeoutSeconds` 0 (no limit). The runspace engine's names, `-MaxParallel` and `-DrainTimeoutSeconds`, work here too, and on the dispatcher.

`-TimeoutSeconds 600` kills any program still running after 10 minutes. Its result has `TimedOut = $true`, still goes through `-OnComplete`, and the rest of the batch carries on. The default, 0, means no limit. Any programs it started are killed with it. It can be fractional. A command's own `TimeoutSeconds` key overrides this for that command; 0 opts it out of the limit. A bad value there gives that command a `StartError`.

Expand Down Expand Up @@ -194,7 +194,7 @@ Each worker runs in its own runspace: a separate PowerShell session inside the s
}
```
- Don't call `exit` in a worker. It ends that job quietly (not as a crash) and sets your whole script's exit code.
- `-MaxParallel` has no default; you choose it. `-DrainTimeoutSeconds` (how long Ctrl+C waits for running workers) defaults to 60.
- `-MaxParallel` has no default; you choose it. `-DrainTimeoutSeconds` (how long Ctrl+C waits for running workers) defaults to 60. The process engines' names, `-MaxConcurrent` and `-AbortTimeoutSeconds`, work here too.
- If `-ResultHandler`, `-PollCallback` or `-NextJobProvider` throws, the run stops: workers still going are told to stop and the exception comes out of the call.
- Returns an object with `ForceStoppedCount`, `CrashedCount`, `Aborted` and `Crashed`, which lists the `Job` and `Error` of each worker that threw.

Expand Down Expand Up @@ -286,7 +286,7 @@ Inside `Invoke-PkRun { ... }`, Ctrl+C:

1. stops new jobs from starting,
2. cancels the run's token (`Test-PkCancelRequested` turns true in workers),
3. waits for in-flight work (`-DrainTimeoutSeconds` / `-AbortTimeoutSeconds`, default 60),
3. waits for in-flight work (`-AbortTimeoutSeconds`, default 60; every engine also takes it as `-DrainTimeoutSeconds`),
4. force-stops whatever is left, calling `-JobAbortCleanup` for each killed or never-started process job.

Press Ctrl+C again to skip step 3: whatever is still running is force-stopped straight away, and your script carries on. A third press stops the script the way Ctrl+C normally does. Presses less than a second apart count as one, and Ctrl+Break works the same as Ctrl+C.
Expand Down
Loading