Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 98 additions & 0 deletions ParallelKit.Harness.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -2617,6 +2617,104 @@ Test-Assert ((Get-StartMessageCount "*$missing*") -eq 2) ("and collapsed there (
Reset-Harness
Write-Host ""

Write-Host "=== AE: a list in Arguments is quoted, one argument per element ===" -ForegroundColor Cyan

$argvExe = Join-Path $env:TEMP ("pk-argv-" + [guid]::NewGuid().ToString("N") + ".exe")
$argvSrc = [System.IO.Path]::ChangeExtension($argvExe, '.cs')
[System.IO.File]::WriteAllText($argvSrc, 'public static class PkArgv { public static void Main(string[] args) { foreach (string a in args) { System.Console.WriteLine("[" + a + "]"); } } }')
$csc = $null
try { $csc = Invoke-PkProcessWithTimeout -FilePath $cscExe -Arguments (ConvertTo-PkArgumentString @('/nologo', ('/out:' + $argvExe), $argvSrc)) } catch { Write-Host " csc.exe failed to run: $_" }
[System.IO.File]::Delete($argvSrc)
$compiled = ($null -ne $csc -and $csc.ExitCode -eq 0 -and [System.IO.File]::Exists($argvExe))
Test-Assert $compiled "the argument-printing test program compiles"
if ($compiled) {
function Get-ArgvText { param($Stdout) return (@(([string]$Stdout).Split([string[]]@("`r`n"), [System.StringSplitOptions]::RemoveEmptyEntries)) -join '') }
function Get-BatchArgvText {
param($ArgumentValue)
$batch = Invoke-PkProcessBatch -Commands @(@{ FilePath = $argvExe; Arguments = $ArgumentValue }) -MaxConcurrent 1
return (Get-ArgvText $batch.Results[0].Stdout)
}
$spaced = @('C:\My Files\a.png', 'out.jxl')
$spacedText = '[C:\My Files\a.png][out.jxl]'

# All four entry points.
$text = Get-BatchArgvText $spaced
Test-Assert ($text -eq $spacedText) ("batch: an array is one argument per element (" + $text + ")")
$feedArgs = @{ Given = $false }
$gotArgs = @{ Text = $null }
$null = Invoke-PkProcessDispatch -MaxConcurrent 1 `
-GroupProvider { if ($feedArgs.Given) { return $null }; $feedArgs.Given = $true; return 'g' }.GetNewClosure() `
-JobBuilder { param($g) return @(@{ FilePath = $argvExe; Arguments = $spaced }) }.GetNewClosure() `
-JobFinalize { param($job, $q) $gotArgs.Text = (Complete-PkProcess -Job $job).Stdout }.GetNewClosure()
$text = Get-ArgvText $gotArgs.Text
Test-Assert ($text -eq $spacedText) ("dispatcher: an array is one argument per element (" + $text + ")")
$text = $null
try { $text = Get-ArgvText (Invoke-PkProcessWithTimeout -FilePath $argvExe -Arguments $spaced).Stdout } catch { $text = "threw: $_" }
Test-Assert ($text -eq $spacedText) ("Invoke-PkProcessWithTimeout takes an array (" + $text + ")")
$text = $null
try {
$started = Start-PkProcess -FilePath $argvExe -Arguments $spaced
$null = Wait-PkProcess -Process $started['Proc'] -TimeoutMs 10000
$text = Get-ArgvText (Complete-PkProcess -Job $started).Stdout
} catch { $text = "threw: $_" }
Test-Assert ($text -eq $spacedText) ("Start-PkProcess takes an array (" + $text + ")")

# The kinds of list.
$list = [System.Collections.Generic.List[string]]::new(); $list.Add('C:\My Files\a.png'); $list.Add('out.jxl')
$started = Start-PkProcess -FilePath $argvExe -Arguments $list
$null = Wait-PkProcess -Process $started['Proc'] -TimeoutMs 10000
$text = Get-ArgvText (Complete-PkProcess -Job $started).Stdout
Test-Assert ($text -eq $spacedText) ("a List[string] is quoted too (" + $text + ")")
$arrayList = [System.Collections.ArrayList]::new(); $null = $arrayList.Add('two words')
Test-Assert ((Get-BatchArgvText $arrayList) -eq '[two words]') "so is an ArrayList, even of one element"
Test-Assert ((Get-BatchArgvText ([string[]]@('two words', 'x'))) -eq '[two words][x]') "and a [string[]]"
Test-Assert ((Get-BatchArgvText @('two words')) -eq '[two words]') "a single-element array stays one argument"
Test-Assert ((Get-BatchArgvText @(1, 2.5, 'x')) -eq '[1][2.5][x]') "numbers in a list become arguments"
Test-Assert ((Get-BatchArgvText @()) -eq '') "an empty list is no arguments"
Test-Assert ((Get-BatchArgvText $null) -eq '') "`$null is no arguments"
Test-Assert ((Get-BatchArgvText 7) -eq '[7]') "a single number is one argument"

# $null elements are dropped, the same from every route; '' is an argument.
Test-Assert ((Get-BatchArgvText @('a', $null, 'b')) -eq '[a][b]') "a `$null element is dropped"
Test-Assert ((Get-BatchArgvText @('a', '', 'b')) -eq '[a][][b]') "an empty string stays an argument"
Test-Assert ((ConvertTo-PkArgumentString @('a', $null, 'b')) -eq 'a b') "ConvertTo-PkArgumentString drops a `$null element too"
$withNull = @('a', $null, 'b')
Test-Assert ((ConvertTo-PkArgumentString $withNull) -eq 'a b') "and from a variable, where binding used to turn it into `"`""
Test-Assert ((ConvertTo-PkArgumentString $null) -eq '') "and takes `$null itself"
Test-Assert ((ConvertTo-PkArgumentString 'two words') -eq '"two words"' -and (ConvertTo-PkArgumentString @(1, 'x y')) -eq '1 "x y"') "it still takes one string, and non-strings"

# A string is still exactly as written.
Test-Assert ((Get-BatchArgvText '"C:\My Files\a.png" out.jxl') -eq $spacedText) "a string is still passed exactly as written"
# The documented change: list elements are values, not pieces of a command line.
Test-Assert ((Get-BatchArgvText @('"two words"', 'x')) -eq '["two words"][x]') "quotes already in a list element reach the program as quotes"

# The join used to follow $OFS.
$OFS = ','
try { $text = Get-BatchArgvText $spaced } finally { Remove-Variable -Name OFS }
Test-Assert ($text -eq $spacedText) ("`$OFS doesn't change a list (" + $text + ")")

# A hashtable is enumerable, but it isn't a list of arguments.
Reset-Harness
$hashBatch = Invoke-PkProcessBatch -Commands @(@{ FilePath = $argvExe; Arguments = @{ a = 1 } }) -MaxConcurrent 1
Test-Assert ($hashBatch.Results[0].StartError -like '*Arguments must be*' -and $null -eq $hashBatch.Results[0].ExitCode) "a hashtable in Arguments is that command's StartError"
$thrown = $null
try { $null = Invoke-PkProcessWithTimeout -FilePath $argvExe -Arguments @{ a = 1 } } catch { $thrown = [string]$_ }
Test-Assert ($thrown -like '*Arguments must be*') "and a clear error from the direct functions"

# Inside a worker, where the kit's functions are plain copies.
Reset-Harness
$argWorker = [System.Collections.Generic.List[object]]::new()
$argOut = Invoke-PkRunspaceJobs -MaxParallel 1 -JobParams (New-JobList -Jobs @(@{ Exe = $argvExe })) -ResultHandler { param($r) $argWorker.Add($r) }.GetNewClosure() -WorkerScript {
param([string]$Exe)
$batch = Invoke-PkProcessBatch -Commands @(@{ FilePath = $Exe; Arguments = @('two words', 'x') }) -MaxConcurrent 1
return @{ Stdout = $batch.Results[0].Stdout; StartError = $batch.Results[0].StartError }
}
Test-Assert ($argOut.CrashedCount -eq 0 -and $argWorker.Count -eq 1 -and (Get-ArgvText $argWorker[0]['Stdout']) -eq '[two words][x]') "a list works inside a worker"
try { [System.IO.File]::Delete($argvExe) } catch { } # temp file only; best effort
}
Reset-Harness
Write-Host ""

Set-PkMessageHandler -Handler $null
Reset-PkAbortState

Expand Down
60 changes: 42 additions & 18 deletions ParallelKit.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -682,9 +682,11 @@ function ConvertTo-PkArgumentString {
# An argument with a space, tab, newline or quote is wrapped in quotes, its
# quotes become \", and backslashes are doubled where they come before a quote.
# These are the C runtime's rules, which most Windows programs use; cmd.exe /c
# parses its own way.
# parses its own way. The kit's -Arguments and Arguments values do this
# themselves when given a list.
# .PARAMETER Arguments
# The arguments, in order.
# The arguments, in order. A $null one is left out; an empty string is passed
# as "".
# .OUTPUTS
# [string]
# .EXAMPLE
Expand All @@ -695,13 +697,17 @@ function ConvertTo-PkArgumentString {
# .NET Framework's ProcessStartInfo has no ArgumentList, hence this.
param(
[Parameter(Mandatory, Position = 0)]
[AllowNull()]
[AllowEmptyCollection()]
[AllowEmptyString()]
[string[]]$Arguments
# Not [string[]]: binding an array held in a variable to that turns a
# $null element into '' before it can be left out.
[object[]]$Arguments
)
$parts = [System.Collections.Generic.List[string]]::new()
foreach ($arg in $Arguments) {
if ($null -eq $arg) { $arg = '' }
foreach ($item in $Arguments) {
if ($null -eq $item) { continue }
$arg = [string]$item
if ($arg.Length -gt 0 -and $arg.IndexOfAny([char[]]@(' ', "`t", "`n", "`v", '"')) -lt 0) {
$parts.Add($arg)
continue
Expand Down Expand Up @@ -729,6 +735,20 @@ function ConvertTo-PkArgumentString {
return ($parts -join ' ')
}

function Resolve-PkArguments {
# -Arguments / Arguments as given -> the command line. A string is used as
# written; a list is quoted one argument per element, so a path with a space
# stays whole (a plain [string] cast would join it with spaces, or $OFS).
param($Arguments)
if ($null -eq $Arguments) { return '' }
if ($Arguments -is [string]) { return $Arguments }
if ($Arguments -is [System.Collections.IDictionary]) {
throw ("Arguments must be one string or a list of arguments (got a " + $Arguments.GetType().Name + ").")
}
if ($Arguments -is [System.Collections.IEnumerable]) { return (ConvertTo-PkArgumentString -Arguments @($Arguments)) }
return [string]$Arguments
}

function Resolve-PkEncoding {
# -OutputEncoding as given (an Encoding, a code page number, or a name
# like 'utf-8' or 'utf8') -> an Encoding, or $null for "not set".
Expand Down Expand Up @@ -804,7 +824,8 @@ function Start-PkProcess {
# drives like Data:\ or ~ work. A bare name (tool.exe) is found the way Windows
# finds programs, mostly on PATH.
# .PARAMETER Arguments
# Its command line as one string (see ConvertTo-PkArgumentString).
# Its command line: one string, passed exactly as written, or a list, each
# element one argument, quoted for you (see ConvertTo-PkArgumentString).
# .PARAMETER WorkingDirectory
# Where to run it. Default: your current location. Read as PowerShell reads
# paths, so a relative path is relative to your current location, and drives
Expand All @@ -817,7 +838,7 @@ function Start-PkProcess {
param(
[Parameter(Mandatory)]
[string]$FilePath,
[string]$Arguments = "",
$Arguments = "",
[string]$WorkingDirectory = "",
$OutputEncoding = $null
)
Expand All @@ -833,12 +854,13 @@ function Start-PkProcessCore {
param(
[Parameter(Mandatory)]
[string]$FilePath,
[string]$Arguments = "",
$Arguments = "",
[string]$WorkingDirectory = "",
$OutputEncoding = $null,
[System.Management.Automation.SessionState]$CallerState
)
$encoding = Resolve-PkEncoding -Encoding $OutputEncoding
$commandLine = Resolve-PkArguments -Arguments $Arguments
# Self-starting: a caller who never ran Initialize-ParallelKit still
# gets children that die with the script, and output reads that
# aren't starved of threads. Both are cheap no-ops once done.
Expand All @@ -847,7 +869,7 @@ function Start-PkProcessCore {

$psi = [System.Diagnostics.ProcessStartInfo]::new()
$psi.FileName = $FilePath
$psi.Arguments = $Arguments
$psi.Arguments = $commandLine
$psi.UseShellExecute = $false
if ($null -ne $encoding) {
$psi.StandardOutputEncoding = $encoding
Expand Down Expand Up @@ -1116,7 +1138,8 @@ function Invoke-PkProcessWithTimeout {
# .PARAMETER FilePath
# The program to run.
# .PARAMETER Arguments
# Its command line as one string (see ConvertTo-PkArgumentString).
# Its command line: one string, passed exactly as written, or a list, each
# element one argument, quoted for you (see ConvertTo-PkArgumentString).
# .PARAMETER TimeoutMs
# The time limit in milliseconds (default 60000).
# .PARAMETER StreamTimeoutMs
Expand All @@ -1135,7 +1158,7 @@ function Invoke-PkProcessWithTimeout {
param(
[Parameter(Mandatory)]
[string]$FilePath,
[string]$Arguments = "",
$Arguments = "",
[int]$TimeoutMs = 60000,
[int]$StreamTimeoutMs = 15000,
[string]$WorkingDirectory = "",
Expand Down Expand Up @@ -1187,10 +1210,11 @@ function Invoke-PkProcessDispatch {
#
# The dispatcher keeps its own state on each job hashtable. Don't use these keys
# for anything else: GroupKey, SlotReleased, Proc, StdoutTask, StderrTask,
# Stopwatch, TimeLimit, TimedOut. It also sets Group to the job's group, unless
# the job already has a Group of its own. The default starter reads FilePath,
# Arguments, WorkingDirectory and OutputEncoding from the job. A job's
# TimeoutSeconds key overrides -TimeoutSeconds.
# Stopwatch, TimeLimit, TimedOut, StartFailureReported. It also sets Group to
# the job's group, unless the job already has a Group of its own. The default
# starter reads FilePath, Arguments (a string as written, or a list quoted one
# argument per element), WorkingDirectory and OutputEncoding from the job. A
# job's TimeoutSeconds key overrides -TimeoutSeconds.
# .PARAMETER GroupProvider
# { } returning the next group, or $null when there are no more.
# .PARAMETER JobBuilder
Expand Down Expand Up @@ -1300,7 +1324,7 @@ function Invoke-PkProcessDispatch {
$pkCallerState = $PSCmdlet.SessionState
$pkJobStarter = {
param($pkJob)
$pkStarted = Start-PkProcessCore -CallerState $pkCallerState -FilePath $pkJob['FilePath'] -Arguments ([string]$pkJob['Arguments']) -WorkingDirectory ([string]$pkJob['WorkingDirectory']) -OutputEncoding $pkJob['OutputEncoding']
$pkStarted = Start-PkProcessCore -CallerState $pkCallerState -FilePath $pkJob['FilePath'] -Arguments $pkJob['Arguments'] -WorkingDirectory ([string]$pkJob['WorkingDirectory']) -OutputEncoding $pkJob['OutputEncoding']
$pkJob['Proc'] = $pkStarted.Proc; $pkJob['StdoutTask'] = $pkStarted.StdoutTask; $pkJob['StderrTask'] = $pkStarted.StderrTask
$pkJob['Stopwatch'] = $pkStarted.Stopwatch
return $pkJob
Expand Down Expand Up @@ -1925,7 +1949,7 @@ function Invoke-PkProcessBatch {
& $pkReportStartFailure $pkC $_
continue
}
$pkJobs.Add(@{ FilePath = $pkC['FilePath']; Arguments = [string]$pkC['Arguments']; WorkingDirectory = [string]$pkC['WorkingDirectory']; OutputEncoding = $pkEncoding; TimeoutSeconds = $pkC['TimeoutSeconds']; Command = $pkC })
$pkJobs.Add(@{ FilePath = $pkC['FilePath']; Arguments = $pkC['Arguments']; WorkingDirectory = [string]$pkC['WorkingDirectory']; OutputEncoding = $pkEncoding; TimeoutSeconds = $pkC['TimeoutSeconds']; Command = $pkC })
}
return $pkJobs
}.GetNewClosure()
Expand Down Expand Up @@ -1984,7 +2008,7 @@ function Get-PkWorkerSafeFunctionNames {
return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool', 'Stop-PkProcessTree', 'Stop-PkJobProcesses', 'Resolve-PkEncoding',
'Start-PkProcess', 'Wait-PkProcess', 'Wait-PkRunningJobs', 'Complete-PkProcess', 'Invoke-PkProcessWithTimeout',
'Invoke-PkProcessDispatch', 'Invoke-PkProcessDispatchCore', 'Invoke-PkProcessBatch',
'Register-PkCancellationSource', 'Unregister-PkCancellationSource', 'New-PkBatchResult', 'ConvertTo-PkArgumentString', 'Get-PkForceStopReason', 'Resolve-PkFileSystemPath', 'Add-PkDispatchGroupJobs', 'Start-PkProcessCore', 'Get-PkJobTimeLimit', 'Stop-PkTimedOutJob', 'Get-PkStartFailureCause')
'Register-PkCancellationSource', 'Unregister-PkCancellationSource', 'New-PkBatchResult', 'ConvertTo-PkArgumentString', 'Get-PkForceStopReason', 'Resolve-PkFileSystemPath', 'Add-PkDispatchGroupJobs', 'Start-PkProcessCore', 'Get-PkJobTimeLimit', 'Stop-PkTimedOutJob', 'Get-PkStartFailureCause', 'Resolve-PkArguments')
}

function Get-PkHandlerParamCount {
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ $batch = Invoke-PkRun { Invoke-PkProcessBatch @batchParams }

Long parameter lists in these examples use splatting (`@batchParams`) instead of trailing backticks: it's easier to read, and a stray space after a backtick can't break it.

Each command is a hashtable with `FilePath`, `Arguments` and optionally `WorkingDirectory`, plus any keys of your own. `Arguments` is passed to the program as one string, exactly as written. `ConvertTo-PkArgumentString @(...)` builds that string from a list, quoting each argument so the program gets exactly that list back, spaces, quotes and trailing backslashes included. It follows the rules most Windows programs use; `cmd.exe /c` parses its own way. Programs run hidden, with their output captured.
Each command is a hashtable with `FilePath`, `Arguments` and optionally `WorkingDirectory`, plus any keys of your own. `Arguments` is either one string, passed to the program exactly as written, or a list, where each element is one argument: `Arguments = @($in, $out)` works with spaces in the paths. A list is quoted the way `ConvertTo-PkArgumentString @(...)` quotes it, so the program gets exactly that list back, spaces, quotes and trailing backslashes included, and `$null` elements are left out. Quotes inside an element are passed on as quote characters, so don't pre-quote list elements. It follows the rules most Windows programs use; `cmd.exe /c` parses its own way, so give it one string. Programs run hidden, with their output captured.

Programs run in your current location (`Get-Location`) unless a command says otherwise, and a relative `WorkingDirectory`, or a `FilePath` with a folder in it like `.\tools\cjxl.exe`, is relative to that location too. Drives PowerShell knows, like `~` or one made with `New-PSDrive`, work in both. A bare name like `cjxl.exe` is found the way Windows finds programs, mostly on `PATH`. A `.bat` or `.cmd` program can't run in a network-share (`\\server\share`) folder: `cmd.exe` falls back to `C:\Windows` there, so give such commands a local `WorkingDirectory`.

Expand Down Expand Up @@ -260,7 +260,7 @@ Callbacks: a plain `{ ... }` callback sees your variables and functions from whe
| Function | Use |
|---|---|
| `Invoke-PkProcessWithTimeout -FilePath -Arguments -TimeoutMs` | Run one program and wait, with a hard time limit (default 60s). Returns an object with `ExitCode`, `Stdout`, `Stderr`, `StreamTimedOut` and `StreamError`. If it overruns, or the run is aborted, the program is killed and this **throws**; add `-IgnoreAbort` for a cleanup step that must finish anyway. |
| `ConvertTo-PkArgumentString @(args)` | Join a list of arguments into one correctly quoted command line, for any `-Arguments` / `Arguments` value. |
| `ConvertTo-PkArgumentString @(args)` | Join a list of arguments into one correctly quoted command line. `-Arguments` / `Arguments` also take the list directly and quote it the same way. |
| `Start-PkProcess -FilePath -Arguments` | Start one hidden, with output captured in the background. Returns `@{ Proc; StdoutTask; StderrTask; Stopwatch }`; throws if it can't start. |
| `Wait-PkProcess -Process -TimeoutMs` | Wait for exit with a timeout that can't overrun. Returns `$true` if it exited. |
| `Complete-PkProcess -Job` | Collect exit code and output from a finished one (`-Job` is what `Start-PkProcess` returned). Same result as `Invoke-PkProcessWithTimeout`. |
Expand Down
Loading