Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,18 @@ All notable changes to this project will be documented in this file.

## Unreleased

- N/A
### Changed

- A request refused with 401 because the session token has expired now reports `Session expired at <time>:
authenticate again with New-IDSession or New-IDPlatformToken`. Previously, it reported `Unauthorized. Unauthorized.`.
Companion modules get the same message.

### Fixed

- Companion module commands which page their results now page correctly from a starting offset or cursor.
Each later request carries one offset or cursor, in place of the starting one, and pages on from the
starting offset. Previously, the starting value was sent again with each later request, and paging
restarted from zero, so items were skipped or returned twice.

## [1.0.0] - 2026-10-12

Expand Down
44 changes: 35 additions & 9 deletions IdentityCommand/Private/Get-PagedResult.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@
The already-fetched first page of results.

.PARAMETER URI
The URI used for the initial request (without any paging query parameter appended).
Subsequent pages are requested against this URI.
The URI used for the initial request. Subsequent pages are requested against this URI, with any
paging query parameter it already holds (OffsetRequestKey or CursorRequestKey) replaced by the
next page's value. Every other query parameter is resent exactly as given. In Offset style with
Method GET, an offset already in the URI is the position paging starts from.

.PARAMETER Style
'Cursor' to follow a continuation token, 'Offset' to page via an incrementing offset
Expand Down Expand Up @@ -67,7 +69,8 @@

.PARAMETER BodyTemplate
Offset style only, required with Method POST. The body sent with the initial request. It is
cloned per page with the offset replaced, so filters and search terms carry across pages.
cloned per page with the offset replaced, so filters and search terms carry across pages. An
offset already in the body is the position paging starts from.

.PARAMETER BodyPagingProperty
Offset style only. Name of a nested object in the body which holds the paging values, for
Expand Down Expand Up @@ -152,6 +155,20 @@
if ($ResultProperty) { $PageResult.$ResultProperty } else { $PageResult }
}

#Splits the query string into its pairs, each kept exactly as sent so the encoding of every
#other key survives, and separates out any value already held for the paging key.
$PagingKey = if ($Style -eq 'Cursor') { $CursorRequestKey } else { $OffsetRequestKey }
$UriParts = $URI -split '\?', 2
$QueryPairs = if ($UriParts.Count -gt 1) { @($UriParts[1] -split '&' | Where-Object { $_ }) } else { @() }
$IsPagingPair = {
param($Pair)
[Uri]::UnescapeDataString(($Pair -split '=', 2)[0]) -eq $PagingKey
}
$StartValue = $QueryPairs | Where-Object { & $IsPagingPair $_ } | Select-Object -Last 1 |
ForEach-Object { [Uri]::UnescapeDataString("$(($_ -split '=', 2)[1])") }
$KeptPairs = @($QueryPairs | Where-Object { -not (& $IsPagingPair $_) })
$BaseURI = if ($KeptPairs.Count -gt 0) { "$($UriParts[0])?$($KeptPairs -join '&')" } else { $UriParts[0] }

$Items = [Collections.Generic.List[Object]]::New()

$InitialItems = & $GetItems $InitialResult
Expand All @@ -167,7 +184,7 @@

while (-not [String]::IsNullOrEmpty($NextCursor)) {

$PageURI = Add-QueryString -URI $URI -Parameter @{ $CursorRequestKey = $NextCursor }
$PageURI = Add-QueryString -URI $BaseURI -Parameter @{ $CursorRequestKey = $NextCursor }

$PageResult = Invoke-IDRestMethod -Uri $PageURI -Method GET
$PageItems = & $GetItems $PageResult
Expand Down Expand Up @@ -201,6 +218,15 @@

$Total = if ($PSBoundParameters.ContainsKey('TotalCount')) { $TotalCount } else { $InitialResult.$TotalResponseKey }

#The position the initial request started from: the offset in the body when paging a POST,
#otherwise the offset in the URI, or zero when neither holds one.
if ($Method -eq 'POST') {
$BodyPaging = if ($BodyPagingProperty) { $BodyTemplate[$BodyPagingProperty] } else { $BodyTemplate }
$StartValue = if ($null -ne $BodyPaging) { $BodyPaging[$OffsetRequestKey] }
}
$StartOffset = 0
$null = [int]::TryParse("$StartValue", [ref]$StartOffset)

#Resolves a dotted property path, so a next-offset reported as a nested value
#(CDS reports paging.offset) can be read the same way as a top level one.
$GetPathValue = {
Expand All @@ -223,7 +249,7 @@
} elseif ($OffsetResponseKey) {
-not [string]::IsNullOrEmpty((& $GetPathValue $PageResult $OffsetResponseKey))
} else {
($null -ne $Total) -and ($Items.Count -lt $Total)
($null -ne $Total) -and (($StartOffset + $Items.Count) -lt $Total)
}
}

Expand All @@ -237,11 +263,11 @@
while (& $HasMorePages $LastPage) {

#An echoed next offset is the server's own idea of where the next page starts;
#otherwise pick up from however many items have been collected so far.
#otherwise move on from the starting offset by however many items have been collected so far.
$NextOffset = if ($OffsetResponseKey) {
& $GetPathValue $LastPage $OffsetResponseKey
} else {
$Items.Count
$StartOffset + $Items.Count
}

$PageResult = if ($Method -eq 'POST') {
Expand All @@ -259,7 +285,7 @@

} else {

$PageURI = Add-QueryString -URI $URI -Parameter @{ $OffsetRequestKey = $NextOffset }
$PageURI = Add-QueryString -URI $BaseURI -Parameter @{ $OffsetRequestKey = $NextOffset }
Invoke-IDRestMethod -Uri $PageURI -Method GET

}
Expand All @@ -269,7 +295,7 @@

if (($null -eq $PageItems) -or (@($PageItems).Count -eq 0)) {
#Defends against a reported total the server never actually delivers
break
break
}

$CurrentPageFirstItem = @($PageItems)[0] | ConvertTo-Json -Compress -Depth 5
Expand Down
15 changes: 15 additions & 0 deletions IdentityCommand/Private/Invoke-IDRestMethod.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@
$ISPSSSession.LastErrorTime = Get-Date

$ErrorID = $PSItem | Select-Object -ExpandProperty FullyQualifiedErrorId
$StatusCode = [int]$PSItem.Exception.Response.StatusCode

try {

Expand Down Expand Up @@ -296,6 +297,20 @@
$ErrorMessage = $ISPSSSession.LastError.Exception.Message
}

#A 401 sent with an expired token reports the expiry; the token carries no refresh token
if ($StatusCode -eq 401) {

$Expiry = try {
$Exp = (ConvertFrom-JwtPayload -Token $PSBoundParameters['WebSession'].Headers['Authorization']).exp
if ($null -ne $Exp) { [DateTimeOffset]::FromUnixTimeSeconds([long]$Exp) }
} catch { $null }

if (($null -ne $Expiry) -and ($Expiry -le [DateTimeOffset]::UtcNow)) {
$ErrorMessage = "Session expired at $($Expiry.LocalDateTime.ToString()): authenticate again with New-IDSession or New-IDPlatformToken"
}

}

#throw the error
$PSCmdlet.ThrowTerminatingError(

Expand Down
217 changes: 217 additions & 0 deletions Tests/Get-PagedResult.Tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,223 @@

}

Context 'Starting position supplied by the caller' {

BeforeAll {
#Values of a query key, in order, as the raw (still encoded) strings sent
$Script:GetQueryValue = {
param($Target, $Key)
$Query = ([uri]$Target).Query.TrimStart('?')
@($Query -split '&' | Where-Object { $_ -like "$Key=*" } | ForEach-Object { $_.Substring($Key.Length + 1) })
}

#A page of the fake data set, starting at Offset, as a server would return it
$Script:GetPage = {
param($Offset)
$Last = [Math]::Min($Offset + $Script:PageLimit, $Script:SetSize) - 1
$Slice = if ($Offset -le $Last) { @($Offset..$Last) } else { @() }
[pscustomobject]@{
items = $Slice
totalCount = $Script:SetSize
isLastPage = ($Offset + $Script:PageLimit -ge $Script:SetSize)
}
}
}

BeforeEach {
$Script:PageLimit = 10
$Script:SetSize = 25
$Script:Requests = [Collections.Generic.List[string]]::New()
$Script:Bodies = [Collections.Generic.List[object]]::New()

#Like the live servers, uses the last offset value when a key is repeated
Mock -CommandName Invoke-IDRestMethod -MockWith {
$null = $Script:Requests.Add($URI)
if ($Body) {
$Parsed = $Body | ConvertFrom-Json
$null = $Script:Bodies.Add($Parsed)
$Offset = if ($Parsed.paging) { $Parsed.paging.offset } else { $Parsed.offset }
} else {
$Offset = @(& $Script:GetQueryValue $URI 'offset') | Select-Object -Last 1
}
& $Script:GetPage ([int]$Offset)
}
}

Context 'Offset style, GET' {

It 'pages from zero when the URI holds no offset' {
$Initial = & $Script:GetPage 0
$Result = Get-PagedResult -InitialResult $Initial -URI "$($Script:PageURI)?limit=10" -Style Offset -ResultProperty items

$Result | Should -Be @(0..24)
$Script:Requests | Should -Be @(
"$($Script:PageURI)?limit=10&offset=10"
"$($Script:PageURI)?limit=10&offset=20"
)
}

It 'advances from the offset in the URI' {
$Initial = & $Script:GetPage 5
$Result = Get-PagedResult -InitialResult $Initial -URI "$($Script:PageURI)?limit=10&offset=5" -Style Offset -ResultProperty items

$Result | Should -Be @(5..24)
$Script:Requests | Should -Be @(
"$($Script:PageURI)?limit=10&offset=15"
)
}

It 'sends exactly one offset on each later request' {
$Script:SetSize = 45
$Initial = & $Script:GetPage 5
$null = Get-PagedResult -InitialResult $Initial -URI "$($Script:PageURI)?offset=5&limit=10" -Style Offset -ResultProperty items

$Script:Requests | Should -HaveCount 3
foreach ($Request in $Script:Requests) {
@(& $Script:GetQueryValue $Request 'offset') | Should -HaveCount 1
}
$Script:Requests | ForEach-Object { & $Script:GetQueryValue $_ 'offset' } | Should -Be @('15', '25', '35')
}

It 'drops a URI holding only the offset back to the bare path' {
$Initial = & $Script:GetPage 5
$null = Get-PagedResult -InitialResult $Initial -URI "$($Script:PageURI)?offset=5" -Style Offset -ResultProperty items

$Script:Requests | Should -Be @("$($Script:PageURI)?offset=15")
}

It 'keeps the other query keys byte for byte' {
$Initial = & $Script:GetPage 5
$null = Get-PagedResult -InitialResult $Initial -URI "$($Script:PageURI)?severity=CRITICAL%2CHIGH&limit=10&offset=5&name=a%20b%2Bc" -Style Offset -ResultProperty items

$Script:Requests | Should -Be @(
"$($Script:PageURI)?severity=CRITICAL%2CHIGH&limit=10&name=a%20b%2Bc&offset=15"
)
}

It 'collects the expected items with a last page flag and a starting offset' {
#Mirrors Get-RMFinding -offset 5 -limit 10 against 115 findings
$Script:SetSize = 115
$Initial = & $Script:GetPage 5
$Result = Get-PagedResult -InitialResult $Initial -URI "$($Script:PageURI)?offset=5&limit=10" -Style Offset -ResultProperty items -LastPageKey isLastPage

$Result | Should -Be @(5..114)
$Script:Requests | Should -HaveCount 10
$Script:Requests | ForEach-Object { & $Script:GetQueryValue $_ 'offset' } | Should -Be @('15', '25', '35', '45', '55', '65', '75', '85', '95', '105')
}

It 'honours a non-default offset request key already in the URI' {
Mock -CommandName Invoke-IDRestMethod -MockWith {
$null = $Script:Requests.Add($URI)
& $Script:GetPage ([int](@(& $Script:GetQueryValue $URI 'skip') | Select-Object -Last 1))
}

$Initial = & $Script:GetPage 5
$Result = Get-PagedResult -InitialResult $Initial -URI "$($Script:PageURI)?skip=5&top=10" -Style Offset -ResultProperty items -OffsetRequestKey skip

$Result | Should -Be @(5..24)
$Script:Requests | Should -Be @("$($Script:PageURI)?top=10&skip=15")
}

It 'leaves the server reported next offset authoritative' {
Mock -CommandName Invoke-IDRestMethod -MockWith {
$null = $Script:Requests.Add($URI)
[pscustomobject]@{ items = @('two'); paging = [pscustomobject]@{} }
}

$Initial = [pscustomobject]@{ items = @('one'); paging = [pscustomobject]@{ offset = 500 } }
$null = Get-PagedResult -InitialResult $Initial -URI "$($Script:PageURI)?offset=5&limit=10" -Style Offset -ResultProperty items -OffsetResponseKey 'paging.offset'

$Script:Requests | Should -Be @("$($Script:PageURI)?limit=10&offset=500")
}

It 'still stops when the server ignores the offset' {
Mock -CommandName Invoke-IDRestMethod -MockWith {
$null = $Script:Requests.Add($URI)
& $Script:GetPage 5
}

$Initial = & $Script:GetPage 5
$Result = Get-PagedResult -InitialResult $Initial -URI "$($Script:PageURI)?offset=5&limit=10" -Style Offset -ResultProperty items

$Result | Should -Be @(5..14)
$Script:Requests | Should -HaveCount 1
}

}

Context 'Offset style, POST' {

It 'pages from zero when the body holds no offset' {
$Initial = & $Script:GetPage 0
$Result = Get-PagedResult -InitialResult $Initial -URI $Script:PageURI -Style Offset -ResultProperty items -Method POST -BodyTemplate @{ limit = 10 }

$Result | Should -Be @(0..24)
$Script:Bodies | ForEach-Object { $_.offset } | Should -Be @(10, 20)
$Script:Requests | Should -Be @($Script:PageURI, $Script:PageURI)
}

It 'advances from a top level offset in the body' {
$Script:SetSize = 35
$Initial = & $Script:GetPage 5
$Result = Get-PagedResult -InitialResult $Initial -URI $Script:PageURI -Style Offset -ResultProperty items -Method POST -BodyTemplate @{ limit = 10; offset = 5; filter = 'x' }

$Result | Should -Be @(5..34)
$Script:Bodies | ForEach-Object { $_.offset } | Should -Be @(15, 25)
$Script:Bodies | ForEach-Object { $_.filter } | Should -Be @('x', 'x')
}

It 'advances from an offset in the nested paging property' {
$Script:SetSize = 35
$Initial = & $Script:GetPage 5
$Template = [ordered]@{ paging = [ordered]@{ limit = 10; offset = 5 } }
$Result = Get-PagedResult -InitialResult $Initial -URI $Script:PageURI -Style Offset -ResultProperty items -Method POST -BodyTemplate $Template -BodyPagingProperty paging

$Result | Should -Be @(5..34)
$Script:Bodies | ForEach-Object { $_.paging.offset } | Should -Be @(15, 25)
$Template.paging.offset | Should -Be 5
}

}

Context 'Cursor style' {

BeforeEach {
#Cursor N returns item N and points at N+1, up to item 3
Mock -CommandName Invoke-IDRestMethod -MockWith {
$null = $Script:Requests.Add($URI)
$Cursor = [int](@(& $Script:GetQueryValue $URI 'cursor') | Select-Object -Last 1)
[pscustomobject]@{ items = @($Cursor); nextCursor = $(if ($Cursor -lt 3) { $Cursor + 1 }) }
}
}

It 'sends the next cursor when the URI holds none' {
$Initial = [pscustomobject]@{ items = @(0); nextCursor = 1 }
$Result = Get-PagedResult -InitialResult $Initial -URI "$($Script:PageURI)?limit=1" -Style Cursor -ResultProperty items

$Result | Should -Be @(0, 1, 2, 3)
$Script:Requests | Should -Be @(
"$($Script:PageURI)?limit=1&cursor=1"
"$($Script:PageURI)?limit=1&cursor=2"
"$($Script:PageURI)?limit=1&cursor=3"
)
}

It 'replaces a starting cursor in the URI, keeping the other keys byte for byte' {
$Initial = [pscustomobject]@{ items = @(1); nextCursor = 2 }
$Result = Get-PagedResult -InitialResult $Initial -URI "$($Script:PageURI)?cursor=1&q=a%2Cb%20c&limit=1" -Style Cursor -ResultProperty items

$Result | Should -Be @(1, 2, 3)
$Script:Requests | Should -Be @(
"$($Script:PageURI)?q=a%2Cb%20c&limit=1&cursor=2"
"$($Script:PageURI)?q=a%2Cb%20c&limit=1&cursor=3"
)
}

}

}

}

}
Loading
Loading