From fbd7ec46c636ee5f9d940cb81e5abbc1df90f055 Mon Sep 17 00:00:00 2001 From: Prangshuman Das Date: Wed, 7 Oct 2026 15:32:23 +0200 Subject: [PATCH] Add isolated method-exclusion and evidence-gated SQL retry experiments AB#646383. Diagnostic only: twenty manual W1/DE trials, maximum five parallel; immutable source and runtime pins. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3952f078-a881-4da8-ad96-13b727e48a91 --- .github/workflows/CICD.yaml | 18 +- .github/workflows/PullRequestHandler.yaml | 2 +- build/scripts/ParallelTestExecution.psm1 | 124 +++++- build/scripts/SqlApiTestRetry.psm1 | 201 +++++++++ .../SqlResetPilot/CompaniesProbe.Tests.ps1 | 12 + .../SqlResetPilot/ContainerPreflight.ps1 | 6 +- .../scripts/SqlResetPilot/Finalize.Tests.ps1 | 9 +- build/scripts/SqlResetPilot/Finalize.ps1 | 5 +- build/scripts/SqlResetPilot/Lifecycle.psm1 | 7 +- .../scripts/SqlResetPilot/Preflight.Tests.ps1 | 10 +- build/scripts/SqlResetPilot/Prepare.ps1 | 14 +- build/scripts/SqlResetPilot/README.md | 174 ++++---- build/scripts/SqlResetPilot/TwoArm.Tests.ps1 | 164 ++++++++ build/scripts/tests/SqlApiTestRetry.Test.ps1 | 385 ++++++++++++++++++ 14 files changed, 1017 insertions(+), 114 deletions(-) create mode 100644 build/scripts/SqlApiTestRetry.psm1 create mode 100644 build/scripts/SqlResetPilot/TwoArm.Tests.ps1 create mode 100644 build/scripts/tests/SqlApiTestRetry.Test.ps1 diff --git a/.github/workflows/CICD.yaml b/.github/workflows/CICD.yaml index 3db952d48d6..0c7873932b6 100644 --- a/.github/workflows/CICD.yaml +++ b/.github/workflows/CICD.yaml @@ -1,4 +1,4 @@ -name: 'SQL API warmup experiment - DO NOT MERGE' +name: 'SQL API two-arm experiment - DO NOT MERGE' on: workflow_dispatch: @@ -11,13 +11,14 @@ permissions: jobs: Pilot: - if: github.ref == 'refs/heads/features/646383-sql-api-warmup-experiment' && github.run_attempt == 1 + if: github.ref == 'refs/heads/features/646383-sql-api-two-arm-experiment' && github.run_attempt == 1 strategy: fail-fast: false max-parallel: 5 matrix: country: [W1, DE] trial: [1, 2, 3, 4, 5] + experiment: [A, B] runs-on: GitHub-BCAppsPL timeout-minutes: 120 defaults: @@ -27,6 +28,7 @@ jobs: ALGoOrgSettings: ${{ vars.ALGoOrgSettings }} ALGoRepoSettings: ${{ vars.ALGoRepoSettings }} BC_SQL_PILOT_ARM: control + BC_SQL_API_EXPERIMENT: ${{ matrix.experiment }} BC_SQL_PILOT_COUNTRY: ${{ matrix.country }} BC_SQL_PILOT_TRIAL: ${{ matrix.trial }} BuildMode: IntegrationTests @@ -62,7 +64,7 @@ jobs: uses: microsoft/AL-Go/Actions/ReadSettings@91b96c2b294be6f823277dafe6f03350abfb9d23 with: shell: pwsh - project: build/projects/Test Apps ${{ matrix.country }} Trial${{ matrix.trial }} + project: build/projects/Test Apps ${{ matrix.country }} Trial${{ matrix.trial }}${{ matrix.experiment }} buildMode: IntegrationTests - name: Read normal disposable CI license and package secrets @@ -79,7 +81,7 @@ jobs: Secrets: '${{ steps.secrets.outputs.Secrets }}' with: shell: pwsh - project: build/projects/Test Apps ${{ matrix.country }} Trial${{ matrix.trial }} + project: build/projects/Test Apps ${{ matrix.country }} Trial${{ matrix.trial }}${{ matrix.experiment }} buildMode: IntegrationTests artifact: ${{ steps.packages.outputs.artifact }} installAppsJson: ${{ steps.packages.outputs.apps }} @@ -94,11 +96,11 @@ jobs: if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: sql-api-warmup-${{ matrix.country }}-${{ matrix.trial }}-${{ github.run_id }} + name: sql-api-two-arm-${{ matrix.experiment }}-${{ matrix.country }}-${{ matrix.trial }}-${{ github.run_id }} retention-days: 7 if-no-files-found: error path: | sql-reset-pilot-output/** - build/projects/Test Apps ${{ matrix.country }} Trial${{ matrix.trial }}/TestResults*.xml - build/projects/Test Apps ${{ matrix.country }} Trial${{ matrix.trial }}/ContainerEventLog.evtx - build/projects/Test Apps ${{ matrix.country }} Trial${{ matrix.trial }}/BuildOutput.txt + build/projects/Test Apps ${{ matrix.country }} Trial${{ matrix.trial }}${{ matrix.experiment }}/TestResults*.xml + build/projects/Test Apps ${{ matrix.country }} Trial${{ matrix.trial }}${{ matrix.experiment }}/ContainerEventLog.evtx + build/projects/Test Apps ${{ matrix.country }} Trial${{ matrix.trial }}${{ matrix.experiment }}/BuildOutput.txt diff --git a/.github/workflows/PullRequestHandler.yaml b/.github/workflows/PullRequestHandler.yaml index 142d13c1cfd..599330b4467 100644 --- a/.github/workflows/PullRequestHandler.yaml +++ b/.github/workflows/PullRequestHandler.yaml @@ -35,7 +35,7 @@ jobs: Initialization: needs: [ PregateCheck ] - if: (!failure() && !cancelled()) && github.head_ref != 'features/646383-sql-api-warmup-experiment' + if: (!failure() && !cancelled()) && github.head_ref != 'features/646383-sql-api-warmup-experiment' && github.head_ref != 'features/646383-sql-api-two-arm-experiment' runs-on: [ windows-latest ] outputs: projects: ${{ steps.determineProjectsToBuild.outputs.ProjectsJson }} diff --git a/build/scripts/ParallelTestExecution.psm1 b/build/scripts/ParallelTestExecution.psm1 index a5a532273bc..6e43f662b06 100644 --- a/build/scripts/ParallelTestExecution.psm1 +++ b/build/scripts/ParallelTestExecution.psm1 @@ -13,6 +13,7 @@ if (-not (Get-Command Write-Log -ErrorAction SilentlyContinue)) { } } Import-Module (Join-Path $PSScriptRoot "ALAppBuild.psm1" -Resolve) +Import-Module (Join-Path $PSScriptRoot 'SqlApiTestRetry.psm1' -Resolve) if ($env:BC_SQL_PILOT_ARM) { Import-Module (Join-Path $PSScriptRoot 'SqlResetPilot\Lifecycle.psm1') } @@ -47,9 +48,26 @@ function Get-DisabledTestsForApp { } } + if ((Test-SqlApiExperiment) -and $env:BC_SQL_API_EXPERIMENT -eq 'A') { + $disabledTests += [PSCustomObject]@{ codeunitId = 148318; method = 'CapabilitiesProjectsEnabledViaAPI' } + } return @($disabledTests) } +function Test-SqlApiExperiment { + return ($env:GITHUB_REPOSITORY -eq 'microsoft/BCApps' -and + $env:GITHUB_REF -eq 'refs/heads/features/646383-sql-api-two-arm-experiment' -and + $env:GITHUB_EVENT_NAME -eq 'workflow_dispatch' -and $env:GITHUB_RUN_ATTEMPT -eq '1' -and + $env:BC_SQL_PILOT_ARM -eq 'control' -and $env:BC_SQL_API_EXPERIMENT -in @('A', 'B')) +} + +function Get-TestJobResultFileName { + param([string]$Path, [string]$Suffix) + $dir = [System.IO.Path]::GetDirectoryName($Path) + if (-not $dir) { $dir = '.' } + Join-Path $dir "$([System.IO.Path]::GetFileNameWithoutExtension($Path))-$Suffix$([System.IO.Path]::GetExtension($Path))" +} + function Get-ParametersForCommand { param( [Parameter(Mandatory=$true)] @@ -646,14 +664,57 @@ function Receive-TestJobResult { $output = $sb.ToString() $outcome = 'Passed' + $sqlRetryEvidence = $null if ($Job.State -eq 'Failed' -or $Job.State -eq 'Stopped') { if (-not $env:BC_SQL_PILOT_ARM -and (Test-TransientTestFailure $output) -and -not $Retried.ContainsKey($Entry.appName)) { $outcome = 'Transient' } else { $outcome = 'Failed' + if ($Job.State -eq 'Failed' -and -not $Retried.ContainsKey($Entry.appName) -and + $Entry.PSObject.Properties['SqlRetryContext'] -and $Entry.SqlRetryContext -and + $env:BC_SQL_API_EXPERIMENT -eq 'B' -and (Test-SqlApiExperiment)) { + $sqlRetryEvidence = Get-SqlApiTestRetryEvidence -Context $Entry.SqlRetryContext + if ($sqlRetryEvidence) { $outcome = 'SqlPoolRetry' } + } } } + if ($Entry.PSObject.Properties['SqlRetryContext'] -and $Entry.SqlRetryContext) { + $context = $Entry.SqlRetryContext + if (-not (Test-SqlApiExperimentSelection -Context $context -Experiment $env:BC_SQL_API_EXPERIMENT)) { + $outcome = 'Failed' + $sqlRetryEvidence = $null + } + if ($Entry.SqlRetryExpectedTests.Count -gt 0 -and $outcome -eq 'Passed') { + if (-not (Test-SqlApiRetryResult -Context $context -ExpectedTests $Entry.SqlRetryExpectedTests)) { + $outcome = 'Failed' + } + } + $attemptDirectory = Join-Path $env:BC_SQL_PILOT_OUTPUT "test-attempts\$($context.Tenant)-$($context.CodeunitId)-$($context.Attempt)" + New-Item -ItemType Directory -Path $attemptDirectory -Force -ErrorAction Stop | Out-Null + foreach ($key in $context.ResultFiles.Keys) { + $path = $context.ResultFiles[$key] + if (Test-Path -LiteralPath $path) { + Copy-Item -LiteralPath $path -Destination (Join-Path $attemptDirectory $key) -ErrorAction Stop + } + } + @{ + codeunitId = $context.CodeunitId; tenant = $context.Tenant; attempt = $context.Attempt + outcome = $outcome; jobState = [string]$Job.State; startedUtc = $context.StartedUtc.ToString('o') + completedUtc = [datetime]::UtcNow.ToString('o'); experiment = $env:BC_SQL_API_EXPERIMENT + retryEligible = [bool]$sqlRetryEvidence; expectedCases = $context.TestCount + } | ConvertTo-Json | Set-Content (Join-Path $attemptDirectory 'outcome.json') -ErrorAction Stop + if ($context.Attempt -eq 2 -and $outcome -eq 'Passed') { + foreach ($key in $context.ResultFiles.Keys) { + $first = Join-Path $env:BC_SQL_PILOT_OUTPUT "test-attempts\$($context.Tenant)-$($context.CodeunitId)-1\$key" + if (-not (Test-Path $first) -or -not (Test-Path $context.ResultFiles[$key])) { + throw 'Cannot merge retry without both original and retry result files.' + } + Merge-TestResultFiles -targetFile $context.FinalResultFiles[$key] ` + -sourceFiles @($first, $context.ResultFiles[$key]) + } + } + } if ($env:BC_SQL_PILOT_ARM) { $output | Set-Content (Join-Path $env:BC_SQL_PILOT_OUTPUT "worker-$($Job.Id).log") -Encoding UTF8 } @@ -664,6 +725,7 @@ function Receive-TestJobResult { AppName = $Entry.appName Tenant = $Entry.tenant JobState = $Job.State + SqlRetryEvidence = $sqlRetryEvidence } } @@ -755,13 +817,44 @@ function Start-RequiredDisabledDispatch { } } + $sqlRetryContext = $null + $sqlRetryExpectedTests = @() + $fileSuffix = $TenantInfo.Id + if (Test-SqlApiExperiment) { + if (-not $Parameters.JUnitResultFileName) { throw 'Two-arm experiment requires JUnit evidence.' } + $attempt = 1 + $finalFiles = @{} + if ($State.sqlRetryEvidence.ContainsKey($WorkItem.Key)) { + $attempt = 2 + $fileSuffix = "$($TenantInfo.Id)-sqlretry-$($WorkItem.CodeunitId)" + $sqlRetryExpectedTests = @($State.sqlRetryEvidence[$WorkItem.Key].TestCases) + # BCH ReRun selects only failing methods. This experiment retries the entire codeunit. + $codeunitParameters.Remove('ReRun') | Out-Null + } + $files = @{} + foreach ($key in $appendKeys.Keys) { + if ($Parameters.ContainsKey($key) -and $Parameters[$key]) { + $files[$key] = Get-TestJobResultFileName $Parameters[$key] $fileSuffix + $finalFiles[$key] = Get-TestJobResultFileName $Parameters[$key] $TenantInfo.Id + if ($attempt -eq 2 -and (Test-Path $files[$key])) { throw 'Retry result file already exists.' } + } + } + $sqlRetryContext = [PSCustomObject]@{ + ContainerName = $Parameters.containerName; Tenant = $TenantInfo.Id + CodeunitId = $WorkItem.CodeunitId; TestCount = $WorkItem.TestCount + StartedUtc = [datetime]::UtcNow; ResultFiles = $files; FinalResultFiles = $finalFiles + Attempt = $attempt; OutputDirectory = $env:BC_SQL_PILOT_OUTPUT + } + } $job = Start-TestJob -parameters $codeunitParameters -tenant $TenantInfo.Id -scriptPath $ScriptPath ` - -testType $TestType -skipAutomaticDisabledPass + -testType $TestType -skipAutomaticDisabledPass -fileSuffix $fileSuffix $State.jobs = @($State.jobs) + @( [PSCustomObject]@{ jobId = $job.Id tenant = $TenantInfo.Id appName = $WorkItem.Key + SqlRetryContext = $sqlRetryContext + SqlRetryExpectedTests = $sqlRetryExpectedTests } ) Start-Sleep -Seconds 1 @@ -798,6 +891,7 @@ function Invoke-RequiredDisabledTestExecution { transient = @() retried = @{} retryTenant = @{} + sqlRetryEvidence = @{} } $pending = @($WorkItems) @@ -857,7 +951,7 @@ function Invoke-RequiredDisabledTestExecution { -ScriptPath $ScriptPath -TestType $TestType -State $state -Verb $dispatch.Verb } $null = Wait-ForAllTestJobs -state $state - if ($env:BC_SQL_PILOT_ARM -and ($state.hasFailures -or $state.transient.Count -gt 0)) { + if ($env:BC_SQL_PILOT_ARM -and $state.hasFailures) { $state.hasFailures = $true break } @@ -885,6 +979,16 @@ function Register-TestJobOutcome { $State ) + if ($Result.Outcome -eq 'SqlPoolRetry') { + if (-not (Test-SqlApiExperiment) -or $env:BC_SQL_API_EXPERIMENT -ne 'B' -or + -not $Result.SqlRetryEvidence -or $State.retried.ContainsKey($Result.AppName)) { + $State.hasFailures = $true + return + } + $State.sqlRetryEvidence[$Result.AppName] = $Result.SqlRetryEvidence + $State.transient = @($State.transient) + @([PSCustomObject]@{ Key = $Result.AppName; Tenant = $Result.Tenant }) + return + } if ($env:BC_SQL_PILOT_ARM -and $Result.Outcome -in @('Failed', 'Transient')) { $State.hasFailures = $true return @@ -1104,6 +1208,9 @@ function Wait-ForAllTestJobs { $result = Receive-TestJobResult -Entry $entry -Job $pendingJob -Retried $state.retried Register-TestJobOutcome -Result $result -State $state + } elseif (Test-SqlApiExperiment) { + $state.hasFailures = $true + Write-Host "::warning::Expected experiment worker job $($entry.jobId) is missing; failing closed." } } # All jobs in $state.jobs have been received and removed; clear the list so any later @@ -1510,6 +1617,19 @@ function Invoke-ParallelTestExecution { if ($env:BC_SQL_PILOT_ARM) { Merge-TenantTestResults -parameters $parameters -tenants $tenants -workItems $requiredDisabledWorkItems -rerunSuffixes @() + if ((Test-SqlApiExperiment) -and (Test-Path $parameters.JUnitResultFileName)) { + [xml]$finalXml = Get-Content $parameters.JUnitResultFileName -Raw + $finalCases = @($finalXml.SelectNodes('/testsuites/testsuite/testcase')) + @{ + total = $finalCases.Count + failed = @($finalCases | Where-Object { $_.SelectSingleNode('failure | error') }).Count + skipped = @($finalCases | Where-Object { $_.SelectSingleNode('skipped') }).Count + passed = @($finalCases | Where-Object { -not $_.SelectSingleNode('failure | error | skipped') }).Count + suites = $finalXml.SelectNodes('/testsuites/testsuite').Count + experiment = $env:BC_SQL_API_EXPERIMENT + firstAttemptsAndRetries = 'See test-attempts; final XML replaces only validated recovered CUs.' + } | ConvertTo-Json | Set-Content (Join-Path $env:BC_SQL_PILOT_OUTPUT 'final-counts.json') + } $state.finalResult = -not $state.hasFailures $state.completed = $true $state | ConvertTo-Json -Depth 5 | Set-Content $stateFile -Force diff --git a/build/scripts/SqlApiTestRetry.psm1 b/build/scripts/SqlApiTestRetry.psm1 new file mode 100644 index 00000000000..c548b5cd693 --- /dev/null +++ b/build/scripts/SqlApiTestRetry.psm1 @@ -0,0 +1,201 @@ +<# +.SYNOPSIS + Correlates clean-codeunit HTTP failures with NST SQL-pool exceptions. +.DESCRIPTION + This is a manual diagnostic experiment, not a generic HTTP retry or a pool-lifecycle fix. + Every failed case must have a request CorrelationId matching an NST ClientSessionId, + a NullReferenceException in AcquireSqlConnectionFromPool, and the observed OData + metadata stack, within this dispatch's time window. Missing evidence fails closed. + The scheduler permits only one retry, restores the tenant first, and reruns the entire + codeunit. Original results and matching events are retained separately from final results. + The scheduler enables this only on the exact manual experiment branch, arm B, attempt one. + SqlApiRetryEvidence JSON files under the diagnostic output are uploaded with raw outcomes, + including on successful runs, so recovered failures remain visible without duplicating cases. +#> +function Get-SqlApiRetrySuite { + param($Context) + + [xml]$document = Get-Content -LiteralPath $Context.ResultFiles.JUnitResultFileName -Raw -ErrorAction Stop + $suites = @($document.SelectNodes('/testsuites/testsuite') | Where-Object { + ($_.GetAttribute('name') -split ' ', 2)[0] -eq [string]$Context.CodeunitId + }) + if ($suites.Count -ne 1) { + throw "Expected exactly one JUnit suite for codeunit $($Context.CodeunitId), found $($suites.Count)." + } + return ,$suites[0] +} + +function Get-SqlApiRetryEvents { + param( + [string]$ContainerName, + [datetime]$StartedUtc, + [datetime]$FinishedUtc + ) + + return @(Invoke-ScriptInBcContainer -containerName $ContainerName -scriptblock { + param($start, $end) + try { + $events = @(Get-WinEvent -FilterHashtable @{ + LogName = 'Application' + ProviderName = 'MicrosoftDynamicsNavServer*' + Id = 701 + StartTime = [datetime]::Parse($start).ToUniversalTime() + EndTime = [datetime]::Parse($end).ToUniversalTime() + } -MaxEvents 1000 -ErrorAction Stop) + } catch { + if ($_.FullyQualifiedErrorId -like 'NoMatchingEventsFound*') { + return + } + throw + } + foreach ($eventRecord in $events) { + $eventRecord.ToXml() + } + } -argumentList $StartedUtc.ToString('o'), $FinishedUtc.ToString('o')) +} + +<# +.SYNOPSIS + Persists original JUnit and correlated NST events before authorizing one clean-codeunit retry. +#> +function Get-SqlApiTestRetryEvidence { + param($Context) + + try { + $suite = Get-SqlApiRetrySuite -Context $Context + $cases = @($suite.SelectNodes('testcase')) + if ($cases.Count -ne $Context.TestCount -or $cases.Count -eq 0) { + throw "Incomplete JUnit codeunit: expected $($Context.TestCount) cases, found $($cases.Count)." + } + $failures = @($suite.SelectNodes('testcase/failure')) + if ($failures.Count -eq 0 -or $suite.SelectNodes('.//error').Count -gt 0) { + return + } + $requestIds = @() + foreach ($failure in $failures) { + $message = $failure.GetAttribute('message') + if ($message -notmatch 'GET request failed\. Response code is 500 \(InternalServerError\)' -or + $message -notmatch 'Object reference not set to an instance of an object\.' -or + $message -notmatch 'CorrelationId:\s*([0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12})(?![0-9a-f-])') { + return + } + $requestIds += $Matches[1] + } + # Refuse malformed/partial results rather than overlooking an unrepresented failure. + foreach ($case in $cases) { + if ($case.GetAttribute('result') -eq 'Fail' -and $case.SelectNodes('failure').Count -ne 1) { + return + } + } + + $finishedUtc = [datetime]::UtcNow + $events = @(Get-SqlApiRetryEvents -ContainerName $Context.ContainerName ` + -StartedUtc $Context.StartedUtc -FinishedUtc $finishedUtc) + $matchedEvents = @() + foreach ($requestId in $requestIds) { + $matchesForRequest = @() + foreach ($eventText in $events) { + [xml]$eventXml = $eventText + $ns = [System.Xml.XmlNamespaceManager]::new($eventXml.NameTable) + $ns.AddNamespace('e', 'http://schemas.microsoft.com/win/2004/08/events/event') + $system = $eventXml.SelectSingleNode('/e:Event/e:System', $ns) + $provider = $system.SelectSingleNode('e:Provider', $ns).GetAttribute('Name') + $eventId = $system.SelectSingleNode('e:EventID', $ns).InnerText + $time = [datetime]::Parse($system.SelectSingleNode('e:TimeCreated', $ns).GetAttribute('SystemTime')).ToUniversalTime() + $text = ($eventXml.SelectNodes('/e:Event/e:EventData/e:Data', $ns) | + ForEach-Object { $_.InnerText }) -join "`n" + if ($provider -match '^MicrosoftDynamicsNavServer\$' -and $eventId -eq '701' -and + $time -ge $Context.StartedUtc -and $time -le $finishedUtc -and + $text -match "(?m)^ClientSessionId:\s*$([regex]::Escape($requestId))\s*$" -and + $text -match 'RootException: NullReferenceException' -and + $text -match 'at Microsoft\.Dynamics\.Nav\.Runtime\.NavSqlConnectionScope\.AcquireSqlConnectionFromPool\(' -and + $text -match 'at Microsoft\.Dynamics\.Nav\.XmlMetadata\.MetadataProvider\.GetRelativeHelpUrl\(' -and + $text -match 'at Microsoft\.Dynamics\.Nav\.Service\.OData\.V4\.PageDataProvider\.GetNavRecordDataAsync\(') { + $matchesForRequest += [string]$eventText + } + } + if ($matchesForRequest.Count -eq 0) { + Write-Host "::warning::SQL API retry not eligible: no matching NST event for request '$requestId'." + return + } + $matchedEvents += $matchesForRequest + } + $evidence = [PSCustomObject]@{ + ContainerName = $Context.ContainerName + Tenant = $Context.Tenant + CodeunitId = $Context.CodeunitId + StartedUtc = $Context.StartedUtc.ToString('o') + FinishedUtc = $finishedUtc.ToString('o') + RequestIds = @($requestIds | Select-Object -Unique) + OriginalSuite = $suite.OuterXml + Events = @($matchedEvents | Select-Object -Unique) + TestCases = @($cases | ForEach-Object { + [PSCustomObject]@{ Name = $_.GetAttribute('name'); Skipped = ($null -ne $_.SelectSingleNode('skipped')) } + }) + } + $folder = Join-Path ([System.IO.Path]::GetDirectoryName($Context.ResultFiles.JUnitResultFileName)) 'SqlApiRetryEvidence' + if ($Context.PSObject.Properties['OutputDirectory'] -and $Context.OutputDirectory) { + $folder = Join-Path $Context.OutputDirectory 'SqlApiRetryEvidence' + } + $null = New-Item -ItemType Directory -Path $folder -Force -ErrorAction Stop + $evidenceFile = Join-Path $folder "$($Context.Tenant)-$($Context.CodeunitId)-$([guid]::NewGuid().ToString('N')).json" + $evidence | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $evidenceFile -Encoding utf8 -ErrorAction Stop + Write-Host "::warning::SQL API pool failure confirmed for codeunit $($Context.CodeunitId) on '$($Context.Tenant)'; requests: $($evidence.RequestIds -join ', '). Retrying once after a tenant reset. Original evidence: $evidenceFile" + return $evidence + } catch { + Write-Host "::warning::SQL API retry evidence unavailable; preserving test failure: $($_.Exception.Message)" + return + } +} + +<# +.SYNOPSIS + Requires a SQL retry to preserve the original test names and skip status without failures. +#> +function Test-SqlApiRetryResult { + param( + $Context, + [array]$ExpectedTests + ) + + try { + $suite = Get-SqlApiRetrySuite -Context $Context + $cases = @($suite.SelectNodes('testcase')) + if ($cases.Count -ne $ExpectedTests.Count -or $suite.SelectNodes('.//failure | .//error').Count -gt 0) { + throw 'The retry did not produce a complete, passing codeunit.' + } + foreach ($expected in $ExpectedTests) { + $actual = @($cases | Where-Object { $_.GetAttribute('name') -ceq $expected.Name }) + if ($actual.Count -ne 1 -or + ($null -ne $actual[0].SelectSingleNode('skipped')) -ne $expected.Skipped -or + $actual[0].GetAttribute('result') -eq 'Fail') { + throw "Retry result missing, duplicated, failed or changed skip status for '$($expected.Name)'." + } + } + return $true + } catch { + Write-Host "::warning::SQL API retry remains failed: $($_.Exception.Message)" + return $false + } +} + +function Test-SqlApiExperimentSelection { + param($Context, [ValidateSet('A', 'B')][string]$Experiment) + if ([string]$Context.CodeunitId -ne '148318') { return $true } + try { + $suite = Get-SqlApiRetrySuite -Context $Context + $target = @($suite.SelectNodes('testcase') | Where-Object { + ($_.GetAttribute('name') -split '\.')[-1] -ceq 'CapabilitiesProjectsEnabledViaAPI' + }) + if ($target.Count -ne 1) { throw 'Expected exactly one target method in Capabilities results.' } + if (($null -ne $target[0].SelectSingleNode('skipped')) -ne ($Experiment -eq 'A')) { + throw 'Target method skip status differs from the experiment arm.' + } + return $true + } catch { + Write-Host "::warning::Experiment selection verification failed: $($_.Exception.Message)" + return $false + } +} + +Export-ModuleMember -Function Get-SqlApiTestRetryEvidence, Test-SqlApiRetryResult, Test-SqlApiExperimentSelection diff --git a/build/scripts/SqlResetPilot/CompaniesProbe.Tests.ps1 b/build/scripts/SqlResetPilot/CompaniesProbe.Tests.ps1 index 9ede785e853..d2ed24c07d6 100644 --- a/build/scripts/SqlResetPilot/CompaniesProbe.Tests.ps1 +++ b/build/scripts/SqlResetPilot/CompaniesProbe.Tests.ps1 @@ -44,6 +44,18 @@ Describe 'External authenticated companies probe' { } Should -Invoke Start-Sleep -ModuleName Lifecycle -Times 0 } + It 'uses exactly one attempt in experiment arm even for HTTP500' -ForEach @( + @{ Arm = 'A' }, @{ Arm = 'B' } + ) { + $old = $env:BC_SQL_API_EXPERIMENT + try { + $env:BC_SQL_API_EXPERIMENT = $Arm + Mock Invoke-WebRequest -ModuleName Lifecycle { @{ StatusCode = 500; Headers = @{}; Content = '' } } + { Invoke-SqlPilotCompaniesProbe 'owned' 'tenant2' $script:credential 'My Company' } | Should -Throw '*after 1 attempt*' + Should -Invoke Invoke-WebRequest -ModuleName Lifecycle -Times 1 -Exactly + Should -Invoke Start-Sleep -ModuleName Lifecycle -Times 0 + } finally { $env:BC_SQL_API_EXPERIMENT = $old } + } It 'bounds transient HTTP to three attempts and records each failure safely' -ForEach @( @{ Code = 500 }, @{ Code = 502 }, @{ Code = 503 }, @{ Code = 504 } ) { diff --git a/build/scripts/SqlResetPilot/ContainerPreflight.ps1 b/build/scripts/SqlResetPilot/ContainerPreflight.ps1 index 25e895d1da9..4ab3eebc661 100644 --- a/build/scripts/SqlResetPilot/ContainerPreflight.ps1 +++ b/build/scripts/SqlResetPilot/ContainerPreflight.ps1 @@ -1,13 +1,14 @@ param([hashtable]$Parameters) $ErrorActionPreference = 'Stop' -if ($env:GITHUB_REF -ne 'refs/heads/features/646383-sql-api-warmup-experiment' -or +if ($env:GITHUB_REF -ne 'refs/heads/features/646383-sql-api-two-arm-experiment' -or + $env:BC_SQL_API_EXPERIMENT -notin @('A', 'B') -or $env:GITHUB_EVENT_NAME -ne 'workflow_dispatch' -or $env:GITHUB_RUN_ATTEMPT -ne '1' -or $env:BC_SQL_PILOT_ARM -ne 'control' -or -not $env:BC_SQL_PILOT_OUTPUT -or $env:BC_SQL_PILOT_COUNTRY -notin @('W1', 'DE') -or $env:BC_SQL_PILOT_TRIAL -notmatch '^[1-5]$' -or $env:GITHUB_RUN_ID -notmatch '^\d{1,20}$') { throw 'Refusing SQL pilot outside its disposable manually dispatched CI job.' } -$expectedName = "bcbuildprojectsTestApps$($env:BC_SQL_PILOT_COUNTRY)Trial$($env:BC_SQL_PILOT_TRIAL)$($env:GITHUB_RUN_ID)" +$expectedName = "bcbuildprojectsTestApps$($env:BC_SQL_PILOT_COUNTRY)Trial$($env:BC_SQL_PILOT_TRIAL)$($env:BC_SQL_API_EXPERIMENT)$($env:GITHUB_RUN_ID)" if ($Parameters.ContainerName -ne $expectedName -or (Test-BcContainer -containerName $expectedName)) { throw 'Pilot container name is unexpected or already exists. No replacement is authorized.' } @@ -30,6 +31,7 @@ if ($Parameters.platformArtifactUrl -ne 'https://bcinsider-fvh2ekdjecfjd6gk.b02. @{ container = $expectedName; runId = $env:GITHUB_RUN_ID; arm = $env:BC_SQL_PILOT_ARM country = $env:BC_SQL_PILOT_COUNTRY; trial = $env:BC_SQL_PILOT_TRIAL + experiment = $env:BC_SQL_API_EXPERIMENT helperVersion = $helperVersion; image = $Parameters.useGenericImage imageGenericTag = '1.0.2.128'; platform = '30.0.55665.0' hostOS = [Environment]::OSVersion.Version.ToString() diff --git a/build/scripts/SqlResetPilot/Finalize.Tests.ps1 b/build/scripts/SqlResetPilot/Finalize.Tests.ps1 index 72502044d1c..412ac9ac889 100644 --- a/build/scripts/SqlResetPilot/Finalize.Tests.ps1 +++ b/build/scripts/SqlResetPilot/Finalize.Tests.ps1 @@ -10,15 +10,17 @@ Describe 'SQL pilot final cleanup ownership' { $script:oldHelper = $env:BcContainerHelperPath $script:oldCountry = $env:BC_SQL_PILOT_COUNTRY $script:oldTrial = $env:BC_SQL_PILOT_TRIAL + $script:oldExperiment = $env:BC_SQL_API_EXPERIMENT $env:GITHUB_WORKSPACE = $PSScriptRoot $env:GITHUB_RUN_ID = '123' $env:BC_SQL_PILOT_ARM = 'control' $env:BC_SQL_PILOT_COUNTRY = 'DE' $env:BC_SQL_PILOT_TRIAL = '2' + $env:BC_SQL_API_EXPERIMENT = 'B' $env:BcContainerHelperPath = Join-Path $PSScriptRoot 'BcContainerHelper.ps1' Mock Import-Module {} Mock Test-Path { $true } - Mock Get-Content { '{"runId":"123","arm":"control","country":"DE","trial":"2","container":"bcbuildprojectsTestAppsDETrial2123"}' } + Mock Get-Content { '{"runId":"123","arm":"control","country":"DE","trial":"2","experiment":"B","container":"bcbuildprojectsTestAppsDETrial2B123"}' } Mock Get-ChildItem {} Mock New-Item {} Mock Copy-Item {} @@ -35,12 +37,13 @@ Describe 'SQL pilot final cleanup ownership' { $env:BcContainerHelperPath = $script:oldHelper $env:BC_SQL_PILOT_COUNTRY = $script:oldCountry $env:BC_SQL_PILOT_TRIAL = $script:oldTrial + $env:BC_SQL_API_EXPERIMENT = $script:oldExperiment Remove-Variable sqlPilotTestContainerExists -Scope Global } It 'exports before disposing only the registered container' { & (Join-Path $PSScriptRoot 'Finalize.ps1') Should -Invoke Get-BcContainerEventLog -Times 1 -Exactly -Scope It - Should -Invoke Remove-BcContainer -Times 1 -Exactly -Scope It -ParameterFilter { $containerName -eq 'bcbuildprojectsTestAppsDETrial2123' } + Should -Invoke Remove-BcContainer -Times 1 -Exactly -Scope It -ParameterFilter { $containerName -eq 'bcbuildprojectsTestAppsDETrial2B123' } } It 'still tears down if exporting fails and keeps that failure visible' { Mock Get-BcContainerEventLog { throw 'event export failed' } @@ -51,7 +54,7 @@ Describe 'SQL pilot final cleanup ownership' { Mock Get-ChildItem { [PSCustomObject]@{ FullName = 'TestResults.xml' } } & (Join-Path $PSScriptRoot 'Finalize.ps1') Should -Invoke Get-ChildItem -Times 1 -Exactly -Scope It -ParameterFilter { - $Path -like '*Test Apps DE Trial2\.buildartifacts' -and $Filter -eq 'TestResults*.xml' -and $File + $Path -like '*Test Apps DE Trial2B\.buildartifacts' -and $Filter -eq 'TestResults*.xml' -and $File } Should -Invoke Copy-Item -Times 1 -Exactly -Scope It -ParameterFilter { $Destination -like '*clean-results\buildartifacts' diff --git a/build/scripts/SqlResetPilot/Finalize.ps1 b/build/scripts/SqlResetPilot/Finalize.ps1 index fc5c99261a7..ec5e15bf5d0 100644 --- a/build/scripts/SqlResetPilot/Finalize.ps1 +++ b/build/scripts/SqlResetPilot/Finalize.ps1 @@ -7,9 +7,10 @@ if (-not (Test-Path $ownershipPath)) { } $ownership = Get-Content $ownershipPath -Raw | ConvertFrom-Json if ($ownership.runId -ne $env:GITHUB_RUN_ID -or $ownership.arm -ne $env:BC_SQL_PILOT_ARM -or + $env:BC_SQL_API_EXPERIMENT -notin @('A', 'B') -or $ownership.experiment -ne $env:BC_SQL_API_EXPERIMENT -or $env:BC_SQL_PILOT_COUNTRY -notin @('W1', 'DE') -or $env:BC_SQL_PILOT_TRIAL -notmatch '^[1-5]$' -or $ownership.country -ne $env:BC_SQL_PILOT_COUNTRY -or $ownership.trial -ne $env:BC_SQL_PILOT_TRIAL -or - $ownership.container -ne "bcbuildprojectsTestApps$($env:BC_SQL_PILOT_COUNTRY)Trial$($env:BC_SQL_PILOT_TRIAL)$($env:GITHUB_RUN_ID)") { + $ownership.container -ne "bcbuildprojectsTestApps$($env:BC_SQL_PILOT_COUNTRY)Trial$($env:BC_SQL_PILOT_TRIAL)$($env:BC_SQL_API_EXPERIMENT)$($env:GITHUB_RUN_ID)") { throw 'Cleanup ownership mismatch.' } Import-Module (Join-Path (Split-Path $env:BcContainerHelperPath -Parent) 'BcContainerHelper.psm1') -ErrorAction Stop @@ -18,7 +19,7 @@ try { $events = Get-BcContainerEventLog -containerName $ownership.container -doNotOpen Copy-Item $events (Join-Path $output 'final-container.evtx') } - $project = Join-Path $env:GITHUB_WORKSPACE "build\projects\Test Apps $($env:BC_SQL_PILOT_COUNTRY) Trial$($env:BC_SQL_PILOT_TRIAL)" + $project = Join-Path $env:GITHUB_WORKSPACE "build\projects\Test Apps $($env:BC_SQL_PILOT_COUNTRY) Trial$($env:BC_SQL_PILOT_TRIAL)$($env:BC_SQL_API_EXPERIMENT)" $resultLocations = @{ 'project-root' = $project 'buildartifacts' = Join-Path $project '.buildartifacts' diff --git a/build/scripts/SqlResetPilot/Lifecycle.psm1 b/build/scripts/SqlResetPilot/Lifecycle.psm1 index f59cf3d3036..db975202184 100644 --- a/build/scripts/SqlResetPilot/Lifecycle.psm1 +++ b/build/scripts/SqlResetPilot/Lifecycle.psm1 @@ -150,13 +150,14 @@ function Invoke-SqlPilotCompaniesProbe { $script:pilot.Workers[$Tenant].Generation -lt 1) { throw 'Companies probe requires an owned, freshly restored worker.' } + $maximumAttempts = if ($env:BC_SQL_API_EXPERIMENT -in @('A', 'B')) { 1 } else { 3 } $trace = Join-Path $script:pilot.OutputDirectory 'companies-probes.jsonl' $observation = @{ tenant = $Tenant; generation = $script:pilot.Workers[$Tenant].Generation startedUtc = [DateTime]::UtcNow.ToString('o'); status = $null; passed = $false clientRequestId = [guid]::NewGuid().ToString(); serverRequestId = $null uri = $null; retries = 0; attempts = 0; recordType = 'outcome' - outcome = 'failed'; recovered = $false; maximumAttempts = 3 + outcome = 'failed'; recovered = $false; maximumAttempts = $maximumAttempts } $clock = [Diagnostics.Stopwatch]::StartNew() try { @@ -173,7 +174,7 @@ function Invoke-SqlPilotCompaniesProbe { $uri = [UriBuilder]::new($scheme, $ip, [int]$config.ODataServicesPort, "$($config.ServerInstance)/api/v2.0/companies", "?tenant=$Tenant").Uri.AbsoluteUri $observation.uri = $uri - for ($attemptNumber = 1; $attemptNumber -le 3; $attemptNumber++) { + for ($attemptNumber = 1; $attemptNumber -le $maximumAttempts; $attemptNumber++) { $attempt = @{ recordType = 'attempt'; probeId = $observation.clientRequestId tenant = $Tenant; generation = $observation.generation; uri = $uri @@ -243,7 +244,7 @@ function Invoke-SqlPilotCompaniesProbe { $observation.outcome = if ($observation.recovered) { 'recovered' } else { 'first-try-success' } break } - if (-not $attempt.retryable -or $attemptNumber -eq 3) { break } + if (-not $attempt.retryable -or $attemptNumber -eq $maximumAttempts) { break } Start-Sleep -Seconds 2 } if (-not $observation.passed) { throw 'Companies probe exhausted eligible attempts or encountered a permanent failure.' } diff --git a/build/scripts/SqlResetPilot/Preflight.Tests.ps1 b/build/scripts/SqlResetPilot/Preflight.Tests.ps1 index 110f3d90abf..0b8e806fce1 100644 --- a/build/scripts/SqlResetPilot/Preflight.Tests.ps1 +++ b/build/scripts/SqlResetPilot/Preflight.Tests.ps1 @@ -6,10 +6,11 @@ Describe 'Experiment country and trial container ownership' { BeforeEach { $script:saved = @{} foreach ($name in @('GITHUB_REF','GITHUB_EVENT_NAME','GITHUB_RUN_ATTEMPT','GITHUB_RUN_ID', - 'BC_SQL_PILOT_ARM','BC_SQL_PILOT_COUNTRY','BC_SQL_PILOT_TRIAL','BC_SQL_PILOT_OUTPUT','Settings')) { + 'BC_SQL_PILOT_ARM','BC_SQL_PILOT_COUNTRY','BC_SQL_PILOT_TRIAL','BC_SQL_PILOT_OUTPUT','BC_SQL_API_EXPERIMENT','Settings')) { $script:saved[$name] = [Environment]::GetEnvironmentVariable($name) } - $env:GITHUB_REF = 'refs/heads/features/646383-sql-api-warmup-experiment' + $env:GITHUB_REF = 'refs/heads/features/646383-sql-api-two-arm-experiment' + $env:BC_SQL_API_EXPERIMENT = 'B' $env:GITHUB_EVENT_NAME = 'workflow_dispatch' $env:GITHUB_RUN_ATTEMPT = '1' $env:GITHUB_RUN_ID = '123' @@ -22,7 +23,7 @@ Describe 'Experiment country and trial container ownership' { companyName = 'My Company'; enableCleanTestCodeunitExecution = $true; enableTaskScheduler = $false } | ConvertTo-Json $script:parameters = @{ - ContainerName = 'bcbuildprojectsTestAppsDETrial2123' + ContainerName = 'bcbuildprojectsTestAppsDETrial2B123' platformArtifactUrl = 'https://bcinsider-fvh2ekdjecfjd6gk.b02.azurefd.net/platform/30.0.55665.0/platform' } Mock Get-Module { { '6.1.19-preview2811389' } } -ParameterFilter { $Name -eq 'BcContainerHelper' } @@ -36,7 +37,8 @@ Describe 'Experiment country and trial container ownership' { $record = Get-Content (Join-Path $TestDrive 'container-ownership.json') -Raw | ConvertFrom-Json $record.country | Should -Be 'DE' $record.trial | Should -Be '2' - $record.container | Should -Be 'bcbuildprojectsTestAppsDETrial2123' + $record.container | Should -Be 'bcbuildprojectsTestAppsDETrial2B123' + $record.experiment | Should -Be 'B' $script:parameters.useGenericImage | Should -Be 'mcr.microsoft.com/businesscentral@sha256:c899d12093ad7bbdbfd08ccc0e6294e0f98c682c7e35db4ecfbca345fb068492' } It 'refuses a preexisting container rather than replacing it' { diff --git a/build/scripts/SqlResetPilot/Prepare.ps1 b/build/scripts/SqlResetPilot/Prepare.ps1 index 3d72bc76f33..ccdeac21bca 100644 --- a/build/scripts/SqlResetPilot/Prepare.ps1 +++ b/build/scripts/SqlResetPilot/Prepare.ps1 @@ -1,7 +1,8 @@ $ErrorActionPreference = 'Stop' $base = 'c4953dceffe02a017adad34973e1955017bf5d20' if ($env:GITHUB_REPOSITORY -ne 'microsoft/BCApps' -or - $env:GITHUB_REF -ne 'refs/heads/features/646383-sql-api-warmup-experiment' -or + $env:GITHUB_REF -ne 'refs/heads/features/646383-sql-api-two-arm-experiment' -or + $env:BC_SQL_API_EXPERIMENT -notin @('A', 'B') -or $env:GITHUB_EVENT_NAME -ne 'workflow_dispatch' -or $env:GITHUB_RUN_ATTEMPT -ne '1' -or $env:BC_SQL_PILOT_ARM -ne 'control' -or $env:BC_SQL_PILOT_COUNTRY -notin @('W1', 'DE') -or $env:BC_SQL_PILOT_TRIAL -notmatch '^[1-5]$' -or $env:GITHUB_RUN_ID -notmatch '^\d{1,20}$') { @@ -13,7 +14,7 @@ git diff --exit-code $base HEAD -- src if ($LASTEXITCODE -ne 0) { throw 'AL/package source differs from the pinned compiled PR2 source.' } $project = "build\projects\Test Apps $($env:BC_SQL_PILOT_COUNTRY)" -$trialProject = "$project Trial$($env:BC_SQL_PILOT_TRIAL)" +$trialProject = "$project Trial$($env:BC_SQL_PILOT_TRIAL)$($env:BC_SQL_API_EXPERIMENT)" if (Test-Path $trialProject) { throw 'Trial project must not already exist.' } # AL-Go derives its container name from the project path. Preserve wrapper depth and country settings. Copy-Item $project $trialProject -Recurse @@ -62,10 +63,13 @@ foreach ($artifact in $artifacts) { } @{ experimentHead = $env:GITHUB_SHA; run = $env:GITHUB_RUN_ID; arm = $env:BC_SQL_PILOT_ARM + experiment = $env:BC_SQL_API_EXPERIMENT; baseline = '8a7ebc7d665c0a812c2b3b4902c63df570959954' country = $env:BC_SQL_PILOT_COUNTRY; trial = $env:BC_SQL_PILOT_TRIAL; project = $trialProject - warmup = 'original-first-app-before-clean-lane'; companiesProbe = 'per-restored-worker-readiness-retries' - companiesProbeMaximumAttempts = 3; companiesProbeTimeoutSeconds = 20; companiesProbeRetryDelaySeconds = 2 - warmupRetries = 0; testRetries = 0; schedulerRetries = 0; ciRetries = 0 + warmup = 'original-first-app-before-clean-lane'; companiesProbe = 'per-restored-worker-single-attempt' + companiesProbeMaximumAttempts = 1; companiesProbeTimeoutSeconds = 20; companiesProbeRetryDelaySeconds = 0 + additionalDisabledTest = $(if ($env:BC_SQL_API_EXPERIMENT -eq 'A') { '148318.CapabilitiesProjectsEnabledViaAPI' } else { $null }) + warmupRetries = 0; maximumEvidenceGatedRetriesPerCodeunit = $(if ($env:BC_SQL_API_EXPERIMENT -eq 'B') { 1 } else { 0 }) + genericTestRetries = 0; schedulerRetries = 0; ciRetries = 0 budgetMinutes = 120; startedUtc = [DateTime]::UtcNow.ToString('o') artifacts = $manifest } | ConvertTo-Json -Depth 8 | Set-Content (Join-Path $output 'provenance.json') -Encoding UTF8 diff --git a/build/scripts/SqlResetPilot/README.md b/build/scripts/SqlResetPilot/README.md index e28eb693e3f..7a4a52293ec 100644 --- a/build/scripts/SqlResetPilot/README.md +++ b/build/scripts/SqlResetPilot/README.md @@ -1,93 +1,99 @@ -# API warmup experiment — diagnostic only, do not merge +# SQL API two-arm diagnostic — AB#646383 — DO NOT MERGE -One manual `CICD.yaml` dispatch on `features/646383-sql-api-warmup-experiment` -runs **W1 and DE, five independent trials each**. Maximum parallelism is five; -each trial is capped at 120 minutes (1,200 runner-minutes total maximum). -Only attempt one is allowed. No automatic rerun, replacement dispatch, PR, -merge queue, local/shared NST operation, or fresh-database-name treatment. +One manual `CICD.yaml` dispatch on `features/646383-sql-api-two-arm-experiment` +runs **A/B × W1/DE × five trials = 20 independent disposable containers**. +Maximum parallelism is **five globally**, timeout 120 minutes per cell (2,400 +runner-minutes maximum). Exact branch and run-attempt-one guards reject reruns. +The exact experiment PR head is excluded from automatic PR Initialization; +there is no full-country pipeline, job retry, replacement run, or local NST work. -## Fixed baseline +## Shared immutable baseline -The branch starts from SQL reset pilot `d7dd4dddcd49da397ee42de1677d1b7d44622149`. -AL source remains exactly PR2 `c4953dceffe02a017adad34973e1955017bf5d20`. -Both countries consume their own unchanged compiled packages from run -`37372848860`; this does not claim country package binaries are identical. -`Prepare.ps1` verifies the source ancestry/diff, artifact run/head/digests, -downloaded archive SHA256 and per-package hashes. Missing or expired artifacts -fail closed; there is no moving-build fallback. +Both arms start at warmup head `8a7ebc7d665c0a812c2b3b4902c63df570959954`. +AL/package source remains **c4953dceffe02a017adad34973e1955017bf5d20**. +`Prepare.ps1` verifies ancestry, no source diff, exact source run **37372848860**, +artifact metadata/digests, ZIP SHA256 and package hashes. Missing/expired +artifacts fail closed; there is no rebuild or moving-package fallback. | Country | App artifact | Test artifact | | --- | --- | --- | | W1 | 11374170356 | 11374145359 | | DE | 11376415128 | 11375174896 | -The pinned runtime is NST **30.0.55665.0**, application **30.0.55683.0**, -BCH **6.1.19-preview2811389**, AL-Go -`91b96c2b294be6f823277dafe6f03350abfb9d23`, and generic image -`sha256:c899d12093ad7bbdbfd08ccc0e6294e0f98c682c7e35db4ecfbca345fb068492`. -The image matches the pilot; historic source logs did not record its digest. -PowerShell 7, four tenants, original password-file authentication and company -`My Company` are retained. - -## Experiment order and failure policy - -1. Original discovery runs on a worker, which is then restored from default. -2. The immutable pristine template is frozen **before** app warmup fixtures. -3. The original `Invoke-WarmupDispatch` runs the ordinary first test app alone - on default, with the original `SkipAutomaticDisabledPass` behavior. - Results live under `sql-reset-pilot-output/warmup`, outside prefix coverage. - Failure, a transient queue, a skipped warmup or an incomplete job stops the trial. -4. For each of seven clean batches, restore all three workers under their - original `tenant2/3/4` database names, then probe all three, then dispatch. -5. Each probe makes **up to three external host HTTP GET attempts** to - `/BC/api/v2.0/companies?tenant=tenantN`, using the actual server instance, - port/IP and existing container credential. Basic auth is sent preemptively. - Each attempt has 20-second connection and read timeouts; eligible attempts are separated by two - seconds. Only HTTP 500/502/503/504 and transport connection/timeout failures - may retry. Authentication failures, other HTTP statuses (including all 4xx), - invalid JSON and missing expected company are terminal. Redirects and hidden - HTTP-client retries remain disabled. Exhaustion stops the trial before dispatch. -6. Test failures drain the current three-worker batch and stop later batches. - No test, transient, warmup or restore retries are enabled. Existing bounded - mount-state polling and ordinary scheduler waits are unchanged. - -The cohort remains the same ordered **21-codeunit prefix**, ending with Expense -Users (148315), Capabilities (148318), and Activity Log (148343). The previous W1 -control yielded **253 cases: 234 passed, 19 skipped**, across 20 codeunits with -cases. This is a quick cohort, **not all API paths or the full suite**. Warmup -counts must be reported separately. Discovery-order drift fails closed for both -countries; a DE-specific prefix requires evidence from the original DE baseline, -must include all three Expense codeunits, and must be recorded in full. - -## Isolation, evidence and interpretation - -At runtime only, the selected project descriptor is copied to -`build/projects/Test Apps Trial` at the same directory depth. -AL-Go derives a unique container name: -`bcbuildprojectsTestAppsTrial`. -Country settings and wrapper targets stay unchanged. Preflight refuses an -existing container; final cleanup checks country, trial, run, arm and exact -registered name before removing only that owned disposable container. - -Artifacts preserve provenance, shell/runtime identity, worklist, SQL reset -timeline, warmup outcome/results, worker output, prefix JUnit and EVTX. -Finalization collects project-root `TestResults*.xml` into `clean-results/project-root` -and `.buildartifacts/TestResults*.xml` into `clean-results/buildartifacts`, preserving -both without overwrites. Warmup results remain separately labeled under `warmup`. -`companies-probes.jsonl` records sanitized URI, tenant, generation, HTTP status, -UTC start/end, duration and safe client/server correlation IDs **for every attempt**. -Separate outcome records distinguish `first-try-success`, `recovered` and `failed`, -and record actual attempt/retry counts. Report this treatment as **companies-probe -retry allowed**, not blanket no retries; warmup, AL tests, scheduler and CI still -have zero retries. The earlier run `37608087700` used the previous no-probe-retry -commit and cannot acquire this change in flight. No password, -authorization header, response body or raw HTTP exception is added to logs. -Runner loss or cancellation can prevent final export/cleanup and is inconclusive. - -This experiment combines two warmup interventions; it cannot distinguish their -individual effects. A pass/non-reproduction is not proof of a production fix or -SQL-name reuse safety. Report every trial, including preflight/probe failures, -timeouts and missing evidence. Do not launch another matrix without authorization. - -Local validation requires Pester 5+ (validated with 6.1.0), not an NST: -`Invoke-Pester .\build\scripts\SqlResetPilot` +NST **30.0.55665.0**, application **30.0.55683.0**, BCH +**6.1.19-preview2811389**, AL-Go `91b96c2b294be6f823277dafe6f03350abfb9d23`, +generic image `sha256:c899d12093ad7bbdbfd08ccc0e6294e0f98c682c7e35db4ecfbca345fb068492`, +PowerShell 7, four tenants, existing password-file authentication and `My Company` +are unchanged. No auth/provider changes or AL assertion changes. + +Both arms freeze the pristine template before ordinary first-app warmup, then +restore each clean batch under the original tenant database names, probe every +restored worker, and only then dispatch. Companies probes use **one attempt**, +20-second connect/read timeouts, no hidden HTTP retries or redirects. This +disables the optional three-attempt readiness behavior added after the original +warmup run `37608087700` at `fcc1776c6b165199dd66e4227675b1cc31da7a8f`. +The newer baseline's nested `.buildartifacts` collection is retained. + +## Treatments + +**A — exclude one method:** the existing `Get-DisabledTestsForApp` runner config +adds only `{codeunitId:148318, method:CapabilitiesProjectsEnabledViaAPI}`. +No source or package is edited; the whole Capabilities codeunit stays selected. +All existing baseline disabled tests remain unchanged. No test retries. + +**B — all baseline-enabled tests remain enabled:** no additional disabled entry. +At most **one whole-codeunit retry**, and only after every failed JUnit case has: + +* The observed GET HTTP500/null-reference response and a valid CorrelationId. +* An NST Application event 701 from `MicrosoftDynamicsNavServer$...`, obtained + from this container during this dispatch's UTC time window. +* Matching `ClientSessionId`, `RootException: NullReferenceException`, and frames + `NavSqlConnectionScope.AcquireSqlConnectionFromPool`, + `MetadataProvider.GetRelativeHelpUrl`, and `PageDataProvider.GetNavRecordDataAsync`. + +Generic HTTP500, generic NRE, real/mixed assertions, stale/unmatched events, +partial XML, stopped jobs, missing evidence or persistence errors never qualify. +No automatic polling/wait for evidence. This conservative additional observed +metadata-stack requirement can reject other SQL-pool shapes. + +The entire current batch drains before a retry. The retry uses the same worker +after another pristine-template restore and fresh single-attempt companies probe. +`ReRun` is explicitly removed: BCH must run **every method**, not just failures. +Retry files use a unique suffix. Recovery requires the exact original test names, +count and skip states with no failures/errors. Only verified recovery replaces +that CU in the final merged result; original and retry files remain separately +preserved. A failed/incomplete retry stays failed and cannot retry again. +Any unrelated terminal batch failure stops later batches (including queued retries). + +## Cohort and interpretation + +The exact ordered 21-CU prefix remains: +139700,139702,139703,139706,139725,139726,139732,139739,139742,139745, +139802,139803,139806,139826,139832,139854,139972,139780,148315,148318,148343. +Order drift fails closed. This is not the full API suite. +Previous complete W1 coverage was 253 cases (234 passing, 19 skipped). +A should turn only the target method into an additional skip; B retains baseline +coverage. Actual counts and any missing/incomplete coverage must be reported per +trial; do not count recovered first failures as first-attempt passes. + +Containers derive from `Test Apps Trial` at unchanged +descriptor depth. Preflight refuses existing containers; cleanup verifies +country/trial/experiment/run/exact name before deleting only the owned container. + +Artifacts preserve provenance, pins, ordered worklist, SQL reset timeline, +single-attempt probe records, worker logs, EVTX, separately labeled warmup, +project-root and nested final XML, plus: +* `test-attempts/--1/`: original result snapshot and outcome. +* `test-attempts/--2/`: retry result snapshot and outcome. +* `SqlApiRetryEvidence/`: original CU XML and correlated NST events. + +Per-CU snapshots may include previously completed suites on that worker; count +only the outcome's CU when aggregating first attempts. Final merged XML contains +one result per CU, with recovery replacing rather than duplicating cases. +Runner loss/cancellation can prevent final export and is **inconclusive**. +Passes are non-reproductions, not proof of a production fix or SQL-name reuse safety. + +Local validation uses Pester 5+ and PSScriptAnalyzer, with all live container +operations mocked. Tests: `build\scripts\SqlResetPilot`, +`build\scripts\tests\SqlApiTestRetry.Test.ps1`, and +`build\scripts\tests\ParallelTestExecution.Test.ps1`. diff --git a/build/scripts/SqlResetPilot/TwoArm.Tests.ps1 b/build/scripts/SqlResetPilot/TwoArm.Tests.ps1 new file mode 100644 index 00000000000..4e2a19610a1 --- /dev/null +++ b/build/scripts/SqlResetPilot/TwoArm.Tests.ps1 @@ -0,0 +1,164 @@ +$discoveryArm = $env:BC_SQL_PILOT_ARM +try { + $env:BC_SQL_PILOT_ARM = 'control' + Import-Module (Join-Path $PSScriptRoot '..\ParallelTestExecution.psm1') -Force +} finally { + $env:BC_SQL_PILOT_ARM = $discoveryArm +} +Describe 'Exact manual two-arm experiment isolation' { + BeforeEach { + $script:saved = @{} + foreach ($key in @('GITHUB_REPOSITORY','GITHUB_REF','GITHUB_EVENT_NAME','GITHUB_RUN_ATTEMPT', + 'BC_SQL_PILOT_ARM','BC_SQL_API_EXPERIMENT','BC_SQL_PILOT_OUTPUT')) { + $script:saved[$key] = [Environment]::GetEnvironmentVariable($key) + } + $env:GITHUB_REPOSITORY = 'microsoft/BCApps' + $env:GITHUB_REF = 'refs/heads/features/646383-sql-api-two-arm-experiment' + $env:GITHUB_EVENT_NAME = 'workflow_dispatch' + $env:GITHUB_RUN_ATTEMPT = '1' + $env:BC_SQL_PILOT_ARM = 'control' + $env:BC_SQL_API_EXPERIMENT = 'B' + $env:BC_SQL_PILOT_OUTPUT = $TestDrive + } + AfterEach { + foreach ($key in $script:saved.Keys) { [Environment]::SetEnvironmentVariable($key, $script:saved[$key]) } + } + InModuleScope ParallelTestExecution { + Context 'Runner and scheduler hooks' { + BeforeEach { + Mock Get-ChildItem { @() } + Mock Start-Sleep {} + Mock Start-TestJob { [PSCustomObject]@{ Id = 1 } } + $script:state = [PSCustomObject]@{ + jobs = @(); hasFailures = $false; transient = @(); retried = @{}; retryTenant = @{}; sqlRetryEvidence = @{} + } + $script:item = [PSCustomObject]@{ + Key = 'Test App::148318'; AppName = 'Test App'; AppId = 'id' + CodeunitId = '148318'; CodeunitName = 'Capabilities'; TestCount = 3 + } + $script:tenant = [PSCustomObject]@{ Id = 'tenant2'; DatabaseName = 'tenant2' } + $script:parameters = @{ + containerName = 'owned'; JUnitResultFileName = (Join-Path $TestDrive 'TestResults.xml') + credential = [PSCredential]::Empty; companyName = 'My Company' + } + } + It 'adds exactly the one named method only in A and does not mutate baseline config' { + $env:BC_SQL_API_EXPERIMENT = 'A' + $disabled = @(Get-DisabledTestsForApp 'Test App') + $disabled.Count | Should -Be 1 + $disabled[0].codeunitId | Should -Be 148318 + $disabled[0].method | Should -BeExactly 'CapabilitiesProjectsEnabledViaAPI' + $env:BC_SQL_API_EXPERIMENT = 'B' + @(Get-DisabledTestsForApp 'Test App').Count | Should -Be 0 + } + It 'cannot activate on ' -ForEach @( + @{ Kind = 'PR'; Key = 'GITHUB_EVENT_NAME'; Value = 'pull_request' } + @{ Kind = 'another branch'; Key = 'GITHUB_REF'; Value = 'refs/heads/main' } + @{ Kind = 'rerun'; Key = 'GITHUB_RUN_ATTEMPT'; Value = '2' } + @{ Kind = 'other repository'; Key = 'GITHUB_REPOSITORY'; Value = 'other/BCApps' } + @{ Kind = 'fresh SQL'; Key = 'BC_SQL_PILOT_ARM'; Value = 'fresh' } + ) { + [Environment]::SetEnvironmentVariable($Key, $Value) + Test-SqlApiExperiment | Should -BeFalse + } + It 'queues only a confirmed B failure and never a generic transient' { + $result = [PSCustomObject]@{ Outcome = 'SqlPoolRetry'; AppName = $script:item.Key; Tenant = 'tenant2'; SqlRetryEvidence = @{ TestCases = @('one') } } + Register-TestJobOutcome $result $script:state + $script:state.transient.Count | Should -Be 1 + $script:state.hasFailures | Should -BeFalse + $script:state.retried[$script:item.Key] = $true + Register-TestJobOutcome $result $script:state + $script:state.transient.Count | Should -Be 1 + $script:state.hasFailures | Should -BeTrue + } + It 'never queues SQL retry in A' { + $env:BC_SQL_API_EXPERIMENT = 'A' + Register-TestJobOutcome ([PSCustomObject]@{ Outcome = 'SqlPoolRetry' }) $script:state + $script:state.hasFailures | Should -BeTrue + $script:state.transient.Count | Should -Be 0 + } + It 'retries the full CU without BCH ReRun and writes to a separate result file' { + $script:state.sqlRetryEvidence[$script:item.Key] = @{ TestCases = @(@{ Name = 'one'; Skipped = $false }) } + Start-RequiredDisabledDispatch $script:parameters $script:item $script:tenant 'runner.ps1' 'IntegrationTest' $script:state 'Re-dispatching' + Should -Invoke Start-TestJob -Times 1 -Exactly -ParameterFilter { + -not $parameters.ContainsKey('ReRun') -and $parameters.testCodeunit -eq '148318' -and + $fileSuffix -eq 'tenant2-sqlretry-148318' -and $skipAutomaticDisabledPass + } + $script:state.jobs[0].SqlRetryContext.Attempt | Should -Be 2 + $script:state.jobs[0].SqlRetryContext.ResultFiles.JUnitResultFileName | Should -Match 'tenant2-sqlretry-148318' + } + It 'restores and probes again before a once-only retry and drains failures' { + $script:order = [Collections.Generic.List[string]]::new() + $script:waits = 0 + Mock Reset-BcTestTenant { $script:order.Add('restore') } + Mock Invoke-SqlPilotCompaniesProbe { $script:order.Add('probe') } + Mock Start-RequiredDisabledDispatch { $script:order.Add("dispatch:$Verb") } + Mock Wait-ForAllTestJobs { + $script:waits++ + if ($script:waits -eq 1) { + $state.sqlRetryEvidence[$script:item.Key] = @{ TestCases = @('one') } + $state.transient = @([PSCustomObject]@{ Key = $script:item.Key; Tenant = 'tenant2' }) + } else { $state.hasFailures = $true } + } + Invoke-RequiredDisabledTestExecution $script:parameters @($script:item) @($script:tenant) ` + 'default-test-template' 'runner.ps1' 'IntegrationTest' | Should -BeFalse + ($script:order -join ',') | Should -Be 'restore,probe,dispatch:Dispatching,restore,probe,dispatch:Re-dispatching' + $script:waits | Should -Be 2 + } + It 'preserves first failure and separate full retry, merging only validated recovery' { + Mock Receive-Job {} -RemoveParameterType Job + Mock Remove-Job {} -RemoveParameterType Job + Mock Get-SqlApiTestRetryEvidence { @{ TestCases = @(@{ Name = 'CapabilitiesProjectsEnabledViaAPI'; Skipped = $false }) } } + $originalPath = Join-Path $TestDrive 'TestResults-tenant2.xml' + $retryPath = Join-Path $TestDrive 'TestResults-tenant2-sqlretry-148318.xml' + $original = '' + $passed = '' + $original | Set-Content $originalPath + $context = [PSCustomObject]@{ + CodeunitId = 148318; Tenant = 'tenant2'; StartedUtc = [datetime]::UtcNow + TestCount = 1; Attempt = 1; ResultFiles = @{ JUnitResultFileName = $originalPath } + FinalResultFiles = @{ JUnitResultFileName = $originalPath }; OutputDirectory = $TestDrive + } + $entry = [PSCustomObject]@{ appName = $script:item.Key; tenant = 'tenant2'; SqlRetryContext = $context; SqlRetryExpectedTests = @() } + $firstResult = Receive-TestJobResult $entry @{ Id = 91; State = 'Failed' } @{} + $firstResult.Outcome | Should -Be 'SqlPoolRetry' + $firstSaved = Join-Path $TestDrive 'test-attempts\tenant2-148318-1\JUnitResultFileName' + (Get-Content $firstSaved -Raw).Trim() | Should -Be $original + $passed | Set-Content $retryPath + $context.Attempt = 2 + $context.ResultFiles = @{ JUnitResultFileName = $retryPath } + $entry.SqlRetryExpectedTests = $firstResult.SqlRetryEvidence.TestCases + (Receive-TestJobResult $entry @{ Id = 92; State = 'Completed' } @{ $script:item.Key = $true }).Outcome | Should -Be 'Passed' + ([xml](Get-Content $originalPath -Raw)).SelectNodes('//failure').Count | Should -Be 0 + (Get-Content $firstSaved -Raw).Trim() | Should -Be $original + Test-Path (Join-Path $TestDrive 'test-attempts\tenant2-148318-2\JUnitResultFileName') | Should -BeTrue + Should -Invoke Get-SqlApiTestRetryEvidence -Times 1 -Exactly + } + It 'does not classify stopped or already-retried failures as eligible' { + Mock Receive-Job {} -RemoveParameterType Job + Mock Remove-Job {} -RemoveParameterType Job + Mock Get-SqlApiTestRetryEvidence { throw 'Evidence classifier must not be called.' } + Mock Test-SqlApiExperimentSelection { $true } + $context = [PSCustomObject]@{ + CodeunitId = 148318; Tenant = 'tenant2'; StartedUtc = [datetime]::UtcNow + TestCount = 1; Attempt = 2; ResultFiles = @{} + } + $entry = [PSCustomObject]@{ appName = $script:item.Key; tenant = 'tenant2'; SqlRetryContext = $context; SqlRetryExpectedTests = @() } + foreach ($jobState in @('Stopped','Failed')) { + (Receive-TestJobResult $entry @{ Id = 91; State = $jobState } @{ $script:item.Key = $true }).Outcome | Should -Be 'Failed' + } + Should -Invoke Get-SqlApiTestRetryEvidence -Times 0 + } + It 'rejects silently reenabled A and inherited B exclusions in actual JUnit' { + $path = Join-Path $TestDrive 'selection.xml' + $context = [PSCustomObject]@{ CodeunitId = 148318; ResultFiles = @{ JUnitResultFileName = $path } } + '' | Set-Content $path + Test-SqlApiExperimentSelection $context 'A' | Should -BeTrue + Test-SqlApiExperimentSelection $context 'B' | Should -BeFalse + '' | Set-Content $path + Test-SqlApiExperimentSelection $context 'A' | Should -BeFalse + Test-SqlApiExperimentSelection $context 'B' | Should -BeTrue + } + } + } +} diff --git a/build/scripts/tests/SqlApiTestRetry.Test.ps1 b/build/scripts/tests/SqlApiTestRetry.Test.ps1 new file mode 100644 index 00000000000..d1c7f33f53f --- /dev/null +++ b/build/scripts/tests/SqlApiTestRetry.Test.ps1 @@ -0,0 +1,385 @@ +Describe 'SQL API bounded retry evidence and result validation' { + BeforeAll { + Import-Module (Join-Path $PSScriptRoot '..\SqlApiTestRetry.psm1') -Force + $script:createdContainerStub = -not (Get-Command Invoke-ScriptInBcContainer -ListImported -ErrorAction SilentlyContinue) + if ($script:createdContainerStub) { + function global:Invoke-ScriptInBcContainer { + param([string]$containerName, [scriptblock]$scriptblock, [object[]]$argumentList) + $null = $containerName, $scriptblock, $argumentList + throw 'Container execution must be mocked.' + } + } + function New-FailureMessage { + param([string]$CorrelationId = $script:requestId) + "Assert.Fail failed. GET request failed. Response code is 500 (InternalServerError), expected code is 200. Error code: Unknown. Error message: Object reference not set to an instance of an object. CorrelationId: $CorrelationId." + } + function New-CaseXml { + param([string]$Name, [string]$Message, [switch]$Skipped, [string]$Result = 'Pass') + $body = '' + if ($Message) { + $body = '' + $Result = 'Fail' + } + if ($Skipped) { $body += '' } + '' + $body + '' + } + function Write-Suite { + param([string[]]$Cases = $script:originalCases) + '' + + ($Cases -join '') + '' | + Set-Content -LiteralPath $script:context.ResultFiles.JUnitResultFileName -Encoding utf8 + } + function New-NstEvent { + param( + [string]$CorrelationId = $script:requestId, + [datetime]$Time = $script:eventTime, + [string]$Provider = 'MicrosoftDynamicsNavServer$BC', + [string]$EventId = '701', + [string]$Stack = "RootException: NullReferenceException`nat Microsoft.Dynamics.Nav.Runtime.NavSqlConnectionScope.AcquireSqlConnectionFromPool(...)`nat Microsoft.Dynamics.Nav.XmlMetadata.MetadataProvider.GetRelativeHelpUrl(...)`nat Microsoft.Dynamics.Nav.Service.OData.V4.PageDataProvider.GetNavRecordDataAsync(...)" + ) + $text = [System.Security.SecurityElement]::Escape("ClientSessionId: $CorrelationId`n$Stack") + '' + $EventId + '' + $text + '' + } + } + + AfterAll { + if ($script:createdContainerStub) { Remove-Item function:global:Invoke-ScriptInBcContainer } + } + + BeforeEach { + $script:fixtureRoot = Join-Path $PSScriptRoot ('.sql-api-retry-fixture-' + [guid]::NewGuid().ToString('N')) + $null = New-Item -ItemType Directory -Path $script:fixtureRoot + $script:requestId = '11111111-2222-4333-8444-555555555555' + $script:eventTime = [datetime]::UtcNow.AddSeconds(-10) + $script:context = [pscustomobject]@{ + ContainerName = 'synthetic-container' + Tenant = 'synthetic-tenant' + CodeunitId = 70001 + TestCount = 3 + StartedUtc = $script:eventTime.AddSeconds(-10) + ResultFiles = [pscustomobject]@{ JUnitResultFileName = Join-Path $script:fixtureRoot 'results.xml' } + } + $script:expectedTests = @( + [pscustomobject]@{ Name = 'Synthetic API Codeunit.Read Item'; Skipped = $false } + [pscustomobject]@{ Name = 'Synthetic API Codeunit.Read Customer'; Skipped = $false } + [pscustomobject]@{ Name = 'Synthetic API Codeunit.Disabled Case'; Skipped = $true } + ) + $script:originalCases = @( + New-CaseXml -Name $script:expectedTests[0].Name -Message (New-FailureMessage) + New-CaseXml -Name $script:expectedTests[1].Name + New-CaseXml -Name $script:expectedTests[2].Name -Skipped + ) + $script:passingCases = @( + New-CaseXml -Name $script:expectedTests[0].Name + New-CaseXml -Name $script:expectedTests[1].Name + New-CaseXml -Name $script:expectedTests[2].Name -Skipped + ) + Write-Suite + Mock -ModuleName SqlApiTestRetry Write-Host {} + Mock -ModuleName SqlApiTestRetry Invoke-ScriptInBcContainer { throw 'Live container execution is forbidden.' } + } + + AfterEach { + Remove-Item -LiteralPath $script:fixtureRoot -Recurse -Force + } + + Context 'Evidence classification and preservation' { + BeforeEach { + $script:events = @(New-NstEvent) + Mock -ModuleName SqlApiTestRetry Get-SqlApiRetryEvents { $script:events } + } + + It 'recognizes the observed namespaced event and failure attribute shapes without changing results' { + $before = [System.IO.File]::ReadAllBytes($script:context.ResultFiles.JUnitResultFileName) + $evidence = Get-SqlApiTestRetryEvidence -Context $script:context + $evidence | Should -Not -BeNullOrEmpty + $evidence.RequestIds | Should -Be @($script:requestId) + $evidence.Events | Should -Be $script:events + $evidence.TestCases.Name | Should -Be $script:expectedTests.Name + $evidence.TestCases.Skipped | Should -Be @($false, $false, $true) + [Convert]::ToBase64String([System.IO.File]::ReadAllBytes($script:context.ResultFiles.JUnitResultFileName)) | + Should -Be ([Convert]::ToBase64String($before)) + Should -Invoke -ModuleName SqlApiTestRetry Get-SqlApiRetryEvents -Times 1 -Exactly -ParameterFilter { + $ContainerName -eq 'synthetic-container' -and $StartedUtc -eq $script:context.StartedUtc -and + $FinishedUtc -ge $script:eventTime -and $FinishedUtc -le [datetime]::UtcNow + } + Should -Invoke -ModuleName SqlApiTestRetry Invoke-ScriptInBcContainer -Times 0 + } + + It 'persists the first full suite and events before the result file is replaced by the rerun' { + $original = ([xml](Get-Content -LiteralPath $script:context.ResultFiles.JUnitResultFileName -Raw)).testsuites.testsuite.OuterXml + $evidence = Get-SqlApiTestRetryEvidence -Context $script:context + $files = @(Get-ChildItem -LiteralPath (Join-Path $script:fixtureRoot 'SqlApiRetryEvidence') -Filter '*.json') + $files.Count | Should -Be 1 + $savedBeforeRerun = Get-Content -LiteralPath $files[0].FullName -Raw + ($savedBeforeRerun | ConvertFrom-Json).OriginalSuite | Should -Be $original + Write-Suite -Cases $script:passingCases + Test-SqlApiRetryResult -Context $script:context -ExpectedTests $evidence.TestCases | Should -BeTrue + Get-Content -LiteralPath $files[0].FullName -Raw | Should -Be $savedBeforeRerun + $persisted = $savedBeforeRerun | ConvertFrom-Json + $persisted.Events | Should -Be $script:events + $persisted.RequestIds | Should -Be @($script:requestId) + $persisted.Tenant | Should -Be 'synthetic-tenant' + $persisted.CodeunitId | Should -Be 70001 + $persisted.TestCases.Name | Should -Be $script:expectedTests.Name + } + + It 'rejects without querying events' -ForEach @( + @{ Kind = 'generic 500'; Message = 'Assert.Fail failed. GET request failed. Response code is 500 (InternalServerError), expected code is 200.' } + @{ Kind = 'authentication 401'; Message = 'Assert.Fail failed. GET request failed. Response code is 401 (Unauthorized), expected code is 200.' } + @{ Kind = 'real assertion'; Message = 'Assert.AreEqual failed. Expected quantity 5, actual quantity 4.' } + ) { + $script:originalCases[0] = New-CaseXml -Name $script:expectedTests[0].Name -Message $Message + Write-Suite + Get-SqlApiTestRetryEvidence -Context $script:context | Should -BeNullOrEmpty + Should -Invoke -ModuleName SqlApiTestRetry Get-SqlApiRetryEvents -Times 0 + Test-Path (Join-Path $script:fixtureRoot 'SqlApiRetryEvidence') | Should -BeFalse + } + + It 'rejects an eligible HTTP failure mixed with a genuine assertion failure' { + $script:originalCases[1] = New-CaseXml -Name $script:expectedTests[1].Name -Message 'Assert.AreEqual failed. Expected true, actual false.' + Write-Suite + Get-SqlApiTestRetryEvidence -Context $script:context | Should -BeNullOrEmpty + Should -Invoke -ModuleName SqlApiTestRetry Get-SqlApiRetryEvents -Times 0 + } + + It 'rejects 401 even when the null-reference text and correlation match' { + $message = (New-FailureMessage).Replace('500 (InternalServerError)', '401 (Unauthorized)') + $script:originalCases[0] = New-CaseXml -Name $script:expectedTests[0].Name -Message $message + Write-Suite + Get-SqlApiTestRetryEvidence -Context $script:context | Should -BeNullOrEmpty + Should -Invoke -ModuleName SqlApiTestRetry Get-SqlApiRetryEvents -Times 0 + } + + It 'requires matching events for every failed request' { + $secondRequest = '66666666-7777-4888-8999-aaaaaaaaaaaa' + $script:originalCases[1] = New-CaseXml -Name $script:expectedTests[1].Name -Message (New-FailureMessage -CorrelationId $secondRequest) + Write-Suite + Get-SqlApiTestRetryEvidence -Context $script:context | Should -BeNullOrEmpty + $script:events += New-NstEvent -CorrelationId $secondRequest + $evidence = Get-SqlApiTestRetryEvidence -Context $script:context + $evidence.RequestIds | Should -Be @($script:requestId, $secondRequest) + $evidence.Events.Count | Should -Be 2 + } + + It 'rejects correlation in the failure message' -ForEach @( + @{ Kind = 'missing'; Correlation = '' } + @{ Kind = 'malformed'; Correlation = 'not-a-guid' } + @{ Kind = 'overlong'; Correlation = '11111111-2222-4333-8444-5555555555556' } + ) { + $script:originalCases[0] = New-CaseXml -Name $script:expectedTests[0].Name -Message (New-FailureMessage -CorrelationId $Correlation) + Write-Suite + Get-SqlApiTestRetryEvidence -Context $script:context | Should -BeNullOrEmpty + Should -Invoke -ModuleName SqlApiTestRetry Get-SqlApiRetryEvents -Times 0 + } + + It 'rejects event evidence with ' -ForEach @( + @{ Kind = 'wrong correlation' } + @{ Kind = 'missing correlation' } + @{ Kind = 'stale time' } + @{ Kind = 'future time' } + @{ Kind = 'wrong provider' } + @{ Kind = 'wrong event ID' } + @{ Kind = 'wrong exception' } + @{ Kind = 'wrong pool frame' } + @{ Kind = 'wrong metadata frame' } + @{ Kind = 'wrong OData frame' } + ) { + switch ($Kind) { + 'wrong correlation' { $script:events = @(New-NstEvent -CorrelationId '66666666-7777-4888-8999-aaaaaaaaaaaa') } + 'missing correlation' { $script:events = @((New-NstEvent) -replace 'ClientSessionId:', 'OtherSessionId:') } + 'stale time' { $script:events = @(New-NstEvent -Time $script:context.StartedUtc.AddTicks(-1)) } + 'future time' { $script:events = @(New-NstEvent -Time ([datetime]::UtcNow.AddHours(1))) } + 'wrong provider' { $script:events = @(New-NstEvent -Provider 'UnrelatedProvider') } + 'wrong event ID' { $script:events = @(New-NstEvent -EventId '702') } + 'wrong exception' { $script:events = @((New-NstEvent) -replace 'NullReferenceException', 'InvalidOperationException') } + 'wrong pool frame' { $script:events = @((New-NstEvent) -replace 'AcquireSqlConnectionFromPool', 'OpenConnection') } + 'wrong metadata frame' { $script:events = @((New-NstEvent) -replace 'GetRelativeHelpUrl', 'ReadMetadata') } + 'wrong OData frame' { $script:events = @((New-NstEvent) -replace 'GetNavRecordDataAsync', 'ReadSomethingElse') } + } + Get-SqlApiTestRetryEvidence -Context $script:context | Should -BeNullOrEmpty + Test-Path (Join-Path $script:fixtureRoot 'SqlApiRetryEvidence') | Should -BeFalse + } + + It 'accepts an event exactly at the dispatch start boundary' { + $script:events = @(New-NstEvent -Time $script:context.StartedUtc) + Get-SqlApiTestRetryEvidence -Context $script:context | Should -Not -BeNullOrEmpty + } + + It 'fails closed with a warning for ' -ForEach @( + @{ Kind = 'missing result file' } + @{ Kind = 'malformed result XML' } + @{ Kind = 'wrong suite' } + @{ Kind = 'missing test count' } + @{ Kind = 'partial result count' } + @{ Kind = 'malformed event XML' } + @{ Kind = 'missing event system' } + @{ Kind = 'event collection access error' } + @{ Kind = 'evidence persistence failure' } + ) { + switch ($Kind) { + 'missing result file' { Remove-Item -LiteralPath $script:context.ResultFiles.JUnitResultFileName } + 'malformed result XML' { '' | Set-Content -LiteralPath $script:context.ResultFiles.JUnitResultFileName } + 'wrong suite' { + (Get-Content -LiteralPath $script:context.ResultFiles.JUnitResultFileName -Raw).Replace('70001', '70002') | + Set-Content -LiteralPath $script:context.ResultFiles.JUnitResultFileName + } + 'missing test count' { $script:context.PSObject.Properties.Remove('TestCount') } + 'partial result count' { $script:context.TestCount = 4 } + 'malformed event XML' { $script:events = @('') } + 'missing event system' { $script:events = @('') } + 'event collection access error' { Mock -ModuleName SqlApiTestRetry Get-SqlApiRetryEvents { throw 'Synthetic access denied' } } + 'evidence persistence failure' { Mock -ModuleName SqlApiTestRetry Set-Content { throw 'Synthetic write denied' } } + } + Get-SqlApiTestRetryEvidence -Context $script:context | Should -BeNullOrEmpty + Should -Invoke -ModuleName SqlApiTestRetry Write-Host -Times 1 -Exactly -ParameterFilter { + $Object -like '::warning::*evidence unavailable*preserving test failure*' + } + } + + It 'fails closed with a warning when event collection returns no events' { + $script:events = @() + Get-SqlApiTestRetryEvidence -Context $script:context | Should -BeNullOrEmpty + Should -Invoke -ModuleName SqlApiTestRetry Write-Host -Times 1 -Exactly -ParameterFilter { + $Object -like '::warning::*no matching NST event*' + } + } + + It 'does not retry an error element or an unrepresented failed case' -ForEach @( + @{ CaseXml = '' } + @{ CaseXml = '' } + ) { + $script:originalCases[1] = $CaseXml + Write-Suite + Get-SqlApiTestRetryEvidence -Context $script:context | Should -BeNullOrEmpty + Should -Invoke -ModuleName SqlApiTestRetry Get-SqlApiRetryEvents -Times 0 + } + + It 'does not retry an already passing suite' { + Write-Suite -Cases $script:passingCases + Get-SqlApiTestRetryEvidence -Context $script:context | Should -BeNullOrEmpty + Should -Invoke -ModuleName SqlApiTestRetry Get-SqlApiRetryEvents -Times 0 + } + } + + Context 'Rerun result shape' { + It 'accepts all original full names and skip states in any order without mutating results' { + Write-Suite -Cases @($script:passingCases[2], $script:passingCases[0], $script:passingCases[1]) + $before = Get-Content -LiteralPath $script:context.ResultFiles.JUnitResultFileName -Raw + Test-SqlApiRetryResult -Context $script:context -ExpectedTests $script:expectedTests | Should -BeTrue + Get-Content -LiteralPath $script:context.ResultFiles.JUnitResultFileName -Raw | Should -Be $before + } + + It 'rejects rerun with ' -ForEach @( + @{ Kind = 'new skipped case' } + @{ Kind = 'removed original skip' } + @{ Kind = 'empty suite' } + @{ Kind = 'duplicate testcase' } + @{ Kind = 'missing testcase' } + @{ Kind = 'extra testcase' } + @{ Kind = 'renamed full testcase' } + @{ Kind = 'empty testcase name' } + @{ Kind = 'changed case in name' } + @{ Kind = 'failure element' } + @{ Kind = 'error element' } + @{ Kind = 'failed result without failure element' } + @{ Kind = 'missing result file' } + @{ Kind = 'malformed result XML' } + ) { + switch ($Kind) { + 'new skipped case' { $script:passingCases[0] = New-CaseXml -Name $script:expectedTests[0].Name -Skipped } + 'removed original skip' { $script:passingCases[2] = New-CaseXml -Name $script:expectedTests[2].Name } + 'empty suite' { $script:passingCases = @() } + 'duplicate testcase' { $script:passingCases[1] = $script:passingCases[0] } + 'missing testcase' { $script:passingCases = @($script:passingCases[0], $script:passingCases[2]) } + 'extra testcase' { $script:passingCases += New-CaseXml -Name 'Synthetic API Codeunit.Extra Case' } + 'renamed full testcase' { $script:passingCases[0] = New-CaseXml -Name 'Read Item' } + 'empty testcase name' { $script:passingCases[0] = New-CaseXml -Name '' } + 'changed case in name' { $script:passingCases[0] = New-CaseXml -Name $script:expectedTests[0].Name.ToLowerInvariant() } + 'failure element' { $script:passingCases[0] = $script:originalCases[0] } + 'error element' { $script:passingCases[0] = $script:passingCases[0].Replace('', '') } + 'failed result without failure element' { $script:passingCases[0] = New-CaseXml -Name $script:expectedTests[0].Name -Result 'Fail' } + } + Write-Suite -Cases $script:passingCases + if ($Kind -eq 'missing result file') { Remove-Item -LiteralPath $script:context.ResultFiles.JUnitResultFileName } + if ($Kind -eq 'malformed result XML') { '' | Set-Content -LiteralPath $script:context.ResultFiles.JUnitResultFileName } + Test-SqlApiRetryResult -Context $script:context -ExpectedTests $script:expectedTests | Should -BeFalse + Should -Invoke -ModuleName SqlApiTestRetry Write-Host -Times 1 -Exactly -ParameterFilter { + $Object -like '::warning::SQL API retry remains failed:*' + } + } + } + + Context 'Bounded container event collection' { + BeforeEach { + Mock -ModuleName SqlApiTestRetry Invoke-ScriptInBcContainer { + param($containerName, $scriptblock, $argumentList) + $null = $containerName + & $scriptblock @argumentList + } + Mock -ModuleName SqlApiTestRetry Get-WinEvent { @() } + } + + It 'bounds the application query by NST provider, event ID, dispatch time and maximum event count' { + $start = $script:context.StartedUtc + $end = $script:eventTime + $null = & (Get-Module SqlApiTestRetry) { + param($start, $end) + Get-SqlApiRetryEvents -ContainerName 'synthetic-container' -StartedUtc $start -FinishedUtc $end + } $start $end + Should -Invoke -ModuleName SqlApiTestRetry Invoke-ScriptInBcContainer -Times 1 -Exactly -ParameterFilter { + $containerName -eq 'synthetic-container' -and $argumentList.Count -eq 2 -and + $argumentList[0] -eq $script:context.StartedUtc.ToString('o') -and $argumentList[1] -eq $script:eventTime.ToString('o') + } + Should -Invoke -ModuleName SqlApiTestRetry Get-WinEvent -Times 1 -Exactly -ParameterFilter { + $FilterHashtable.LogName -eq 'Application' -and + $FilterHashtable.ProviderName -eq 'MicrosoftDynamicsNavServer*' -and + $FilterHashtable.Id -eq 701 -and + $FilterHashtable.StartTime -eq $script:context.StartedUtc -and + $FilterHashtable.EndTime -eq $script:eventTime -and + $MaxEvents -eq 1000 -and $ErrorAction -eq 'Stop' + } + } + + It 'returns original XML from collected events' { + $script:collectedXml = New-NstEvent + Mock -ModuleName SqlApiTestRetry Get-WinEvent { + $eventRecord = [pscustomobject]@{ Xml = $script:collectedXml } + $eventRecord | Add-Member ScriptMethod ToXml { $this.Xml } + $eventRecord + } + $actual = & (Get-Module SqlApiTestRetry) { + param($context) + Get-SqlApiRetryEvents -ContainerName $context.ContainerName -StartedUtc $context.StartedUtc -FinishedUtc ([datetime]::UtcNow) + } $script:context + $actual | Should -Be $script:collectedXml + } + + It 'treats only NoMatchingEventsFound as an empty collection' { + Mock -ModuleName SqlApiTestRetry Get-WinEvent { + $errorRecord = [System.Management.Automation.ErrorRecord]::new( + [System.Exception]::new('Synthetic no matching events'), + 'NoMatchingEventsFound,Microsoft.PowerShell.Commands.GetWinEventCommand', + [System.Management.Automation.ErrorCategory]::ObjectNotFound, $null) + throw $errorRecord + } + $actual = & (Get-Module SqlApiTestRetry) { + param($context) + Get-SqlApiRetryEvents -ContainerName $context.ContainerName -StartedUtc $context.StartedUtc -FinishedUtc ([datetime]::UtcNow) + } $script:context + $actual | Should -BeNullOrEmpty + } + + It 'propagates event-log access errors instead of reporting no matches' { + Mock -ModuleName SqlApiTestRetry Get-WinEvent { throw [System.UnauthorizedAccessException]::new('Synthetic event log access denied') } + { + & (Get-Module SqlApiTestRetry) { + param($context) + Get-SqlApiRetryEvents -ContainerName $context.ContainerName -StartedUtc $context.StartedUtc -FinishedUtc ([datetime]::UtcNow) + } $script:context + } | Should -Throw '*Synthetic event log access denied*' + } + } +}