From 384bc8d74ce83b66886dc2990d3f18dd174fbbeb Mon Sep 17 00:00:00 2001 From: Artem Lysenko Date: Thu, 20 Aug 2026 17:57:02 +0200 Subject: [PATCH 1/2] Retry transient DynamicDependency bootstrap flake in foundation test pass (AB#63230809) DynamicDependency Elevated bootstrap tests intermittently fail on Win10 19044/19045 with MddBootstrapInitialize 0x80270254. Add an opt-in, targeted rerun-on-failure to the shared foundation test path so a transient failure is retried and reported as unreliable instead of failing the PR/official run, without masking genuine regressions. - TestAll.ps1: testdefs may set MaxReruns; a failing TAEF testdef is rerun up to that many times into separate rerun logs (Te.rerun1/2/3.wtl). - DynamicDependency Win32/WinRT testdefs opt in with MaxReruns: 3. - WindowsAppSDK-RunTests-Steps.yml: pass the rerun logs plus useRetryLogic=true and requiredPassRateThreshold=1 to ConvertWttLogToXUnit so failed-then-passed tests are marked unreliable (skipped), while tests that fail every attempt still fail. --- TestAll.ps1 | 67 +++++++++++++++---- .../WindowsAppSDK-RunTests-Steps.yml | 5 ++ .../DynamicDependency_API_Win32.testdef | 3 +- .../DynamicDependency_API_WinRT.testdef | 3 +- 4 files changed, 63 insertions(+), 15 deletions(-) diff --git a/TestAll.ps1 b/TestAll.ps1 index e4709ef81b..033093e04b 100644 --- a/TestAll.ps1 +++ b/TestAll.ps1 @@ -18,6 +18,7 @@ "Parameters": { "type": "string" }, "Architectures": { "type": "array", "items": { "type": "string" } }, "Status": { "enum": ["Enabled", "Disabled"] }, + "MaxReruns": { "type": "integer", "default": 0 }, }, "required": ["Description", "Filename", "Architectures", "Status"] } @@ -118,6 +119,19 @@ function Get-Tests $t | Add-Member -MemberType NoteProperty -Name 'TestDef' -Value $testdef.FullName $t | Add-Member -MemberType NoteProperty -Name 'Type' -Value $testType + # Optional: number of times to rerun this test (into separate rerun logs) if it fails the primary pass. + # Used to tolerate known transient/flaky failures without masking genuine regressions - a test that + # fails the primary pass but passes on rerun is later reported as "unreliable" (skipped) instead of failed. + if ($testConfig.PSObject.Properties.Name -contains 'MaxReruns') + { + $maxReruns = [int]$testConfig.MaxReruns + } + else + { + $maxReruns = 0 + } + $t | Add-Member -MemberType NoteProperty -Name 'MaxReruns' -Value $maxReruns + $tests += $t $count += 1 } @@ -143,16 +157,17 @@ function List-Tests function Run-TaefTest { - param($test) + param($test, $logFile) $testFolder = Split-Path -parent $test.TestDef $tePath = Join-Path $testFolder "te.exe" $dllFile = Join-Path $testFolder $test.Filename - $teLogFile = (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.wtl") - $teLogPathTo = (Join-Path $env:Build_SourcesDirectory "TestOutput\$Configuration\$Platform") + # Pipe te.exe console output to the host so it is not captured as this function's return value + # (the caller assigns the return value to read te.exe's exit code). WTT results still go to $logFile. + & $tePath $dllFile $test.Parameters /enableWttLogging /appendWttLogging /screenCaptureOnError /logFile:$logFile /testMode:EtwLogger /EtwLogger:WprProfile=WDGDEPAdex /EtwLogger:SavePoint=TestFailure /EtwLogger:RecordingScope=Execution /EtwLogger:WprProfileFile=$wprProfilePath | Out-Host - & $tePath $dllFile $test.Parameters /enableWttLogging /appendWttLogging /screenCaptureOnError /logFile:$teLogFile /testMode:EtwLogger /EtwLogger:WprProfile=WDGDEPAdex /EtwLogger:SavePoint=TestFailure /EtwLogger:RecordingScope=Execution /EtwLogger:WprProfileFile=$wprProfilePath + return $LASTEXITCODE } function Run-PowershellTest @@ -164,6 +179,14 @@ function Run-PowershellTest function Run-Tests { + $teLogFile = (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.wtl") + # Rerun logs consumed by WindowsAppSDK-ConvertWttLogToXUnit-Steps.yml (wttSingleRerunInputPath / wttMultipleRerunInputPath / wttMoreRerunInputPath). + $rerunLogFiles = @( + (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.rerun1.wtl"), + (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.rerun2.wtl"), + (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.rerun3.wtl") + ) + $tests = Get-Tests foreach ($test in $tests) { @@ -174,7 +197,18 @@ function Run-Tests { if ($test.Type -eq 'TAEF') { - Run-TaefTest $test + $exitCode = Run-TaefTest $test $teLogFile + + # Tolerate known transient failures: rerun the failing testdef into separate rerun logs. + # The WTT-to-XUnit conversion (useRetryLogic=true) reports a test that failed the primary + # pass but passed on rerun as "unreliable" rather than failed. A test that fails every + # attempt is still reported as failed, so genuine regressions are not masked. + $maxReruns = [Math]::Min([int]$test.MaxReruns, $rerunLogFiles.Count) + for ($attempt = 1; ($exitCode -ne 0) -and ($attempt -le $maxReruns); $attempt++) + { + Write-Host "$($test.Filename) failed the previous run (exit code $exitCode); rerun attempt $attempt of $maxReruns" + $exitCode = Run-TaefTest $test $rerunLogFiles[$attempt - 1] + } } elseif ($test.Type -eq 'Powershell') { @@ -247,19 +281,26 @@ if ($Test -eq $true) { $teLogFile = (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.wtl") $teLogPathTo = (Join-Path $env:Build_SourcesDirectory "TestOutput\$Configuration\$Platform") - remove-item -Path $teLogFile -ErrorAction Ignore - remove-item -Path (Join-path $teLogPathTo "Te.wtl") -ErrorAction Ignore + # Primary log plus rerun logs produced for flaky testdefs (see Run-Tests / Run-TaefTest). + $teLogFileNames = @("Te.wtl", "Te.rerun1.wtl", "Te.rerun2.wtl", "Te.rerun3.wtl") + foreach ($logName in $teLogFileNames) { + remove-item -Path (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\$logName") -ErrorAction Ignore + remove-item -Path (Join-Path $teLogPathTo $logName) -ErrorAction Ignore + } Run-Tests - # copy test log to TestOutput folder - if (Test-Path -Path $teLogFile) { - Write-Host "Starting copy test log from '$teLogFile'" + # copy test logs (primary + any rerun logs) to TestOutput folder + foreach ($logName in $teLogFileNames) { + $logSource = (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\$logName") + if (Test-Path -Path $logSource) { + Write-Host "Starting copy test log from '$logSource'" - New-Item -ItemType Directory -Path $teLogPathTo -Force - copy-item -Path $teLogFile -Destination $teLogPathTo -Force + New-Item -ItemType Directory -Path $teLogPathTo -Force + copy-item -Path $logSource -Destination $teLogPathTo -Force - Write-Host "Test log copied to '$teLogPathTo'" + Write-Host "Test log copied to '$teLogPathTo'" + } } # copy screenshots to TestOutput folder diff --git a/build/AzurePipelinesTemplates/WindowsAppSDK-RunTests-Steps.yml b/build/AzurePipelinesTemplates/WindowsAppSDK-RunTests-Steps.yml index 54eebdc144..08cfb07b32 100644 --- a/build/AzurePipelinesTemplates/WindowsAppSDK-RunTests-Steps.yml +++ b/build/AzurePipelinesTemplates/WindowsAppSDK-RunTests-Steps.yml @@ -234,6 +234,11 @@ steps: - template: AzurePipelinesTemplates\WindowsAppSDK-ConvertWttLogToXUnit-Steps.yml@WindowsAppSDKConfig parameters: WttInputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\Te.wtl' + wttSingleRerunInputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\Te.rerun1.wtl' + wttMultipleRerunInputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\Te.rerun2.wtl' + wttMoreRerunInputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\Te.rerun3.wtl' + useRetryLogic: true + requiredPassRateThreshold: 1 xunitOutputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\testResults-$(buildConfiguration)_$(buildPlatform)_${{ parameters.ImageName }}.xml' TestNamePrefix: '$(buildConfiguration)_$(buildPlatform)_${{ parameters.ImageName }}' BypassTests: '$(Build.SourcesDirectory)\test\BypassTests.json' diff --git a/test/DynamicDependency/Test_Win32/DynamicDependency_API_Win32.testdef b/test/DynamicDependency/Test_Win32/DynamicDependency_API_Win32.testdef index 4833dbb0ac..47137ae30f 100644 --- a/test/DynamicDependency/Test_Win32/DynamicDependency_API_Win32.testdef +++ b/test/DynamicDependency/Test_Win32/DynamicDependency_API_Win32.testdef @@ -5,7 +5,8 @@ "Filename": "DynamicDependency_Test_Win32.dll", "Parameters": "", "Architectures": ["x64", "arm64"], - "Status": "Enabled" + "Status": "Enabled", + "MaxReruns": 3 } ] } diff --git a/test/DynamicDependency/Test_WinRT/DynamicDependency_API_WinRT.testdef b/test/DynamicDependency/Test_WinRT/DynamicDependency_API_WinRT.testdef index dec20224a4..22d89499cf 100644 --- a/test/DynamicDependency/Test_WinRT/DynamicDependency_API_WinRT.testdef +++ b/test/DynamicDependency/Test_WinRT/DynamicDependency_API_WinRT.testdef @@ -5,7 +5,8 @@ "Filename": "DynamicDependency_Test_WinRT.dll", "Parameters": "", "Architectures": ["x64", "arm64"], - "Status": "Enabled" + "Status": "Enabled", + "MaxReruns": 3 } ] } From c85f51ac14ea478a4600dd1e206bd9cb14d4f17b Mon Sep 17 00:00:00 2001 From: Artem Lysenko Date: Thu, 20 Aug 2026 18:16:28 +0200 Subject: [PATCH 2/2] Validate/clamp MaxReruns range when reading testdef (PR feedback) --- TestAll.ps1 | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/TestAll.ps1 b/TestAll.ps1 index 033093e04b..eff4b24316 100644 --- a/TestAll.ps1 +++ b/TestAll.ps1 @@ -122,13 +122,24 @@ function Get-Tests # Optional: number of times to rerun this test (into separate rerun logs) if it fails the primary pass. # Used to tolerate known transient/flaky failures without masking genuine regressions - a test that # fails the primary pass but passes on rerun is later reported as "unreliable" (skipped) instead of failed. + # Only 0..3 are supported (there are three rerun logs, see Run-Tests); non-integer or out-of-range + # values are coerced into that range with a warning so the testdef value matches actual behavior. + $maxSupportedReruns = 3 + $maxReruns = 0 if ($testConfig.PSObject.Properties.Name -contains 'MaxReruns') { - $maxReruns = [int]$testConfig.MaxReruns - } - else - { - $maxReruns = 0 + $parsedMaxReruns = 0 + if (-not [int]::TryParse("$($testConfig.MaxReruns)", [ref]$parsedMaxReruns)) + { + Write-Warning "Invalid MaxReruns value '$($testConfig.MaxReruns)' in '$($testdef.FullName)' for '$id'. Using 0." + $parsedMaxReruns = 0 + } + elseif (($parsedMaxReruns -lt 0) -or ($parsedMaxReruns -gt $maxSupportedReruns)) + { + Write-Warning "MaxReruns value '$parsedMaxReruns' in '$($testdef.FullName)' for '$id' is outside the supported range 0..$maxSupportedReruns. Clamping to that range." + $parsedMaxReruns = [Math]::Max(0, [Math]::Min($parsedMaxReruns, $maxSupportedReruns)) + } + $maxReruns = $parsedMaxReruns } $t | Add-Member -MemberType NoteProperty -Name 'MaxReruns' -Value $maxReruns