diff --git a/TestAll.ps1 b/TestAll.ps1 index e4709ef81b..eff4b24316 100644 --- a/TestAll.ps1 +++ b/TestAll.ps1 @@ -18,6 +18,7 @@ "Parameters": { "type": "string" }, "Architectures": { "type": "array", "items": { "type": "string" } }, "Status": { "enum": ["Enabled", "Disabled"] }, + "MaxReruns": { "type": "integer", "default": 0 }, }, "required": ["Description", "Filename", "Architectures", "Status"] } @@ -118,6 +119,30 @@ function Get-Tests $t | Add-Member -MemberType NoteProperty -Name 'TestDef' -Value $testdef.FullName $t | Add-Member -MemberType NoteProperty -Name 'Type' -Value $testType + # Optional: number of times to rerun this test (into separate rerun logs) if it fails the primary pass. + # Used to tolerate known transient/flaky failures without masking genuine regressions - a test that + # fails the primary pass but passes on rerun is later reported as "unreliable" (skipped) instead of failed. + # Only 0..3 are supported (there are three rerun logs, see Run-Tests); non-integer or out-of-range + # values are coerced into that range with a warning so the testdef value matches actual behavior. + $maxSupportedReruns = 3 + $maxReruns = 0 + if ($testConfig.PSObject.Properties.Name -contains 'MaxReruns') + { + $parsedMaxReruns = 0 + if (-not [int]::TryParse("$($testConfig.MaxReruns)", [ref]$parsedMaxReruns)) + { + Write-Warning "Invalid MaxReruns value '$($testConfig.MaxReruns)' in '$($testdef.FullName)' for '$id'. Using 0." + $parsedMaxReruns = 0 + } + elseif (($parsedMaxReruns -lt 0) -or ($parsedMaxReruns -gt $maxSupportedReruns)) + { + Write-Warning "MaxReruns value '$parsedMaxReruns' in '$($testdef.FullName)' for '$id' is outside the supported range 0..$maxSupportedReruns. Clamping to that range." + $parsedMaxReruns = [Math]::Max(0, [Math]::Min($parsedMaxReruns, $maxSupportedReruns)) + } + $maxReruns = $parsedMaxReruns + } + $t | Add-Member -MemberType NoteProperty -Name 'MaxReruns' -Value $maxReruns + $tests += $t $count += 1 } @@ -143,16 +168,17 @@ function List-Tests function Run-TaefTest { - param($test) + param($test, $logFile) $testFolder = Split-Path -parent $test.TestDef $tePath = Join-Path $testFolder "te.exe" $dllFile = Join-Path $testFolder $test.Filename - $teLogFile = (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.wtl") - $teLogPathTo = (Join-Path $env:Build_SourcesDirectory "TestOutput\$Configuration\$Platform") + # Pipe te.exe console output to the host so it is not captured as this function's return value + # (the caller assigns the return value to read te.exe's exit code). WTT results still go to $logFile. + & $tePath $dllFile $test.Parameters /enableWttLogging /appendWttLogging /screenCaptureOnError /logFile:$logFile /testMode:EtwLogger /EtwLogger:WprProfile=WDGDEPAdex /EtwLogger:SavePoint=TestFailure /EtwLogger:RecordingScope=Execution /EtwLogger:WprProfileFile=$wprProfilePath | Out-Host - & $tePath $dllFile $test.Parameters /enableWttLogging /appendWttLogging /screenCaptureOnError /logFile:$teLogFile /testMode:EtwLogger /EtwLogger:WprProfile=WDGDEPAdex /EtwLogger:SavePoint=TestFailure /EtwLogger:RecordingScope=Execution /EtwLogger:WprProfileFile=$wprProfilePath + return $LASTEXITCODE } function Run-PowershellTest @@ -164,6 +190,14 @@ function Run-PowershellTest function Run-Tests { + $teLogFile = (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.wtl") + # Rerun logs consumed by WindowsAppSDK-ConvertWttLogToXUnit-Steps.yml (wttSingleRerunInputPath / wttMultipleRerunInputPath / wttMoreRerunInputPath). + $rerunLogFiles = @( + (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.rerun1.wtl"), + (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.rerun2.wtl"), + (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.rerun3.wtl") + ) + $tests = Get-Tests foreach ($test in $tests) { @@ -174,7 +208,18 @@ function Run-Tests { if ($test.Type -eq 'TAEF') { - Run-TaefTest $test + $exitCode = Run-TaefTest $test $teLogFile + + # Tolerate known transient failures: rerun the failing testdef into separate rerun logs. + # The WTT-to-XUnit conversion (useRetryLogic=true) reports a test that failed the primary + # pass but passed on rerun as "unreliable" rather than failed. A test that fails every + # attempt is still reported as failed, so genuine regressions are not masked. + $maxReruns = [Math]::Min([int]$test.MaxReruns, $rerunLogFiles.Count) + for ($attempt = 1; ($exitCode -ne 0) -and ($attempt -le $maxReruns); $attempt++) + { + Write-Host "$($test.Filename) failed the previous run (exit code $exitCode); rerun attempt $attempt of $maxReruns" + $exitCode = Run-TaefTest $test $rerunLogFiles[$attempt - 1] + } } elseif ($test.Type -eq 'Powershell') { @@ -247,19 +292,26 @@ if ($Test -eq $true) { $teLogFile = (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.wtl") $teLogPathTo = (Join-Path $env:Build_SourcesDirectory "TestOutput\$Configuration\$Platform") - remove-item -Path $teLogFile -ErrorAction Ignore - remove-item -Path (Join-path $teLogPathTo "Te.wtl") -ErrorAction Ignore + # Primary log plus rerun logs produced for flaky testdefs (see Run-Tests / Run-TaefTest). + $teLogFileNames = @("Te.wtl", "Te.rerun1.wtl", "Te.rerun2.wtl", "Te.rerun3.wtl") + foreach ($logName in $teLogFileNames) { + remove-item -Path (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\$logName") -ErrorAction Ignore + remove-item -Path (Join-Path $teLogPathTo $logName) -ErrorAction Ignore + } Run-Tests - # copy test log to TestOutput folder - if (Test-Path -Path $teLogFile) { - Write-Host "Starting copy test log from '$teLogFile'" + # copy test logs (primary + any rerun logs) to TestOutput folder + foreach ($logName in $teLogFileNames) { + $logSource = (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\$logName") + if (Test-Path -Path $logSource) { + Write-Host "Starting copy test log from '$logSource'" - New-Item -ItemType Directory -Path $teLogPathTo -Force - copy-item -Path $teLogFile -Destination $teLogPathTo -Force + New-Item -ItemType Directory -Path $teLogPathTo -Force + copy-item -Path $logSource -Destination $teLogPathTo -Force - Write-Host "Test log copied to '$teLogPathTo'" + Write-Host "Test log copied to '$teLogPathTo'" + } } # copy screenshots to TestOutput folder diff --git a/build/AzurePipelinesTemplates/WindowsAppSDK-RunTests-Steps.yml b/build/AzurePipelinesTemplates/WindowsAppSDK-RunTests-Steps.yml index 54eebdc144..08cfb07b32 100644 --- a/build/AzurePipelinesTemplates/WindowsAppSDK-RunTests-Steps.yml +++ b/build/AzurePipelinesTemplates/WindowsAppSDK-RunTests-Steps.yml @@ -234,6 +234,11 @@ steps: - template: AzurePipelinesTemplates\WindowsAppSDK-ConvertWttLogToXUnit-Steps.yml@WindowsAppSDKConfig parameters: WttInputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\Te.wtl' + wttSingleRerunInputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\Te.rerun1.wtl' + wttMultipleRerunInputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\Te.rerun2.wtl' + wttMoreRerunInputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\Te.rerun3.wtl' + useRetryLogic: true + requiredPassRateThreshold: 1 xunitOutputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\testResults-$(buildConfiguration)_$(buildPlatform)_${{ parameters.ImageName }}.xml' TestNamePrefix: '$(buildConfiguration)_$(buildPlatform)_${{ parameters.ImageName }}' BypassTests: '$(Build.SourcesDirectory)\test\BypassTests.json' diff --git a/test/DynamicDependency/Test_Win32/DynamicDependency_API_Win32.testdef b/test/DynamicDependency/Test_Win32/DynamicDependency_API_Win32.testdef index 4833dbb0ac..47137ae30f 100644 --- a/test/DynamicDependency/Test_Win32/DynamicDependency_API_Win32.testdef +++ b/test/DynamicDependency/Test_Win32/DynamicDependency_API_Win32.testdef @@ -5,7 +5,8 @@ "Filename": "DynamicDependency_Test_Win32.dll", "Parameters": "", "Architectures": ["x64", "arm64"], - "Status": "Enabled" + "Status": "Enabled", + "MaxReruns": 3 } ] } diff --git a/test/DynamicDependency/Test_WinRT/DynamicDependency_API_WinRT.testdef b/test/DynamicDependency/Test_WinRT/DynamicDependency_API_WinRT.testdef index dec20224a4..22d89499cf 100644 --- a/test/DynamicDependency/Test_WinRT/DynamicDependency_API_WinRT.testdef +++ b/test/DynamicDependency/Test_WinRT/DynamicDependency_API_WinRT.testdef @@ -5,7 +5,8 @@ "Filename": "DynamicDependency_Test_WinRT.dll", "Parameters": "", "Architectures": ["x64", "arm64"], - "Status": "Enabled" + "Status": "Enabled", + "MaxReruns": 3 } ] }