Retry transient DynamicDependency Elevated bootstrap flake in Foundation test pass (AB#63230809)

TestAll.ps1

@@ -18,6 +18,7 @@
             "Parameters": { "type": "string" },
             "Architectures": { "type": "array", "items": { "type": "string" } },
             "Status": { "enum": ["Enabled", "Disabled"] },
+            "MaxReruns": { "type": "integer", "default": 0 },
           },
           "required": ["Description", "Filename", "Architectures", "Status"]
         }
@@ -118,6 +119,30 @@ function Get-Tests
             $t | Add-Member -MemberType NoteProperty -Name 'TestDef' -Value $testdef.FullName
             $t | Add-Member -MemberType NoteProperty -Name 'Type' -Value $testType
 
+            # Optional: number of times to rerun this test (into separate rerun logs) if it fails the primary pass.
+            # Used to tolerate known transient/flaky failures without masking genuine regressions - a test that
+            # fails the primary pass but passes on rerun is later reported as "unreliable" (skipped) instead of failed.
+            # Only 0..3 are supported (there are three rerun logs, see Run-Tests); non-integer or out-of-range
+            # values are coerced into that range with a warning so the testdef value matches actual behavior.
+            $maxSupportedReruns = 3
+            $maxReruns = 0
+            if ($testConfig.PSObject.Properties.Name -contains 'MaxReruns')
+            {
+                $parsedMaxReruns = 0
+                if (-not [int]::TryParse("$($testConfig.MaxReruns)", [ref]$parsedMaxReruns))
+                {
+                    Write-Warning "Invalid MaxReruns value '$($testConfig.MaxReruns)' in '$($testdef.FullName)' for '$id'. Using 0."
+                    $parsedMaxReruns = 0
+                }
+                elseif (($parsedMaxReruns -lt 0) -or ($parsedMaxReruns -gt $maxSupportedReruns))
+                {
+                    Write-Warning "MaxReruns value '$parsedMaxReruns' in '$($testdef.FullName)' for '$id' is outside the supported range 0..$maxSupportedReruns. Clamping to that range."
+                    $parsedMaxReruns = [Math]::Max(0, [Math]::Min($parsedMaxReruns, $maxSupportedReruns))
+                }
+                $maxReruns = $parsedMaxReruns
+            }
+            $t | Add-Member -MemberType NoteProperty -Name 'MaxReruns' -Value $maxReruns
+
             $tests += $t
             $count += 1
         }
@@ -143,16 +168,17 @@ function List-Tests
 
 function Run-TaefTest
 {
-    param($test)
+    param($test, $logFile)
 
     $testFolder = Split-Path -parent $test.TestDef
     $tePath = Join-Path $testFolder "te.exe"
     $dllFile = Join-Path $testFolder $test.Filename
 
-    $teLogFile = (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.wtl")
-    $teLogPathTo = (Join-Path $env:Build_SourcesDirectory "TestOutput\$Configuration\$Platform")
+    # Pipe te.exe console output to the host so it is not captured as this function's return value
+    # (the caller assigns the return value to read te.exe's exit code). WTT results still go to $logFile.
+    & $tePath $dllFile $test.Parameters /enableWttLogging /appendWttLogging /screenCaptureOnError /logFile:$logFile /testMode:EtwLogger /EtwLogger:WprProfile=WDGDEPAdex /EtwLogger:SavePoint=TestFailure /EtwLogger:RecordingScope=Execution /EtwLogger:WprProfileFile=$wprProfilePath | Out-Host
 
-    & $tePath $dllFile $test.Parameters /enableWttLogging /appendWttLogging /screenCaptureOnError /logFile:$teLogFile /testMode:EtwLogger /EtwLogger:WprProfile=WDGDEPAdex /EtwLogger:SavePoint=TestFailure /EtwLogger:RecordingScope=Execution /EtwLogger:WprProfileFile=$wprProfilePath
+    return $LASTEXITCODE
 }
 
 function Run-PowershellTest
@@ -164,6 +190,14 @@ function Run-PowershellTest
 
 function Run-Tests
 {
+    $teLogFile = (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.wtl")
+    # Rerun logs consumed by WindowsAppSDK-ConvertWttLogToXUnit-Steps.yml (wttSingleRerunInputPath / wttMultipleRerunInputPath / wttMoreRerunInputPath).
+    $rerunLogFiles = @(
+        (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.rerun1.wtl"),
+        (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.rerun2.wtl"),
+        (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.rerun3.wtl")
+    )
+
     $tests = Get-Tests
     foreach ($test in $tests)
     {
@@ -174,7 +208,18 @@ function Run-Tests
         {
             if ($test.Type -eq 'TAEF')
             {
-                Run-TaefTest $test
+                $exitCode = Run-TaefTest $test $teLogFile
+
+                # Tolerate known transient failures: rerun the failing testdef into separate rerun logs.
+                # The WTT-to-XUnit conversion (useRetryLogic=true) reports a test that failed the primary
+                # pass but passed on rerun as "unreliable" rather than failed. A test that fails every
+                # attempt is still reported as failed, so genuine regressions are not masked.
+                $maxReruns = [Math]::Min([int]$test.MaxReruns, $rerunLogFiles.Count)
+                for ($attempt = 1; ($exitCode -ne 0) -and ($attempt -le $maxReruns); $attempt++)
+                {
+                    Write-Host "$($test.Filename) failed the previous run (exit code $exitCode); rerun attempt $attempt of $maxReruns"
+                    $exitCode = Run-TaefTest $test $rerunLogFiles[$attempt - 1]
+                }
             }
             elseif ($test.Type -eq 'Powershell')
             {
@@ -247,19 +292,26 @@ if ($Test -eq $true)
 {
     $teLogFile = (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\Te.wtl")
     $teLogPathTo = (Join-Path $env:Build_SourcesDirectory "TestOutput\$Configuration\$Platform")
-    remove-item -Path $teLogFile -ErrorAction Ignore
-    remove-item -Path (Join-path $teLogPathTo "Te.wtl") -ErrorAction Ignore
+    # Primary log plus rerun logs produced for flaky testdefs (see Run-Tests / Run-TaefTest).
+    $teLogFileNames = @("Te.wtl", "Te.rerun1.wtl", "Te.rerun2.wtl", "Te.rerun3.wtl")
+    foreach ($logName in $teLogFileNames) {
+        remove-item -Path (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\$logName") -ErrorAction Ignore
+        remove-item -Path (Join-Path $teLogPathTo $logName) -ErrorAction Ignore
+    }
 
     Run-Tests
 
-    # copy test log to TestOutput folder
-    if (Test-Path -Path $teLogFile) {
-        Write-Host "Starting copy test log from '$teLogFile'"
+    # copy test logs (primary + any rerun logs) to TestOutput folder
+    foreach ($logName in $teLogFileNames) {
+        $logSource = (Join-Path $env:Build_SourcesDirectory "BuildOutput\$Configuration\$Platform\$logName")
+        if (Test-Path -Path $logSource) {
+            Write-Host "Starting copy test log from '$logSource'"
 
-        New-Item -ItemType Directory -Path $teLogPathTo -Force
-        copy-item -Path $teLogFile -Destination $teLogPathTo -Force
+            New-Item -ItemType Directory -Path $teLogPathTo -Force
+            copy-item -Path $logSource -Destination $teLogPathTo -Force
 
-        Write-Host "Test log copied to '$teLogPathTo'"
+            Write-Host "Test log copied to '$teLogPathTo'"
+        }
     }
 
     # copy screenshots to TestOutput folder

build/AzurePipelinesTemplates/WindowsAppSDK-RunTests-Steps.yml

@@ -234,6 +234,11 @@ steps:
   - template: AzurePipelinesTemplates\WindowsAppSDK-ConvertWttLogToXUnit-Steps.yml@WindowsAppSDKConfig
     parameters:
       WttInputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\Te.wtl'
+      wttSingleRerunInputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\Te.rerun1.wtl'
+      wttMultipleRerunInputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\Te.rerun2.wtl'
+      wttMoreRerunInputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\Te.rerun3.wtl'
+      useRetryLogic: true
+      requiredPassRateThreshold: 1
       xunitOutputPath: '$(Build.SourcesDirectory)\TestOutput\$(buildConfiguration)\$(buildPlatform)\testResults-$(buildConfiguration)_$(buildPlatform)_${{ parameters.ImageName }}.xml'
       TestNamePrefix: '$(buildConfiguration)_$(buildPlatform)_${{ parameters.ImageName }}'
       BypassTests: '$(Build.SourcesDirectory)\test\BypassTests.json'

test/DynamicDependency/Test_Win32/DynamicDependency_API_Win32.testdef

@@ -5,7 +5,8 @@
             "Filename": "DynamicDependency_Test_Win32.dll",
             "Parameters": "",
             "Architectures": ["x64", "arm64"],
-            "Status": "Enabled"
+            "Status": "Enabled",
+            "MaxReruns": 3
         }
     ]
 }

test/DynamicDependency/Test_WinRT/DynamicDependency_API_WinRT.testdef

@@ -5,7 +5,8 @@
             "Filename": "DynamicDependency_Test_WinRT.dll",
             "Parameters": "",
             "Architectures": ["x64", "arm64"],
-            "Status": "Enabled"
+            "Status": "Enabled",
+            "MaxReruns": 3
         }
     ]
 }