From 56464295f9890a4e924e6344f75d2d18496c0a7d Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sat, 7 Mar 2026 16:43:10 +0900 Subject: [PATCH 01/49] chore: update xunit related package version to v3 with mtp v2 --- .../Runners/UnitTestRunner.cs | 11 ++++++++++- .../BenchmarkDotNet.Analyzers.Tests.csproj | 15 ++++++--------- .../Fixtures/Serializable/ValueTupleDouble.cs | 2 -- .../Fixtures/Serializable/ValueTupleTriple.cs | 4 +--- ...nchmarkDotNet.Exporters.Plotting.Tests.csproj | 9 ++------- .../ScottPlotExporterTests.cs | 3 +++ ...Net.IntegrationTests.ConfigPerAssembly.csproj | 2 +- ...IntegrationTests.DisabledOptimizations.csproj | 4 ++-- ....IntegrationTests.EnabledOptimizations.csproj | 4 ++-- ...Tests.ManualRunning.MultipleFrameworks.csproj | 12 ++++-------- ...kDotNet.IntegrationTests.ManualRunning.csproj | 10 ++-------- .../DotMemoryTests.cs | 1 + .../DotTraceTests.cs | 1 + .../MsBuildArgumentTests.cs | 1 + .../AsyncBenchmarksTests.cs | 4 ---- .../BenchmarkDotNet.IntegrationTests.csproj | 15 ++++++--------- .../CopyToOutputTests.cs | 1 + .../CustomBuildConfigurationTests.cs | 4 +--- .../GcModeTests.cs | 3 ++- .../InProcessTest.cs | 1 - .../MemoryDiagnoserTests.cs | 0 .../ReferencesTests.cs | 1 - .../Shared/BenchmarkTestExecutor.cs | 1 + .../TailCallDiagnoserTests.cs | 1 + .../WasmTests.cs | 2 +- .../BenchmarkDotNet.Tests.csproj | 16 ++++------------ .../Detectors/Cpu/CpuInfoParserTests_Cim.cs | 6 +++--- .../Detectors/Cpu/LinuxCpuInfoParserTests.cs | 2 +- .../Shared/Loggers/OutputLogger.cs | 1 + .../Shared/XUnit/InlineDataEnvSpecific.cs | 1 + .../XUnit/InlineDataEnvSpecificDiscoverer.cs | 1 + tests/Directory.Build.props | 4 ++++ 32 files changed, 64 insertions(+), 79 deletions(-) mode change 100755 => 100644 tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj mode change 100755 => 100644 tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs mode change 100755 => 100644 tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj diff --git a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs index ab3833d129..f06f9b3132 100644 --- a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs @@ -44,7 +44,16 @@ private DotNetTestSettings GetTestSettingsParameters(FilePath logFile, string tf EnvironmentVariables = { ["Platform"] = "" // force the tool to not look for the .dll in platform-specific directory - } + }, + PathType = DotNetTestPathType.Auto, + ArgumentCustomization = args + => args.Append("--report-xunit-trx") + .AppendSwitchQuoted("--report-xunit-trx-filename", System.IO.Path.GetFileName(logFile.FullPath)) + .Append("--no-ansi") + .AppendSwitch("--output", "Detailed") + .AppendSwitch("--show-stdout", "Failed") + .Append("--diagnostic") + .AppendSwitch("--diagnostic-verbosity", "Trace") }; return settings; } diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/BenchmarkDotNet.Analyzers.Tests.csproj b/tests/BenchmarkDotNet.Analyzers.Tests/BenchmarkDotNet.Analyzers.Tests.csproj index 91297cfd14..725fa7ff05 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/BenchmarkDotNet.Analyzers.Tests.csproj +++ b/tests/BenchmarkDotNet.Analyzers.Tests/BenchmarkDotNet.Analyzers.Tests.csproj @@ -2,6 +2,7 @@ false + Exe BenchmarkDotNet.Analyzers.Tests net472;net10.0 true @@ -12,7 +13,6 @@ - @@ -21,13 +21,10 @@ - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - + + + + @@ -36,4 +33,4 @@ - \ No newline at end of file + diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleDouble.cs b/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleDouble.cs index 5642d4c615..4802092df4 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleDouble.cs +++ b/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleDouble.cs @@ -1,5 +1,3 @@ -using Xunit.Abstractions; - namespace BenchmarkDotNet.Analyzers.Tests.Fixtures; public class ValueTupleDouble : IXunitSerializable diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleTriple.cs b/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleTriple.cs index 0b19f5c211..7bc8fe044c 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleTriple.cs +++ b/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleTriple.cs @@ -1,5 +1,3 @@ -using Xunit.Abstractions; - namespace BenchmarkDotNet.Analyzers.Tests.Fixtures; public class ValueTupleTriple : IXunitSerializable @@ -29,4 +27,4 @@ public static implicit operator ValueTupleTriple((T1, T2, T3) valueT public override string ToString() => Value1 == null || Value2 == null || Value3 == null ? "" : $"{Value1} · {Value2} · {Value3}"; -} \ No newline at end of file +} diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj index d18c1d327c..14ff835dae 100644 --- a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj @@ -2,12 +2,12 @@ net10.0;net472 + Exe true false - @@ -15,12 +15,7 @@ - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs index bb481d3ea8..1d5118dc69 100644 --- a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs @@ -3,6 +3,9 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Tests.Mocks; using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using Xunit; namespace BenchmarkDotNet.Exporters.Plotting.Tests { diff --git a/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj b/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj index 24ce7ba476..462d92aa72 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.IntegrationTests.ConfigPerAssembly - net472;net10.0 + net472;net8.0;net10.0 true BenchmarkDotNet.IntegrationTests.ConfigPerAssembly BenchmarkDotNet.IntegrationTests.ConfigPerAssembly diff --git a/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj b/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj index 55839c1d79..9ad2394109 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj @@ -2,14 +2,14 @@ BenchmarkDotNet.IntegrationTests.DisabledOptimizations - net472;net10.0 + net472;net8.0;net10.0 true BenchmarkDotNet.IntegrationTests.DisabledOptimizations BenchmarkDotNet.IntegrationTests.DisabledOptimizations true false AnyCPU - + false diff --git a/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj b/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj index f237f23791..cd2db5690c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj @@ -2,14 +2,14 @@ BenchmarkDotNet.IntegrationTests.EnabledOptimizations - net472;net10.0 + net472;net8.0;net10.0 true BenchmarkDotNet.IntegrationTests.EnabledOptimizations BenchmarkDotNet.IntegrationTests.EnabledOptimizations true false AnyCPU - + true diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj index 96d84f4943..1378b38bb9 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj @@ -1,8 +1,9 @@ - + BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks net462;net48;net8.0;net10.0 + Exe $(MSBuildWarningsAsMessages);MSB3277 false @@ -17,7 +18,6 @@ - @@ -34,12 +34,8 @@ - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - + + diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj old mode 100755 new mode 100644 index 72dd42c0f4..db77746136 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj @@ -14,7 +14,6 @@ - @@ -24,7 +23,7 @@ $(DefineConstants);$(ExtraDefineConstants) - + @@ -43,12 +42,7 @@ - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/DotMemoryTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/DotMemoryTests.cs index efb02f0aea..0020481687 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/DotMemoryTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/DotMemoryTests.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Xunit; namespace BenchmarkDotNet.IntegrationTests.ManualRunning { diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/DotTraceTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/DotTraceTests.cs index dc11404f9f..4a240100c4 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/DotTraceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/DotTraceTests.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Xunit; namespace BenchmarkDotNet.IntegrationTests.ManualRunning { diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/MsBuildArgumentTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/MsBuildArgumentTests.cs index 6592e76047..434af9df6c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/MsBuildArgumentTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/MsBuildArgumentTests.cs @@ -2,6 +2,7 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using Xunit; namespace BenchmarkDotNet.IntegrationTests.ManualRunning { diff --git a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs index 63ccd02838..b41110a1af 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs @@ -1,8 +1,4 @@ using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Toolchains.InProcess.Emit; -using BenchmarkDotNet.Toolchains.InProcess.NoEmit; using System.Threading.Tasks.Sources; namespace BenchmarkDotNet.IntegrationTests diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj index 9fde386d87..7ad4db9121 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj @@ -4,6 +4,7 @@ BenchmarkDotNet.IntegrationTests net10.0 $(TargetFrameworks);net472 + Exe true BenchmarkDotNet.IntegrationTests BenchmarkDotNet.IntegrationTests @@ -20,7 +21,6 @@ - @@ -35,6 +35,8 @@ + + @@ -43,18 +45,13 @@ - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - + + + diff --git a/tests/BenchmarkDotNet.IntegrationTests/CopyToOutputTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CopyToOutputTests.cs index bd8da25d01..3d88695806 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CopyToOutputTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CopyToOutputTests.cs @@ -1,5 +1,6 @@ #if NETFRAMEWORK using BenchmarkDotNet.IntegrationTests.CustomPaths; +using Xunit; namespace BenchmarkDotNet.IntegrationTests { diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomBuildConfigurationTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomBuildConfigurationTests.cs index a08fb99903..e165df196b 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CustomBuildConfigurationTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CustomBuildConfigurationTests.cs @@ -1,12 +1,10 @@ -using System.Reflection; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Tests.XUnit; -#if !DEBUG -#endif +using System.Reflection; namespace BenchmarkDotNet.IntegrationTests { diff --git a/tests/BenchmarkDotNet.IntegrationTests/GcModeTests.cs b/tests/BenchmarkDotNet.IntegrationTests/GcModeTests.cs index 8d2558ebe3..ccd5f365fe 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/GcModeTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/GcModeTests.cs @@ -1,7 +1,8 @@ -using System.Runtime; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Jobs; +using System.Runtime; + #if NETFRAMEWORK using BenchmarkDotNet.Environments; #endif diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs index d0cd8ecaed..69a73294b5 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs @@ -17,7 +17,6 @@ using System.Reflection; using System.Runtime.CompilerServices; - namespace BenchmarkDotNet.IntegrationTests { public class InProcessTest : BenchmarkTestExecutor diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs old mode 100755 new mode 100644 diff --git a/tests/BenchmarkDotNet.IntegrationTests/ReferencesTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ReferencesTests.cs index 9daa2db80a..3adec4f6cb 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ReferencesTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ReferencesTests.cs @@ -1,4 +1,3 @@ - namespace BenchmarkDotNet.IntegrationTests { public class ReferencesTests : BenchmarkTestExecutor diff --git a/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs b/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs index e3413c5e4a..795127c130 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs @@ -6,6 +6,7 @@ using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; +using Xunit; namespace BenchmarkDotNet.IntegrationTests { diff --git a/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs index c224cbd47f..8d34d90724 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs @@ -11,6 +11,7 @@ using BenchmarkDotNet.Tests.XUnit; using System.Collections.Generic; using System.Linq; +using Xunit; namespace BenchmarkDotNet.IntegrationTests { diff --git a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs index 78ff33566f..a3b63bbe83 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs @@ -91,4 +91,4 @@ private ManualConfig GetConfig(string runtimeMoniker, string javaScriptEngine, F .WithOption(ConfigOptions.GenerateMSBuildBinLog, false); } } -} +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj old mode 100755 new mode 100644 index 40240b4ec5..5fd5275d39 --- a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj +++ b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj @@ -2,7 +2,8 @@ BenchmarkDotNet.Tests - net10.0;net472 + net8.0;net10.0;net472 + Exe BenchmarkDotNet.Tests BenchmarkDotNet.Tests true @@ -11,7 +12,6 @@ - @@ -21,13 +21,8 @@ - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - + + @@ -36,9 +31,6 @@ - - - diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/CpuInfoParserTests_Cim.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/CpuInfoParserTests_Cim.cs index 1b6a6ada95..1e6ed0d7bc 100644 --- a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/CpuInfoParserTests_Cim.cs +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/CpuInfoParserTests_Cim.cs @@ -34,13 +34,13 @@ public void RealTwoProcessorEightCoresTest() Name:Intel(R) Xeon(R) CPU E5-2630 v3 NumberOfCores:8 NumberOfLogicalProcessors:16 - - + + MaxClockSpeed:2400 Name:Intel(R) Xeon(R) CPU E5-2630 v3 NumberOfCores:8 NumberOfLogicalProcessors:16 - + """; CpuInfo? actual = CpuInfoParser.ParseCimOutput(cpuInfo); diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/LinuxCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/LinuxCpuInfoParserTests.cs index 393bd2798b..e907ffa9aa 100644 --- a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/LinuxCpuInfoParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/LinuxCpuInfoParserTests.cs @@ -138,7 +138,7 @@ public void AmdRyzen9_7950X() CPU max MHz: 5881.0000 CPU min MHz: 400.0000 BogoMIPS: 8983.23 - Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl + Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512 cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean diff --git a/tests/BenchmarkDotNet.Tests/Shared/Loggers/OutputLogger.cs b/tests/BenchmarkDotNet.Tests/Shared/Loggers/OutputLogger.cs index 2a83781f14..e9020d6e24 100644 --- a/tests/BenchmarkDotNet.Tests/Shared/Loggers/OutputLogger.cs +++ b/tests/BenchmarkDotNet.Tests/Shared/Loggers/OutputLogger.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Loggers; using System.Runtime.CompilerServices; +using Xunit; namespace BenchmarkDotNet.Tests.Loggers { diff --git a/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecific.cs b/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecific.cs index 65acf07295..79a720b6b1 100644 --- a/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecific.cs +++ b/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecific.cs @@ -1,5 +1,6 @@ using System.Reflection; using Xunit.Sdk; +using Xunit.v3; namespace BenchmarkDotNet.Tests.XUnit; diff --git a/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecificDiscoverer.cs b/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecificDiscoverer.cs index dba7caa4c2..a9a73f20a3 100644 --- a/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecificDiscoverer.cs +++ b/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecificDiscoverer.cs @@ -1,3 +1,4 @@ +using Xunit; using Xunit.Sdk; namespace BenchmarkDotNet.Tests.XUnit; diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index 5ae2fbba84..f884b70545 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -10,6 +10,10 @@ $([System.IO.Path]::GetDirectoryName($(MSBuildThisFileDirectory))) + + true + + From a0bcd8222f1ae7ccc5ea226c05bcf626a5f77597 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sat, 7 Mar 2026 16:58:34 +0900 Subject: [PATCH 02/49] chore: add using Xunit.Sdk statement for XunitSerializable --- .../Fixtures/Serializable/ValueTupleDouble.cs | 2 ++ .../Fixtures/Serializable/ValueTupleTriple.cs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleDouble.cs b/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleDouble.cs index 4802092df4..56819e9cf2 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleDouble.cs +++ b/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleDouble.cs @@ -1,3 +1,5 @@ +using Xunit.Sdk; + namespace BenchmarkDotNet.Analyzers.Tests.Fixtures; public class ValueTupleDouble : IXunitSerializable diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleTriple.cs b/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleTriple.cs index 7bc8fe044c..82b871d3a3 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleTriple.cs +++ b/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Serializable/ValueTupleTriple.cs @@ -1,3 +1,5 @@ +using Xunit.Sdk; + namespace BenchmarkDotNet.Analyzers.Tests.Fixtures; public class ValueTupleTriple : IXunitSerializable From 7169a3fddd24c33b217979b1a1c837a1b9030ea3 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sat, 7 Mar 2026 16:59:12 +0900 Subject: [PATCH 03/49] chore: fix xUnit3003 analyzer errors --- .../BenchmarkDotNet.Tests/Shared/XUnit/FactEnvSpecific.cs | 7 +++++++ .../Shared/XUnit/TheoryEnvSpecific.cs | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/tests/BenchmarkDotNet.Tests/Shared/XUnit/FactEnvSpecific.cs b/tests/BenchmarkDotNet.Tests/Shared/XUnit/FactEnvSpecific.cs index 6c57f08078..a0e02d4c07 100644 --- a/tests/BenchmarkDotNet.Tests/Shared/XUnit/FactEnvSpecific.cs +++ b/tests/BenchmarkDotNet.Tests/Shared/XUnit/FactEnvSpecific.cs @@ -1,3 +1,5 @@ +using System.Runtime.CompilerServices; +using Xunit; namespace BenchmarkDotNet.Tests.XUnit; @@ -14,4 +16,9 @@ public FactEnvSpecificAttribute(string reason, params EnvRequirement[] requireme if (skip != null) Skip = $"{skip} ({reason})"; } + + public FactEnvSpecificAttribute([CallerFilePath] string? sourceFilePath = null, [CallerLineNumber] int sourceLineNumber = -1) + : base(sourceFilePath, sourceLineNumber) + { + } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Shared/XUnit/TheoryEnvSpecific.cs b/tests/BenchmarkDotNet.Tests/Shared/XUnit/TheoryEnvSpecific.cs index 3303797884..92f039f030 100644 --- a/tests/BenchmarkDotNet.Tests/Shared/XUnit/TheoryEnvSpecific.cs +++ b/tests/BenchmarkDotNet.Tests/Shared/XUnit/TheoryEnvSpecific.cs @@ -1,3 +1,5 @@ +using System.Runtime.CompilerServices; +using Xunit; namespace BenchmarkDotNet.Tests.XUnit; @@ -14,4 +16,9 @@ public TheoryEnvSpecificAttribute(string reason, params EnvRequirement[] require if (skip != null) Skip = $"{skip} ({reason})"; } + + public TheoryEnvSpecificAttribute([CallerFilePath] string? sourceFilePath = null, [CallerLineNumber] int sourceLineNumber = -1) + : base(sourceFilePath, sourceLineNumber) + { + } } \ No newline at end of file From 5c2e7f4a3514f01105d32d4cd1dbd96ec66dec39 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sat, 7 Mar 2026 17:50:12 +0900 Subject: [PATCH 04/49] chore: modify theorydata related incompatible codes --- .../ArgumentsAttributeAnalyzerTests.cs | 4 +- ...GeneralParameterAttributesAnalyzerTests.cs | 44 +++++++++---------- .../ParamsAllValuesAttributeAnalyzerTests.cs | 12 ++--- .../ParamsAttributeAnalyzerTests.cs | 38 +++++++--------- .../BenchmarkRunner/RunAnalyzerTests.cs | 4 +- .../Extensions/TheoryDataExtensions.cs | 2 +- .../AsyncBenchmarksTests.cs | 4 ++ .../AsyncEnumerableBenchmarksTests.cs | 8 ++-- .../CallerThreadTests.cs | 4 +- .../EngineTests.cs | 4 +- .../RunAsyncTests.cs | 4 +- 11 files changed, 57 insertions(+), 71 deletions(-) diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ArgumentsAttributeAnalyzerTests.cs b/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ArgumentsAttributeAnalyzerTests.cs index 7cad72b679..f4a937bc62 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ArgumentsAttributeAnalyzerTests.cs +++ b/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ArgumentsAttributeAnalyzerTests.cs @@ -633,7 +633,7 @@ public async Task Providing_expected_value_type_should_not_trigger_diagnostic( { var testCode = /* lang=c#-test */ $$""" using DifferentNamespace; - + using BenchmarkDotNet.Attributes; public class BenchmarkClass @@ -1188,7 +1188,7 @@ public void BenchmarkMethod({{integerValueAndType.Value2}} a) } public static IEnumerable DummyAttributeUsage - => DummyAttributeUsageTheoryData; + => DummyAttributeUsageTheoryData.Select(x => x.Data); public static TheoryData EmptyArgumentsAttributeUsagesWithMismatchingValueCount() { diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/GeneralParameterAttributesAnalyzerTests.cs b/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/GeneralParameterAttributesAnalyzerTests.cs index fc7455faa8..fd544e0731 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/GeneralParameterAttributesAnalyzerTests.cs +++ b/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/GeneralParameterAttributesAnalyzerTests.cs @@ -1,4 +1,4 @@ -using BenchmarkDotNet.Analyzers; +using BenchmarkDotNet.Analyzers; using BenchmarkDotNet.Analyzers.Attributes; using BenchmarkDotNet.Analyzers.Tests.Fixtures; using Microsoft.CodeAnalysis; @@ -191,7 +191,7 @@ public class BenchmarkClass } public static TheoryData UniqueParameterAttributeUsages - => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)]; + => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)]; public static TheoryData DuplicateSameParameterAttributeUsages => DuplicateSameAttributeUsagesTheoryData; @@ -353,7 +353,7 @@ public class BenchmarkClass await RunAsync(); } - public static TheoryData UniqueParameterAttributeUsages => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)]; + public static TheoryData UniqueParameterAttributeUsages => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)]; public static TheoryData DuplicateSameParameterAttributeUsages => DuplicateSameAttributeUsagesTheoryData; @@ -512,21 +512,19 @@ public static IEnumerable DuplicateAttributeUsageCountsAndNonPublicCla => CombinationsGenerator.CombineArguments(DuplicateParameterAttributeUsageCounts, NonPublicClassMemberAccessModifiers); public static TheoryData UniqueParameterAttributeUsages - => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)]; + => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)]; public static IEnumerable<(string AttributeName, string AttributeUsage)> UniqueParameterAttributes - => UniqueParameterAttributesTheoryData.Select(tdr => ((tdr[0] as string)!, (tdr[1] as string)!)); + => UniqueParameterAttributesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2)); public static IEnumerable NonPublicClassMemberAccessModifiers -#pragma warning disable IDE0028 // Simplify collection initialization - => new NonPublicClassMemberAccessModifiersTheoryData(); -#pragma warning restore IDE0028 // Simplify collection initialization + => new NonPublicClassMemberAccessModifiersTheoryData().Select(x => x.Data); public static IEnumerable<(string CurrentUniqueAttributeUsage, int CurrentUniqueAttributeUsagePosition, int[] Counts)> DuplicateSameParameterAttributeUsages - => DuplicateSameAttributeUsagesTheoryData.Select(tdr => ((tdr[0] as string)!, (int)tdr[1], (tdr[2] as int[])!)); + => DuplicateSameAttributeUsagesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2, tdr.Data.Item3)); public static IEnumerable DuplicateParameterAttributeUsageCounts - => DuplicateAttributeUsageCountsTheoryData; + => DuplicateAttributeUsageCountsTheoryData.Select(x => x.Data); } public class PropertyMustBePublic : AnalyzerTestFixture @@ -681,18 +679,16 @@ public static IEnumerable DuplicateAttributeUsageCountsAndNonPublicCla => CombinationsGenerator.CombineArguments(DuplicateParameterAttributeUsageCounts, NonPublicClassMemberAccessModifiers); public static TheoryData UniqueParameterAttributeUsages - => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)]; + => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr.Data.Item2 as string)!)]; public static IEnumerable<(string AttributeName, string AttributeUsage)> UniqueParameterAttributes - => UniqueParameterAttributesTheoryData.Select(tdr => ((tdr[0] as string)!, (tdr[1] as string)!)); + => UniqueParameterAttributesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2)); public static IEnumerable NonPublicClassMemberAccessModifiers -#pragma warning disable IDE0028 // Simplify collection initialization - => new NonPublicClassMemberAccessModifiersTheoryData(); -#pragma warning restore IDE0028 // Simplify collection initialization + => new NonPublicClassMemberAccessModifiersTheoryData().Select(x => x.Data); public static IEnumerable<(string CurrentUniqueAttributeUsage, int CurrentUniqueAttributeUsagePosition, int[] Counts)> DuplicateSameParameterAttributeUsages - => DuplicateSameAttributeUsagesTheoryData.Select(tdr => ((tdr[0] as string)!, (int)tdr[1], (tdr[2] as int[])!)); + => DuplicateSameAttributeUsagesTheoryData.Select(tdr => (tdr.Data.Item1!, tdr.Data.Item2, tdr.Data.Item3)); public static TheoryData DuplicateParameterAttributeUsageCounts => DuplicateAttributeUsageCountsTheoryData; @@ -846,7 +842,7 @@ public class BenchmarkClass } public static TheoryData UniqueParameterAttributeUsages - => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)]; + => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)]; public static TheoryData UniqueParameterAttributes => UniqueParameterAttributesTheoryData; @@ -984,7 +980,7 @@ public class BenchmarkClass } public static TheoryData UniqueParameterAttributeUsages - => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)]; + => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)]; public static TheoryData UniqueParameterAttributes => UniqueParameterAttributesTheoryData; @@ -1149,20 +1145,20 @@ public static IEnumerable DuplicateAttributeUsageCountsAndNonPublicPro => CombinationsGenerator.CombineArguments(DuplicateParameterAttributeUsageCounts, NonPublicPropertySetters()); public static TheoryData UniqueParameterAttributeUsages - => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)]; + => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)]; public static IEnumerable<(string AttributeName, string AttributeUsage)> UniqueParameterAttributes - => UniqueParameterAttributesTheoryData.Select(tdr => ((tdr[0] as string)!, (tdr[1] as string)!)); + => UniqueParameterAttributesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2)); public static IEnumerable<(string CurrentUniqueAttributeUsage, int CurrentUniqueAttributeUsagePosition, int[] Counts)> DuplicateSameParameterAttributeUsages - => DuplicateSameAttributeUsagesTheoryData.Select(tdr => ((tdr[0] as string)!, (int)tdr[1], (tdr[2] as int[])!)); + => DuplicateSameAttributeUsagesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2, tdr.Data.Item3)); public static TheoryData DuplicateParameterAttributeUsageCounts => DuplicateAttributeUsageCountsTheoryData; public static IEnumerable NonPublicPropertySetters() - => new NonPublicPropertySetterAccessModifiersTheoryData() - .Select(m => $"{{ get; {m} set; }}") + => new NonPublicPropertySetterAccessModifiersTheoryData().Select(x => x.Data) + .Select(m => $"{{ get; {m} set; }}") .Concat(["{ get; }", "=> 0;"]); public static TheoryData NonPublicPropertySettersTheoryData() @@ -1904,7 +1900,7 @@ private static IEnumerable GenerateDuplicateAttributeUsageCombinations(Th private static ReadOnlyCollection<(string CurrentUniqueAttributeUsage, int CurrentUniqueAttributeUsagePosition, int[] Counts)> GenerateDuplicateSameAttributeUsageCombinations(TheoryData uniqueAttributeUsages) { var uniqueAttributeUsagesList = uniqueAttributeUsages - .Select(tdr => (tdr[1] as string)!) + .Select(tdr => tdr.Data.Item2) .ToList() .AsReadOnly(); diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ParamsAllValuesAttributeAnalyzerTests.cs b/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ParamsAllValuesAttributeAnalyzerTests.cs index ce92329620..68fcfcefa4 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ParamsAllValuesAttributeAnalyzerTests.cs +++ b/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ParamsAllValuesAttributeAnalyzerTests.cs @@ -32,9 +32,7 @@ public class BenchmarkClass } public static IEnumerable FieldOrPropertyDeclarations -#pragma warning disable IDE0028 // Simplify collection initialization - => new FieldOrPropertyDeclarationsTheoryData(); -#pragma warning restore IDE0028 // Simplify collection initialization + => new FieldOrPropertyDeclarationsTheoryData().Select(x => x.Data); public static IEnumerable InvalidTypes => [ @@ -149,9 +147,7 @@ public class BenchmarkClass } public static IEnumerable FieldOrPropertyDeclarations -#pragma warning disable IDE0028 // Simplify collection initialization - => new FieldOrPropertyDeclarationsTheoryData(); -#pragma warning restore IDE0028 // Simplify collection initialization + => new FieldOrPropertyDeclarationsTheoryData().Select(x => x.Data); public static IEnumerable NonEnumStructs => [ @@ -249,9 +245,7 @@ public class BenchmarkClass } public static IEnumerable FieldOrPropertyDeclarations -#pragma warning disable IDE0028 // Simplify collection initialization - => new FieldOrPropertyDeclarationsTheoryData(); -#pragma warning restore IDE0028 // Simplify collection initialization + => new FieldOrPropertyDeclarationsTheoryData().Select(x => x.Data); public static IEnumerable NonEnumOrBoolStructs => [ diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ParamsAttributeAnalyzerTests.cs b/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ParamsAttributeAnalyzerTests.cs index 1070aaa466..da10426467 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ParamsAttributeAnalyzerTests.cs +++ b/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ParamsAttributeAnalyzerTests.cs @@ -32,9 +32,7 @@ public string {{fieldOrPropertyDeclaration}} } public static IEnumerable FieldOrPropertyDeclarations -#pragma warning disable IDE0028 // Simplify collection initialization - => new FieldOrPropertyDeclarationsTheoryData(); -#pragma warning restore IDE0028 // Simplify collection initialization + => new FieldOrPropertyDeclarationsTheoryData().Select(x => x.Data); } public class MustHaveValues : AnalyzerTestFixture @@ -166,12 +164,10 @@ public string {{fieldOrPropertyDeclaration}} } public static IEnumerable FieldOrPropertyDeclarations -#pragma warning disable IDE0028 // Simplify collection initialization - => new FieldOrPropertyDeclarationsTheoryData(); -#pragma warning restore IDE0028 // Simplify collection initialization + => new FieldOrPropertyDeclarationsTheoryData().Select(x => x.Data); public static IEnumerable DummyAttributeUsage - => DummyAttributeUsageTheoryData; + => DummyAttributeUsageTheoryData.Select(x => x.Data); public static IEnumerable ScalarValuesListLength => Enumerable.Range(1, ScalarValues.Count); @@ -183,7 +179,7 @@ private static ReadOnlyCollection ScalarValues .AsReadOnly(); public static IEnumerable ScalarValuesContainerAttributeArgument - => ScalarValuesContainerAttributeArgumentTheoryData(); + => ScalarValuesContainerAttributeArgumentTheoryData().Select(x => x.Data); public static IEnumerable EmptyParamsAttributeUsagesWithLocationMarker() { @@ -305,7 +301,7 @@ public async Task Providing_expected_value_type_should_not_trigger_diagnostic( { var testCode = /* lang=c#-test */ $$""" using DifferentNamespace; - + using BenchmarkDotNet.Attributes; public class BenchmarkClass @@ -386,7 +382,7 @@ public async Task Providing_expected_enum_value_type_array_using_not_fully_quali { var testCode = /* lang=c#-test */ $$""" using DifferentNamespace; - + using BenchmarkDotNet.Attributes; public class BenchmarkClass @@ -444,7 +440,7 @@ public async Task Providing_expected_type_using_not_fully_qualified_name_located using BenchmarkDotNet.Attributes; namespace DifferentNamespace; - + public class BenchmarkClass { [{{dummyAttributeUsage}}Params({{string.Format(scalarValuesContainerAttributeArgument, valueAndType.Value1)}})] @@ -468,7 +464,7 @@ public async Task Providing_expected_type_using_not_fully_qualified_name_located { var testCode = /* lang=c#-test */ $$""" using DifferentNamespace; - + using BenchmarkDotNet.Attributes; public class BenchmarkClass @@ -822,7 +818,7 @@ public async Task Providing_an_unexpected_array_value_type_to_params_attribute_s var testCode = /* lang=c#-test */ $$""" using DifferentNamespace; - + using BenchmarkDotNet.Attributes; public class BenchmarkClass @@ -891,12 +887,10 @@ public object[] {{fieldOrPropertyDeclaration}} } public static IEnumerable FieldOrPropertyDeclarations -#pragma warning disable IDE0028 // Simplify collection initialization - => new FieldOrPropertyDeclarationsTheoryData(); -#pragma warning restore IDE0028 // Simplify collection initialization + => new FieldOrPropertyDeclarationsTheoryData().Select(x => x.Data); public static IEnumerable DummyAttributeUsage - => DummyAttributeUsageTheoryData; + => DummyAttributeUsageTheoryData.Select(x => x.Data); public static IEnumerable> IntegerValuesAndTypesWithinTargetTypeRange => [ @@ -1020,7 +1014,7 @@ public static IEnumerable EmptyValuesAttributeArgument() } public static IEnumerable ScalarValuesContainerAttributeArgumentEnumerable - => ScalarValuesContainerAttributeArgumentTheoryData(); + => ScalarValuesContainerAttributeArgumentTheoryData().Select(x => x.Data); public static IEnumerable ArrayValuesContainerAttributeArgumentEnumerableLocal() { @@ -1261,12 +1255,10 @@ public string {{fieldOrPropertyDeclaration}} } public static IEnumerable FieldOrPropertyDeclarations -#pragma warning disable IDE0028 // Simplify collection initialization - => new FieldOrPropertyDeclarationsTheoryData(); -#pragma warning restore IDE0028 // Simplify collection initialization + => new FieldOrPropertyDeclarationsTheoryData().Select(x => x.Data); public static IEnumerable DummyAttributeUsage - => DummyAttributeUsageTheoryData; + => DummyAttributeUsageTheoryData.Select(x => x.Data); public static IEnumerable ScalarValuesListLength => Enumerable.Range(2, ScalarValues.Count); @@ -1277,7 +1269,7 @@ private static ReadOnlyCollection ScalarValues .ToList() .AsReadOnly(); public static IEnumerable ScalarValuesContainerAttributeArgument - => ScalarValuesContainerAttributeArgumentTheoryData(); + => ScalarValuesContainerAttributeArgumentTheoryData().Select(x => x.Data); public static IEnumerable ScalarValuesContainerAttributeArgumentWithLocationMarker() { diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/BenchmarkRunner/RunAnalyzerTests.cs b/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/BenchmarkRunner/RunAnalyzerTests.cs index 458e8bd326..f50cbeaf8b 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/BenchmarkRunner/RunAnalyzerTests.cs +++ b/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/BenchmarkRunner/RunAnalyzerTests.cs @@ -61,7 +61,7 @@ public async Task Invoking_with_type_argument_class_having_at_least_one_public_m var testCode = /* lang=c#-test */ $$""" using BenchmarkDotNet.Running; - + public class Program { public static void Main(string[] args) { @@ -548,7 +548,7 @@ public void BenchmarkMethod() } public static IEnumerable NonPublicClassAccessModifiersExceptFile - => new NonPublicClassAccessModifiersTheoryData().Where(m => m != "file "); + => new NonPublicClassAccessModifiersTheoryData().Where(m => m.Data != "file ").Select(x => x.Data); } public class TypeArgumentClassMustBeUnsealed : AnalyzerTestFixture diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Extensions/TheoryDataExtensions.cs b/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Extensions/TheoryDataExtensions.cs index bc5fcc258c..1970ab058d 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Extensions/TheoryDataExtensions.cs +++ b/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/Extensions/TheoryDataExtensions.cs @@ -6,5 +6,5 @@ namespace BenchmarkDotNet.Analyzers.Tests.Fixtures; public static class TheoryDataExtensions { public static ReadOnlyCollection AsReadOnly(this TheoryData theoryData) - => (theoryData as IEnumerable).ToList().AsReadOnly(); + => theoryData.Select(x => x.Data).ToList().AsReadOnly(); } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs index b41110a1af..63ccd02838 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs @@ -1,4 +1,8 @@ using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using BenchmarkDotNet.Toolchains.InProcess.NoEmit; using System.Threading.Tasks.Sources; namespace BenchmarkDotNet.IntegrationTests diff --git a/tests/BenchmarkDotNet.IntegrationTests/AsyncEnumerableBenchmarksTests.cs b/tests/BenchmarkDotNet.IntegrationTests/AsyncEnumerableBenchmarksTests.cs index e90eebe53f..c6d1dc8d01 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AsyncEnumerableBenchmarksTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AsyncEnumerableBenchmarksTests.cs @@ -9,22 +9,22 @@ namespace BenchmarkDotNet.IntegrationTests; public class AsyncEnumerableBenchmarksTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) { - public static TheoryData GetAllToolchains() => + public static TheoryData GetAllToolchains() => new( [ InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = true }), InProcessNoEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), InProcessNoEmitToolchain.From(new() { ExecuteOnSeparateThread = true }), Job.Default.GetToolchain() - ]; + ]); // InProcessNoEmitToolchain does not support custom async enumerables or [AsyncCallerType]. - public static TheoryData GetCustomSupportedToolchains() => + public static TheoryData GetCustomSupportedToolchains() => new( [ InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = true }), Job.Default.GetToolchain() - ]; + ]); [Theory] [MemberData(nameof(GetAllToolchains), DisableDiscoveryEnumeration = true)] diff --git a/tests/BenchmarkDotNet.IntegrationTests/CallerThreadTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CallerThreadTests.cs index 3130471618..77155524cf 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CallerThreadTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CallerThreadTests.cs @@ -19,12 +19,12 @@ namespace BenchmarkDotNet.IntegrationTests; public class CallerThreadTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) { - public static TheoryData GetToolchains() => + public static TheoryData GetToolchains() => new( [ InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), InProcessNoEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), Job.Default.GetToolchain() - ]; + ]); [Theory] [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] diff --git a/tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs index e8b054c200..6a824768bd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs @@ -73,12 +73,12 @@ public class FooBench public void Foo() => Thread.Sleep(10); } - public static TheoryData GetToolchains() => + public static TheoryData GetToolchains() => new( [ InProcessEmitToolchain.Default, InProcessNoEmitToolchain.Default, Job.Default.GetToolchain() - ]; + ]); // https://github.com/dotnet/BenchmarkDotNet/issues/1120 [TheoryEnvSpecific(EnvRequirement.NonGitHubDraftPR)] diff --git a/tests/BenchmarkDotNet.IntegrationTests/RunAsyncTests.cs b/tests/BenchmarkDotNet.IntegrationTests/RunAsyncTests.cs index 89e943f4ae..3b618974d6 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/RunAsyncTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/RunAsyncTests.cs @@ -9,14 +9,14 @@ namespace BenchmarkDotNet.IntegrationTests; public class RunAsyncTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) { - public static TheoryData GetToolchains() => + public static TheoryData GetToolchains() => new( [ InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = true }), InProcessNoEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), InProcessNoEmitToolchain.From(new() { ExecuteOnSeparateThread = true }), Job.Default.GetToolchain() - ]; + ]); [Theory] [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] From 47058e3ca00dad51dd73e1b840f85fe15990f325 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sun, 29 Mar 2026 03:52:23 +0900 Subject: [PATCH 05/49] chore: add global.json to use Microsoft.Testing.Platform by dotnet test --- global.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 global.json diff --git a/global.json b/global.json new file mode 100644 index 0000000000..3140116df3 --- /dev/null +++ b/global.json @@ -0,0 +1,5 @@ +{ + "test": { + "runner": "Microsoft.Testing.Platform" + } +} From bb05d4691906fb14759eebcec45914b3357289e7 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sun, 29 Mar 2026 11:39:03 +0900 Subject: [PATCH 06/49] chore: add tests/.editorconfig --- BenchmarkDotNet.slnx | 1 + tests/.editorconfig | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 tests/.editorconfig diff --git a/BenchmarkDotNet.slnx b/BenchmarkDotNet.slnx index f9557849db..d1eb909584 100644 --- a/BenchmarkDotNet.slnx +++ b/BenchmarkDotNet.slnx @@ -29,6 +29,7 @@ + diff --git a/tests/.editorconfig b/tests/.editorconfig new file mode 100644 index 0000000000..64a64c01c8 --- /dev/null +++ b/tests/.editorconfig @@ -0,0 +1,7 @@ +root = false + +# C# files +[*.cs] +# xUnit1051: Calls to methods which accept CancellationToken should use TestContext.Current.CancellationToken to allow test cancellation to be more responsive. +dotnet_diagnostic.xUnit1051.severity = suggestion + From e6a627d81474beb5c8f64ec85da3bc3ba822de20 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sun, 29 Mar 2026 19:51:06 +0900 Subject: [PATCH 07/49] chore: fix custom inlinedata attribute --- .../Shared/XUnit/InlineDataEnvSpecific.cs | 40 ++++++++++++++++--- .../XUnit/InlineDataEnvSpecificDiscoverer.cs | 33 --------------- 2 files changed, 34 insertions(+), 39 deletions(-) delete mode 100644 tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecificDiscoverer.cs diff --git a/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecific.cs b/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecific.cs index 79a720b6b1..8f9419f5c0 100644 --- a/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecific.cs +++ b/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecific.cs @@ -1,25 +1,53 @@ using System.Reflection; +using Xunit; using Xunit.Sdk; using Xunit.v3; namespace BenchmarkDotNet.Tests.XUnit; -[DataDiscoverer("BenchmarkDotNet.Tests.XUnit.InlineDataEnvSpecificDiscoverer", "BenchmarkDotNet.IntegrationTests")] [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] public class InlineDataEnvSpecific : DataAttribute { - readonly object[] data; + /// + /// Gets the data to be passed to the test. + /// + // If the user passes null to the constructor, we assume what they meant was a + // single null value to be passed to the test. + public object?[] Data { get; } - public InlineDataEnvSpecific(object data, string reason, params EnvRequirement[] requirements) + public InlineDataEnvSpecific(object? data, string reason, params EnvRequirement[] requirements) : this([data], reason, requirements) { } - public InlineDataEnvSpecific(object[] data, string reason, params EnvRequirement[] requirements) + public InlineDataEnvSpecific(object?[] data, string reason, params EnvRequirement[] requirements) { - this.data = data; + // If the user passes null to the constructor, we assume what they meant was a + // single null value to be passed to the test. + Data = data ?? [null]; string? skip = EnvRequirementChecker.GetSkip(requirements); if (skip != null) Skip = $"{skip} ({reason})"; } - public override IEnumerable GetData(MethodInfo testMethod) => [data]; + public override ValueTask> GetData( + MethodInfo testMethod, + DisposalTracker disposalTracker) + { + var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); + TestIntrospectionHelper.MergeTraitsInto(traits, Traits); + + return new([ + new TheoryDataRow(Data) + { + Explicit = ExplicitAsNullable, + Label = Label, + Skip = Skip, + TestDisplayName = TestDisplayName, + Timeout = TimeoutAsNullable, + Traits = traits, + } + ]); + } + + public override bool SupportsDiscoveryEnumeration() + => true; } diff --git a/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecificDiscoverer.cs b/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecificDiscoverer.cs deleted file mode 100644 index a9a73f20a3..0000000000 --- a/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecificDiscoverer.cs +++ /dev/null @@ -1,33 +0,0 @@ -using Xunit; -using Xunit.Sdk; - -namespace BenchmarkDotNet.Tests.XUnit; - -public class InlineDataEnvSpecificDiscoverer : IDataDiscoverer -{ - public IEnumerable GetData(IAttributeInfo dataAttribute, IMethodInfo testMethod) - { - // InlineDataEnvSpecific has two constructors: - // 1. InlineDataEnvSpecific(object data, string reason, params EnvRequirement[] requirements) - // 2. InlineDataEnvSpecific(object[] data, string reason, params EnvRequirement[] requirements) - // GetConstructorArguments returns arguments from the constructor that was actually called - - var args = dataAttribute.GetConstructorArguments().ToArray(); - if (args.Length == 0) - yield break; - - // First argument is either a single object or object[] - wrap accordingly - if (args[0] is object[] dataArray) - { - // Array constructor was used - yield return dataArray; - } - else - { - // Single object constructor was used - wrap in array - yield return new[] { args[0] }; - } - } - - public bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) => true; -} From 67750123148208c62fb547c190661b2e1c7dbaec Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 30 Mar 2026 05:40:06 +0900 Subject: [PATCH 08/49] chore: fix wasm related tests chore: fix wasm related tests --- tests/Directory.Build.targets | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/Directory.Build.targets b/tests/Directory.Build.targets index 100d75c087..c08869ed02 100644 --- a/tests/Directory.Build.targets +++ b/tests/Directory.Build.targets @@ -1,4 +1,14 @@ + + + + + + + From d6b8ec01ca92caedc219efba6b3603299827e239 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 30 Mar 2026 14:04:09 +0900 Subject: [PATCH 09/49] chore: suppress MSB3277 warnings --- .../BenchmarkDotNet.IntegrationTests.ManualRunning.csproj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj index db77746136..7641efa817 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj @@ -3,6 +3,9 @@ BenchmarkDotNet.IntegrationTests.ManualRunning net472;net10.0 + + $(MSBuildWarningsAsMessages);MSB3277 + Exe true BenchmarkDotNet.IntegrationTests.ManualRunning BenchmarkDotNet.IntegrationTests.ManualRunning From 9778e4683864916c6b91368c4e505304d9bb260e Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 30 Mar 2026 13:35:29 +0900 Subject: [PATCH 10/49] chore: migrate multipleframeworks tests to tunit --- ...ts.ManualRunning.MultipleFrameworks.csproj | 10 +- .../MultipleFrameworksTest.cs | 17 ++-- .../Properties/AssemblyInfo.cs | 5 + .../ModuleInitializerAttribute.cs | 11 +++ .../Shared/TUnit/BenchmarkTestExecutor.cs | 93 +++++++++++++++++++ .../Shared/TUnit/OutputLogger.cs | 39 ++++++++ ...tNet.IntegrationTests.ManualRunning.csproj | 8 +- .../AllSetupAndCleanupTest.cs | 2 +- .../Shared/BenchmarkTestExecutor.cs | 1 - .../Shared/Extensions.cs | 2 +- .../MisconfiguredEnvironmentException.cs | 2 +- .../Shared/{XUnit => }/SmartAssert.cs | 2 +- 12 files changed, 168 insertions(+), 24 deletions(-) create mode 100644 tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Properties/AssemblyInfo.cs create mode 100644 tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Shared/System.Runtime.CompilerServices/ModuleInitializerAttribute.cs create mode 100644 tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Shared/TUnit/BenchmarkTestExecutor.cs create mode 100644 tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Shared/TUnit/OutputLogger.cs rename tests/BenchmarkDotNet.Tests/Shared/{XUnit => }/SmartAssert.cs (96%) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj index 1378b38bb9..17e9fc606d 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj @@ -17,12 +17,10 @@ true - - - - - - + + + + diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs index bdc47e5e9c..eb4f587658 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs @@ -2,6 +2,7 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Tests; namespace BenchmarkDotNet.IntegrationTests.ManualRunning { @@ -9,15 +10,11 @@ public class MultipleFrameworksTest : BenchmarkTestExecutor { private const string TfmEnvVarName = "TfmEnvVarName"; - public MultipleFrameworksTest(ITestOutputHelper output) : base(output) - { - } - - [Theory] - [InlineData(RuntimeMoniker.Net462)] - [InlineData(RuntimeMoniker.Net48)] - [InlineData(RuntimeMoniker.Net80)] - [InlineData(RuntimeMoniker.Net10_0)] + [Test] + [TUnit.Core.Arguments(RuntimeMoniker.Net462)] + [TUnit.Core.Arguments(RuntimeMoniker.Net48)] + [TUnit.Core.Arguments(RuntimeMoniker.Net80)] + [TUnit.Core.Arguments(RuntimeMoniker.Net10_0)] public void EachFrameworkIsRebuilt(string runtime) { var config = ManualConfig.CreateEmpty().AddJob(Job.Dry.WithRuntime(Runtime.Parse(runtime)).WithEnvironmentVariable(TfmEnvVarName, runtime)); @@ -42,7 +39,7 @@ public class ValuePerTfm [Benchmark] public void ThrowWhenWrong() { - if (Environment.GetEnvironmentVariable(TfmEnvVarName) != moniker) + if (Environment.GetEnvironmentVariable(TfmEnvVarName) != moniker.ToString()) { throw new InvalidOperationException($"Has not been recompiled, the value was {moniker}, expected {Environment.GetEnvironmentVariable(TfmEnvVarName)}"); } diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Properties/AssemblyInfo.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..84dd4b071d --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Properties/AssemblyInfo.cs @@ -0,0 +1,5 @@ +using System.Runtime.InteropServices; + +[assembly: Guid("f8203913-4b95-4fd9-b640-08d58dd563e2")] + +[assembly: NotInParallel] diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Shared/System.Runtime.CompilerServices/ModuleInitializerAttribute.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Shared/System.Runtime.CompilerServices/ModuleInitializerAttribute.cs new file mode 100644 index 0000000000..874ad6d08a --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Shared/System.Runtime.CompilerServices/ModuleInitializerAttribute.cs @@ -0,0 +1,11 @@ +#if NETFRAMEWORK +namespace System.Runtime.CompilerServices; + +[AttributeUsage(AttributeTargets.Method, Inherited = false)] +public sealed class ModuleInitializerAttribute : Attribute +{ + public ModuleInitializerAttribute() + { + } +} +#endif diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Shared/TUnit/BenchmarkTestExecutor.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Shared/TUnit/BenchmarkTestExecutor.cs new file mode 100644 index 0000000000..a4a367b256 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Shared/TUnit/BenchmarkTestExecutor.cs @@ -0,0 +1,93 @@ +using AwesomeAssertions; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.IntegrationTests; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using TUnit.Core.Interfaces; + +namespace BenchmarkDotNet.Tests; + +public class BenchmarkTestExecutor +{ + protected ITestOutput Output => TestContext.Current!.Output!; + + /// + /// Runs Benchmarks with the most simple config (SingleRunFastConfig) + /// combined with any benchmark config applied to TBenchmark (via an attribute) + /// By default will verify if every benchmark was successfully executed + /// + /// type that defines Benchmarks + /// Optional custom config to be used instead of the default + /// Optional: disable validation (default = true/enabled) + /// The summary from the benchmark run + public Reports.Summary CanExecute(IConfig? config = null, bool fullValidation = true) + { + return CanExecute(typeof(TBenchmark), config, fullValidation); + } + + /// + /// Runs Benchmarks with the most simple config (SingleRunFastConfig) + /// combined with any benchmark config applied to Type (via an attribute) + /// By default will verify if every benchmark was successfully executed + /// + /// type that defines Benchmarks + /// Optional custom config to be used instead of the default + /// Optional: disable validation (default = true/enabled) + /// The summary from the benchmark run + public Reports.Summary CanExecute(Type type, IConfig? config = null, bool fullValidation = true) + { + // Add logging, so the Benchmark execution is in the TestRunner output (makes Debugging easier) + if (config == null) + config = CreateSimpleConfig(); + + if (!config.GetLoggers().OfType().Any()) + config = config.AddLogger(Output != null ? new OutputLogger(Output) : ConsoleLogger.Default); + if (!config.GetLoggers().OfType().Any()) + config = config.AddLogger(ConsoleLogger.Default); + + if (!config.GetColumnProviders().Any()) + config = config.AddColumnProvider(DefaultColumnProviders.Instance); + + // Make sure we ALWAYS combine the Config (default or passed in) with any Config applied to the Type/Class + var summary = BenchmarkRunner.Run(type, config); + + if (fullValidation) + { + summary.HasCriticalValidationErrors.Should().BeFalse("The \"Summary\" should have NOT \"HasCriticalValidationErrors\""); + summary.Reports.Any().Should().BeTrue("The \"Summary\" should contain at least one \"BenchmarkReport\" in the \"Reports\" collection"); + + summary.CheckPlatformLinkerIssues(); + + summary.Reports.Should().OnlyContain(r => r.BuildResult.IsBuildSuccess, + "The following benchmarks have failed to build: " + + string.Join(", ", summary.Reports.Where(r => !r.BuildResult.IsBuildSuccess).Select(r => r.BenchmarkCase.DisplayInfo))); + + summary.Reports.Should().OnlyContain(r => r.ExecuteResults != null, + "The following benchmarks have failed to execute: " + + string.Join(", ", summary.Reports.Where(r => r.ExecuteResults == null || r.ExecuteResults.Any(er => !er.IsSuccess)).Select(r => r.BenchmarkCase.DisplayInfo))); ; + + summary.Reports.SelectMany(x => x.ExecuteResults).Should().OnlyContain(r => r.IsSuccess, + "All reports should have succeeded to execute"); + } + + return summary; + } + + protected IConfig CreateSimpleConfig(OutputLogger? logger = null, Job? job = null) + { + var baseConfig = job == null ? (IConfig)new SingleRunFastConfig() : new SingleJobConfig(job); + return baseConfig + .AddLogger(logger ?? (Output != null ? new OutputLogger(Output) : ConsoleLogger.Default)) + .AddColumnProvider(DefaultColumnProviders.Instance) + .AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()); + } + + protected static IReadOnlyList GetSingleStandardOutput(Summary summary) + => summary.Reports.Single().ExecuteResults.Single().StandardOutput; + + protected static IReadOnlyList GetCombinedStandardOutput(Summary summary) + => summary.Reports.SelectMany(r => r.ExecuteResults).SelectMany(e => e.StandardOutput).ToArray(); +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Shared/TUnit/OutputLogger.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Shared/TUnit/OutputLogger.cs new file mode 100644 index 0000000000..5350d2d9f9 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Shared/TUnit/OutputLogger.cs @@ -0,0 +1,39 @@ +using BenchmarkDotNet.Loggers; +using System.Runtime.CompilerServices; +using TUnit.Core.Interfaces; + +namespace BenchmarkDotNet.Tests; + +public class OutputLogger : AccumulationLogger +{ + private readonly ITestOutput Output; + private string currentLine = ""; + + public OutputLogger(ITestOutput output) + { + Output = output ?? throw new ArgumentNullException(nameof(Output)); + } + + [MethodImpl(MethodImplOptions.Synchronized)] + public override void Write(LogKind logKind, string text) + { + currentLine += text; + base.Write(logKind, text); + } + + [MethodImpl(MethodImplOptions.Synchronized)] + public override void WriteLine() + { + Output.WriteLine(currentLine); + currentLine = ""; + base.WriteLine(); + } + + [MethodImpl(MethodImplOptions.Synchronized)] + public override void WriteLine(LogKind logKind, string text) + { + Output.WriteLine(currentLine + text); + currentLine = ""; + base.WriteLine(logKind, text); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj index 7641efa817..eea825a7d2 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj @@ -28,9 +28,11 @@ - - - + + + + + diff --git a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs index 54934cd6a2..94965c50f6 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs @@ -2,7 +2,7 @@ using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Tests.XUnit; +using BenchmarkDotNet.Tests; namespace BenchmarkDotNet.IntegrationTests { diff --git a/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs b/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs index 795127c130..b608fdc1ec 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs @@ -1,6 +1,5 @@ using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; -using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; diff --git a/tests/BenchmarkDotNet.IntegrationTests/Shared/Extensions.cs b/tests/BenchmarkDotNet.IntegrationTests/Shared/Extensions.cs index e6f27f54eb..bbee466722 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/Shared/Extensions.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/Shared/Extensions.cs @@ -1,6 +1,6 @@ using BenchmarkDotNet.Reports; -namespace BenchmarkDotNet.IntegrationTests.Xunit +namespace BenchmarkDotNet.IntegrationTests { public static class Extensions { diff --git a/tests/BenchmarkDotNet.IntegrationTests/Shared/MisconfiguredEnvironmentException.cs b/tests/BenchmarkDotNet.IntegrationTests/Shared/MisconfiguredEnvironmentException.cs index 8f5d236d9f..c397c9e339 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/Shared/MisconfiguredEnvironmentException.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/Shared/MisconfiguredEnvironmentException.cs @@ -1,4 +1,4 @@ -namespace BenchmarkDotNet.IntegrationTests.Xunit +namespace BenchmarkDotNet.IntegrationTests { public class MisconfiguredEnvironmentException : Exception { diff --git a/tests/BenchmarkDotNet.Tests/Shared/XUnit/SmartAssert.cs b/tests/BenchmarkDotNet.Tests/Shared/SmartAssert.cs similarity index 96% rename from tests/BenchmarkDotNet.Tests/Shared/XUnit/SmartAssert.cs rename to tests/BenchmarkDotNet.Tests/Shared/SmartAssert.cs index 3cbe6b3f82..c1cad4e86a 100644 --- a/tests/BenchmarkDotNet.Tests/Shared/XUnit/SmartAssert.cs +++ b/tests/BenchmarkDotNet.Tests/Shared/SmartAssert.cs @@ -1,6 +1,6 @@ using System.Text; -namespace BenchmarkDotNet.Tests.XUnit +namespace BenchmarkDotNet.Tests { public static class SmartAssert { From d66316f9b9739000e6677a8e886f8059fbf7306b Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 30 Mar 2026 14:27:35 +0900 Subject: [PATCH 11/49] chore: update build project to support mtp --- build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs index f06f9b3132..2e6e2bbbd5 100644 --- a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs @@ -2,7 +2,9 @@ using Cake.Common.Diagnostics; using Cake.Common.Tools.DotNet; using Cake.Common.Tools.DotNet.Test; +using Cake.Core; using Cake.Core.IO; +using System.Linq; using System.Runtime.InteropServices; namespace BenchmarkDotNet.Build.Runners; @@ -40,7 +42,6 @@ private DotNetTestSettings GetTestSettingsParameters(FilePath logFile, string tf Framework = tfm, NoBuild = true, NoRestore = true, - Loggers = new[] { "trx", $"trx;LogFileName={logFile.FullPath}", "console;verbosity=detailed" }, EnvironmentVariables = { ["Platform"] = "" // force the tool to not look for the .dll in platform-specific directory From 53ba00e680419fd1b9815a304edcef8a5047cfbf Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Thu, 2 Apr 2026 13:03:05 +0900 Subject: [PATCH 12/49] chore: fix run-tests-selected workflow --- .github/workflows/run-tests-selected.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests-selected.yaml b/.github/workflows/run-tests-selected.yaml index 0e2f436810..688df34ea4 100644 --- a/.github/workflows/run-tests-selected.yaml +++ b/.github/workflows/run-tests-selected.yaml @@ -89,12 +89,13 @@ jobs: run: | $PSNativeCommandUseErrorActionPreference = $true $iterationCount = ${{ inputs.iteration_count }} + + Write-Host ("OSArchitecture" + [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture) foreach($i in 1..$iterationCount) { Write-Output ('##[group]Executing Iteration: {0}/${{ inputs.iteration_count }}' -f $i) - dotnet test -c Release --framework ${{ inputs.framework }} --filter ${{ inputs.filter }} -tl:off --no-build --logger "console;verbosity=normal" - + dotnet test -c Release --framework ${{ inputs.framework }} --no-build --filter "${{ inputs.filter }}" --no-ansi Write-Output '##[endgroup]' } From 43b1a5ac4ec573b17f7d51a0c4f8f989964ddeb1 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Fri, 3 Apr 2026 15:02:10 +0900 Subject: [PATCH 13/49] chore: add PreferNativeArm64 setting to Directory.Build.targets --- tests/Directory.Build.targets | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/Directory.Build.targets b/tests/Directory.Build.targets index c08869ed02..f4df154447 100644 --- a/tests/Directory.Build.targets +++ b/tests/Directory.Build.targets @@ -1,4 +1,9 @@ + + + true + + Date: Mon, 6 Apr 2026 15:14:17 +0900 Subject: [PATCH 14/49] chore: add diagnostic logs and collect as artifacts --- .github/workflows/run-tests.yaml | 16 ++++++++++++---- .../Runners/UnitTestRunner.cs | 12 ++++++------ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index b3ea72e473..39be091cc2 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -57,7 +57,9 @@ jobs: if: always() with: name: test-windows-core-trx-${{ github.run_id }}-${{ matrix.os }} - path: "**/*.trx" + path: | + **/*.trx + **/log_*.diag test-windows-full: strategy: @@ -100,7 +102,9 @@ jobs: if: always() with: name: test-windows-full-trx-${{ github.run_id }}-${{ matrix.os }} - path: "**/*.trx" + path: | + **/*.trx + **/log_*.diag test-linux: strategy: @@ -136,7 +140,9 @@ jobs: if: always() with: name: test-linux-trx-${{ github.run_id }}-${{ matrix.os }} - path: "**/*.trx" + path: | + **/*.trx + **/log_*.diag test-macos: name: test-macos (${{ matrix.os.arch }}) @@ -177,7 +183,9 @@ jobs: if: always() with: name: test-macos(${{ matrix.os.arch }})-trx-${{ github.run_id }} - path: "**/*.trx" + path: | + **/*.trx + **/log_*.diag test-pack: runs-on: ubuntu-latest diff --git a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs index 2e6e2bbbd5..c7c5caa1d8 100644 --- a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs @@ -49,12 +49,12 @@ private DotNetTestSettings GetTestSettingsParameters(FilePath logFile, string tf PathType = DotNetTestPathType.Auto, ArgumentCustomization = args => args.Append("--report-xunit-trx") - .AppendSwitchQuoted("--report-xunit-trx-filename", System.IO.Path.GetFileName(logFile.FullPath)) - .Append("--no-ansi") - .AppendSwitch("--output", "Detailed") - .AppendSwitch("--show-stdout", "Failed") - .Append("--diagnostic") - .AppendSwitch("--diagnostic-verbosity", "Trace") + .AppendSwitchQuoted("--report-xunit-trx-filename", System.IO.Path.GetFileName(logFile.FullPath)) + .Append("--no-ansi") + .AppendSwitch("--output", "Detailed") + .AppendSwitch("--show-stdout", "Failed") + .Append("--diagnostic") + .AppendSwitch("--diagnostic-verbosity", "Trace") }; return settings; } From 8db42ead605e59037f1aa3e345a6eeb767dbdb71 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 6 Apr 2026 15:41:00 +0900 Subject: [PATCH 15/49] chore: update generate-coverage-report.yaml --- .../workflows/generate-coverage-report.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/generate-coverage-report.yaml b/.github/workflows/generate-coverage-report.yaml index 9656efd86e..c689cb4ce3 100644 --- a/.github/workflows/generate-coverage-report.yaml +++ b/.github/workflows/generate-coverage-report.yaml @@ -8,7 +8,7 @@ on: type: boolean description: Set `true` to skip integration tests. default: true - + concurrency: group: ${{ github.workflow }}-${{ github.event.inputs.skip_integration_tests }}-${{ github.head_ref || github.ref || github.run_id }} cancel-in-progress: true @@ -49,13 +49,13 @@ jobs: - name: Collect Code Coverage run: | - dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Tests -c Release --no-build --framework net10.0" - dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Analyzers.Tests -c Release --no-build --framework net10.0" + dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Tests -c Release --no-build --framework net10.0" + dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Analyzers.Tests -c Release --no-build --framework net10.0" - name: Collect Code Coverage for BenchmarkDotNet.IntegrationTests if: ${{ github.event.inputs.skip_integration_tests == 'false'}} run: | - dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net10.0" + dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net10.0" - name: Shutdown dotnet-coverage server. run: dotnet coverage shutdown bdn_coverage --timeout 60000 @@ -77,7 +77,7 @@ jobs: - name: Setup additional tools for Wasm/NativeAot tests if: ${{ github.event.inputs.skip_integration_tests == 'false'}} uses: ./.github/actions/setup-additional-tools - + - name: Install dotnet-coverage run: dotnet tool install --global dotnet-coverage @@ -89,13 +89,13 @@ jobs: - name: Collect Code Coverage run: | - dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Tests -c Release --no-build --framework net472" - dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Analyzers.Tests -c Release --no-build --framework net472" + dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Tests -c Release --no-build --framework net472" + dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Analyzers.Tests -c Release --no-build --framework net472" - - name: Collect Code Coverage for BenchmarkDotNet.IntegrationTests - if: ${{ github.event.inputs.skip_integration_tests == 'false'}} + - name: Collect Code Coverage for BenchmarkDotNet.IntegrationTests + if: ${{ github.event.inputs.skip_integration_tests == 'false'}} run: | - dotnet coverage connect bdn_coverage 'dotnet test tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net472' + dotnet coverage connect bdn_coverage 'dotnet test --project tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net472 --filter-query "/*/*/(!*DotMemoryTests)&(!*DotTraceTests)/(!*WasmIsSupported)&(!*WasmSupportsInProcessDiagnosers)"' - name: Shutdown dotnet-coverage server. run: dotnet coverage shutdown bdn_coverage --timeout 60000 From 54f990bb21224bcdbd2a393e7b22048cf4d22416 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 6 Apr 2026 10:37:56 +0900 Subject: [PATCH 16/49] chore: temporary skip disassemble tests for macos --- .../DisassemblyDiagnoserTests.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs index b6a9ac1129..42d5623e5a 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs @@ -97,6 +97,9 @@ public void Recursive() [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, IToolchain toolchain) { + if (OsDetector.IsMacOS()) + Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/3076"); + var disassemblyDiagnoser = new DisassemblyDiagnoser( new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); @@ -117,6 +120,9 @@ public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, IToolchain [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, IToolchain toolchain) { + if (OsDetector.IsMacOS()) + Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/3076"); + var disassemblyDiagnoser = new DisassemblyDiagnoser( new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 1, filters: ["*WithCalls*"])); @@ -143,6 +149,9 @@ public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleGenericTypes(Jit jit, Platform platform, IToolchain toolchain) { + if (OsDetector.IsMacOS()) + Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/3076"); + var disassemblyDiagnoser = new DisassemblyDiagnoser( new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); @@ -164,6 +173,9 @@ [Benchmark] public void JustReturn() { } [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleInlinableBenchmarks(Jit jit, Platform platform, IToolchain toolchain) { + if (OsDetector.IsMacOS()) + Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/3076"); + var disassemblyDiagnoser = new DisassemblyDiagnoser( new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); From d5a08d4b450e7175082f0390013a052a00e6a7fe Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Thu, 9 Apr 2026 09:49:54 +0900 Subject: [PATCH 17/49] chore: add isinprocess condition to skip tests --- .../DisassemblyDiagnoserTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs index 42d5623e5a..efaa266d74 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs @@ -97,7 +97,7 @@ public void Recursive() [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, IToolchain toolchain) { - if (OsDetector.IsMacOS()) + if (OsDetector.IsMacOS() && toolchain.IsInProcess) Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/3076"); var disassemblyDiagnoser = new DisassemblyDiagnoser( From 4676b5143645d1a7038ab6c6bee4f694df247743 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 1 Jun 2026 18:50:15 +0900 Subject: [PATCH 18/49] chore: add consolecapture setting and parallel setting --- .../Properties/AssemblyInfo.cs | 5 +++++ .../Properties/AssemblyInfo.cs | 5 +++++ .../Properties/AssemblyInfo.cs | 6 ++++++ .../Properties/AssemblyInfo.cs | 3 ++- tests/BenchmarkDotNet.Tests/Properties/AssemblyInfo.cs | 2 ++ 5 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 tests/BenchmarkDotNet.Analyzers.Tests/Properties/AssemblyInfo.cs create mode 100644 tests/BenchmarkDotNet.Exporters.Plotting.Tests/Properties/AssemblyInfo.cs create mode 100644 tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Properties/AssemblyInfo.cs diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/Properties/AssemblyInfo.cs b/tests/BenchmarkDotNet.Analyzers.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..22131f407f --- /dev/null +++ b/tests/BenchmarkDotNet.Analyzers.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,5 @@ +using System.Runtime.InteropServices; + +[assembly: Guid("dfaa2493-0fd1-4d10-83b1-6014693da3d7")] + +[assembly: CaptureConsole] diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/Properties/AssemblyInfo.cs b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..24d36fdb3a --- /dev/null +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,5 @@ +using System.Runtime.InteropServices; + +[assembly: Guid("bde280f4-7829-4095-8385-a73f5f7acdc3")] + +[assembly: CaptureConsole] diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Properties/AssemblyInfo.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..8c41d34c8b --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Properties/AssemblyInfo.cs @@ -0,0 +1,6 @@ +using System.Runtime.InteropServices; + +[assembly: Guid("4d7c0994-2750-45dc-bbe8-750b158ea826")] + +[assembly: CaptureConsole] +[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly, DisableTestParallelization = true)] diff --git a/tests/BenchmarkDotNet.IntegrationTests/Properties/AssemblyInfo.cs b/tests/BenchmarkDotNet.IntegrationTests/Properties/AssemblyInfo.cs index b1e42e7c4c..32cfe6b1c7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/Properties/AssemblyInfo.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/Properties/AssemblyInfo.cs @@ -2,4 +2,5 @@ [assembly: Guid("74362bb1-9f64-4be5-b079-b4ac19dae5db")] -[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly, DisableTestParallelization = true)] \ No newline at end of file +[assembly: CaptureConsole] +[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly, DisableTestParallelization = true)] diff --git a/tests/BenchmarkDotNet.Tests/Properties/AssemblyInfo.cs b/tests/BenchmarkDotNet.Tests/Properties/AssemblyInfo.cs index 6d531ab196..7f3f80e5ef 100644 --- a/tests/BenchmarkDotNet.Tests/Properties/AssemblyInfo.cs +++ b/tests/BenchmarkDotNet.Tests/Properties/AssemblyInfo.cs @@ -1,3 +1,5 @@ using System.Runtime.InteropServices; [assembly: Guid("16c47abf-43e0-4db4-8151-36ca7a4082ae")] + +[assembly: CaptureConsole] From 30f82d0e2c6488a1caa87ccf4ce791ebc8f6844f Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Wed, 3 Jun 2026 03:53:06 +0900 Subject: [PATCH 19/49] chore: modify test to use dotnet run for simple output --- build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs index c7c5caa1d8..ada4459bbe 100644 --- a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs @@ -1,7 +1,7 @@ using BenchmarkDotNet.Build.Helpers; using Cake.Common.Diagnostics; using Cake.Common.Tools.DotNet; -using Cake.Common.Tools.DotNet.Test; +using Cake.Common.Tools.DotNet.Run; using Cake.Core; using Cake.Core.IO; using System.Linq; @@ -34,9 +34,9 @@ public class UnitTestRunner(BuildContext context) private DirectoryPath TestOutputDirectory { get; } = context.RootDirectory .Combine("TestResults"); - private DotNetTestSettings GetTestSettingsParameters(FilePath logFile, string tfm) + private DotNetRunSettings GetTestSettingsParameters(FilePath logFile, string tfm) { - var settings = new DotNetTestSettings + var settings = new DotNetRunSettings { Configuration = context.BuildConfiguration, Framework = tfm, @@ -46,7 +46,6 @@ private DotNetTestSettings GetTestSettingsParameters(FilePath logFile, string tf { ["Platform"] = "" // force the tool to not look for the .dll in platform-specific directory }, - PathType = DotNetTestPathType.Auto, ArgumentCustomization = args => args.Append("--report-xunit-trx") .AppendSwitchQuoted("--report-xunit-trx-filename", System.IO.Path.GetFileName(logFile.FullPath)) @@ -68,7 +67,7 @@ private void RunTests(FilePath projectFile, string alias, string tfm) var settings = GetTestSettingsParameters(trxFile, tfm); context.Information($"Run tests for {projectFile} ({tfm}), result file: '{trxFile}'"); - context.DotNetTest(projectFile.FullPath, settings); + context.DotNetRun(projectFile.FullPath, settings); } private void RunUnitTests(string tfm) From 7608656a04bce37b14ae71d8c330ff39e624ec5f Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Wed, 3 Jun 2026 09:20:45 +0900 Subject: [PATCH 20/49] chore: set default diag log level to warning and add --no-launchprofile option --- build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs index ada4459bbe..ce19395da7 100644 --- a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs @@ -36,6 +36,12 @@ public class UnitTestRunner(BuildContext context) private DotNetRunSettings GetTestSettingsParameters(FilePath logFile, string tfm) { + // Enabled `Trace` level logging when debug logging is enabled on GitHub Actions. + // https://docs.github.com/en/actions/how-tos/monitor-workflows/enable-debug-logging + var diagnosticVerbosity = context.EnvironmentVariable("ACTIONS_STEP_DEBUG") == "true" + ? "Trace" + : "Warning"; // Logging Warning/Error/Critical level logs by default. + var settings = new DotNetRunSettings { Configuration = context.BuildConfiguration, @@ -53,7 +59,8 @@ private DotNetRunSettings GetTestSettingsParameters(FilePath logFile, string tfm .AppendSwitch("--output", "Detailed") .AppendSwitch("--show-stdout", "Failed") .Append("--diagnostic") - .AppendSwitch("--diagnostic-verbosity", "Trace") + .AppendSwitch("--diagnostic-verbosity", diagnosticVerbosity) + .Append("--no-launch-profile"), }; return settings; } From 2e223a347e8841421fa1da5f16c86acac9a00553 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sun, 19 Jul 2026 17:25:53 +0900 Subject: [PATCH 21/49] chore: update parallel settings that introduced on xunit v3 4.0.0-pre.154 --- .../Properties/AssemblyInfo.cs | 5 ++++- .../Properties/AssemblyInfo.cs | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Properties/AssemblyInfo.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Properties/AssemblyInfo.cs index 8c41d34c8b..bf52573154 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Properties/AssemblyInfo.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Properties/AssemblyInfo.cs @@ -1,6 +1,9 @@ using System.Runtime.InteropServices; +using Xunit.Sdk; +using Xunit.v3; [assembly: Guid("4d7c0994-2750-45dc-bbe8-750b158ea826")] [assembly: CaptureConsole] -[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly, DisableTestParallelization = true)] +[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly)] +[assembly: Parallelization(Mode = ParallelMode.None)] diff --git a/tests/BenchmarkDotNet.IntegrationTests/Properties/AssemblyInfo.cs b/tests/BenchmarkDotNet.IntegrationTests/Properties/AssemblyInfo.cs index 32cfe6b1c7..e9f7b3c195 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/Properties/AssemblyInfo.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/Properties/AssemblyInfo.cs @@ -1,6 +1,9 @@ using System.Runtime.InteropServices; +using Xunit.Sdk; +using Xunit.v3; [assembly: Guid("74362bb1-9f64-4be5-b079-b4ac19dae5db")] [assembly: CaptureConsole] -[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly, DisableTestParallelization = true)] +[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly)] +[assembly: Parallelization(Mode = ParallelMode.None)] From 08ef532ac4887fb41babff503ee92ebe1ba95595 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sun, 19 Jul 2026 18:36:23 +0900 Subject: [PATCH 22/49] chore: fix build error on build project --- build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs index ce19395da7..0aca17fca1 100644 --- a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs @@ -38,7 +38,7 @@ private DotNetRunSettings GetTestSettingsParameters(FilePath logFile, string tfm { // Enabled `Trace` level logging when debug logging is enabled on GitHub Actions. // https://docs.github.com/en/actions/how-tos/monitor-workflows/enable-debug-logging - var diagnosticVerbosity = context.EnvironmentVariable("ACTIONS_STEP_DEBUG") == "true" + var diagnosticVerbosity = context.Environment.GetEnvironmentVariable("ACTIONS_STEP_DEBUG") == "true" ? "Trace" : "Warning"; // Logging Warning/Error/Critical level logs by default. From 890a59497ce7a2e0d45284a6eb81ae7b3c91479c Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sun, 19 Jul 2026 21:41:48 +0900 Subject: [PATCH 23/49] chore: remove ConsoleLogger to avoid dupicated log lines if [assembly:CaptureOutput] is enabled --- .../Shared/BenchmarkTestExecutor.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs b/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs index b608fdc1ec..e562516c90 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs @@ -76,8 +76,6 @@ private IConfig GetMinimalConfig(IConfig? config) if (!config.GetLoggers().OfType().Any()) config = config.AddLogger(Output != null ? new OutputLogger(Output) : ConsoleLogger.Default); - if (!config.GetLoggers().OfType().Any()) - config = config.AddLogger(ConsoleLogger.Default); if (!config.GetColumnProviders().Any()) config = config.AddColumnProvider(DefaultColumnProviders.Instance); From 355e704005e8f47efe80165463a3a52c6ad0aecc Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sun, 19 Jul 2026 17:26:21 +0900 Subject: [PATCH 24/49] chore: add launchsettings.json per test projects --- .../Properties/launchSettings.json | 20 ++++++++++++++++ .../Properties/launchSettings.json | 24 +++++++++++++++++++ .../Properties/launchSettings.json | 12 ++++++++++ .../Properties/launchSettings.json | 24 +++++++++++++++++++ .../Properties/launchSettings.json | 24 +++++++++++++++++++ .../Properties/launchSettings.json | 24 +++++++++++++++++++ 6 files changed, 128 insertions(+) create mode 100644 tests/BenchmarkDotNet.Analyzers.Tests/Properties/launchSettings.json create mode 100644 tests/BenchmarkDotNet.Exporters.Plotting.Tests/Properties/launchSettings.json create mode 100644 tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Properties/launchSettings.json create mode 100644 tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Properties/launchSettings.json create mode 100644 tests/BenchmarkDotNet.IntegrationTests/Properties/launchSettings.json create mode 100644 tests/BenchmarkDotNet.Tests/Properties/launchSettings.json diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/Properties/launchSettings.json b/tests/BenchmarkDotNet.Analyzers.Tests/Properties/launchSettings.json new file mode 100644 index 0000000000..88b9648915 --- /dev/null +++ b/tests/BenchmarkDotNet.Analyzers.Tests/Properties/launchSettings.json @@ -0,0 +1,20 @@ +{ + "profiles": { + "Default": { + "commandName": "Project", + "commandLineArgs": "" + }, + "--list-test json": { + "commandName": "Project", + "commandLineArgs": "--list-test json" + }, + "--info": { + "commandName": "Project", + "commandLineArgs": "--info" + }, + "--help": { + "commandName": "Project", + "commandLineArgs": "--help" + } + } +} diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/Properties/launchSettings.json b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/Properties/launchSettings.json new file mode 100644 index 0000000000..6b34abfe26 --- /dev/null +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/Properties/launchSettings.json @@ -0,0 +1,24 @@ +{ + "profiles": { + "Default": { + "commandName": "Project", + "commandLineArgs": "" + }, + "Run filtered tests": { + "commandName": "Project", + "commandLineArgs": "--filter ScottPlotExporterTests --output Detailed" + }, + "--list-test json": { + "commandName": "Project", + "commandLineArgs": "--list-test json" + }, + "--info": { + "commandName": "Project", + "commandLineArgs": "--info" + }, + "--help": { + "commandName": "Project", + "commandLineArgs": "--help" + } + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Properties/launchSettings.json b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Properties/launchSettings.json new file mode 100644 index 0000000000..7563185165 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "profiles": { + "Default": { + "commandName": "Project", + "commandLineArgs": "" + }, + "--help": { + "commandName": "Project", + "commandLineArgs": "--help" + } + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Properties/launchSettings.json b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Properties/launchSettings.json new file mode 100644 index 0000000000..c3c6453192 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Properties/launchSettings.json @@ -0,0 +1,24 @@ +{ + "profiles": { + "Default": { + "commandName": "Project", + "commandLineArgs": "" + }, + "Run filtered tests": { + "commandName": "Project", + "commandLineArgs": "--filter DotMemoryTests --output Detailed" + }, + "--list-test json": { + "commandName": "Project", + "commandLineArgs": "--list-test json" + }, + "--info": { + "commandName": "Project", + "commandLineArgs": "--info" + }, + "--help": { + "commandName": "Project", + "commandLineArgs": "--help" + } + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/Properties/launchSettings.json b/tests/BenchmarkDotNet.IntegrationTests/Properties/launchSettings.json new file mode 100644 index 0000000000..6f8f831200 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/Properties/launchSettings.json @@ -0,0 +1,24 @@ +{ + "profiles": { + "Default": { + "commandName": "Project", + "commandLineArgs": "" + }, + "Run filtered tests": { + "commandName": "Project", + "commandLineArgs": "--filter AppConfigTests --output Detailed" + }, + "--list-test json": { + "commandName": "Project", + "commandLineArgs": "--list-test json" + }, + "--info": { + "commandName": "Project", + "commandLineArgs": "--info" + }, + "--help": { + "commandName": "Project", + "commandLineArgs": "--help" + } + } +} diff --git a/tests/BenchmarkDotNet.Tests/Properties/launchSettings.json b/tests/BenchmarkDotNet.Tests/Properties/launchSettings.json new file mode 100644 index 0000000000..70ba352178 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Properties/launchSettings.json @@ -0,0 +1,24 @@ +{ + "profiles": { + "Default": { + "commandName": "Project", + "commandLineArgs": "" + }, + "Run filtered tests": { + "commandName": "Project", + "commandLineArgs": "--filter ConfigParserTests --output Detailed" + }, + "--list-test json": { + "commandName": "Project", + "commandLineArgs": "--list-test json" + }, + "--info": { + "commandName": "Project", + "commandLineArgs": "--info" + }, + "--help": { + "commandName": "Project", + "commandLineArgs": "--help" + } + } +} From e3398890dfc25ebb3347d71ca0722b5c282da296 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sun, 19 Jul 2026 23:59:44 +0900 Subject: [PATCH 25/49] chore: fix ci error with line ending differences --- .gitattributes | 4 ++++ src/BenchmarkDotNet/Exporters/HtmlExporter.cs | 8 ++++---- tests/.editorconfig | 4 ++++ .../Exporters/CommonExporterVerifyTests.cs | 4 +++- ...onExporterVerifyTests.Exporters_Invariant.verified.txt | 8 ++++---- ...CommonExporterVerifyTests.Exporters_en-US.verified.txt | 8 ++++---- ...CommonExporterVerifyTests.Exporters_ru-RU.verified.txt | 8 ++++---- tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs | 2 +- 8 files changed, 28 insertions(+), 18 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..367bc806eb --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Verify snapshot files must keep LF line endings on every platform. +# Verify.XunitV3 rejects a verified file that contains a CR with core.autocrlf. +*.verified.* text eol=lf +*.received.* text eol=lf diff --git a/src/BenchmarkDotNet/Exporters/HtmlExporter.cs b/src/BenchmarkDotNet/Exporters/HtmlExporter.cs index c335ac75a0..b4eca64101 100644 --- a/src/BenchmarkDotNet/Exporters/HtmlExporter.cs +++ b/src/BenchmarkDotNet/Exporters/HtmlExporter.cs @@ -9,10 +9,10 @@ public class HtmlExporter : ExporterBase { private const string CssDefinition = @" "; protected override string FileExtension => "html"; diff --git a/tests/.editorconfig b/tests/.editorconfig index 64a64c01c8..a0bc3cface 100644 --- a/tests/.editorconfig +++ b/tests/.editorconfig @@ -5,3 +5,7 @@ root = false # xUnit1051: Calls to methods which accept CancellationToken should use TestContext.Current.CancellationToken to allow test cancellation to be more responsive. dotnet_diagnostic.xUnit1051.severity = suggestion +# Verify settings +[*.{received,verified}.{txt}] +charset = utf-8-bom +trim_trailing_whitespace = false diff --git a/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs b/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs index 2b2c446eb0..b102fa339d 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs +++ b/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs @@ -58,9 +58,11 @@ [new Metric(new FakeMetricDescriptor("CacheMisses", "Hardware counter 'CacheMiss logger, CancellationToken.None); } + var result = logger.GetLog().Replace("\r\n", "\n"); + var settings = VerifyHelper.Create(); settings.UseTextForParameters(GetName(cultureInfo)); - await Verifier.Verify(logger.GetLog(), settings); + await Verifier.Verify(result, settings); } private static void PrintTitle(AccumulationLogger logger, IExporter exporter) diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt index 7b0a36970f..08a5fe76dc 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt @@ -27,10 +27,10 @@ HtmlExporter MockSummary diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt index 7b0a36970f..08a5fe76dc 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt @@ -27,10 +27,10 @@ HtmlExporter MockSummary diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_ru-RU.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_ru-RU.verified.txt index 7b0a36970f..08a5fe76dc 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_ru-RU.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_ru-RU.verified.txt @@ -27,10 +27,10 @@ HtmlExporter MockSummary diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs b/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs index c33564d125..975dda77fc 100644 --- a/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs +++ b/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs @@ -24,7 +24,7 @@ public Task PerfonarTableTest(string key) { var table = TableDataMap[key]; string markdown = table.ToMarkdown(new PerfonarTableStyle()); - string json = LightJsonSerializer.Serialize(table.RootEntry, new LightJsonSettings { Indent = true }); + string json = LightJsonSerializer.Serialize(table.RootEntry, new LightJsonSettings { Indent = true }).Replace("\r\n","\n"); return VerifyString(key, markdown + "\n" + json); } From 6e02dcf0ae238d9065814a7512eaf0262adf559e Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 20 Jul 2026 01:34:53 +0900 Subject: [PATCH 26/49] chore: add setting to skip test when no test data --- .../MemoryDiagnoserTests.cs | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 5371dc8dd1..c351c8dbe3 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -53,7 +53,8 @@ public class AccurateAllocations [Benchmark] public Task AllocateTask() => Task.FromResult(-12345); } - [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + [Theory(SkipTestWithoutData = true)] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurate(IToolchain toolchain) { @@ -141,7 +142,8 @@ private void AllocateUntilGcWakesUp() } } - [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + [Theory(SkipTestWithoutData = true)] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup(IToolchain toolchain) { @@ -156,7 +158,8 @@ public class EmptyBenchmark [Benchmark] public void EmptyMethod() { } } - [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + [Theory(SkipTestWithoutData = true)] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void EngineShouldNotInterfereAllocationResults(IToolchain toolchain) { @@ -181,7 +184,8 @@ public ulong TimeConsuming() } // #1542 - [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + [Theory(SkipTestWithoutData = true)] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void TieredJitShouldNotInterfereAllocationResults(IToolchain toolchain) { @@ -197,7 +201,8 @@ public class NoBoxing [Benchmark] public ValueTuple ReturnsValueType() => new ValueTuple(0); } - [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + [Theory(SkipTestWithoutData = true)] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void EngineShouldNotIntroduceBoxing(IToolchain toolchain) { @@ -218,7 +223,8 @@ public class NonAllocatingAsynchronousBenchmarks [Benchmark] public ValueTask CompletedValueTaskOfT() => new ValueTask(default(int)); } - [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + [Theory(SkipTestWithoutData = true)] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AwaitingTasksShouldNotInterfereAllocationResults(IToolchain toolchain) { @@ -243,7 +249,8 @@ public void WithOperationsPerInvoke() private void DoNotInline(object left, object right) { } } - [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + [Theory(SkipTestWithoutData = true)] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocatedMemoryShouldBeScaledForOperationsPerInvoke(IToolchain toolchain) { @@ -269,7 +276,7 @@ public byte[] SixtyFourBytesArray() } } - [TheoryEnvSpecific("Full Framework cannot measure precisely enough for low invocation counts.", EnvRequirement.DotNetCoreOnly)] + [TheoryEnvSpecific("Full Framework cannot measure precisely enough for low invocation counts.", EnvRequirement.DotNetCoreOnly, SkipTestWithoutData = true)] [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) @@ -339,7 +346,7 @@ public void Allocate() } } - [TheoryEnvSpecific("Full Framework cannot measure precisely enough", EnvRequirement.DotNetCoreOnly)] + [TheoryEnvSpecific("Full Framework cannot measure precisely enough", EnvRequirement.DotNetCoreOnly, SkipTestWithoutData = true)] [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolchain) From ad6d28ccf085c20b8ff9bdbc805f8ee056d8fad8 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 20 Jul 2026 04:00:16 +0900 Subject: [PATCH 27/49] chore: disable mtp and xunit entrypoint generation on benchmark project --- tests/Directory.Build.targets | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/Directory.Build.targets b/tests/Directory.Build.targets index f4df154447..e282f8c222 100644 --- a/tests/Directory.Build.targets +++ b/tests/Directory.Build.targets @@ -3,7 +3,13 @@ true - + + + + false + false + + Date: Mon, 20 Jul 2026 21:35:32 +0900 Subject: [PATCH 28/49] chore: add testconfig.json --- tests/BenchmarkDotNet.Analyzers.Tests/testconfig.json | 10 ++++++++++ .../testconfig.json | 10 ++++++++++ .../testconfig.json | 10 ++++++++++ .../testconfig.json | 10 ++++++++++ .../MemoryDiagnoserTests.cs | 1 + tests/BenchmarkDotNet.IntegrationTests/testconfig.json | 10 ++++++++++ tests/BenchmarkDotNet.Tests/testconfig.json | 10 ++++++++++ 7 files changed, 61 insertions(+) create mode 100644 tests/BenchmarkDotNet.Analyzers.Tests/testconfig.json create mode 100644 tests/BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json create mode 100644 tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/testconfig.json create mode 100644 tests/BenchmarkDotNet.IntegrationTests.ManualRunning/testconfig.json create mode 100644 tests/BenchmarkDotNet.IntegrationTests/testconfig.json create mode 100644 tests/BenchmarkDotNet.Tests/testconfig.json diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/testconfig.json b/tests/BenchmarkDotNet.Analyzers.Tests/testconfig.json new file mode 100644 index 0000000000..ca4d9057fa --- /dev/null +++ b/tests/BenchmarkDotNet.Analyzers.Tests/testconfig.json @@ -0,0 +1,10 @@ +{ + "platformOptions": { + }, + "environmentVariables": { + "TESTINGPLATFORM_TELEMETRY_OPTOUT": "true", + "TESTINGPLATFORM_UI_LANGUAGE": "en-us" + }, + "xUnit": { + } +} diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json new file mode 100644 index 0000000000..ca4d9057fa --- /dev/null +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json @@ -0,0 +1,10 @@ +{ + "platformOptions": { + }, + "environmentVariables": { + "TESTINGPLATFORM_TELEMETRY_OPTOUT": "true", + "TESTINGPLATFORM_UI_LANGUAGE": "en-us" + }, + "xUnit": { + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/testconfig.json b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/testconfig.json new file mode 100644 index 0000000000..ca4d9057fa --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/testconfig.json @@ -0,0 +1,10 @@ +{ + "platformOptions": { + }, + "environmentVariables": { + "TESTINGPLATFORM_TELEMETRY_OPTOUT": "true", + "TESTINGPLATFORM_UI_LANGUAGE": "en-us" + }, + "xUnit": { + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/testconfig.json b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/testconfig.json new file mode 100644 index 0000000000..ca4d9057fa --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/testconfig.json @@ -0,0 +1,10 @@ +{ + "platformOptions": { + }, + "environmentVariables": { + "TESTINGPLATFORM_TELEMETRY_OPTOUT": "true", + "TESTINGPLATFORM_UI_LANGUAGE": "en-us" + }, + "xUnit": { + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index c351c8dbe3..cbb9146ddd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -11,6 +11,7 @@ using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; diff --git a/tests/BenchmarkDotNet.IntegrationTests/testconfig.json b/tests/BenchmarkDotNet.IntegrationTests/testconfig.json new file mode 100644 index 0000000000..ca4d9057fa --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/testconfig.json @@ -0,0 +1,10 @@ +{ + "platformOptions": { + }, + "environmentVariables": { + "TESTINGPLATFORM_TELEMETRY_OPTOUT": "true", + "TESTINGPLATFORM_UI_LANGUAGE": "en-us" + }, + "xUnit": { + } +} diff --git a/tests/BenchmarkDotNet.Tests/testconfig.json b/tests/BenchmarkDotNet.Tests/testconfig.json new file mode 100644 index 0000000000..ca4d9057fa --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/testconfig.json @@ -0,0 +1,10 @@ +{ + "platformOptions": { + }, + "environmentVariables": { + "TESTINGPLATFORM_TELEMETRY_OPTOUT": "true", + "TESTINGPLATFORM_UI_LANGUAGE": "en-us" + }, + "xUnit": { + } +} From d42f983814d02a464572978355b9307444336bac Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Thu, 23 Jul 2026 00:50:30 +0900 Subject: [PATCH 29/49] chore: add telemetry/progress disable verification for some tests --- .../MemoryDiagnoserTests.cs | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index cbb9146ddd..115bcca923 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -11,7 +11,6 @@ using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; @@ -24,6 +23,7 @@ using BenchmarkDotNet.Validators; using System.Reflection; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace BenchmarkDotNet.IntegrationTests { @@ -190,11 +190,34 @@ public ulong TimeConsuming() [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void TieredJitShouldNotInterfereAllocationResults(IToolchain toolchain) { + if (toolchain.IsInProcess) + ValidateMtpProgressDisabled(); + AssertAllocations(toolchain, typeof(TimeConsumingBenchmark), new Dictionary { { nameof(TimeConsumingBenchmark.TimeConsuming), 0 } }, iterationCount: 10); // 1 iteration is not enough to repro the problem + + static void ValidateMtpProgressDisabled() + { + if (!OsDetector.IsWindows() || RuntimeInformation.OSArchitecture != Architecture.Arm64) + return; + + // On Windows(arm64) environment following test failed with extra memory allocations (1 or 2 bytes) + // TieredJitShouldNotInterfereAllocationResults + var args = Environment.GetCommandLineArgs(); + for (int i = 0; i < args.Length; i++) + { + if (args[i] == "--progress=off") + return; + + if (i < args.Length - 1 && args[i] == "--progress" && args[i + 1] == "off") + return; + } + + throw new NotSupportedException("Measure memory allocation requires setting `--progress off`"); + } } public class NoBoxing @@ -286,12 +309,32 @@ public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.OSX)) return; + if (toolchain.IsInProcess) + ValidateTelemetryOptOut(); + long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length AssertAllocations(toolchain, typeof(TimeConsuming), new Dictionary { { nameof(TimeConsuming.SixtyFourBytesArray), 64 + objectAllocationOverhead + arraySizeOverhead } }); + + static void ValidateTelemetryOptOut() + { + if (!OsDetector.IsWindows()) + return; + + // When MTP telemetry feature is enabled, extra allocation (792 bytes) occurred randomly on Windows. + var optout = Environment.GetEnvironmentVariable("TESTINGPLATFORM_TELEMETRY_OPTOUT"); + switch (optout) + { + case "1": + case "true": + break; + default: + throw new NotSupportedException("Measure memory allocation requires setting `TESTINGPLATFORM_TELEMETRY_OPTOUT=true`"); + } + } } public class MultiThreadedAllocation From 67d5cefbd99ad8ff24c0c1f24f614b94c5d8cad3 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Thu, 23 Jul 2026 00:51:59 +0900 Subject: [PATCH 30/49] chore: disable progress on CI execution to avoid extra memory allocations --- .github/workflows/run-tests-selected.yaml | 2 +- build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-tests-selected.yaml b/.github/workflows/run-tests-selected.yaml index 688df34ea4..fac15e347c 100644 --- a/.github/workflows/run-tests-selected.yaml +++ b/.github/workflows/run-tests-selected.yaml @@ -95,7 +95,7 @@ jobs: foreach($i in 1..$iterationCount) { Write-Output ('##[group]Executing Iteration: {0}/${{ inputs.iteration_count }}' -f $i) - dotnet test -c Release --framework ${{ inputs.framework }} --no-build --filter "${{ inputs.filter }}" --no-ansi + dotnet test -c Release --framework ${{ inputs.framework }} --no-build --filter "${{ inputs.filter }}" --no-ansi --progress off --output Detailed Write-Output '##[endgroup]' } diff --git a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs index 0aca17fca1..07aff4737a 100644 --- a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs @@ -56,6 +56,7 @@ private DotNetRunSettings GetTestSettingsParameters(FilePath logFile, string tfm => args.Append("--report-xunit-trx") .AppendSwitchQuoted("--report-xunit-trx-filename", System.IO.Path.GetFileName(logFile.FullPath)) .Append("--no-ansi") + .AppendSwitch("--progress", "off") .AppendSwitch("--output", "Detailed") .AppendSwitch("--show-stdout", "Failed") .Append("--diagnostic") From c84eb072802e6b2549e58650477949f05042351d Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Thu, 23 Jul 2026 03:41:40 +0900 Subject: [PATCH 31/49] chore: remove unused test filter and add parameters --- .github/workflows/generate-coverage-report.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/generate-coverage-report.yaml b/.github/workflows/generate-coverage-report.yaml index c689cb4ce3..40a43ea345 100644 --- a/.github/workflows/generate-coverage-report.yaml +++ b/.github/workflows/generate-coverage-report.yaml @@ -49,13 +49,13 @@ jobs: - name: Collect Code Coverage run: | - dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Tests -c Release --no-build --framework net10.0" - dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Analyzers.Tests -c Release --no-build --framework net10.0" + dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Tests -c Release --no-build --framework net10.0 --no-ansi --output Detailed" + dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Analyzers.Tests -c Release --no-build --framework net10.0 --no-ansi --output Detailed" - name: Collect Code Coverage for BenchmarkDotNet.IntegrationTests if: ${{ github.event.inputs.skip_integration_tests == 'false'}} run: | - dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net10.0" + dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net10.0 --no-ansi --output Detailed" - name: Shutdown dotnet-coverage server. run: dotnet coverage shutdown bdn_coverage --timeout 60000 @@ -89,13 +89,13 @@ jobs: - name: Collect Code Coverage run: | - dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Tests -c Release --no-build --framework net472" - dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Analyzers.Tests -c Release --no-build --framework net472" + dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Tests -c Release --no-build --framework net472 --no-ansi --output Detailed" + dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Analyzers.Tests -c Release --no-build --framework net472 --no-ansi --output Detailed" - name: Collect Code Coverage for BenchmarkDotNet.IntegrationTests if: ${{ github.event.inputs.skip_integration_tests == 'false'}} run: | - dotnet coverage connect bdn_coverage 'dotnet test --project tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net472 --filter-query "/*/*/(!*DotMemoryTests)&(!*DotTraceTests)/(!*WasmIsSupported)&(!*WasmSupportsInProcessDiagnosers)"' + dotnet coverage connect bdn_coverage 'dotnet test --project tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net472 --no-ansi --output Detailed" - name: Shutdown dotnet-coverage server. run: dotnet coverage shutdown bdn_coverage --timeout 60000 From 56175a1288bdc61f24945d9745149d2e134ea999 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Fri, 24 Jul 2026 04:24:25 +0900 Subject: [PATCH 32/49] chore: add setting to suppress dotnet cli telemetry --- tests/BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json | 1 + .../testconfig.json | 1 + .../testconfig.json | 1 + tests/BenchmarkDotNet.IntegrationTests/testconfig.json | 1 + tests/BenchmarkDotNet.Tests/testconfig.json | 1 + 5 files changed, 5 insertions(+) diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json index ca4d9057fa..38f5be2e55 100644 --- a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json @@ -2,6 +2,7 @@ "platformOptions": { }, "environmentVariables": { + "DOTNET_CLI_TELEMETRY_OPTOUT": "true", "TESTINGPLATFORM_TELEMETRY_OPTOUT": "true", "TESTINGPLATFORM_UI_LANGUAGE": "en-us" }, diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/testconfig.json b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/testconfig.json index ca4d9057fa..38f5be2e55 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/testconfig.json +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/testconfig.json @@ -2,6 +2,7 @@ "platformOptions": { }, "environmentVariables": { + "DOTNET_CLI_TELEMETRY_OPTOUT": "true", "TESTINGPLATFORM_TELEMETRY_OPTOUT": "true", "TESTINGPLATFORM_UI_LANGUAGE": "en-us" }, diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/testconfig.json b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/testconfig.json index ca4d9057fa..38f5be2e55 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/testconfig.json +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/testconfig.json @@ -2,6 +2,7 @@ "platformOptions": { }, "environmentVariables": { + "DOTNET_CLI_TELEMETRY_OPTOUT": "true", "TESTINGPLATFORM_TELEMETRY_OPTOUT": "true", "TESTINGPLATFORM_UI_LANGUAGE": "en-us" }, diff --git a/tests/BenchmarkDotNet.IntegrationTests/testconfig.json b/tests/BenchmarkDotNet.IntegrationTests/testconfig.json index ca4d9057fa..38f5be2e55 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/testconfig.json +++ b/tests/BenchmarkDotNet.IntegrationTests/testconfig.json @@ -2,6 +2,7 @@ "platformOptions": { }, "environmentVariables": { + "DOTNET_CLI_TELEMETRY_OPTOUT": "true", "TESTINGPLATFORM_TELEMETRY_OPTOUT": "true", "TESTINGPLATFORM_UI_LANGUAGE": "en-us" }, diff --git a/tests/BenchmarkDotNet.Tests/testconfig.json b/tests/BenchmarkDotNet.Tests/testconfig.json index ca4d9057fa..38f5be2e55 100644 --- a/tests/BenchmarkDotNet.Tests/testconfig.json +++ b/tests/BenchmarkDotNet.Tests/testconfig.json @@ -2,6 +2,7 @@ "platformOptions": { }, "environmentVariables": { + "DOTNET_CLI_TELEMETRY_OPTOUT": "true", "TESTINGPLATFORM_TELEMETRY_OPTOUT": "true", "TESTINGPLATFORM_UI_LANGUAGE": "en-us" }, From e90e7564fa26846bc2f53cd952d4f9e0979393a0 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Fri, 24 Jul 2026 07:54:11 +0900 Subject: [PATCH 33/49] chore: add skip setting for flaky test --- .../MemoryDiagnoserTests.cs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 115bcca923..06b3d45b97 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -306,12 +306,18 @@ public byte[] SixtyFourBytesArray() public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) { // TODO: Skip test on macos. Temporary workaround for https://github.com/dotnet/BenchmarkDotNet/issues/2779 - if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.OSX)) - return; + if (OsDetector.IsMacOS()) + Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/2779"); - if (toolchain.IsInProcess) + if (OsDetector.IsWindows() && toolchain.IsInProcess) + { ValidateTelemetryOptOut(); + // Skip test on `windows(arm64)`. + if (RuntimeInformation.OSArchitecture == Architecture.Arm64 && Portability.RuntimeInformation.IsNetCore) + Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/2779"); + } + long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length AssertAllocations(toolchain, typeof(TimeConsuming), new Dictionary @@ -321,9 +327,6 @@ public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) static void ValidateTelemetryOptOut() { - if (!OsDetector.IsWindows()) - return; - // When MTP telemetry feature is enabled, extra allocation (792 bytes) occurred randomly on Windows. var optout = Environment.GetEnvironmentVariable("TESTINGPLATFORM_TELEMETRY_OPTOUT"); switch (optout) From c73d5bd009c41e7f8b17918aaf0dcf6800a62467 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Fri, 24 Jul 2026 09:46:12 +0900 Subject: [PATCH 34/49] chore: temporary disable longRunningTest detection background worker --- tests/BenchmarkDotNet.IntegrationTests/xunit.runner.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/xunit.runner.json b/tests/BenchmarkDotNet.IntegrationTests/xunit.runner.json index 53b385245e..8ff7a75dc8 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/xunit.runner.json +++ b/tests/BenchmarkDotNet.IntegrationTests/xunit.runner.json @@ -1,8 +1,8 @@ -{ +{ "shadowCopy": false, "methodDisplay": "method", "diagnosticMessages": true, - "longRunningTestSeconds": 60, + "longRunningTestSeconds": 0, "parallelizeAssembly": false, "parallelizeTestCollections": false } \ No newline at end of file From b4501372656a8b372485990da04f2351d6e5c1bd Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sat, 25 Jul 2026 08:33:07 +0900 Subject: [PATCH 35/49] chore: modify test condition for flaky tests --- .../MemoryDiagnoserTests.cs | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 06b3d45b97..9a2998b146 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -35,10 +35,7 @@ public class MemoryDiagnoserTests public static IEnumerable GetToolchains() { - // xunit v2 allocates every 100ms on a background timer that makes the tests flaky on Mac/Linux (MSTestEnableParentProcessQuery). - // TODO: remove the guard when the test framework is updated. - if (OsDetector.IsWindows()) - yield return [InProcessEmitToolchain.Default]; + yield return [InProcessEmitToolchain.Default]; if (ContinuousIntegration.IsGitHubDraftPR()) yield break; @@ -230,6 +227,12 @@ public class NoBoxing [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void EngineShouldNotIntroduceBoxing(IToolchain toolchain) { + if (toolchain.IsInProcess && OsDetector.IsWindows() && !Portability.RuntimeInformation.IsNetCore) + { + // Randomly failed on Windows(x64)+.NET Framework. + Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/pull/3065#issuecomment-5064984074"); + } + AssertAllocations(toolchain, typeof(NoBoxing), new Dictionary { { nameof(NoBoxing.ReturnsValueType), 0 } @@ -252,6 +255,12 @@ public class NonAllocatingAsynchronousBenchmarks [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AwaitingTasksShouldNotInterfereAllocationResults(IToolchain toolchain) { + if (toolchain.IsInProcess && OsDetector.IsWindows() && !Portability.RuntimeInformation.IsNetCore) + { + // Randomly failed on Windows(x64/arm64)+.NET Framework. + Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/3203"); + } + AssertAllocations(toolchain, typeof(NonAllocatingAsynchronousBenchmarks), new Dictionary { { nameof(NonAllocatingAsynchronousBenchmarks.CompletedTask), 0 }, @@ -305,15 +314,11 @@ public byte[] SixtyFourBytesArray() [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) { - // TODO: Skip test on macos. Temporary workaround for https://github.com/dotnet/BenchmarkDotNet/issues/2779 - if (OsDetector.IsMacOS()) - Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/2779"); - if (OsDetector.IsWindows() && toolchain.IsInProcess) { ValidateTelemetryOptOut(); - // Skip test on `windows(arm64)`. + // Skip test on `windows(arm64)` because extra memory allocation still occurred. if (RuntimeInformation.OSArchitecture == Architecture.Arm64 && Portability.RuntimeInformation.IsNetCore) Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/2779"); } From c94d8c399b401b2e5c588995c3a2688f9dad60ef Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sat, 25 Jul 2026 08:46:08 +0900 Subject: [PATCH 36/49] chore: remove temporary workaround setting for macos from ThreadingDiagnoserTests --- .../ThreadingDiagnoserTests.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs index 3812a0c034..ddab46b218 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs @@ -185,10 +185,6 @@ private void AssertStats(Summary summary, IToolchain toolchain, Dictionary Date: Sat, 25 Jul 2026 10:41:41 +0900 Subject: [PATCH 37/49] chore: remove validation logics because flaky test is reproduced on windows(x64) also --- .../MemoryDiagnoserTests.cs | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 9a2998b146..11f671758c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -315,13 +315,7 @@ public byte[] SixtyFourBytesArray() public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) { if (OsDetector.IsWindows() && toolchain.IsInProcess) - { - ValidateTelemetryOptOut(); - - // Skip test on `windows(arm64)` because extra memory allocation still occurred. - if (RuntimeInformation.OSArchitecture == Architecture.Arm64 && Portability.RuntimeInformation.IsNetCore) - Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/2779"); - } + Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/2779"); long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length @@ -329,20 +323,6 @@ public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) { { nameof(TimeConsuming.SixtyFourBytesArray), 64 + objectAllocationOverhead + arraySizeOverhead } }); - - static void ValidateTelemetryOptOut() - { - // When MTP telemetry feature is enabled, extra allocation (792 bytes) occurred randomly on Windows. - var optout = Environment.GetEnvironmentVariable("TESTINGPLATFORM_TELEMETRY_OPTOUT"); - switch (optout) - { - case "1": - case "true": - break; - default: - throw new NotSupportedException("Measure memory allocation requires setting `TESTINGPLATFORM_TELEMETRY_OPTOUT=true`"); - } - } } public class MultiThreadedAllocation From 431e2561bc1174706190b7beea8c43fad6dda713 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Wed, 29 Jul 2026 01:32:06 +0900 Subject: [PATCH 38/49] chore: update PerfonarTest based on latest update --- tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs b/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs index 975dda77fc..b42584734b 100644 --- a/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs +++ b/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs @@ -24,7 +24,7 @@ public Task PerfonarTableTest(string key) { var table = TableDataMap[key]; string markdown = table.ToMarkdown(new PerfonarTableStyle()); - string json = LightJsonSerializer.Serialize(table.RootEntry, new LightJsonSettings { Indent = true }).Replace("\r\n","\n"); + string json = LightJsonSerializer.Serialize(table.RootEntry, new LightJsonSettings { Indent = true, NewLine = "\n" }); return VerifyString(key, markdown + "\n" + json); } From 370765d53de1bdeb6f394b5b6d1a42043a9462ef Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:20:50 +0900 Subject: [PATCH 39/49] chore: pass cancellationtoken to benchmark executor --- .../BenchmarkSwitcherTest.cs | 38 +++++++++---------- .../EventProcessorTests.cs | 2 +- .../ExceptionDiagnoserTests.cs | 2 +- .../InProcessTest.cs | 2 +- .../LargeAddressAwareTest.cs | 4 +- .../MemoryDiagnoserTests.cs | 2 +- .../RunningEmptyBenchmarkTests.cs | 32 ++++++++-------- .../Shared/BenchmarkTestExecutor.cs | 4 +- .../ThreadingDiagnoserTests.cs | 4 +- .../Shared/Mocks/MockRunner.cs | 2 +- 10 files changed, 46 insertions(+), 46 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs index ff0fd75f22..c8b7644653 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs @@ -30,7 +30,7 @@ public void WhenInvalidCommandLineArgumentIsPassedAnErrorMessageIsDisplayedAndNo var summaries = BenchmarkSwitcher .FromTypes([]) - .Run(["--DOES_NOT_EXIST"], config); + .Run(["--DOES_NOT_EXIST"], config, TestContext.Current.CancellationToken); Assert.Empty(summaries); Assert.Contains("Option 'DOES_NOT_EXIST' is unknown.", logger.GetLog()); @@ -44,7 +44,7 @@ public void WhenUserAsksForInfoAnInfoIsDisplayedAndNoBenchmarksAreExecuted() var summaries = BenchmarkSwitcher .FromTypes([]) - .Run(["--info"], config); + .Run(["--info"], config, TestContext.Current.CancellationToken); Assert.Empty(summaries); Assert.Contains(HostEnvironmentInfo.GetInformation(), logger.GetLog()); @@ -58,7 +58,7 @@ public void WhenInvalidTypeIsProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAre var summaries = BenchmarkSwitcher .FromTypes([typeof(ClassC)]) - .Run(["--filter", "*"], config); + .Run(["--filter", "*"], config, TestContext.Current.CancellationToken); Assert.Empty(summaries); Assert.Contains(GetValidationErrorForType(typeof(ClassC)), logger.GetLog()); @@ -72,7 +72,7 @@ public void WhenNoTypesAreProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAreExe var summaries = BenchmarkSwitcher .FromTypes([]) - .Run(["--filter", "*"], config); + .Run(["--filter", "*"], config, TestContext.Current.CancellationToken); Assert.Empty(summaries); Assert.Contains("No benchmarks were found.", logger.GetLog()); @@ -87,7 +87,7 @@ public void WhenFilterReturnsNothingAnErrorMessageIsDisplayedAndNoBenchmarksAreE const string filter = "WRONG"; var summaries = BenchmarkSwitcher .FromTypes([typeof(ClassA), typeof(ClassB)]) - .Run(["--filter", filter], config); + .Run(["--filter", filter], config, TestContext.Current.CancellationToken); Assert.Empty(summaries); Assert.Contains($"The filter '{filter}' that you have provided returned 0 benchmarks.", logger.GetLog()); @@ -101,7 +101,7 @@ public void WhenUserAsksToPrintAListWePrintIt() var summaries = BenchmarkSwitcher .FromTypes([typeof(ClassA)]) - .Run(["--list", "flat"], config); + .Run(["--list", "flat"], config, TestContext.Current.CancellationToken); Assert.Empty(summaries); Assert.Contains("BenchmarkDotNet.IntegrationTests.ClassA.Method1", logger.GetLog()); @@ -135,7 +135,7 @@ public void WhenDisableLogFileWeDontWriteToFile() { var summaries = BenchmarkSwitcher .FromTypes([typeof(ClassA)]) - .RunAll(config); + .RunAll(config, cancellationToken: TestContext.Current.CancellationToken); var summary = summaries.Single(); logFilePath = summary.LogFilePath; @@ -161,7 +161,7 @@ public void EnsureLogFileIsWritten() { var summaries = BenchmarkSwitcher .FromTypes([typeof(ClassA)]) - .RunAll(config); + .RunAll(config, cancellationToken: TestContext.Current.CancellationToken); var summary = summaries.Single(); logFilePath = summary.LogFilePath; @@ -185,7 +185,7 @@ public void WhenUserDoesNotProvideFilterOrCategoriesViaCommandLineWeAskToChooseB var summaries = new BenchmarkSwitcher(userInteractionMock) .With([typeof(WithDryAttributeAndCategory)]) - .Run([], config); + .Run([], config, TestContext.Current.CancellationToken); Assert.Empty(summaries); // summaries is empty because the returnValue configured for mock returns 0 types Assert.Equal(1, userInteractionMock.AskUserCalledTimes); @@ -203,7 +203,7 @@ public void WhenUserProvidesCategoriesWithoutFiltersWeDontAskToChooseBenchmarkJu var summaries = new BenchmarkSwitcher(userInteractionMock) .With(types) - .Run([categoriesConsoleLineArgument, TestCategory], config); + .Run([categoriesConsoleLineArgument, TestCategory], config, TestContext.Current.CancellationToken); Assert.Single(summaries); Assert.Equal(0, userInteractionMock.AskUserCalledTimes); @@ -221,7 +221,7 @@ public void WhenUserProvidesCategoriesWithFiltersWeDontAskToChooseBenchmarkJustU var summaries = new BenchmarkSwitcher(userInteractionMock) .With(types) - .Run([categoriesConsoleLineArgument, TestCategory, "--filter", "nothing"], config); + .Run([categoriesConsoleLineArgument, TestCategory, "--filter", "nothing"], config, TestContext.Current.CancellationToken); Assert.Empty(summaries); // the summaries is empty because the provided filter returns nothing Assert.Equal(0, userInteractionMock.AskUserCalledTimes); @@ -239,7 +239,7 @@ public void ValidCommandLineArgumentsAreProperlyHandled() var switcher = new BenchmarkSwitcher(types); // BenchmarkSwitcher only picks up config values via the args passed in, not via class annotations (e.g "[DryConfig]") - var results = switcher.Run(["-j", "Dry", "--filter", "*ClassB.Method4"], config); + var results = switcher.Run(["-j", "Dry", "--filter", "*ClassB.Method4"], config, TestContext.Current.CancellationToken); Assert.Single(results); Assert.Single(results.SelectMany(r => r.BenchmarksCases)); Assert.Single(results.SelectMany(r => r.BenchmarksCases.Select(bc => bc.Job))); @@ -255,7 +255,7 @@ public void WhenJobIsDefinedInTheConfigAndArgumentsDontContainJobArgumentOnlySin MockExporter mockExporter = new MockExporter(); var configWithJobDefined = ManualConfig.CreateEmpty().AddExporter(mockExporter).AddJob(Job.Dry); - var results = switcher.Run(["--filter", "*Method3"], configWithJobDefined); + var results = switcher.Run(["--filter", "*Method3"], configWithJobDefined, TestContext.Current.CancellationToken); Assert.True(mockExporter.exported); @@ -273,7 +273,7 @@ public void WhenJobIsDefinedViaAttributeAndArgumentsDontContainJobArgumentOnlySi MockExporter mockExporter = new MockExporter(); var configWithoutJobDefined = ManualConfig.CreateEmpty().AddExporter(mockExporter); - var results = switcher.Run(["--filter", "*WithDryAttribute*"], configWithoutJobDefined); + var results = switcher.Run(["--filter", "*WithDryAttribute*"], configWithoutJobDefined, TestContext.Current.CancellationToken); Assert.True(mockExporter.exported); @@ -291,7 +291,7 @@ public void JobNotDefinedButStillBenchmarkIsExecuted() MockExporter mockExporter = new MockExporter(); var configWithoutJobDefined = ManualConfig.CreateEmpty().AddExporter(mockExporter); - var results = switcher.Run(["--filter", "*"], configWithoutJobDefined); + var results = switcher.Run(["--filter", "*"], configWithoutJobDefined, TestContext.Current.CancellationToken); Assert.True(mockExporter.exported); @@ -309,7 +309,7 @@ public void WhenUserCreatesStaticBenchmarkMethodWeDisplayAnError_FromTypes() var summariesForType = BenchmarkSwitcher .FromTypes([typeof(Static.BenchmarkClassWithStaticMethodsOnly)]) - .Run(["--filter", "*"], config); + .Run(["--filter", "*"], config, TestContext.Current.CancellationToken); Assert.True(summariesForType.Single().HasCriticalValidationErrors); Assert.Contains("static", logger.GetLog()); @@ -323,7 +323,7 @@ public void WhenUserCreatesStaticBenchmarkMethodWeDisplayAnError_FromAssembly() var summariesForAssembly = BenchmarkSwitcher .FromAssembly(typeof(Static.BenchmarkClassWithStaticMethodsOnly).Assembly) - .Run(["--filter", "*"], config); + .Run(["--filter", "*"], config, TestContext.Current.CancellationToken); Assert.True(summariesForAssembly.Single().HasCriticalValidationErrors); Assert.Contains("static", logger.GetLog()); @@ -339,9 +339,9 @@ public void WhenUserAddTheResumeAttributeAndRunTheBenchmarks() var switcher = new BenchmarkSwitcher(types); // the first run should execute all benchmarks - Assert.Single(switcher.Run(["--filter", "*WithDryAttributeAndCategory*"], config)); + Assert.Single(switcher.Run(["--filter", "*WithDryAttributeAndCategory*"], config, TestContext.Current.CancellationToken)); // resuming after succesfull run should run nothing - Assert.Empty(switcher.Run(["--resume", "--filter", "*WithDryAttributeAndCategory*"], config)); + Assert.Empty(switcher.Run(["--resume", "--filter", "*WithDryAttributeAndCategory*"], config, TestContext.Current.CancellationToken)); } private class UserInteractionMock : IUserInteraction diff --git a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs index d96c75886f..23b582dfa3 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs @@ -175,7 +175,7 @@ public void WhenUsingEventProcessorWithBuildFailures() .AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()); if (validator != null) config = config.AddValidator(validator); - _ = BenchmarkRunner.Run(types, config); + _ = BenchmarkRunner.Run(types, config, cancellationToken: TestContext.Current.CancellationToken); return eventProcessor.Events; } diff --git a/tests/BenchmarkDotNet.IntegrationTests/ExceptionDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ExceptionDiagnoserTests.cs index 766a049ce5..b889b19179 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ExceptionDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ExceptionDiagnoserTests.cs @@ -15,7 +15,7 @@ public void ExceptionCountIsAccurate() { var config = CreateConfig(); - var summary = BenchmarkRunner.Run(config); + var summary = BenchmarkRunner.Run(config, cancellationToken: TestContext.Current.CancellationToken); AssertStats(summary, new Dictionary { diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs index 69a73294b5..ca1070c508 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs @@ -82,7 +82,7 @@ public void BenchmarkDifferentPlatformReturnsValidationError() .AddColumnProvider(DefaultColumnProviders.Instance); var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkAllCases), otherPlatformConfig); - var summary = BenchmarkRunner.Run(runInfo); + var summary = BenchmarkRunner.Run(runInfo, cancellationToken: TestContext.Current.CancellationToken); Assert.NotEmpty(summary.ValidationErrors); } diff --git a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs index bb6a4e8f13..25ef518be9 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs @@ -27,7 +27,7 @@ public void BenchmarkCanAllocateMoreThan2Gb_Core() config.AddColumnProvider(DefaultColumnProviders.Instance) .AddLogger(new OutputLogger(output)); - var summary = BenchmarkRunner.Run(config); + var summary = BenchmarkRunner.Run(config, cancellationToken: TestContext.Current.CancellationToken); Assert.True(summary.Reports .All(report => report.ExecuteResults @@ -56,7 +56,7 @@ public void BenchmarkCanAllocateMoreThan2Gb_Framework() config.AddColumnProvider(DefaultColumnProviders.Instance) .AddLogger(new OutputLogger(output)); - var summary = BenchmarkRunner.Run(config); + var summary = BenchmarkRunner.Run(config, cancellationToken: TestContext.Current.CancellationToken); Assert.True(summary.Reports .All(report => report.ExecuteResults diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 11f671758c..517f8376a7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -398,7 +398,7 @@ private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Diction var config = CreateConfig(toolchain, iterationCount); var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config); - var summary = BenchmarkRunner.Run(benchmarks); + var summary = BenchmarkRunner.Run(benchmarks, cancellationToken: TestContext.Current.CancellationToken); try { summary.CheckPlatformLinkerIssues(); diff --git a/tests/BenchmarkDotNet.IntegrationTests/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.IntegrationTests/RunningEmptyBenchmarkTests.cs index b437ae35d5..02686df83a 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/RunningEmptyBenchmarkTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/RunningEmptyBenchmarkTests.cs @@ -46,7 +46,7 @@ public void GenericTypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBen { GetConfigWithLogger(out var logger, out var config); - var summary = BenchmarkRunner.Run(config, args); + var summary = BenchmarkRunner.Run(config, args, TestContext.Current.CancellationToken); if (args == null) { @@ -71,7 +71,7 @@ public void GenericTypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) { GetConfigWithLogger(out var logger, out var config); - var summary = BenchmarkRunner.Run(config, args); + var summary = BenchmarkRunner.Run(config, args, TestContext.Current.CancellationToken); Assert.False(summary.HasCriticalValidationErrors); Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); @@ -90,7 +90,7 @@ public void TypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkA { GetConfigWithLogger(out var logger, out var config); - var summary = BenchmarkRunner.Run(typeof(EmptyBenchmark), config, args); + var summary = BenchmarkRunner.Run(typeof(EmptyBenchmark), config, args, TestContext.Current.CancellationToken); if (args == null) { @@ -115,7 +115,7 @@ public void TypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) { GetConfigWithLogger(out var logger, out var config); - var summaries = BenchmarkRunner.Run(typeof(NotEmptyBenchmark), config, args); + var summaries = BenchmarkRunner.Run(typeof(NotEmptyBenchmark), config, args, TestContext.Current.CancellationToken); Assert.False(summaries.HasCriticalValidationErrors); Assert.DoesNotContain(summaries.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); @@ -131,7 +131,7 @@ public void TypesWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmark { GetConfigWithLogger(out var logger, out var config); - var summaries = BenchmarkRunner.Run([typeof(EmptyBenchmark), typeof(EmptyBenchmark2)], config, args); + var summaries = BenchmarkRunner.Run([typeof(EmptyBenchmark), typeof(EmptyBenchmark2)], config, args, TestContext.Current.CancellationToken); if (args != null) { Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); @@ -157,7 +157,7 @@ public void TypesWithBenchmarkAttribute_RunsSuccessfully(string[]? args) { GetConfigWithLogger(out var logger, out var config); - var summaries = BenchmarkRunner.Run([typeof(NotEmptyBenchmark)], config, args); + var summaries = BenchmarkRunner.Run([typeof(NotEmptyBenchmark)], config, args, TestContext.Current.CancellationToken); var summary = summaries[0]; Assert.False(summary.HasCriticalValidationErrors); Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); @@ -173,7 +173,7 @@ public void BenchmarkRunInfoWithoutBenchmarkAttribute_ThrowsValidationError_When { GetConfigWithLogger(out var logger, out var config); - var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), config)); + var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), config), TestContext.Current.CancellationToken); Assert.True(summary.HasCriticalValidationErrors); Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); @@ -184,7 +184,7 @@ public void BenchmarkRunInfoWithBenchmarkAttribute_RunsSuccessfully() { GetConfigWithLogger(out var logger, out var config); - var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config)); + var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config), TestContext.Current.CancellationToken); Assert.False(summary.HasCriticalValidationErrors); Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); @@ -201,7 +201,7 @@ public void BenchmarkRunInfosWithoutBenchmarkAttribute_ThrowsValidationError_Whe var summaries = BenchmarkRunner.Run([ BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), config), BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark2), config) - ]); + ], TestContext.Current.CancellationToken); var summary = summaries[0]; Assert.True(summary.HasCriticalValidationErrors); Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); @@ -215,7 +215,7 @@ public void BenchmarkRunInfosWithBenchmarkAttribute_RunsSuccessfully() { GetConfigWithLogger(out var logger, out var config); - var summaries = BenchmarkRunner.Run([BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config)]); + var summaries = BenchmarkRunner.Run([BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config)], TestContext.Current.CancellationToken); var summary = summaries[0]; Assert.False(summary.HasCriticalValidationErrors); Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); @@ -231,7 +231,7 @@ public void MixedTypes_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? { GetConfigWithLogger(out var logger, out var config); - var summaries = BenchmarkRunner.Run([typeof(EmptyBenchmark), typeof(NotEmptyBenchmark)], config, args); + var summaries = BenchmarkRunner.Run([typeof(EmptyBenchmark), typeof(NotEmptyBenchmark)], config, args, TestContext.Current.CancellationToken); if (args != null) { Assert.Contains(GetExpandedValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); @@ -266,12 +266,12 @@ public void AssemblyWithoutBenchmarks_ThrowsValidationError_WhenNoBenchmarksFoun if (args != null) { GetConfigWithLogger(out var logger, out var config); - var summaries = BenchmarkRunner.Run(assemblyBuilder, config, args); + var summaries = BenchmarkRunner.Run(assemblyBuilder, config, args, TestContext.Current.CancellationToken); Assert.Contains(GetAssemblylValidationError(assemblyBuilder), logger.GetLog()); } else { - var summaries = BenchmarkRunner.Run(assemblyBuilder, null, args); + var summaries = BenchmarkRunner.Run(assemblyBuilder, null, args, TestContext.Current.CancellationToken); var summary = summaries[0]; Assert.True(summary.HasCriticalValidationErrors); Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetGeneralValidationError()); @@ -312,12 +312,12 @@ public void AssemblyWithBenchmarks_RunsSuccessfully_WhenBenchmarkAttributePresen if (args != null) { GetConfigWithLogger(out var logger, out var config); - var summaries = BenchmarkRunner.Run(assemblyBuilder, config, args); + var summaries = BenchmarkRunner.Run(assemblyBuilder, config, args, cancellationToken: TestContext.Current.CancellationToken); Assert.DoesNotContain(GetAssemblylValidationError(assemblyBuilder), logger.GetLog()); } else { - var summaries = BenchmarkRunner.Run(assemblyBuilder); + var summaries = BenchmarkRunner.Run(assemblyBuilder, cancellationToken: TestContext.Current.CancellationToken); var summary = summaries[0]; Assert.False(summary.HasCriticalValidationErrors); Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetGeneralValidationError()); @@ -352,7 +352,7 @@ public void AssemblyWithBenchmarks_RunsNothing_WhenAllBenchmarksFilteredOutFromO config.AddFilter(new NameFilter(name => name != "Benchmark")); // Filter out only benchmark method on MockBenchmark - var summaries = BenchmarkRunner.Run(assemblyBuilder, config); + var summaries = BenchmarkRunner.Run(assemblyBuilder, config, cancellationToken: TestContext.Current.CancellationToken); Assert.DoesNotContain(GetValidationErrorForType(benchmarkTypeBuilder), logger.GetLog()); Assert.Contains(GetExporterNoBenchmarksError(), logger.GetLog()); } diff --git a/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs b/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs index e562516c90..63516ba74c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/Shared/BenchmarkTestExecutor.cs @@ -42,7 +42,7 @@ public Summary CanExecute(IConfig? config = null, bool fullValidatio public Summary CanExecute(Type type, IConfig? config = null, bool fullValidation = true) { // Make sure we ALWAYS combine the Config (default or passed in) with any Config applied to the Type/Class - var summary = BenchmarkRunner.Run(type, GetMinimalConfig(config)); + var summary = BenchmarkRunner.Run(type, GetMinimalConfig(config), cancellationToken: TestContext.Current.CancellationToken); if (fullValidation) { @@ -58,7 +58,7 @@ public ValueTask CanExecuteAsync(IConfig? config = null, bo public async ValueTask CanExecuteAsync(Type type, IConfig? config = null, bool fullValidation = true) { // Make sure we ALWAYS combine the Config (default or passed in) with any Config applied to the Type/Class - var summary = await BenchmarkRunner.RunAsync(type, GetMinimalConfig(config)); + var summary = await BenchmarkRunner.RunAsync(type, GetMinimalConfig(config), cancellationToken: TestContext.Current.CancellationToken); if (fullValidation) { diff --git a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs index ddab46b218..c567f8636b 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs @@ -45,7 +45,7 @@ public void CompletedWorkItemCountIsAccurate(IToolchain toolchain) { var config = CreateConfig(toolchain); - var summary = BenchmarkRunner.Run(config); + var summary = BenchmarkRunner.Run(config, cancellationToken: TestContext.Current.CancellationToken); try { summary.CheckPlatformLinkerIssues(); @@ -87,7 +87,7 @@ public void LockContentionCountIsAccurate(IToolchain toolchain) { var config = CreateConfig(toolchain); - var summary = BenchmarkRunner.Run(config); + var summary = BenchmarkRunner.Run(config, cancellationToken: TestContext.Current.CancellationToken); try { summary.CheckPlatformLinkerIssues(); diff --git a/tests/BenchmarkDotNet.Tests/Shared/Mocks/MockRunner.cs b/tests/BenchmarkDotNet.Tests/Shared/Mocks/MockRunner.cs index b5d9788ef2..a11c50eeac 100644 --- a/tests/BenchmarkDotNet.Tests/Shared/Mocks/MockRunner.cs +++ b/tests/BenchmarkDotNet.Tests/Shared/Mocks/MockRunner.cs @@ -33,7 +33,7 @@ public static Summary Run(ITestOutputHelper output, Func(config); + var summary = BenchmarkRunner.Run(config, cancellationToken: TestContext.Current.CancellationToken); var exporter = MarkdownExporter.Mock; ((ExporterBase)exporter).ExportToLogAsync(summary, logger, CancellationToken.None).AsTask().GetAwaiter().GetResult(); From 645343b883deb94fd27211e899e41348f438770d Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:30:01 +0900 Subject: [PATCH 40/49] chore: skip apphost verification on `browser-wasi` runtime also --- tests/Directory.Build.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Directory.Build.targets b/tests/Directory.Build.targets index e282f8c222..035273019a 100644 --- a/tests/Directory.Build.targets +++ b/tests/Directory.Build.targets @@ -15,7 +15,7 @@ + Condition=" '$(UseAppHost)' != 'true' AND '$(TargetFrameworkIdentifier)' == '.NETCoreApp' AND '$(RuntimeIdentifier)' != 'browser-wasm' AND '$(RuntimeIdentifier)' != 'browser-wasi'" /> From 7e6d596c7b908f2a203b6c3b5ffa5c3e6a47fb78 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sat, 15 Aug 2026 01:44:41 +0900 Subject: [PATCH 41/49] chore: add default cli options on testconfig.json --- tests/BenchmarkDotNet.Analyzers.Tests/testconfig.json | 4 ++++ .../BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json | 4 ++++ .../testconfig.json | 3 +++ .../testconfig.json | 4 ++++ tests/BenchmarkDotNet.IntegrationTests/testconfig.json | 4 ++++ tests/BenchmarkDotNet.Tests/testconfig.json | 4 ++++ 6 files changed, 23 insertions(+) diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/testconfig.json b/tests/BenchmarkDotNet.Analyzers.Tests/testconfig.json index ca4d9057fa..f04e503158 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/testconfig.json +++ b/tests/BenchmarkDotNet.Analyzers.Tests/testconfig.json @@ -1,4 +1,8 @@ { + "commandLineOptions": { + "report-xunit-html": true, + "report-xunit-trx": true + }, "platformOptions": { }, "environmentVariables": { diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json index 38f5be2e55..9d8a9fd932 100644 --- a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/testconfig.json @@ -1,4 +1,8 @@ { + "commandLineOptions": { + "report-xunit-html": true, + "report-xunit-trx": true + }, "platformOptions": { }, "environmentVariables": { diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/testconfig.json b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/testconfig.json index 38f5be2e55..18d3492f20 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/testconfig.json +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/testconfig.json @@ -1,4 +1,7 @@ { + "commandLineOptions": { + "report-trx": true + }, "platformOptions": { }, "environmentVariables": { diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/testconfig.json b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/testconfig.json index 38f5be2e55..9d8a9fd932 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/testconfig.json +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/testconfig.json @@ -1,4 +1,8 @@ { + "commandLineOptions": { + "report-xunit-html": true, + "report-xunit-trx": true + }, "platformOptions": { }, "environmentVariables": { diff --git a/tests/BenchmarkDotNet.IntegrationTests/testconfig.json b/tests/BenchmarkDotNet.IntegrationTests/testconfig.json index 38f5be2e55..9d8a9fd932 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/testconfig.json +++ b/tests/BenchmarkDotNet.IntegrationTests/testconfig.json @@ -1,4 +1,8 @@ { + "commandLineOptions": { + "report-xunit-html": true, + "report-xunit-trx": true + }, "platformOptions": { }, "environmentVariables": { diff --git a/tests/BenchmarkDotNet.Tests/testconfig.json b/tests/BenchmarkDotNet.Tests/testconfig.json index 38f5be2e55..9d8a9fd932 100644 --- a/tests/BenchmarkDotNet.Tests/testconfig.json +++ b/tests/BenchmarkDotNet.Tests/testconfig.json @@ -1,4 +1,8 @@ { + "commandLineOptions": { + "report-xunit-html": true, + "report-xunit-trx": true + }, "platformOptions": { }, "environmentVariables": { From 31e317e681c8daaf09ccf6024ff73900d85a51a3 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sat, 15 Aug 2026 15:56:27 +0900 Subject: [PATCH 42/49] chore: update test related package deps --- .../BenchmarkDotNet.Analyzers.Tests.csproj | 2 +- .../BenchmarkDotNet.Exporters.Plotting.Tests.csproj | 2 +- ...IntegrationTests.ManualRunning.MultipleFrameworks.csproj | 4 ++-- .../BenchmarkDotNet.IntegrationTests.ManualRunning.csproj | 2 +- .../BenchmarkDotNet.IntegrationTests.csproj | 4 ++-- tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj | 6 +++--- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/BenchmarkDotNet.Analyzers.Tests.csproj b/tests/BenchmarkDotNet.Analyzers.Tests/BenchmarkDotNet.Analyzers.Tests.csproj index 725fa7ff05..0efb0d8633 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/BenchmarkDotNet.Analyzers.Tests.csproj +++ b/tests/BenchmarkDotNet.Analyzers.Tests/BenchmarkDotNet.Analyzers.Tests.csproj @@ -23,7 +23,7 @@ - + diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj index 14ff835dae..d68b6a4692 100644 --- a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj @@ -15,7 +15,7 @@ - + diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj index 17e9fc606d..f513999a11 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj @@ -32,8 +32,8 @@ - - + + diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj index eea825a7d2..7135235978 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj @@ -47,7 +47,7 @@ - + diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj index 7ad4db9121..4994ec88aa 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj @@ -46,11 +46,11 @@ - + - + diff --git a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj index 5fd5275d39..359afca60f 100644 --- a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj +++ b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj @@ -20,9 +20,9 @@ - - - + + + From 6f70fcd9bbdebc702eb49a1189ea4afd3522606c Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:57:49 +0900 Subject: [PATCH 43/49] chore: add System.Configuration reference for .NET Framework --- .../BenchmarkDotNet.IntegrationTests.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj index 4994ec88aa..68d24f764c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj @@ -58,6 +58,7 @@ + From dc21f3c444faff5041323e8cccc1a448525c78f1 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Wed, 16 Sep 2026 08:09:06 +0900 Subject: [PATCH 44/49] chore: skip flaky memorydiagnoser tests on win-arm64-netfx --- .../MemoryDiagnoserTests.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 517f8376a7..ab9b2675a2 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -188,7 +188,15 @@ public ulong TimeConsuming() public void TieredJitShouldNotInterfereAllocationResults(IToolchain toolchain) { if (toolchain.IsInProcess) + { + // Extra allocation (256 bytes) wheb running on Windows(arm64) + .NET Framework + if (OsDetector.IsWindows() + && RuntimeInformation.ProcessArchitecture == Architecture.Arm64 + && Portability.RuntimeInformation.IsFullFramework) + Assert.Skip("Flaky test on Windows(arm64) + .NET Framework"); + ValidateMtpProgressDisabled(); + } AssertAllocations(toolchain, typeof(TimeConsumingBenchmark), new Dictionary { From 5cd13edf928a99d1542feb27636284bc7df28f9e Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Wed, 16 Sep 2026 10:20:29 +0900 Subject: [PATCH 45/49] chore: cleanup memorydiagnoser tests skip conditions --- .../MemoryDiagnoserTests.cs | 66 ++++--------------- 1 file changed, 13 insertions(+), 53 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index ab9b2675a2..e7ac27434f 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -1,4 +1,3 @@ -using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; @@ -6,12 +5,9 @@ using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; @@ -20,10 +16,8 @@ using BenchmarkDotNet.Toolchains.Mono; using BenchmarkDotNet.Toolchains.Wasm; using BenchmarkDotNet.Toolchains.NativeAot; -using BenchmarkDotNet.Validators; using System.Reflection; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; namespace BenchmarkDotNet.IntegrationTests { @@ -35,7 +29,19 @@ public class MemoryDiagnoserTests public static IEnumerable GetToolchains() { - yield return [InProcessEmitToolchain.Default]; + if (OsDetector.IsWindows() && Portability.RuntimeInformation.IsFullFramework) + { + // Skip InProcessEmitToolchain tests on Windows +.NET Framework environment. + // Because when running tests with xunit v3, memory allocation tests are flaky. + // See following issues. + // https://github.com/dotnet/BenchmarkDotNet/issues/2779 + // https://github.com/dotnet/BenchmarkDotNet/pull/3065#issuecomment-5064984074 + // https://github.com/dotnet/BenchmarkDotNet/issues/3203 + } + else + { + yield return [InProcessEmitToolchain.Default]; + } if (ContinuousIntegration.IsGitHubDraftPR()) yield break; @@ -187,42 +193,11 @@ public ulong TimeConsuming() [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void TieredJitShouldNotInterfereAllocationResults(IToolchain toolchain) { - if (toolchain.IsInProcess) - { - // Extra allocation (256 bytes) wheb running on Windows(arm64) + .NET Framework - if (OsDetector.IsWindows() - && RuntimeInformation.ProcessArchitecture == Architecture.Arm64 - && Portability.RuntimeInformation.IsFullFramework) - Assert.Skip("Flaky test on Windows(arm64) + .NET Framework"); - - ValidateMtpProgressDisabled(); - } - AssertAllocations(toolchain, typeof(TimeConsumingBenchmark), new Dictionary { { nameof(TimeConsumingBenchmark.TimeConsuming), 0 } }, iterationCount: 10); // 1 iteration is not enough to repro the problem - - static void ValidateMtpProgressDisabled() - { - if (!OsDetector.IsWindows() || RuntimeInformation.OSArchitecture != Architecture.Arm64) - return; - - // On Windows(arm64) environment following test failed with extra memory allocations (1 or 2 bytes) - // TieredJitShouldNotInterfereAllocationResults - var args = Environment.GetCommandLineArgs(); - for (int i = 0; i < args.Length; i++) - { - if (args[i] == "--progress=off") - return; - - if (i < args.Length - 1 && args[i] == "--progress" && args[i + 1] == "off") - return; - } - - throw new NotSupportedException("Measure memory allocation requires setting `--progress off`"); - } } public class NoBoxing @@ -235,12 +210,6 @@ public class NoBoxing [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void EngineShouldNotIntroduceBoxing(IToolchain toolchain) { - if (toolchain.IsInProcess && OsDetector.IsWindows() && !Portability.RuntimeInformation.IsNetCore) - { - // Randomly failed on Windows(x64)+.NET Framework. - Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/pull/3065#issuecomment-5064984074"); - } - AssertAllocations(toolchain, typeof(NoBoxing), new Dictionary { { nameof(NoBoxing.ReturnsValueType), 0 } @@ -263,12 +232,6 @@ public class NonAllocatingAsynchronousBenchmarks [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AwaitingTasksShouldNotInterfereAllocationResults(IToolchain toolchain) { - if (toolchain.IsInProcess && OsDetector.IsWindows() && !Portability.RuntimeInformation.IsNetCore) - { - // Randomly failed on Windows(x64/arm64)+.NET Framework. - Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/3203"); - } - AssertAllocations(toolchain, typeof(NonAllocatingAsynchronousBenchmarks), new Dictionary { { nameof(NonAllocatingAsynchronousBenchmarks.CompletedTask), 0 }, @@ -322,9 +285,6 @@ public byte[] SixtyFourBytesArray() [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) { - if (OsDetector.IsWindows() && toolchain.IsInProcess) - Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/2779"); - long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length AssertAllocations(toolchain, typeof(TimeConsuming), new Dictionary From 74e5e7c8a538b14f016cffdfebd3fe21315a0f38 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Tue, 22 Sep 2026 19:53:17 +0900 Subject: [PATCH 46/49] chore: enable some disassembler diagnoser tests on macos --- .github/workflows/generate-coverage-report.yaml | 6 ++++++ .github/workflows/run-tests-selected.yaml | 6 ++++++ .github/workflows/run-tests.yaml | 4 ++++ .../DisassemblyDiagnoserTests.cs | 12 ------------ 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/generate-coverage-report.yaml b/.github/workflows/generate-coverage-report.yaml index 40a43ea345..01b509b9c4 100644 --- a/.github/workflows/generate-coverage-report.yaml +++ b/.github/workflows/generate-coverage-report.yaml @@ -37,6 +37,12 @@ jobs: - name: Setup additional tools for Wasm/NativeAot tests if: ${{ github.event.inputs.skip_integration_tests == 'false'}} uses: ./.github/actions/setup-additional-tools + + # createdump's task_for_pid makes authd evaluate system.privilege.taskport, which prompts for a password on these + # headless runners and blocks forever; pre-authorizing it lets in-process disassembly dump its own process. + - name: Allow task_for_pid without a password prompt + if: runner.os == 'macOS' + run: sudo security authorizationdb write system.privilege.taskport allow - name: Install dotnet-coverage run: dotnet tool install --global dotnet-coverage diff --git a/.github/workflows/run-tests-selected.yaml b/.github/workflows/run-tests-selected.yaml index fac15e347c..e873a46ed8 100644 --- a/.github/workflows/run-tests-selected.yaml +++ b/.github/workflows/run-tests-selected.yaml @@ -66,6 +66,12 @@ jobs: - uses: ./.github/actions/add-windowsdefender-exclusions if: runner.os == 'Windows' + # createdump's task_for_pid makes authd evaluate system.privilege.taskport, which prompts for a password on these + # headless runners and blocks forever; pre-authorizing it lets in-process disassembly dump its own process. + - name: Allow task_for_pid without a password prompt + if: runner.os == 'macOS' + run: sudo security authorizationdb write system.privilege.taskport allow + # Setup - name: Create `artifacts` directory run: | diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 39be091cc2..8c226ecef9 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -160,6 +160,10 @@ jobs: - name: Setup additional tools for Wasm/NativeAot tests if: ${{ github.event.pull_request.draft != true }} uses: ./.github/actions/setup-additional-tools + # createdump's task_for_pid makes authd evaluate system.privilege.taskport, which prompts for a password on these + # headless runners and blocks forever; pre-authorizing it lets in-process disassembly dump its own process. + - name: Allow task_for_pid without a password prompt + run: sudo security authorizationdb write system.privilege.taskport allow # Build and Test - name: Run task 'build' run: ./build.cmd build diff --git a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs index efaa266d74..b6a9ac1129 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs @@ -97,9 +97,6 @@ public void Recursive() [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, IToolchain toolchain) { - if (OsDetector.IsMacOS() && toolchain.IsInProcess) - Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/3076"); - var disassemblyDiagnoser = new DisassemblyDiagnoser( new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); @@ -120,9 +117,6 @@ public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, IToolchain [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, IToolchain toolchain) { - if (OsDetector.IsMacOS()) - Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/3076"); - var disassemblyDiagnoser = new DisassemblyDiagnoser( new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 1, filters: ["*WithCalls*"])); @@ -149,9 +143,6 @@ public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleGenericTypes(Jit jit, Platform platform, IToolchain toolchain) { - if (OsDetector.IsMacOS()) - Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/3076"); - var disassemblyDiagnoser = new DisassemblyDiagnoser( new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); @@ -173,9 +164,6 @@ [Benchmark] public void JustReturn() { } [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleInlinableBenchmarks(Jit jit, Platform platform, IToolchain toolchain) { - if (OsDetector.IsMacOS()) - Assert.Skip("https://github.com/dotnet/BenchmarkDotNet/issues/3076"); - var disassemblyDiagnoser = new DisassemblyDiagnoser( new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); From 40baecb0fb9886e4a6c3df6e7315ed4c25efaa36 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Tue, 22 Sep 2026 21:11:07 +0900 Subject: [PATCH 47/49] chore: add skip inprocess memorydiagnoser tests because it's flaky on windows --- .../BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index e7ac27434f..16f68293bd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -29,9 +29,9 @@ public class MemoryDiagnoserTests public static IEnumerable GetToolchains() { - if (OsDetector.IsWindows() && Portability.RuntimeInformation.IsFullFramework) + if (OsDetector.IsWindows()) { - // Skip InProcessEmitToolchain tests on Windows +.NET Framework environment. + // Skip InProcessEmitToolchain tests on Windows environment. // Because when running tests with xunit v3, memory allocation tests are flaky. // See following issues. // https://github.com/dotnet/BenchmarkDotNet/issues/2779 From 16ed57dbbcc347c696bef849950c7226acecaf46 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:18:51 +0900 Subject: [PATCH 48/49] chore: skip arm64 test that don't works on .NET Framework --- .../Serialization/DisassemblerModelSerializationTests.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/BenchmarkDotNet.Tests/Serialization/DisassemblerModelSerializationTests.cs b/tests/BenchmarkDotNet.Tests/Serialization/DisassemblerModelSerializationTests.cs index 62738eea37..2700607498 100644 --- a/tests/BenchmarkDotNet.Tests/Serialization/DisassemblerModelSerializationTests.cs +++ b/tests/BenchmarkDotNet.Tests/Serialization/DisassemblerModelSerializationTests.cs @@ -1,5 +1,6 @@ using AwesomeAssertions; using BenchmarkDotNet.Disassemblers; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Serialization; using Gee.External.Capstone; using Gee.External.Capstone.Arm64; @@ -107,6 +108,12 @@ public void IntelAsmSerializationTest() [Fact] public void Arm64AsmSerializationTest() { + // TODO: Remove this temporary workaround after migrated to AsmArm64 + if (RuntimeInformation.IsFullFramework) + { + Assert.Skip("Capstone based arm64 disassembler don't works on .NET Framework."); + } + // Arrange byte[] instructionBytes = [0xE1, 0x0B, 0x40, 0xB9]; // ldr w1, [sp, #8] var disassembleSyntax = DisassembleSyntax.Intel; From 8fdedecf28a330dd656686b840ec78a30e603e7b Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:29:14 +0900 Subject: [PATCH 49/49] chore: replace existing InlineDataEnvSpecificAttribute to xUnit build-in attribute --- .../CancellationTokenTests.cs | 50 ++++++----- .../WasmTests.cs | 82 ++++++++++++++----- .../Shared/XUnit/InlineDataEnvSpecific.cs | 53 ------------ 3 files changed, 93 insertions(+), 92 deletions(-) delete mode 100644 tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecific.cs diff --git a/tests/BenchmarkDotNet.IntegrationTests/CancellationTokenTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CancellationTokenTests.cs index 2616ad6bda..f1279835aa 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CancellationTokenTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CancellationTokenTests.cs @@ -12,7 +12,6 @@ using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.InProcess.Emit; using BenchmarkDotNet.Toolchains.InProcess.NoEmit; using BenchmarkDotNet.Toolchains.Wasm; @@ -83,7 +82,7 @@ public void BenchmarkWithCancellationTokenProperty_ReceivesToken_InProcessEmit() } [Theory] - [MemberData(nameof(CancellationToolchains), DisableDiscoveryEnumeration = true)] + [MemberData(nameof(TestData.Toolchains), MemberType = typeof(TestData), DisableDiscoveryEnumeration = true)] public void StaticCancellationTokenOnABaseTypeReceivesToken(IToolchain toolchain) { var config = ManualConfig.CreateEmpty() @@ -93,20 +92,8 @@ public void StaticCancellationTokenOnABaseTypeReceivesToken(IToolchain toolchain CanExecute(config); } - public static IEnumerable CancellationToolchains() - { - yield return [InProcessNoEmitToolchain.Default]; - yield return [InProcessEmitToolchain.Default]; - - if (ContinuousIntegration.IsGitHubDraftPR()) - yield break; - - yield return [Job.Default.GetToolchain()]; - } - - [TheoryEnvSpecific("JSVU does not support ARM on Windows or Linux", EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm, EnvRequirement.NonGitHubDraftPR)] - [InlineData("v8")] - [InlineData("node")] + [Theory] + [MemberData(nameof(TestData.JavaScriptEngines), MemberType = typeof(TestData))] public void BenchmarkWithCancellationTokenProperty_ReceivesToken_Wasm(string javaScriptEngine) { var logger = new OutputLogger(Output); @@ -165,9 +152,8 @@ public async Task RunWithCancellationTokenIsCancelled_InProcessEmit() await Assert.ThrowsAnyAsync(async () => await BenchmarkRunner.RunAsync(config, cancellationToken: cts.Token)); } - [TheoryEnvSpecific(EnvRequirement.NonGitHubDraftPR)] - [InlineDataEnvSpecific("v8", "JSVU does not support ARM on Windows or Linux", [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm, EnvRequirement.NonGitHubDraftPR])] - [InlineData("node")] + [Theory(SkipTestWithoutData = true)] + [MemberData(nameof(TestData.JavaScriptEngines), MemberType = typeof(TestData))] public async Task RunWithCancellationTokenIsCancelled_Wasm(string javaScriptEngine) { var cts = new CancellationTokenSource(); @@ -252,4 +238,30 @@ public ValueTask HandleAsync(HostSignal signal, DiagnoserActionParameters parame public void DisplayResults(ILogger logger) { } } + + public static class TestData + { + public static IEnumerable> JavaScriptEngines() + { + if (EnvRequirementChecker.GetSkip(EnvRequirement.NonGitHubDraftPR) != null) + yield break; + + // JSVU does not support ARM on Windows or Linux + if (EnvRequirementChecker.GetSkip(EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm, EnvRequirement.NonGitHubDraftPR) == null) + yield return "v8"; + + yield return "node"; + } + + public static IEnumerable> Toolchains() + { + yield return new(InProcessNoEmitToolchain.Default); + yield return new(InProcessEmitToolchain.Default); + + if (ContinuousIntegration.IsGitHubDraftPR()) + yield break; + + yield return new(Job.Default.GetToolchain()); + } + } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs index a3b63bbe83..e223b3fed7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs @@ -19,27 +19,16 @@ namespace BenchmarkDotNet.IntegrationTests /// public class WasmTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) { - private const string V8SkipReason = "JSVU does not support ARM on Windows or Linux"; - // WASM AOT does not build on Windows arm64: the mono-aot-cross toolchain runs but does not emit the - // *_compiled_methods.txt token file, so WasmApp.Common.targets fails. It works on Linux arm64. - private const string WasmAotWindowsArmSkipReason = "WASM AOT does not build on Windows arm64"; - - [TheoryEnvSpecific(EnvRequirement.NonGitHubDraftPR)] - [InlineDataEnvSpecific([RuntimeMoniker.MonoWasm10_0, "v8"], V8SkipReason, [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm])] - [InlineData(RuntimeMoniker.MonoWasm10_0, "node")] - [InlineDataEnvSpecific([RuntimeMoniker.MonoWasmAot10_0, "v8"], V8SkipReason, [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm])] - [InlineDataEnvSpecific([RuntimeMoniker.MonoWasmAot10_0, "node"], WasmAotWindowsArmSkipReason, [EnvRequirement.NonWindowsArm])] - // CoreWasm is not tested yet because it is still experimental . - //[InlineDataEnvSpecific([RuntimeMoniker.CoreWasm11_0, "v8"], V8SkipReason, [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm])] - //[InlineData(RuntimeMoniker.CoreWasm11_0, "node")] + [Theory(SkipTestWithoutData = true)] + [MemberData(nameof(TestData.WasmIsSupported), MemberType = typeof(TestData))] + public void WasmIsSupported(string runtimeMoniker, string javaScriptEngine) { CanExecute(GetConfig(runtimeMoniker, javaScriptEngine)); } - [TheoryEnvSpecific(EnvRequirement.NonGitHubDraftPR)] - [InlineDataEnvSpecific(["v8"], V8SkipReason, [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm])] - [InlineData("node")] + [Theory(SkipTestWithoutData = true)] + [MemberData(nameof(TestData.WasmSupportsInProcessDiagnosers), MemberType = typeof(TestData))] public void WasmSupportsInProcessDiagnosers(string javaScriptEngine) { try @@ -58,9 +47,8 @@ public void WasmSupportsInProcessDiagnosers(string javaScriptEngine) } } - [TheoryEnvSpecific(EnvRequirement.NonGitHubDraftPR)] - [InlineDataEnvSpecific(["v8", "custom-main-v8.mjs", WasmIpcType.FileStdOut], V8SkipReason, [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm])] - [InlineData("node", "custom-main-node.mjs", WasmIpcType.WebSocket)] + [Theory(SkipTestWithoutData = true)] + [MemberData(nameof(TestData.WasmSupportsCustomMainJs), MemberType = typeof(TestData))] public void WasmSupportsCustomMainJs(string javaScriptEngine, string customMainJs, WasmIpcType ipcType) { var mainJsTemplate = new FileInfo(Path.Combine("wwwroot", customMainJs)); @@ -91,4 +79,58 @@ private ManualConfig GetConfig(string runtimeMoniker, string javaScriptEngine, F .WithOption(ConfigOptions.GenerateMSBuildBinLog, false); } } -} \ No newline at end of file + + public static class TestData + { + public static IEnumerable> WasmIsSupported() + { + if (EnvRequirementChecker.GetSkip(EnvRequirement.NonGitHubDraftPR) != null) + yield break; + + yield return new(RuntimeMoniker.MonoWasm10_0, "node"); + + if (IsV8Supported()) + { + yield return new(RuntimeMoniker.MonoWasm10_0, "v8"); + yield return new(RuntimeMoniker.MonoWasmAot10_0, "v8"); + + // CoreWasm is not tested yet because it is still experimental. + // yield return new(RuntimeMoniker.CoreWasm11_0, "v8"); + } + + // WASM AOT does not build on Windows arm64: the mono-aot-cross toolchain runs but does not emit the + // *_compiled_methods.txt token file, so WasmApp.Common.targets fails. It works on Linux arm64. + if (EnvRequirementChecker.GetSkip(EnvRequirement.NonWindowsArm) == null) + yield return new(RuntimeMoniker.MonoWasmAot10_0, "node"); + + // CoreWasm is not tested yet because it is still experimental. + // yield return new(RuntimeMoniker.CoreWasm11_0, "node"); + } + + public static IEnumerable> WasmSupportsInProcessDiagnosers() + { + if (EnvRequirementChecker.GetSkip(EnvRequirement.NonGitHubDraftPR) != null) + yield break; + + if (IsV8Supported()) + yield return new("v8"); + + yield return new("node"); + } + + public static IEnumerable> WasmSupportsCustomMainJs() + { + if (EnvRequirementChecker.GetSkip(EnvRequirement.NonGitHubDraftPR) != null) + yield break; + + if (IsV8Supported()) + yield return new("v8", "custom-main-v8.mjs", WasmIpcType.FileStdOut); + + yield return new("node", "custom-main-node.mjs", WasmIpcType.WebSocket); + } + + // JSVU does not support ARM on Windows or Linux + private static bool IsV8Supported() + => EnvRequirementChecker.GetSkip([EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm]) == null; + } +} diff --git a/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecific.cs b/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecific.cs deleted file mode 100644 index 8f9419f5c0..0000000000 --- a/tests/BenchmarkDotNet.Tests/Shared/XUnit/InlineDataEnvSpecific.cs +++ /dev/null @@ -1,53 +0,0 @@ -using System.Reflection; -using Xunit; -using Xunit.Sdk; -using Xunit.v3; - -namespace BenchmarkDotNet.Tests.XUnit; - -[AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] -public class InlineDataEnvSpecific : DataAttribute -{ - /// - /// Gets the data to be passed to the test. - /// - // If the user passes null to the constructor, we assume what they meant was a - // single null value to be passed to the test. - public object?[] Data { get; } - - public InlineDataEnvSpecific(object? data, string reason, params EnvRequirement[] requirements) - : this([data], reason, requirements) { } - - public InlineDataEnvSpecific(object?[] data, string reason, params EnvRequirement[] requirements) - { - // If the user passes null to the constructor, we assume what they meant was a - // single null value to be passed to the test. - Data = data ?? [null]; - string? skip = EnvRequirementChecker.GetSkip(requirements); - if (skip != null) - Skip = $"{skip} ({reason})"; - } - - public override ValueTask> GetData( - MethodInfo testMethod, - DisposalTracker disposalTracker) - { - var traits = new Dictionary>(StringComparer.OrdinalIgnoreCase); - TestIntrospectionHelper.MergeTraitsInto(traits, Traits); - - return new([ - new TheoryDataRow(Data) - { - Explicit = ExplicitAsNullable, - Label = Label, - Skip = Skip, - TestDisplayName = TestDisplayName, - Timeout = TimeoutAsNullable, - Traits = traits, - } - ]); - } - - public override bool SupportsDiscoveryEnumeration() - => true; -}