From 33523cc6d8fcff0ccbd25721e37d7215946b9a70 Mon Sep 17 00:00:00 2001 From: Jeff Lenamon <85593689+lenamonj@users.noreply.github.com> Date: Sun, 6 Sep 2026 21:33:44 -0400 Subject: [PATCH 1/3] Report the complexity aggregates in the family's time unit ComputeBigO left big_o.time_unit at the Run() default, so the BigO row was rendered in nanoseconds whatever unit the family declared; CSVReporter rendered the RMS row from the raw, pre-divided field, so it was smaller by the unit multiplier. Both now go through the family unit and the GetAdjusted*Time() accessors, with a millisecond family in complexity_test.cc that fails on main for both. --- docs/user_guide.md | 5 +++++ src/complexity.cc | 2 ++ src/csv_reporter.cc | 4 +++- test/complexity_test.cc | 40 ++++++++++++++++++++++++++++++++++++---- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/docs/user_guide.md b/docs/user_guide.md index 3fed9261d7..30f3608dea 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -850,6 +850,11 @@ BENCHMARK(BM_StringCompare)->RangeMultiplier(2) ->Range(1<<10, 1<<18)->Complexity([](benchmark::IterationCount n)->double{return n; }); ``` +The `BigO` row reports the coefficient in the same time unit as the rest of the +family, so a family declared with `->Unit(benchmark::kMicrosecond)` reports its +coefficient in microseconds. The `RMS` row is a normalized quantity and is +reported as a percentage, independent of the time unit. + ## Custom Benchmark Name diff --git a/src/complexity.cc b/src/complexity.cc index 8fa3f073af..c1752354d9 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -229,6 +229,8 @@ std::vector ComputeBigO( big_o.cpu_accumulated_time = result_cpu.coef; big_o.report_big_o = true; big_o.complexity = result_cpu.complexity; + // The coefficient is a time; reporters render it in the family's unit. + big_o.time_unit = reports[0].time_unit; // All the time results are reported after being multiplied by the // time unit multiplier. But since RMS is a relative quantity it diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 3e21d11f0a..e29c466922 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -125,7 +125,9 @@ void CSVReporter::PrintRunData(const Run& run) { } Out << ","; - if (run.run_type != Run::RT_Aggregate || + // ComputeBigO stores the RMS pre-divided by the time unit multiplier; the + // GetAdjusted*Time() accessors undo that, as the other reporters rely on. + if (run.report_rms || run.run_type != Run::RT_Aggregate || run.aggregate_unit == StatisticUnit::kTime) { Out << run.GetAdjustedRealTime() << ","; Out << run.GetAdjustedCPUTime() << ","; diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 64a7e72a6d..15ebba69da 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -20,13 +20,17 @@ namespace { int AddComplexityTest(const std::string& test_name, const std::string& big_o_test_name, const std::string& rms_test_name, - const std::string& big_o, int family_index) { + const std::string& big_o, int family_index, + const std::string& time_unit = "ns", + const std::string& csv_rms = "%float") { SetSubstitutions({{"%name", test_name}, {"%bigo_name", big_o_test_name}, {"%rms_name", rms_test_name}, {"%bigo_str", "[ ]* %float " + big_o}, {"%bigo", big_o}, - {"%rms", "[ ]*[0-9]+ %"}}); + {"%rms", "[ ]*[0-9]+ %"}, + {"%unit", time_unit}, + {"%csvrms", csv_rms}}); AddCases( TC_ConsoleOut, {{"^%bigo_name %bigo_str %bigo_str[ ]*$"}, @@ -46,7 +50,7 @@ int AddComplexityTest(const std::string& test_name, {"\"cpu_coefficient\": %float,$", MR_Next}, {"\"real_coefficient\": %float,$", MR_Next}, {"\"big_o\": \"%bigo\",$", MR_Next}, - {"\"time_unit\": \"ns\"$", MR_Next}, + {"\"time_unit\": \"%unit\"$", MR_Next}, {"}", MR_Next}, {"\"name\": \"%rms_name\",$"}, {"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next}, @@ -61,7 +65,7 @@ int AddComplexityTest(const std::string& test_name, {"}", MR_Next}}); AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, {"^\"%bigo_name\"", MR_Not}, - {"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}}); + {"^\"%rms_name\",,%csvrms,%float,,,,,,$", MR_Next}}); return 0; } @@ -266,6 +270,34 @@ const std::string complexity_capture_name = ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", complexity_capture_name + "_RMS", "N", /*family_index=*/9); +// ========================================================================= // +// ------------------- Testing BigO with a declared unit ------------------- // +// ========================================================================= // + +void BM_Complexity_O_N_ms(benchmark::State& state) { + for (auto _ : state) { + // 1us per iteration per entry, half the sizes 50% slower so the RMS is + // well above zero, reported in milliseconds + const double skew = (state.range(0) & (1 << 11)) ? 1.5 : 1.0; + state.SetIterationTime(static_cast(state.range(0)) * 1e-6 * skew); + } + state.SetComplexityN(state.range(0)); +} +BENCHMARK(BM_Complexity_O_N_ms) + ->RangeMultiplier(2) + ->Range(1 << 10, 1 << 16) + ->UseManualTime() + ->Unit(benchmark::kMillisecond) + ->Complexity(benchmark::oN); + +constexpr char n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time"; +constexpr char big_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_BigO"; +constexpr char rms_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_RMS"; + +ADD_COMPLEXITY_CASES(n_ms_test_name, big_o_n_ms_test_name, rms_o_n_ms_test_name, + enum_auto_big_o_n, /*family_index=*/10, "ms", + "0[.][0-9]+"); + } // end namespace // ========================================================================= // From 750d1879bdf637c33e1149822f7db9eff877bbc2 Mon Sep 17 00:00:00 2001 From: Jeff Lenamon <85593689+lenamonj@users.noreply.github.com> Date: Mon, 7 Sep 2026 08:56:49 -0400 Subject: [PATCH 2/3] complexity test: pass the unit and CSV pattern explicitly, drop a comment --- src/complexity.cc | 1 - test/complexity_test.cc | 24 ++++++++++++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/complexity.cc b/src/complexity.cc index c1752354d9..c431a3f035 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -229,7 +229,6 @@ std::vector ComputeBigO( big_o.cpu_accumulated_time = result_cpu.coef; big_o.report_big_o = true; big_o.complexity = result_cpu.complexity; - // The coefficient is a time; reporters render it in the family's unit. big_o.time_unit = reports[0].time_unit; // All the time results are reported after being multiplied by the diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 15ebba69da..8bf5790745 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -21,8 +21,8 @@ int AddComplexityTest(const std::string& test_name, const std::string& big_o_test_name, const std::string& rms_test_name, const std::string& big_o, int family_index, - const std::string& time_unit = "ns", - const std::string& csv_rms = "%float") { + const std::string& time_unit, + const std::string& csv_rms) { SetSubstitutions({{"%name", test_name}, {"%bigo_name", big_o_test_name}, {"%rms_name", rms_test_name}, @@ -108,15 +108,15 @@ constexpr char lambda_big_o_1[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_auto_big_o_1, /*family_index=*/0); + enum_auto_big_o_1, /*family_index=*/0, "ns", "%float"); // Add auto tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_auto_big_o_1, /*family_index=*/1); + enum_auto_big_o_1, /*family_index=*/1, "ns", "%float"); // Add lambda tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - lambda_big_o_1, /*family_index=*/2); + lambda_big_o_1, /*family_index=*/2, "ns", "%float"); // ========================================================================= // // --------------------------- Testing BigO O(N) --------------------------- // @@ -165,15 +165,15 @@ constexpr char lambda_big_o_n[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - enum_auto_big_o_n, /*family_index=*/3); + enum_auto_big_o_n, /*family_index=*/3, "ns", "%float"); // Add auto tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - enum_auto_big_o_n, /*family_index=*/4); + enum_auto_big_o_n, /*family_index=*/4, "ns", "%float"); // Add lambda tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - lambda_big_o_n, /*family_index=*/5); + lambda_big_o_n, /*family_index=*/5, "ns", "%float"); // ========================================================================= // // ------------------------- Testing BigO O(NlgN) ------------------------- // @@ -226,17 +226,17 @@ constexpr char lambda_big_o_n_lg_n[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, - /*family_index=*/6); + /*family_index=*/6, "ns", "%float"); // NOTE: auto big-o is wron.g ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, - /*family_index=*/7); + /*family_index=*/7, "ns", "%float"); //// Add lambda tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, - /*family_index=*/8); + /*family_index=*/8, "ns", "%float"); // ========================================================================= // // -------- Testing formatting of Complexity with captured args ------------ // @@ -269,7 +269,7 @@ const std::string complexity_capture_name = ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", complexity_capture_name + "_RMS", "N", - /*family_index=*/9); + /*family_index=*/9, "ns", "%float"); // ========================================================================= // // ------------------- Testing BigO with a declared unit ------------------- // // ========================================================================= // From 9ab01534b47a629e78675297a2858dde8bcf6196 Mon Sep 17 00:00:00 2001 From: lenamonj Date: Mon, 7 Sep 2026 17:17:40 -0400 Subject: [PATCH 3/3] Drop the CSV reporter change The CSV reporter is deprecated, so its RMS row is left as it is and the test keeps the pattern it had. --- src/csv_reporter.cc | 4 +--- test/complexity_test.cc | 31 ++++++++++++++----------------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index e29c466922..3e21d11f0a 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -125,9 +125,7 @@ void CSVReporter::PrintRunData(const Run& run) { } Out << ","; - // ComputeBigO stores the RMS pre-divided by the time unit multiplier; the - // GetAdjusted*Time() accessors undo that, as the other reporters rely on. - if (run.report_rms || run.run_type != Run::RT_Aggregate || + if (run.run_type != Run::RT_Aggregate || run.aggregate_unit == StatisticUnit::kTime) { Out << run.GetAdjustedRealTime() << ","; Out << run.GetAdjustedCPUTime() << ","; diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 8bf5790745..7551abcb39 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -21,16 +21,14 @@ int AddComplexityTest(const std::string& test_name, const std::string& big_o_test_name, const std::string& rms_test_name, const std::string& big_o, int family_index, - const std::string& time_unit, - const std::string& csv_rms) { + const std::string& time_unit) { SetSubstitutions({{"%name", test_name}, {"%bigo_name", big_o_test_name}, {"%rms_name", rms_test_name}, {"%bigo_str", "[ ]* %float " + big_o}, {"%bigo", big_o}, {"%rms", "[ ]*[0-9]+ %"}, - {"%unit", time_unit}, - {"%csvrms", csv_rms}}); + {"%unit", time_unit}}); AddCases( TC_ConsoleOut, {{"^%bigo_name %bigo_str %bigo_str[ ]*$"}, @@ -65,7 +63,7 @@ int AddComplexityTest(const std::string& test_name, {"}", MR_Next}}); AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, {"^\"%bigo_name\"", MR_Not}, - {"^\"%rms_name\",,%csvrms,%float,,,,,,$", MR_Next}}); + {"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}}); return 0; } @@ -108,15 +106,15 @@ constexpr char lambda_big_o_1[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_auto_big_o_1, /*family_index=*/0, "ns", "%float"); + enum_auto_big_o_1, /*family_index=*/0, "ns"); // Add auto tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_auto_big_o_1, /*family_index=*/1, "ns", "%float"); + enum_auto_big_o_1, /*family_index=*/1, "ns"); // Add lambda tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - lambda_big_o_1, /*family_index=*/2, "ns", "%float"); + lambda_big_o_1, /*family_index=*/2, "ns"); // ========================================================================= // // --------------------------- Testing BigO O(N) --------------------------- // @@ -165,15 +163,15 @@ constexpr char lambda_big_o_n[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - enum_auto_big_o_n, /*family_index=*/3, "ns", "%float"); + enum_auto_big_o_n, /*family_index=*/3, "ns"); // Add auto tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - enum_auto_big_o_n, /*family_index=*/4, "ns", "%float"); + enum_auto_big_o_n, /*family_index=*/4, "ns"); // Add lambda tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - lambda_big_o_n, /*family_index=*/5, "ns", "%float"); + lambda_big_o_n, /*family_index=*/5, "ns"); // ========================================================================= // // ------------------------- Testing BigO O(NlgN) ------------------------- // @@ -226,17 +224,17 @@ constexpr char lambda_big_o_n_lg_n[] = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, - /*family_index=*/6, "ns", "%float"); + /*family_index=*/6, "ns"); // NOTE: auto big-o is wron.g ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, - /*family_index=*/7, "ns", "%float"); + /*family_index=*/7, "ns"); //// Add lambda tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, - /*family_index=*/8, "ns", "%float"); + /*family_index=*/8, "ns"); // ========================================================================= // // -------- Testing formatting of Complexity with captured args ------------ // @@ -269,7 +267,7 @@ const std::string complexity_capture_name = ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", complexity_capture_name + "_RMS", "N", - /*family_index=*/9, "ns", "%float"); + /*family_index=*/9, "ns"); // ========================================================================= // // ------------------- Testing BigO with a declared unit ------------------- // // ========================================================================= // @@ -295,8 +293,7 @@ constexpr char big_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_BigO"; constexpr char rms_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_RMS"; ADD_COMPLEXITY_CASES(n_ms_test_name, big_o_n_ms_test_name, rms_o_n_ms_test_name, - enum_auto_big_o_n, /*family_index=*/10, "ms", - "0[.][0-9]+"); + enum_auto_big_o_n, /*family_index=*/10, "ms"); } // end namespace