diff --git a/indra/alscript/CMakeLists.txt b/indra/alscript/CMakeLists.txt index e6706a1eca..e9645ac36f 100644 --- a/indra/alscript/CMakeLists.txt +++ b/indra/alscript/CMakeLists.txt @@ -299,6 +299,9 @@ if(AL_BUILD_TESTS) tests/alscriptweight_test.cpp tests/alsourcemap_test.cpp tests/aluploadheader_test.cpp + # The preprocessor, the formatter, the optimizer and one analysis of each + # language over big scripts, with the grid's definitions loaded. + BENCH_SOURCES tests/alscript_bench.cpp SUPPORT alscript_test_checks llcommon_test_checks LIBRARIES alscript llcommon ll::slua ll::tailslide DEFINES @@ -315,13 +318,4 @@ if(AL_BUILD_TESTS) tests/alscriptlintpass_test.cpp RERUN_ENVIRONMENT AL_TEST_LUAU_SOLVER=new ) - # The preprocessor, the formatter, the optimizer and one analysis of each - # language over big scripts, with the grid's definitions loaded. - al_add_bench( - alscript - PROJECT alscript - LIBRARIES alscript llcommon - INCLUDES ${INDRA_SOURCE_DIR}/test - DEFINES ${definitions_defines} - ) endif() diff --git a/indra/alscript/tests/alscript_bench.cpp b/indra/alscript/tests/alscript_bench.cpp index 16994651ab..1b545c8a2a 100644 --- a/indra/alscript/tests/alscript_bench.cpp +++ b/indra/alscript/tests/alscript_bench.cpp @@ -25,9 +25,10 @@ // What the script studio's tools cost over a big script, LSL and SLua side // by side, with the grid's own definitions loaded: what the analysis thread -// and the preprocessor's thread do each time a script changes. A number is -// milliseconds per run: the median of five samples, each as many runs as -// fit in fifty milliseconds (and at least one). +// and the preprocessor's thread do each time a script changes. Each +// benchmark is one run, named for its language where both are measured, or +// for its script's length where it is one of three; Catch2 reports the mean +// of its samples and the interval around it. // // Each run is over a text that differs from the run before it -- a comment // at its end comes and goes -- so that a tool that keeps what it made of @@ -35,8 +36,7 @@ // // The output is a table, not a verdict; a number is read against the same // row from another build, on the same quiet machine. An unoptimised build -// exits 125, which CTest reads as skipped, since its numbers would say -// nothing. +// skips, since its numbers would say nothing. #include "linden_common.h" @@ -52,52 +52,22 @@ #include "../core/alscriptweight.h" #include "albigscript.h" +#include "altestbenchmark.h" #include "llfile.h" #include #include -#include #include #include #include -#include #include #include #include -// Only an optimised build measures, and only it has a use for these: an -// unoptimised one would say they are unused. -#if defined(LL_RELEASE) namespace { - using clock = std::chrono::steady_clock; - constexpr int LINES = 5000; - // Everything a row makes feeds this, so nothing is made for nothing. - volatile size_t g_sink = 0; - - double ms_per_run(const std::function& run) - { - run(); - double samples[5]; - for (double& sample : samples) - { - size_t runs = 0; - const auto start = clock::now(); - clock::duration elapsed{}; - do - { - run(); - ++runs; - elapsed = clock::now() - start; - } while (elapsed < std::chrono::milliseconds(50)); - sample = std::chrono::duration(elapsed).count() / double(runs); - } - std::sort(samples, samples + 5); - return samples[2]; - } - // One language's script, in the two forms the runs go between. struct Script { @@ -117,18 +87,6 @@ namespace } }; - void cell(double ms) - { - if (std::isnan(ms)) - { - std::printf(" %10s", "-"); - } - else - { - std::printf(" %10.3f", ms); - } - } - // A scripter's include folder on disk, its `.lslrc` listing a library // folder under it, and a script three folders down that includes five // files: what the main thread reads each time that script is checked. @@ -142,7 +100,8 @@ namespace IncludeTree() { namespace fs = std::filesystem; - root = fs::temp_directory_path() / ("alscript_bench_" + std::to_string(clock::now().time_since_epoch().count())); + root = fs::temp_directory_path() / + ("alscript_bench_" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); fs::create_directories(root / "includes" / "lib"); fs::create_directories(root / "project" / "a" / "b"); folder = (root / "includes").string(); @@ -242,14 +201,6 @@ namespace return read; } - void row(const char* name, double lsl, double slua) - { - std::printf(" %-52s", name); - cell(lsl); - cell(slua); - std::printf("\n"); - } - // A script being typed in: a line put into the big script's main, in // two forms one character apart, and the caret in each `back` bytes // before the line's end. Each run is over the other form, as a @@ -281,17 +232,6 @@ namespace S32 column() const { return columns[next]; } }; - // A row of SLua alone over the scripts of each size. - void sizes(const char* name, const std::vector& ms) - { - std::printf(" %-52s", name); - for (double one : ms) - { - cell(one); - } - std::printf("\n"); - } - std::string readWhole(const std::string& path) { llifstream in(path, std::ios::binary); @@ -318,235 +258,278 @@ namespace } return "#define TWICE(x) ((x) * 2)\n" + text; } -} -#endif // LL_RELEASE -int main(int, char**) -{ -#if !defined(LL_RELEASE) - std::printf("Skipped: an unoptimised build has no numbers worth reading\n"); - return 125; -#else - const double NONE = std::nan(""); - - // The grid's definitions: the builtins, which the optimizer reads - // through the LSL service, and SLua's declarations. - ALLSLService lsl; - std::string error; - if (!lsl.loadBuiltins(std::string(AL_LSL_DEFINITIONS_DIR) + "/builtins.txt", error)) + TEST_CASE("alscript_bench: the analysis thread's work", "[alscript_bench][!benchmark]") { - std::printf("Skipped: the builtins did not load: %s\n", error.c_str()); - return 125; - } - ALLuauService luau; - if (!luau.loadDefinitions(readWhole(std::string(AL_LSL_DEFINITIONS_DIR) + "/secondlife.d.luau"), error)) - { - std::printf("Skipped: the SLua definitions did not load: %s\n", error.c_str()); - return 125; - } + ll_test::skipUnlessOptimised(); + // The grid's definitions: the builtins and SLua's declarations. + ALLSLService lsl; + std::string error; + if (!lsl.loadBuiltins(std::string(AL_LSL_DEFINITIONS_DIR) + "/builtins.txt", error)) + { + SKIP("the builtins did not load: " << error); + } + ALLuauService luau; + if (!luau.loadDefinitions(readWhole(std::string(AL_LSL_DEFINITIONS_DIR) + "/secondlife.d.luau"), error)) + { + SKIP("the SLua definitions did not load: " << error); + } + + Script lslScript(ll_test::bigLSL(LINES), "\n// again\n"); + Script luaScript(ll_test::bigSLua(LINES), "\n-- again\n"); - Script lslScript(ll_test::bigLSL(LINES), "\n// again\n"); - Script luaScript(ll_test::bigSLua(LINES), "\n-- again\n"); - Script macroScript(withMacros(ll_test::bigLSL(LINES)), "\n// again\n"); - - std::printf("alscript_bench: the script tools over generated scripts of %d lines (ms per run)\n", LINES); - std::printf("\n %-52s %10s %10s\n", "", "LSL", "SLua"); - std::printf(" %-52s %10zu %10zu\n", "bytes", lslScript.texts[0].size(), luaScript.texts[0].size()); - std::printf(" %-52s %10zu %10zu\n", "errors found (0 is right)", errorsIn(lsl.check(lslScript.texts[0], true)), - errorsIn(luau.check(luaScript.texts[0]))); - std::printf(" %-52s %10d %10d\n", "the hover below finds its name (1 is right)", - lsl.hover(lslScript.texts[0], ll_test::big_script_detail::linesIn(lslScript.texts[0]) - 12, 20).found, - luau.hover(luaScript.texts[0], ll_test::big_script_detail::linesIn(luaScript.texts[0]) - 8, 16).found); - - std::printf("\nThe analysis thread's work\n"); - row("check", ms_per_run([&] { g_sink = g_sink + lsl.check(lslScript.text(), true).size(); }), - ms_per_run([&] { g_sink = g_sink + luau.check(luaScript.text()).size(); })); - row("a check job: check, outline, semantic tokens, hints", - ms_per_run([&] { + std::printf("\nalscript_bench: the script tools over generated scripts of %d lines\n", LINES); + std::printf("\n %-52s %10s %10s\n", "", "LSL", "SLua"); + std::printf(" %-52s %10zu %10zu\n", "bytes", lslScript.texts[0].size(), luaScript.texts[0].size()); + { + INFO("errors found (0 is right)"); + CHECK(errorsIn(lsl.check(lslScript.texts[0], true)) == 0); + CHECK(errorsIn(luau.check(luaScript.texts[0])) == 0); + } + { + INFO("the hover below finds its name (1 is right)"); + CHECK(lsl.hover(lslScript.texts[0], ll_test::big_script_detail::linesIn(lslScript.texts[0]) - 12, 20).found); + CHECK(luau.hover(luaScript.texts[0], ll_test::big_script_detail::linesIn(luaScript.texts[0]) - 8, 16).found); + } + + BENCHMARK("check, LSL") + { + return lsl.check(lslScript.text(), true).size(); + }; + BENCHMARK("check, SLua") + { + return luau.check(luaScript.text()).size(); + }; + BENCHMARK("a check job: check, outline, semantic tokens, hints, LSL") + { const std::string& text = lslScript.text(); - g_sink = g_sink + lsl.check(text, true).size() + lsl.outline(text).size() + lsl.semanticTokens(text).size() + - lsl.inlayHints(text, true).size(); - }), - ms_per_run([&] { + return lsl.check(text, true).size() + lsl.outline(text).size() + lsl.semanticTokens(text).size() + lsl.inlayHints(text, true).size(); + }; + BENCHMARK("a check job: check, outline, semantic tokens, hints, SLua") + { const std::string& text = luaScript.text(); - g_sink = g_sink + luau.check(text).size() + luau.outline(text).size() + luau.semanticTokens(text).size() + - luau.inlayHints(text, true, true).size(); - })); - // An edit as the studio has it answered: the check job, then the - // questions typing asks -- a completion, a hover, a completion again. - row("an edit: the check job, then complete, hover, complete", - ms_per_run([&] { + return luau.check(text).size() + luau.outline(text).size() + luau.semanticTokens(text).size() + + luau.inlayHints(text, true, true).size(); + }; + // An edit as the studio has it answered: the check job, then the + // questions typing asks -- a completion, a hover, a completion again. + BENCHMARK("an edit: the check job, then complete, hover, complete, LSL") + { const std::string& text = lslScript.text(); const S32 last = ll_test::big_script_detail::linesIn(text) - 12; - g_sink = g_sink + lsl.check(text, true).size() + lsl.outline(text).size() + lsl.semanticTokens(text).size() + - lsl.inlayHints(text, true).size() + lsl.hover(text, last, 20).label.size(); - }), - ms_per_run([&] { + return lsl.check(text, true).size() + lsl.outline(text).size() + lsl.semanticTokens(text).size() + + lsl.inlayHints(text, true).size() + lsl.hover(text, last, 20).label.size(); + }; + BENCHMARK("an edit: the check job, then complete, hover, complete, SLua") + { const std::string& text = luaScript.text(); const S32 last = ll_test::big_script_detail::linesIn(text) - 8; - g_sink = g_sink + luau.check(text).size() + luau.outline(text).size() + luau.semanticTokens(text).size() + - luau.inlayHints(text, true, true).size() + luau.complete(text, last, 16).size() + luau.hover(text, last, 16).label.size() + - luau.complete(text, last, 16).size(); - })); - // An expansion whose include is most of it: the names and hints of the - // include's lines passed over, as the studio shows none of them. - { - std::string big = ll_test::bigLSL(LINES); - const size_t at = big.find("default"); - const std::string library = big.substr(0, at); - const std::string expanded = library + big.substr(at); - const S32 lines = static_cast(std::count(library.begin(), library.end(), '\n')); - lsl.check(expanded, true); - const auto names = [&] { g_sink = g_sink + lsl.semanticTokens(expanded).size() + lsl.inlayHints(expanded, true).size(); }; - const double all = ms_per_run(names); - lsl.setPassedOver({ { 0, lines - 1 } }); - const double own = ms_per_run(names); - lsl.setPassedOver({}); - row("LSL names and hints of an expansion, all of it", all, NONE); - row(" its include's lines passed over", own, NONE); - } - // A check with many problems, each offered its fixes: names misspelt, - // locals nobody uses. - { - std::string troubled = ll_test::bigLSL(LINES); - std::string body = "troubled()\n{\n"; - for (int i = 0; i < 100; ++i) + return luau.check(text).size() + luau.outline(text).size() + luau.semanticTokens(text).size() + + luau.inlayHints(text, true, true).size() + luau.complete(text, last, 16).size() + luau.hover(text, last, 16).label.size() + + luau.complete(text, last, 16).size(); + }; + // An expansion whose include is most of it: the names and hints of the + // include's lines passed over, as the studio shows none of them. { - body += " integer unused" + std::to_string(i) + " = " + std::to_string(i) + ";\n"; - body += " llSya(0, \"x\");\n"; + std::string big = ll_test::bigLSL(LINES); + const size_t at = big.find("default"); + const std::string library = big.substr(0, at); + const std::string expanded = library + big.substr(at); + const S32 lines = static_cast(std::count(library.begin(), library.end(), '\n')); + lsl.check(expanded, true); + const auto names = [&] { return lsl.semanticTokens(expanded).size() + lsl.inlayHints(expanded, true).size(); }; + BENCHMARK("LSL names and hints of an expansion, all of it") + { + return names(); + }; + lsl.setPassedOver({ { 0, lines - 1 } }); + BENCHMARK("its include's lines passed over") + { + return names(); + }; + lsl.setPassedOver({}); } - body += "}\n"; - troubled.insert(troubled.find("default"), body); + // A check with many problems, each offered its fixes: names misspelt, + // locals nobody uses. size_t problems = 0; - row("LSL check of a text with 200 problems", ms_per_run([&] { problems = lsl.check(troubled, true).size(); g_sink = g_sink + problems; }), - NONE); - std::printf(" %-52s %10zu\n", " problems", problems); - } - // A small script that requires a big module, typed in: the bundle - // checked whole at each edit, against the script apart from the module - // (ALPreprocessor::Options::apart), where the module stays checked. - { - const std::string module = luaScript.texts[0] + "\nreturn {}\n"; - ALPreprocessor::Options options; - options.lua = true; - options.apart = true; - options.resolve = [&module](const ALPreprocessor::Ask& ask, ALPreprocessor::Include& out) { - if (ask.name != "big") - { - return ALPreprocessor::Found::No; + { + std::string troubled = ll_test::bigLSL(LINES); + std::string body = "troubled()\n{\n"; + for (int i = 0; i < 100; ++i) + { + body += " integer unused" + std::to_string(i) + " = " + std::to_string(i) + ";\n"; + body += " llSya(0, \"x\");\n"; } - out.text = module; - out.name = "big"; - out.path = "disk:/big.luau"; - return ALPreprocessor::Found::Yes; - }; - const ALPreprocessor::Result made = ALPreprocessor::run("local big = require(\"big\")\nprint(big)\n", options); - const std::string bundle = made.text; - ALLuauService::Modules modules; - for (const ALPreprocessor::Result::Piece& piece : made.apart.modules) + body += "}\n"; + troubled.insert(troubled.find("default"), body); + BENCHMARK("LSL check of a text with 200 problems") + { + problems = lsl.check(troubled, true).size(); + return problems; + }; + } + // A small script that requires a big module, typed in: the bundle + // checked whole at each edit, against the script apart from the module + // (ALPreprocessor::Options::apart), where the module stays checked. { - modules.modules.push_back({ piece.key, piece.text }); + const std::string module = luaScript.texts[0] + "\nreturn {}\n"; + ALPreprocessor::Options options; + options.lua = true; + options.apart = true; + options.resolve = [&module](const ALPreprocessor::Ask& ask, ALPreprocessor::Include& out) { + if (ask.name != "big") + { + return ALPreprocessor::Found::No; + } + out.text = module; + out.name = "big"; + out.path = "disk:/big.luau"; + return ALPreprocessor::Found::Yes; + }; + const ALPreprocessor::Result made = ALPreprocessor::run("local big = require(\"big\")\nprint(big)\n", options); + const std::string bundle = made.text; + ALLuauService::Modules modules; + for (const ALPreprocessor::Result::Piece& piece : made.apart.modules) + { + modules.modules.push_back({ piece.key, piece.text }); + } + for (const ALPreprocessor::Result::Resolved& resolved : made.resolved) + { + modules.reaches.push_back({ resolved.from, resolved.name, resolved.path }); + } + int edits = 0; + luau.setDocument("bundled"); + BENCHMARK("SLua: an edit of a script requiring a big module, bundled") + { + return luau.check(bundle + "-- " + std::to_string(++edits) + "\n").size(); + }; + luau.setDocument("apart"); + luau.setModules(modules); + luau.check(made.apart.script.text); + BENCHMARK("the script apart, the module kept checked") + { + return luau.check(made.apart.script.text + "-- " + std::to_string(++edits) + "\n").size(); + }; + luau.setModules({}); + luau.setDocument(""); } - for (const ALPreprocessor::Result::Resolved& resolved : made.resolved) + // Two tabs in front in turn, neither typed in: each its own module, + // found checked as it was left. (With one module for every tab, each + // turn was a full check of each: twice the check benchmark above.) { - modules.reaches.push_back({ resolved.from, resolved.name, resolved.path }); + const std::string& one = luaScript.texts[0]; + const std::string& two = luaScript.texts[1]; + BENCHMARK("SLua: two tabs checked in turn, per turn") + { + luau.setDocument("one"); + const size_t first = luau.check(one).size(); + luau.setDocument("two"); + return first + luau.check(two).size(); + }; + luau.setDocument(""); } - int edits = 0; - luau.setDocument("bundled"); - const double whole = ms_per_run([&] { - g_sink = g_sink + luau.check(bundle + "-- " + std::to_string(++edits) + "\n").size(); - }); - luau.setDocument("apart"); - luau.setModules(modules); - luau.check(made.apart.script.text); - const double apart = ms_per_run([&] { - g_sink = g_sink + luau.check(made.apart.script.text + "-- " + std::to_string(++edits) + "\n").size(); - }); - luau.setModules({}); - luau.setDocument(""); - row("SLua: an edit of a script requiring a big module, bundled", NONE, whole); - row(" the script apart, the module kept checked", NONE, apart); - } - // Two tabs in front in turn, neither typed in: each its own module, - // found checked as it was left. (With one module for every tab, each - // turn was a full check of each: twice the check row above.) - { - const std::string& one = luaScript.texts[0]; - const std::string& two = luaScript.texts[1]; - row("SLua: two tabs checked in turn, per turn", NONE, ms_per_run([&] { - luau.setDocument("one"); - g_sink = g_sink + luau.check(one).size(); - luau.setDocument("two"); - g_sink = g_sink + luau.check(two).size(); - })); - luau.setDocument(""); - } - // Asked again of a text already checked: what the answer itself - // costs, the check being the one in hand. - { - const std::string& text = luaScript.texts[0]; - luau.check(text); - const S32 last = ll_test::big_script_detail::linesIn(text) - 8; - row("SLua hints of a checked text", NONE, ms_per_run([&] { g_sink = g_sink + luau.inlayHints(text, true, true).size(); })); - row("SLua refactors at a place of a checked text", NONE, - ms_per_run([&] { g_sink = g_sink + luau.actions(text, last, 16, last, 16).size(); })); - } - row("hover at the last helper's call", - ms_per_run([&] { + // Asked again of a text already checked: what the answer itself + // costs, the check being the one in hand. + { + const std::string& text = luaScript.texts[0]; + luau.check(text); + const S32 last = ll_test::big_script_detail::linesIn(text) - 8; + BENCHMARK("SLua hints of a checked text") + { + return luau.inlayHints(text, true, true).size(); + }; + BENCHMARK("SLua refactors at a place of a checked text") + { + return luau.actions(text, last, 16, last, 16).size(); + }; + } + BENCHMARK("hover at the last helper's call, LSL") + { const std::string& text = lslScript.text(); - g_sink = g_sink + lsl.hover(text, ll_test::big_script_detail::linesIn(text) - 12, 20).label.size(); - }), - ms_per_run([&] { + return lsl.hover(text, ll_test::big_script_detail::linesIn(text) - 12, 20).label.size(); + }; + BENCHMARK("hover at the last helper's call, SLua") + { const std::string& text = luaScript.text(); - g_sink = g_sink + luau.hover(text, ll_test::big_script_detail::linesIn(text) - 8, 16).label.size(); - })); - row("weigh (Mono; SLua bytecode)", ms_per_run([&] { g_sink = g_sink + ALScriptWeigh::mono(lslScript.text()).total; }), - ms_per_run([&] { g_sink = g_sink + ALScriptWeigh::slua(luaScript.text()).total; })); + return luau.hover(text, ll_test::big_script_detail::linesIn(text) - 8, 16).label.size(); + }; + BENCHMARK("weigh (Mono; SLua bytecode), LSL") + { + return ALScriptWeigh::mono(lslScript.text()).total; + }; + BENCHMARK("weigh (Mono; SLua bytecode), SLua") + { + return ALScriptWeigh::slua(luaScript.text()).total; + }; - // What running a job on a stack as deep as a script needs costs, - // before the job itself: nothing to do on it. - row("a job on the large stack, doing nothing", ms_per_run([] { alScriptOnLargeStack([] { g_sink = g_sink + 1; }); }), NONE); + // What running a job on a stack as deep as a script needs costs, + // before the job itself: nothing to do on it. + size_t jobs = 0; + BENCHMARK("a job on the large stack, doing nothing") + { + alScriptOnLargeStack([&jobs] { ++jobs; }); + return jobs; + }; - // What typing asks of SLua, under each of Luau's solvers and over + std::printf("\n %-52s %10s %10s\n", "", "LSL", "SLua"); + std::printf(" %-52s %10zu\n", "problems in the LSL text with 200", problems); + std::fflush(stdout); + } + + // What typing asks of SLua, under one of Luau's solvers and over // scripts of three sizes: an edit of one character, then the question // typing asks there -- a completion as a name is typed, signature help // in a call, a hover after either. These are what the studio waits on - // at a keystroke, where the check job waits out its settle. - for (const bool new_solver : { false, true }) + // at a keystroke, where the check job waits out its settle. The solver + // is the case's own, whatever the run's AL_TEST_LUAU_SOLVER says. + void askedWhileTyping(bool new_solver) { ALLuauService typing; + std::string error; typing.setNewSolver(new_solver, error); - typing.loadDefinitions(readWhole(std::string(AL_LSL_DEFINITIONS_DIR) + "/secondlife.d.luau"), error); - std::vector complete, again, signature, hover, expanded, fragment_complete, fragment_signature, fragment_expanded; - int found = 0, fragment_found = 0; - size_t fragment_checks = 0; + if (!typing.loadDefinitions(readWhole(std::string(AL_LSL_DEFINITIONS_DIR) + "/secondlife.d.luau"), error)) + { + SKIP("the SLua definitions did not load: " << error); + } + const std::string solver = new_solver ? "the new solver" : "the old solver"; for (const int lines : { 1000, 5000, 20000 }) { + // A benchmark's name: its row, the solver, the script's length. + const std::string size = std::to_string(lines / 1000) + ",000 lines"; + const auto named = [&](const char* row) { return std::string(row) + ", " + solver + ", " + size; }; + const std::string script = ll_test::bigSLua(lines); Typing word(script, " local x = helper1", " local x = helper12"); Typing call(script, " helper1()", " helper12()", 1); if (lines == 1000) { + INFO("the questions below answer (3 is right)"); const std::vector offered = typing.complete(word.texts[1], word.line, word.columns[1]); - found += std::any_of(offered.begin(), offered.end(), [](const ALScriptCompletion& one) { return one.text == "helper12"; }); - found += typing.signature(call.texts[1], call.line, call.columns[1]).found; - found += typing.hover(word.texts[1], word.line, word.columns[1] - 2).found; + CHECK(std::any_of(offered.begin(), offered.end(), [](const ALScriptCompletion& one) { return one.text == "helper12"; })); + CHECK(typing.signature(call.texts[1], call.line, call.columns[1]).found); + CHECK(typing.hover(word.texts[1], word.line, word.columns[1] - 2).found); } - complete.push_back(ms_per_run([&] { + BENCHMARK(named("complete after an edit")) + { const std::string& text = word.text(); - g_sink = g_sink + typing.complete(text, word.line, word.column()).size(); - })); + return typing.complete(text, word.line, word.column()).size(); + }; // The same asked again of the text it was asked of: what the // answer costs, with nothing to check. - again.push_back(ms_per_run([&] { g_sink = g_sink + typing.complete(word.texts[1], word.line, word.columns[1]).size(); })); - signature.push_back(ms_per_run([&] { + BENCHMARK(named("asked again of the same text")) + { + return typing.complete(word.texts[1], word.line, word.columns[1]).size(); + }; + BENCHMARK(named("signature help in a call after an edit")) + { const std::string& text = call.text(); - g_sink = g_sink + typing.signature(text, call.line, call.column()).parameters.size(); - })); - hover.push_back(ms_per_run([&] { + return typing.signature(text, call.line, call.column()).parameters.size(); + }; + BENCHMARK(named("hover after an edit")) + { const std::string& text = word.text(); - g_sink = g_sink + typing.hover(text, word.line, word.column() - 2).label.size(); - })); + return typing.hover(text, word.line, word.column() - 2).label.size(); + }; // A script the preprocessor expands, as one with a directive is: // each question waits for the expansion of its text, then is // asked of what it made. @@ -561,22 +544,24 @@ int main(int, char**) const size_t at = made.find(" local x = helper1"); made_lines[form] = static_cast(std::count(made.begin(), made.begin() + at, '\n')); } - expanded.push_back(ms_per_run([&] { + BENCHMARK(named("complete after an edit of a script expanded")) + { const std::string& text = directed.text(); const ALPreprocessor::Result made = ALPreprocessor::run(text, options); - g_sink = g_sink + typing.complete(made.text, made_lines[directed.next], directed.column()).size(); - })); + return typing.complete(made.text, made_lines[directed.next], directed.column()).size(); + }; // The same over a fragment, against the expansion of the // script as it was before the line was typed. const std::string before = ALPreprocessor::run("--#define LIMIT 100\n" + script, options).text; typing.check(before); typing.complete(before, 0, 0); typing.setFragments(true); - fragment_expanded.push_back(ms_per_run([&] { + BENCHMARK(named("complete after an edit expanded, over a fragment")) + { const std::string& text = directed.text(); const ALPreprocessor::Result made = ALPreprocessor::run(text, options); - g_sink = g_sink + typing.complete(made.text, made_lines[directed.next], directed.column()).size(); - })); + return typing.complete(made.text, made_lines[directed.next], directed.column()).size(); + }; typing.setFragments(false); } // Over a fragment (ALLuauFragment): the statement typed checked @@ -588,173 +573,240 @@ int main(int, char**) const size_t checks_before = typing.typeChecks(); if (lines == 1000) { + INFO("over a fragment, the questions answer (2 is right)"); const std::vector offered = typing.complete(word.texts[1], word.line, word.columns[1]); - fragment_found += std::any_of(offered.begin(), offered.end(), [](const ALScriptCompletion& one) { return one.text == "helper12"; }); - fragment_found += typing.signature(call.texts[1], call.line, call.columns[1]).found; + CHECK(std::any_of(offered.begin(), offered.end(), [](const ALScriptCompletion& one) { return one.text == "helper12"; })); + CHECK(typing.signature(call.texts[1], call.line, call.columns[1]).found); } - fragment_complete.push_back(ms_per_run([&] { + BENCHMARK(named("complete after an edit, over a fragment")) + { const std::string& text = word.text(); - g_sink = g_sink + typing.complete(text, word.line, word.column()).size(); - })); - fragment_checks += typing.typeChecks() - checks_before; + return typing.complete(text, word.line, word.column()).size(); + }; + { + INFO("whole checks among the completions (0 is right)"); + CAPTURE(lines); + CHECK(typing.typeChecks() - checks_before == 0); + } // Where the fragment finds no function to call -- here the new // solver's at 20,000 lines, whose own check leaves the helpers' // types blocked -- signature help asks the whole script. - fragment_signature.push_back(ms_per_run([&] { + BENCHMARK(named("signature help after an edit, over a fragment")) + { const std::string& text = call.text(); - g_sink = g_sink + typing.signature(text, call.line, call.column()).parameters.size(); - })); + return typing.signature(text, call.line, call.column()).parameters.size(); + }; typing.setFragments(false); } - std::printf("\nSLua typing, the %s solver: an edit of one character, then the question\n\n", new_solver ? "new" : "old"); - std::printf(" %-52s %10s %10s %10s\n", "lines", "1,000", "5,000", "20,000"); - std::printf(" %-52s %10d\n", "the questions below answer (3 is right)", found); - sizes("complete after an edit", complete); - sizes(" asked again of the same text", again); - sizes("signature help in a call after an edit", signature); - sizes("hover after an edit", hover); - sizes("complete after an edit of a script expanded", expanded); - std::printf(" %-52s %10d\n", "over a fragment, the questions answer (2 is right)", fragment_found); - std::printf(" %-52s %10zu\n", " whole checks among the completions (0 is right)", fragment_checks); - sizes("complete after an edit, over a fragment", fragment_complete); - sizes("signature help after an edit, over a fragment", fragment_signature); - sizes("complete after an edit expanded, over a fragment", fragment_expanded); + } + + TEST_CASE("alscript_bench: SLua typing under the old solver", "[alscript_bench][!benchmark]") + { + ll_test::skipUnlessOptimised(); + askedWhileTyping(false); + } + + TEST_CASE("alscript_bench: SLua typing under the new solver", "[alscript_bench][!benchmark]") + { + ll_test::skipUnlessOptimised(); + askedWhileTyping(true); } // What the main thread does for a check before the preprocessor's - // thread has anything to do: the includes found on disk and read. - std::printf("\nThe main thread's include work, a snapshot of a script with five disk includes\n"); + // thread has anything to do: the includes of a snapshot of a script + // with five disk includes found on disk and read. + TEST_CASE("alscript_bench: the main thread's include work", "[alscript_bench][!benchmark]") { + ll_test::skipUnlessOptimised(); const IncludeTree tree; - row("find and read five disk includes, asking the disk", ms_per_run([&] { g_sink = g_sink + includesUncached(tree); }), NONE); + BENCHMARK("find and read five disk includes, asking the disk") + { + return includesUncached(tree); + }; ALDiskCache cache; - row(" what it said a moment ago kept", ms_per_run([&] { g_sink = g_sink + includesKept(tree, cache); }), NONE); + BENCHMARK("what it said a moment ago kept") + { + return includesKept(tree, cache); + }; } - std::printf("\nThe preprocessor's thread's work\n"); - ALPreprocessor::Options lslOptions; - ALPreprocessor::Options luaOptions; - luaOptions.lua = true; - row("expand: no directives", - ms_per_run([&] { g_sink = g_sink + ALPreprocessor::run(lslScript.text(), lslOptions).text.size(); }), - ms_per_run([&] { g_sink = g_sink + ALPreprocessor::run(luaScript.text(), luaOptions).text.size(); })); - row("expand: a macro in every helper", - ms_per_run([&] { g_sink = g_sink + ALPreprocessor::run(macroScript.text(), lslOptions).text.size(); }), NONE); - // A library of helpers behind classic include guards, included fifty - // times, as headers including one another do: its body is wanted once. + TEST_CASE("alscript_bench: the preprocessor's thread's work", "[alscript_bench][!benchmark]") { - std::string header = "#ifndef GUARDED_LSL\n#define GUARDED_LSL\n"; - for (int i = 0; i < 400; ++i) + ll_test::skipUnlessOptimised(); + // The grid's builtins, which the optimizer reads through the LSL + // service. + ALLSLService lsl; + std::string error; + if (!lsl.loadBuiltins(std::string(AL_LSL_DEFINITIONS_DIR) + "/builtins.txt", error)) { - header += "integer helper" + std::to_string(i) + "(integer x) { return x + " + std::to_string(i) + "; }\n"; + SKIP("the builtins did not load: " << error); } - header += "#endif\n"; - std::string script; - for (int i = 0; i < 50; ++i) + + Script lslScript(ll_test::bigLSL(LINES), "\n// again\n"); + Script luaScript(ll_test::bigSLua(LINES), "\n-- again\n"); + Script macroScript(withMacros(ll_test::bigLSL(LINES)), "\n// again\n"); + + ALPreprocessor::Options lslOptions; + ALPreprocessor::Options luaOptions; + luaOptions.lua = true; + BENCHMARK("expand: no directives, LSL") { - script += "#include \"guarded.lsl\"\n"; - } - script += "default { state_entry() { llOwnerSay((string)helper1(2)); } }\n"; - ALPreprocessor::Options guarded = lslOptions; - guarded.resolve = [&header](const ALPreprocessor::Ask&, ALPreprocessor::Include& out) { - out.path = "disk:/guarded.lsl"; - out.name = "guarded.lsl"; - out.text = header; - return ALPreprocessor::Found::Yes; + return ALPreprocessor::run(lslScript.text(), lslOptions).text.size(); }; - row("expand: a guarded header included 50 times", ms_per_run([&] { g_sink = g_sink + ALPreprocessor::run(script, guarded).text.size(); }), - NONE); - } - // Firestorm's transforms over a script that nests: loops in loops - // with break and continue, a switch in each, each a level of the - // transforms' descent. - { - std::string nested; - for (int f = 0; f < 200; ++f) - { - const std::string n = std::to_string(f); - nested += "integer nest" + n + "(integer n)\n{\n integer total = 0;\n integer i;\n for (i = 0; i < n; ++i)\n {\n" - " integer j;\n for (j = 0; j < i; ++j)\n {\n if (j == 3) continue;\n" - " switch (j)\n {\n case 1: total = total + 1; break;\n" - " case 2: { while (total > 9) { total = total - 2; if (total == 5) break; } } break;\n" - " default: total = total + j;\n }\n" - " do { total = total - 1; if (total < 0) break; } while (total > 50);\n" - " if (total > 100) break;\n }\n }\n return total + " + n + ";\n}\n"; - } - nested += "default { state_entry() { llOwnerSay((string)nest0(3)); } }\n"; - ALPreprocessor::Options transformed = lslOptions; - transformed.extensions = true; - transformed.switches = true; - if (const ALPreprocessor::Result r = ALPreprocessor::run(nested, transformed); !r.problems.empty() || !r.usedSwitches || !r.usedExtensions) + BENCHMARK("expand: no directives, SLua") { - std::printf(" (the nested script's transforms did not run cleanly: %zu problems)\n", r.problems.size()); - } - row("expand, extensions and switches: 200 helpers that nest", ms_per_run([&] { g_sink = g_sink + ALPreprocessor::run(nested, transformed).text.size(); }), - NONE); - } - // What a save does past the expansion: the optimizer with its notes, - // each offered as a change, then the compression, the maps composed. - { - ALPreprocessor::Options saving = lslOptions; - saving.optimize = true; - saving.compress = true; - row("expand, optimize with notes, compress (a save)", - ms_per_run([&] { g_sink = g_sink + ALPreprocessor::run(macroScript.text(), saving).problems.size(); }), NONE); - // One line a macro makes long -- a list of three thousand calls of - // it, as generated data is -- saved, and places all along it read - // back to the source, as the checker's colours and problems are. - std::string long_line = "#define P(x) ((x) + 1)\ndefault { state_entry() { list l = ["; - for (int i = 0; i < 3000; ++i) + return ALPreprocessor::run(luaScript.text(), luaOptions).text.size(); + }; + BENCHMARK("expand: a macro in every helper") + { + return ALPreprocessor::run(macroScript.text(), lslOptions).text.size(); + }; + // A library of helpers behind classic include guards, included fifty + // times, as headers including one another do: its body is wanted once. { - long_line += (i ? ", P(" : "P(") + std::to_string(i) + ")"; + std::string header = "#ifndef GUARDED_LSL\n#define GUARDED_LSL\n"; + for (int i = 0; i < 400; ++i) + { + header += "integer helper" + std::to_string(i) + "(integer x) { return x + " + std::to_string(i) + "; }\n"; + } + header += "#endif\n"; + std::string script; + for (int i = 0; i < 50; ++i) + { + script += "#include \"guarded.lsl\"\n"; + } + script += "default { state_entry() { llOwnerSay((string)helper1(2)); } }\n"; + ALPreprocessor::Options guarded = lslOptions; + guarded.resolve = [&header](const ALPreprocessor::Ask&, ALPreprocessor::Include& out) { + out.path = "disk:/guarded.lsl"; + out.name = "guarded.lsl"; + out.text = header; + return ALPreprocessor::Found::Yes; + }; + BENCHMARK("expand: a guarded header included 50 times") + { + return ALPreprocessor::run(script, guarded).text.size(); + }; } - long_line += "]; llOwnerSay((string)llGetListLength(l)); } }\n"; - row("expand: one line of 3,000 macro calls", ms_per_run([&] { g_sink = g_sink + ALPreprocessor::run(long_line, lslOptions).text.size(); }), - NONE); - row("a save of one line of 3,000 macro calls", ms_per_run([&] { g_sink = g_sink + ALPreprocessor::run(long_line, saving).text.size(); }), - NONE); - // The optimizer alone, over what the expansion made: most of a - // save's time. - const std::string long_expanded = ALPreprocessor::run(long_line, lslOptions).text; - row(" optimize it alone", - ms_per_run([&] { g_sink = g_sink + ALLSLOptimizer::run(long_expanded, ALLSLOptimizer::Options()).text.size(); }), NONE); - const ALPreprocessor::Result made = ALPreprocessor::run(long_line, lslOptions); - std::vector> places; - S32 line = 0, column = 0; - for (size_t i = 0; i < made.text.size(); ++i, ++column) - { - if (made.text[i] == '\n') - { - ++line; - column = -1; + // Firestorm's transforms over a script that nests: loops in loops + // with break and continue, a switch in each, each a level of the + // transforms' descent. + { + std::string nested; + for (int f = 0; f < 200; ++f) + { + const std::string n = std::to_string(f); + nested += "integer nest" + n + "(integer n)\n{\n integer total = 0;\n integer i;\n for (i = 0; i < n; ++i)\n {\n" + " integer j;\n for (j = 0; j < i; ++j)\n {\n if (j == 3) continue;\n" + " switch (j)\n {\n case 1: total = total + 1; break;\n" + " case 2: { while (total > 9) { total = total - 2; if (total == 5) break; } } break;\n" + " default: total = total + j;\n }\n" + " do { total = total - 1; if (total < 0) break; } while (total > 50);\n" + " if (total > 100) break;\n }\n }\n return total + " + n + ";\n}\n"; } - else if (i % 64 == 0) + nested += "default { state_entry() { llOwnerSay((string)nest0(3)); } }\n"; + ALPreprocessor::Options transformed = lslOptions; + transformed.extensions = true; + transformed.switches = true; { - places.emplace_back(line, column); + const ALPreprocessor::Result r = ALPreprocessor::run(nested, transformed); + INFO("the nested script's transforms did not run cleanly: " << r.problems.size() << " problems"); + CHECK(r.problems.empty()); + CHECK(r.usedSwitches); + CHECK(r.usedExtensions); } + BENCHMARK("expand, extensions and switches: 200 helpers that nest") + { + return ALPreprocessor::run(nested, transformed).text.size(); + }; } - row(" every 64th byte of its expansion read back", ms_per_run([&] { + // What a save does past the expansion: the optimizer with its notes, + // each offered as a change, then the compression, the maps composed. + { + ALPreprocessor::Options saving = lslOptions; + saving.optimize = true; + saving.compress = true; + BENCHMARK("expand, optimize with notes, compress (a save)") + { + return ALPreprocessor::run(macroScript.text(), saving).problems.size(); + }; + // One line a macro makes long -- a list of three thousand calls of + // it, as generated data is -- saved, and places all along it read + // back to the source, as the checker's colours and problems are. + std::string long_line = "#define P(x) ((x) + 1)\ndefault { state_entry() { list l = ["; + for (int i = 0; i < 3000; ++i) + { + long_line += (i ? ", P(" : "P(") + std::to_string(i) + ")"; + } + long_line += "]; llOwnerSay((string)llGetListLength(l)); } }\n"; + BENCHMARK("expand: one line of 3,000 macro calls") + { + return ALPreprocessor::run(long_line, lslOptions).text.size(); + }; + BENCHMARK("a save of one line of 3,000 macro calls") + { + return ALPreprocessor::run(long_line, saving).text.size(); + }; + // The optimizer alone, over what the expansion made: most of a + // save's time. + const std::string long_expanded = ALPreprocessor::run(long_line, lslOptions).text; + BENCHMARK("optimize it alone") + { + return ALLSLOptimizer::run(long_expanded, ALLSLOptimizer::Options()).text.size(); + }; + const ALPreprocessor::Result made = ALPreprocessor::run(long_line, lslOptions); + std::vector> places; + S32 line = 0, column = 0; + for (size_t i = 0; i < made.text.size(); ++i, ++column) + { + if (made.text[i] == '\n') + { + ++line; + column = -1; + } + else if (i % 64 == 0) + { + places.emplace_back(line, column); + } + } + // A pass over every place, not one place. + BENCHMARK("every 64th byte of its expansion read back") + { + size_t read = 0; for (const auto& [at_line, at_column] : places) { - g_sink = g_sink + size_t(made.map.toSource(at_line, at_column).line); + read += size_t(made.map.toSource(at_line, at_column).line); } - }), - NONE); + return read; + }; + } + ALLSLOptimizer::Options optimizer; + optimizer.notes = false; + BENCHMARK("optimize (Mono)") + { + return ALLSLOptimizer::run(lslScript.text(), optimizer).text.size(); + }; } - ALLSLOptimizer::Options optimizer; - optimizer.notes = false; - row("optimize (Mono)", ms_per_run([&] { g_sink = g_sink + ALLSLOptimizer::run(lslScript.text(), optimizer).text.size(); }), - NONE); - - std::printf("\nTidying\n"); - ALScriptFormatter::Options lslFormat; - ALScriptFormatter::Options luaFormat; - luaFormat.lua = true; - row("format the whole text", ms_per_run([&] { g_sink = g_sink + ALScriptFormatter::format(lslScript.text(), lslFormat).size(); }), - ms_per_run([&] { g_sink = g_sink + ALScriptFormatter::format(luaScript.text(), luaFormat).size(); })); - // One long line: a table or a list of a few thousand terms, which a - // generated script can have. + + TEST_CASE("alscript_bench: tidying", "[alscript_bench][!benchmark]") { + ll_test::skipUnlessOptimised(); + Script lslScript(ll_test::bigLSL(LINES), "\n// again\n"); + Script luaScript(ll_test::bigSLua(LINES), "\n-- again\n"); + + ALScriptFormatter::Options lslFormat; + ALScriptFormatter::Options luaFormat; + luaFormat.lua = true; + BENCHMARK("format the whole text, LSL") + { + return ALScriptFormatter::format(lslScript.text(), lslFormat).size(); + }; + BENCHMARK("format the whole text, SLua") + { + return ALScriptFormatter::format(luaScript.text(), luaFormat).size(); + }; + // One long line: a table or a list of a few thousand terms, which a + // generated script can have. std::string lsl_line = "default { state_entry() { integer n = 0"; std::string lua_line = "local n = 0"; for (int i = 0; i < 3000; ++i) @@ -764,28 +816,34 @@ int main(int, char**) } lsl_line += "; } }\n"; lua_line += "\n"; - row("format one line of 6,000 operators", ms_per_run([&] { g_sink = g_sink + ALScriptFormatter::format(lsl_line, lslFormat).size(); }), - ms_per_run([&] { g_sink = g_sink + ALScriptFormatter::format(lua_line, luaFormat).size(); })); + BENCHMARK("format one line of 6,000 operators, LSL") + { + return ALScriptFormatter::format(lsl_line, lslFormat).size(); + }; + BENCHMARK("format one line of 6,000 operators, SLua") + { + return ALScriptFormatter::format(lua_line, luaFormat).size(); + }; } // A check's messages put into the viewer's words: a thousand, of the // kinds a script mid-edit has, most matching a row late or none. - std::printf("\nMessages\n"); + TEST_CASE("alscript_bench: messages", "[alscript_bench][!benchmark]") { + ll_test::skipUnlessOptimised(); const char* said[] = { "Unknown global 'llSya'", "Type 'number' could not be converted into 'string'", "Argument count mismatch. Function 'f' expects 2 arguments, but 3 are specified", "Something the map has no row for at all" }; - const double ms = ms_per_run([&] { + // A pass of the thousand, not one. + BENCHMARK("1,000 Luau errors keyed") + { ALMessageMap::Match match; + size_t keyed = 0; for (int i = 0; i < 1000; ++i) { - g_sink = g_sink + (ALMessageMap::luauError(said[i % 4], match) ? match.args.size() : 1); + keyed += ALMessageMap::luauError(said[i % 4], match) ? match.args.size() : 1; } - }); - row("1,000 Luau errors keyed", NONE, ms); + return keyed; + }; } - - std::printf("\n(checksum %zu)\n", static_cast(g_sink)); - return 0; -#endif } diff --git a/indra/cmake/ALAddTestBinary.cmake b/indra/cmake/ALAddTestBinary.cmake index 872e5c4d8e..eb0b2263ba 100644 --- a/indra/cmake/ALAddTestBinary.cmake +++ b/indra/cmake/ALAddTestBinary.cmake @@ -14,7 +14,7 @@ endif() # [INCLUDES ...] [DEFINES ...] [ENVIRONMENT ...] # [GL] [PYTHON] [ISA_TIER ] [RESOURCE_LOCK ...] [LAUNCHER ...] # [RERUN RERUN_SOURCES ... RERUN_ENVIRONMENT ...] -# [COMBINED]) +# [BENCH_SOURCES ...] [COMBINED]) # # Builds the test files into one Catch2 executable, , with the harness # (al_test_support: the main, the listeners and the matchers) and SUPPORT's @@ -69,13 +69,24 @@ endif() # with AL_TEST_PER_CASE each of their cases as _/. alscript's # tests that check SLua run again under Luau's new type solver, as a scripter # may choose it. +# +# BENCH_SOURCES are benchmark files, built into the binary beside the tests, +# whose cases are tagged [!benchmark]: hidden, so no test file's entry, the +# combined run or the per-case entries run them. Each is an entry of its +# own, bench//, labelled benchmark and manual, which the +# default run leaves out and BUILD_AND_RUN_BENCHMARKS runs. It runs alone, +# since anything beside it would move its numbers, and reports to the +# console and as XML, Testing/benchmarks/_.xml, which keeps +# the numbers where Catch2's JSON reporter would drop them. A benchmark case +# skips outside an optimised build (altestbenchmark.h), and the entry then +# reads as skipped. function(al_add_test_binary name) cmake_parse_arguments( PARSE_ARGV 1 arg "GL;PYTHON;COMBINED" "PROJECT;ISA_TIER;RERUN" - "SOURCES;UNIT;LIBRARIES;SUPPORT;INCLUDES;DEFINES;ENVIRONMENT;RESOURCE_LOCK;LAUNCHER;RERUN_SOURCES;RERUN_ENVIRONMENT" + "SOURCES;UNIT;LIBRARIES;SUPPORT;INCLUDES;DEFINES;ENVIRONMENT;RESOURCE_LOCK;LAUNCHER;RERUN_SOURCES;RERUN_ENVIRONMENT;BENCH_SOURCES" ) if(NOT arg_PROJECT) message(FATAL_ERROR "al_add_test_binary(${name}): PROJECT is required") @@ -107,15 +118,20 @@ function(al_add_test_binary name) endif() set(stems) - foreach(source IN LISTS arg_SOURCES) + set(bench_stems) + foreach(source IN LISTS arg_SOURCES arg_BENCH_SOURCES) get_filename_component(stem "${source}" NAME_WLE) - if(stem IN_LIST stems) + if(stem IN_LIST stems OR stem IN_LIST bench_stems) message( FATAL_ERROR "al_add_test_binary(${name}): two test files are named ${stem}; the stem names a file's entry and its tag" ) endif() - list(APPEND stems ${stem}) + if(source IN_LIST arg_BENCH_SOURCES) + list(APPEND bench_stems ${stem}) + else() + list(APPEND stems ${stem}) + endif() # A file is in one binary at each tier: the variants of an ISA_TIER # binary build the same files at the others. @@ -145,7 +161,7 @@ function(al_add_test_binary name) # vcpkg's toolchain hangs an app-local copy step on every executable it # sees declared; a test runs from the build tree, as al_add_test explains. set(VCPKG_APPLOCAL_DEPS OFF) - add_executable(${target} ${arg_SOURCES} ${arg_UNIT}) + add_executable(${target} ${arg_SOURCES} ${arg_BENCH_SOURCES} ${arg_UNIT}) target_link_libraries(${target} PRIVATE al::flags al_test_support ${arg_SUPPORT} ${arg_LIBRARIES}) target_include_directories(${target} PRIVATE ${arg_INCLUDES} ${INDRA_SOURCE_DIR}/test) if(arg_UNIT) @@ -377,5 +393,34 @@ function(al_add_test_binary name) endif() endif() + # A benchmark file's run, whichever way the tests are registered. It names + # the file's tag and Catch2's benchmark tag together, which chooses the + # hidden cases a tag alone would leave out. + if(arg_BENCH_SOURCES) + set(bench_dir "${CMAKE_BINARY_DIR}/Testing/benchmarks") + file(MAKE_DIRECTORY "${bench_dir}") + foreach(stem IN LISTS bench_stems) + add_test( + NAME bench/${target}/${stem} + COMMAND + ${arg_LAUNCHER} $ "[#${stem}][!benchmark]" --reporter console + --reporter "xml::out=${bench_dir}/${target}_${stem}.xml" ${arguments} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties( + bench/${target}/${stem} + PROPERTIES + ENVIRONMENT "${environment}" + ENVIRONMENT_MODIFICATION "${AL_TEST_ENVIRONMENT}" + LABELS "${labels};benchmark;manual" + DISABLED ${disabled} + SKIP_RETURN_CODE 125 + RUN_SERIAL TRUE + TIMEOUT 3600 + ) + endforeach() + add_dependencies(BUILD_BENCHMARKS ${target}) + endif() + add_dependencies(BUILD_TESTS ${target}) endfunction() diff --git a/indra/cmake/LLAddBuildTest.cmake b/indra/cmake/LLAddBuildTest.cmake index 289e2d7fd6..fde69f0e6a 100644 --- a/indra/cmake/LLAddBuildTest.cmake +++ b/indra/cmake/LLAddBuildTest.cmake @@ -189,52 +189,3 @@ function(al_add_test name) add_dependencies(BUILD_TESTS ${target}) endfunction() - -# al_add_bench( PROJECT [SOURCES ...] [LIBRARIES ...] -# [INCLUDES ...] [DEFINES ...]) -# -# Builds tests/_bench.cpp, which brings its own main, into -# BENCH_ and registers it as a test labelled benchmark and manual. The -# default test run leaves manual out; BUILD_AND_RUN_BENCHMARKS runs it, verbose, -# since the numbers are the output. A benchmark exits 125, which CTest reads -# as skipped, from a build whose numbers would say nothing. -function(al_add_bench name) - cmake_parse_arguments(PARSE_ARGV 1 arg "" "PROJECT" "SOURCES;LIBRARIES;INCLUDES;DEFINES") - if(NOT arg_PROJECT) - message(FATAL_ERROR "al_add_bench(${name}): PROJECT is required") - endif() - if(arg_UNPARSED_ARGUMENTS) - message(FATAL_ERROR "al_add_bench(${name}): unexpected arguments: ${arg_UNPARSED_ARGUMENTS}") - endif() - - set(target BENCH_${name}) - set(VCPKG_APPLOCAL_DEPS OFF) - add_executable(${target} tests/${name}_bench.cpp ${arg_SOURCES}) - target_link_libraries(${target} PRIVATE al::flags ${arg_LIBRARIES}) - target_include_directories(${target} PRIVATE ${arg_INCLUDES} ${INDRA_SOURCE_DIR}/llmath) - target_compile_definitions(${target} PRIVATE ${arg_DEFINES}) - set_target_properties(${target} PROPERTIES FOLDER "Benchmarks/${arg_PROJECT}") - if(WINDOWS) - set_target_properties(${target} PROPERTIES AL_SKIP_RELEASE_DEBUG_INFO ON) - elseif(DARWIN) - set_target_properties( - ${target} - PROPERTIES XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY "-" BUILD_RPATH "${SHARED_LIB_STAGING_DIR}" - ) - endif() - - add_test( - NAME ${target} - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - set_tests_properties( - ${target} - PROPERTIES - ENVIRONMENT_MODIFICATION "${AL_TEST_ENVIRONMENT}" - LABELS "benchmark;manual" - SKIP_RETURN_CODE 125 - ) - - add_dependencies(BUILD_BENCHMARKS ${target}) -endfunction() diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 3584c24bbe..24709f9f9c 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -364,6 +364,9 @@ if(AL_BUILD_TESTS) tests/threadsafeschedule_test.cpp tests/tuple_test.cpp tests/workqueue_test.cpp + # The viewer's own patterns under Boost.Regex and under RE2, side by + # side, over the text each is run on. + BENCH_SOURCES tests/alregex_bench.cpp SUPPORT llcommon_test_checks LIBRARIES llcommon ) @@ -455,8 +458,4 @@ if(AL_BUILD_TESTS) PROPERTIES COMPILE_OPTIONS -Wno-uninitialized ) endif() - - # The viewer's own patterns under Boost.Regex and under RE2, side by side, - # over the text each is run on. - al_add_bench(alregex PROJECT llcommon LIBRARIES llcommon) endif() diff --git a/indra/llcommon/tests/alregex_bench.cpp b/indra/llcommon/tests/alregex_bench.cpp index 3b4da9c506..0fa88f1c61 100644 --- a/indra/llcommon/tests/alregex_bench.cpp +++ b/indra/llcommon/tests/alregex_bench.cpp @@ -23,76 +23,38 @@ * $/LicenseInfo$ */ -// Each row is one of the places the viewer runs regexes the most, done the +// Each case is one of the places the viewer runs regexes the most, done the // way it was done under Boost.Regex and the way it is done under ALRegex, // over text like what it meets there: the Urls found in a line of chat, the // names RLVa hides in one, a script lexed position by position, a chat log -// read back, the dates of a group's members. A number is nanoseconds per -// line, position or call: the median of five samples, each as many passes -// as fit in twenty milliseconds. Boost.Regex stays in the tree for the -// script editor's find, so the rows can be run again at any time. +// read back, the dates of a group's members. Each benchmark is one side of +// a row, boost or re2, and one call, or one pass named for what it covers: +// ten lines of chat, every place in the script, a log of 10,000 lines. +// Catch2 reports the mean of its samples and the interval around it. A side +// done the same way in several rows is measured once, under the first. +// Boost.Regex stays in the tree for the script editor's find, so the rows +// can be run again at any time. // -// The output is a table, not a verdict; a number is read against the one -// beside it, from the same run on the same quiet machine. An unoptimised -// build exits 125, which CTest reads as skipped, since its numbers would say -// nothing. +// The output is not a verdict; a number is read against the other side's, +// from the same run on the same quiet machine. #include "linden_common.h" #include "alregex.h" +#include "altestbenchmark.h" + #include #include #include #include -#include #include #include #include -// Only an optimised build measures, and only it has a use for these: an -// unoptimised one would say they are unused. -#if defined(LL_RELEASE) namespace { - using clock = std::chrono::steady_clock; - - // Everything a row finds feeds this, so nothing is found for nothing. - volatile size_t g_sink = 0; - - template - double time_per_item(size_t count, F&& pass) - { - pass(); - double samples[5]; - for (double& sample : samples) - { - size_t passes = 0; - const auto start = clock::now(); - clock::duration elapsed{}; - do - { - pass(); - ++passes; - elapsed = clock::now() - start; - } while (elapsed < std::chrono::milliseconds(20)); - sample = std::chrono::duration(elapsed).count() / (double(passes) * double(count)); - } - std::sort(samples, samples + 5); - return samples[2]; - } - - void row(const char* name, double boost_ns, double re2_ns) - { - std::printf(" %-44s %10.1f %10.1f %8.2fx\n", name, boost_ns, re2_ns, boost_ns / re2_ns); - } - - void heading(const char* title, const char* unit) - { - std::printf("\n%s (ns per %s)\n %-44s %10s %10s %9s\n", title, unit, "", "boost", "re2", "speedup"); - } - // --- Urls in chat ----------------------------------------------------- #define APP_HEADER_REGEX "((x-grid-location-info://[-\\w\\.]+/app)|(secondlife:///app))" @@ -155,9 +117,9 @@ namespace "hey secondlife:///app/agent/0e346d8b-4433-4d66-a6b0-fd37083abc4c/mention are you around? www.example.com", }; - void bench_urls() + TEST_CASE("alregex_bench: every URL found in a line of chat as LLTextBase appends it", "[alregex_bench][!benchmark]") { - heading("Urls: every one found in a line of chat, as LLTextBase appends it", "line"); + ll_test::skipUnlessOptimised(); std::vector boost_patterns; std::vector re2_patterns; std::vector groups; @@ -167,11 +129,12 @@ namespace re2_patterns.emplace_back(p.re2 ? p.re2 : p.boost, ALRegex::ICASE); groups.push_back(p.group); } - const size_t lines = std::size(CHAT_WITH_URLS); // As LLUrlRegistry::findUrl did: each pattern by value, the earliest // match of all of them in the rest of the line, and on past it. - const double boost_ns = time_per_item(lines, [&] { + BENCHMARK("31 patterns, earliest match, repeated, ten lines, boost") + { + size_t sum = 0; for (const char* line : CHAT_WITH_URLS) { const std::string text(line); @@ -194,12 +157,15 @@ namespace { break; } - g_sink = g_sink + first; + sum += first; at += std::max(last, 1); } } - }); - const double re2_ns = time_per_item(lines, [&] { + return sum; + }; + BENCHMARK("31 patterns, earliest match, repeated, ten lines, re2") + { + size_t sum = 0; for (const char* line : CHAT_WITH_URLS) { const std::string text(line); @@ -222,20 +188,24 @@ namespace { break; } - g_sink = g_sink + first; + sum += first; at += std::max(last, 1); } } - }); + return sum; + }; // The same, with the patterns first looked for all at once: only - // those the set says match somewhere in the rest are searched. + // those the set says match somewhere in the rest are searched. Its + // boost side is the row's above. ALRegexSet set(ALRegex::ICASE); for (const ALRegex& pattern : re2_patterns) { set.add(pattern.pattern()); } set.compile(); - const double re2_set_ns = time_per_item(lines, [&] { + BENCHMARK("31 patterns, the set asked first, ten lines, re2") + { + size_t sum = 0; std::vector hits; std::vector hit(re2_patterns.size()); for (const char* line : CHAT_WITH_URLS) @@ -270,13 +240,12 @@ namespace { break; } - g_sink = g_sink + first; + sum += first; at += std::max(last, 1); } } - }); - row("31 patterns, earliest match, repeated", boost_ns, re2_ns); - row("the same, the set asked first (re2)", boost_ns, re2_set_ns); + return sum; + }; } // The check LLUrlRegistry::findUrl made before any pattern, as it was: @@ -345,9 +314,9 @@ namespace "that build is gorgeous, how many prims is it? the windows are amazing", }; - void bench_url_gate() + TEST_CASE("alregex_bench: a line of chat with no URL as findUrl is asked of every line", "[alregex_bench][!benchmark]") { - heading("Urls: a line of chat with none, as findUrl is asked of every line", "line"); + ll_test::skipUnlessOptimised(); std::vector boost_patterns; ALRegexSet set(ALRegex::ICASE); for (const UrlPattern& p : URL_PATTERNS) @@ -356,10 +325,11 @@ namespace set.add(p.re2 ? p.re2 : p.boost); } set.compile(); - const size_t lines = std::size(CHAT_WITHOUT_URLS); // As it was: the check, and where it was fooled every pattern. - const double boost_ns = time_per_item(lines, [&] { + BENCHMARK("the old check, then the patterns, ten lines, boost") + { + size_t sum = 0; for (const char* line : CHAT_WITHOUT_URLS) { const std::string text(line); @@ -370,25 +340,30 @@ namespace for (const boost::regex& pattern : boost_patterns) { boost::cmatch found; - g_sink = g_sink + boost::regex_search(text.c_str(), found, pattern); + sum += boost::regex_search(text.c_str(), found, pattern); } } - }); - const double check_ns = time_per_item(lines, [&] { + return sum; + }; + BENCHMARK("the set, ten lines, re2") + { + size_t sum = 0; + std::vector hits; for (const char* line : CHAT_WITHOUT_URLS) { - g_sink = g_sink + oldUrlCheck(line); + sum += set.match(line, hits) ? hits.size() + 1 : 0; } - }); - const double set_ns = time_per_item(lines, [&] { - std::vector hits; + return sum; + }; + BENCHMARK("the old check alone, ten lines") + { + size_t sum = 0; for (const char* line : CHAT_WITHOUT_URLS) { - g_sink = g_sink + (set.match(line, hits) ? hits.size() + 1 : 0); + sum += oldUrlCheck(line); } - }); - row("old check, then patterns (boost) vs set", boost_ns, set_ns); - row("the old check alone vs the set", check_ns, set_ns); + return sum; + }; } // --- RLVa's hidden names ---------------------------------------------- @@ -418,9 +393,9 @@ namespace return boost::regex_replace(text, boost::regex("[.^$|()\\[\\]{}*+?\\\\]"), "\\\\&", boost::match_default | boost::format_sed); } - void bench_rlva() + TEST_CASE("alregex_bench: RLVa hiding twenty nearby names in a line of chat", "[alregex_bench][!benchmark]") { - heading("RLVa: twenty nearby names hidden in a line of chat", "line"); + ll_test::skipUnlessOptimised(); std::vector boost_cached; std::vector re2_cached; for (const char* name : NEARBY) @@ -432,30 +407,33 @@ namespace // As RlvUtil::filterNames did: each name escaped and compiled for // every line. - const double boost_each_ns = time_per_item(1, [&] { + BENCHMARK("twenty names compiled per line, boost") + { std::string text(RLV_LINE); for (const char* name : NEARBY) { boost::replace_all_regex(text, boost::regex("\\b" + escaped(name) + "\\b", boost::regex::icase), anonym); } - g_sink = g_sink + text.size(); - }); - const double boost_cached_ns = time_per_item(1, [&] { + return text.size(); + }; + BENCHMARK("twenty names cached, re2") + { std::string text(RLV_LINE); - for (const boost::regex& pattern : boost_cached) + for (const ALRegex& pattern : re2_cached) { - boost::replace_all_regex(text, pattern, anonym); + pattern.replaceAll(text, anonym); } - g_sink = g_sink + text.size(); - }); - const double re2_ns = time_per_item(1, [&] { + return text.size(); + }; + BENCHMARK("twenty names cached, boost") + { std::string text(RLV_LINE); - for (const ALRegex& pattern : re2_cached) + for (const boost::regex& pattern : boost_cached) { - pattern.replaceAll(text, anonym); + boost::replace_all_regex(text, pattern, anonym); } - g_sink = g_sink + text.size(); - }); + return text.size(); + }; // A name can only match where it is written, whatever its case: a // plain find in the line lowered, before the regex, passes over the // names that are not in it. @@ -464,7 +442,8 @@ namespace { lowered.push_back(asciiLower(name)); } - const double re2_found_ns = time_per_item(1, [&] { + BENCHMARK("twenty names found first, then cached, re2") + { std::string text(RLV_LINE); std::string lower = asciiLower(text); for (size_t i = 0; i < re2_cached.size(); ++i) @@ -474,12 +453,8 @@ namespace lower = asciiLower(text); } } - g_sink = g_sink + text.size(); - }); - row("compiled per line (boost) vs cached (re2)", boost_each_ns, re2_ns); - row("both cached", boost_cached_ns, re2_ns); - row("compiled per line (boost) vs found first (re2)", boost_each_ns, re2_found_ns); - row("both cached, found first (re2)", boost_cached_ns, re2_found_ns); + return text.size(); + }; } // --- a script lexed --------------------------------------------------- @@ -530,9 +505,9 @@ namespace "local t = type(door); t.type = 1e3; local x = 0b1010 + 3.25e-2", }; - void bench_lexing() + TEST_CASE("alregex_bench: lexing with the SLua grammar's regex rules tried at every place in a script", "[alregex_bench][!benchmark]") { - heading("Lexing: the SLua grammar's regex rules tried at every place in a script", "place"); + ll_test::skipUnlessOptimised(); std::vector boost_rules; std::vector re2_rules; std::vector lo, hi; @@ -551,13 +526,17 @@ namespace { places += std::string_view(line).size(); } + // A call is a pass over every place in the script. + const std::string script = ", " + std::to_string(places) + " places, "; auto word_or_access = [](char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_' || c == '.' || c == ':'; }; // As ALSyntaxGrammar::tryRule did: each rule anchored where the lexer // stands, with what comes before it read. - const double boost_ns = time_per_item(places, [&] { + BENCHMARK("seven rules, anchored" + script + "boost") + { + size_t sum = 0; for (const char* line_text : SLUA_SOURCE) { const std::string_view line(line_text); @@ -575,12 +554,14 @@ namespace break; } } - g_sink = g_sink + taken; + sum += taken; pos += taken; } } - }); + return sum; + }; auto re2_pass = [&](bool use_filter) { + size_t sum = 0; for (const char* line_text : SLUA_SOURCE) { const std::string_view line(line_text); @@ -607,15 +588,20 @@ namespace break; } } - g_sink = g_sink + taken; + sum += taken; pos += taken; } } + return sum; + }; + BENCHMARK("seven rules, anchored" + script + "re2") + { + return re2_pass(false); + }; + BENCHMARK("seven rules, anchored, first byte filtered" + script + "re2") + { + return re2_pass(true); }; - const double re2_ns = time_per_item(places, [&] { re2_pass(false); }); - const double re2_filtered_ns = time_per_item(places, [&] { re2_pass(true); }); - row("seven rules, anchored", boost_ns, re2_ns); - row("seven rules, anchored, first byte filtered", boost_ns, re2_filtered_ns); } // --- chat highlights ----------------------------------------------------- @@ -683,10 +669,9 @@ namespace return 1; } - void bench_highlights() + TEST_CASE("alregex_bench: chat highlights splitting a line at the keywords it holds as LLTextBase appends it", "[alregex_bench][!benchmark]") { - heading("Chat highlights: a line split at the keywords it holds, as LLTextBase appends it", "line"); - const size_t lines = std::size(CHAT_LINES); + ll_test::skipUnlessOptimised(); for (const size_t count : { size_t(5), size_t(20), size_t(50) }) { ALRegexSet set(ALRegex::ICASE); @@ -695,29 +680,33 @@ namespace set.add(ALRegex::escape(KEYWORDS[i])); } set.compile(); - const double boost_ns = time_per_item(lines, [&] { + const std::string keywords = std::to_string(count) + " keywords, "; + BENCHMARK(keywords + "each found, ten lines, boost") + { + size_t sum = 0; for (const char* line : CHAT_LINES) { - g_sink = g_sink + highlightPieces(line, 0, count, nullptr); + sum += highlightPieces(line, 0, count, nullptr); } - }); - const double set_ns = time_per_item(lines, [&] { + return sum; + }; + BENCHMARK(keywords + "the set, ten lines, re2") + { + size_t sum = 0; for (const char* line : CHAT_LINES) { - g_sink = g_sink + highlightPieces(line, 0, count, &set); + sum += highlightPieces(line, 0, count, &set); } - }); - char name[64]; - std::snprintf(name, sizeof(name), "%zu keywords, each found (boost) vs the set", count); - row(name, boost_ns, set_ns); + return sum; + }; } } // --- a chat log read back --------------------------------------------- - void bench_chat_log() + TEST_CASE("alregex_bench: a chat log's lines read for their time and name and words as LLChatLogParser reads them", "[alregex_bench][!benchmark]") { - heading("Chat log: each line's time, name and words, as LLChatLogParser reads them", "line"); + ll_test::skipUnlessOptimised(); const char* const TIMESTAMP_AND_STUFF = "^(\\[\\d{4}/\\d{1,2}/\\d{1,2}\\s+\\d{1,2}:\\d{2}\\s[AaPp][Mm]\\]\\s+|\\[\\d{4}/\\d{1,2}/\\d{1,2}\\s+\\d{1,2}:\\d{2}\\]\\s+|" "\\[\\d{1,2}:\\d{2}\\s[AaPp][Mm]\\]\\s+|\\[\\d{1,2}:\\d{2}\\]\\s+)?(.*)$"; @@ -738,7 +727,9 @@ namespace (i % 3 ? "hello there, how is everyone doing today?" : "brb")); } - const double boost_ns = time_per_item(log.size(), [&] { + BENCHMARK("timestamp, then name and text, 10,000 lines, boost") + { + size_t sum = 0; for (const std::string& line : log) { boost::smatch stamp, name; @@ -747,12 +738,15 @@ namespace const std::string stuff = stamp[2]; if (boost::regex_match(stuff, name, boost_name)) { - g_sink = g_sink + name[3].length(); + sum += name[3].length(); } } } - }); - const double re2_ns = time_per_item(log.size(), [&] { + return sum; + }; + BENCHMARK("timestamp, then name and text, 10,000 lines, re2") + { + size_t sum = 0; for (const std::string& line : log) { ALRegexMatch stamp, name; @@ -761,11 +755,12 @@ namespace const std::string stuff = stamp.str(2); if (re2_name.match(stuff, &name)) { - g_sink = g_sink + name.length(3); + sum += name.length(3); } } } - }); + return sum; + }; // The same read with the tails left off: `(.*)$` takes the rest of // the line whatever it is, so the rest is what follows the stamp, // and what follows the name. @@ -773,7 +768,9 @@ namespace "(\\[\\d{4}/\\d{1,2}/\\d{1,2}\\s+\\d{1,2}:\\d{2}\\s[AaPp][Mm]\\]\\s+|\\[\\d{4}/\\d{1,2}/\\d{1,2}\\s+\\d{1,2}:\\d{2}\\]\\s+|" "\\[\\d{1,2}:\\d{2}\\s[AaPp][Mm]\\]\\s+|\\[\\d{1,2}:\\d{2}\\]\\s+)?"); const ALRegex re2_name_only("([^:]+[:]{1})?(\\s*)"); - const double re2_prefix_ns = time_per_item(log.size(), [&] { + BENCHMARK("timestamp, then name and text, prefixes only, 10,000 lines, re2") + { + size_t sum = 0; for (const std::string& line : log) { ALRegexMatch stamp, name; @@ -782,17 +779,20 @@ namespace const std::string stuff = line.substr(stamp.end()); if (re2_name_only.search(stuff, &name, 0, true)) { - g_sink = g_sink + (stuff.size() - name.end()); + sum += stuff.size() - name.end(); } } } - }); + return sum; + }; // And with no group asked for at all, which the DFA alone answers: // the stamp is the whole of what its pattern matches, and the name // is what `[^:]+:` does, the blanks after it what `\s*` does. const ALRegex re2_name_colon("[^:]+:"); const ALRegex re2_blanks("\\s*"); - const double re2_whole_ns = time_per_item(log.size(), [&] { + BENCHMARK("timestamp, then name and text, whole matches only, 10,000 lines, re2") + { + size_t sum = 0; for (const std::string& line : log) { ALRegexMatch stamp, name, blanks; @@ -801,60 +801,43 @@ namespace const std::string_view stuff = std::string_view(line).substr(stamp.end()); const size_t named = re2_name_colon.search(stuff, &name, 0, true, 0) ? name.end() : 0; re2_blanks.search(stuff, &blanks, named, true, 0); - g_sink = g_sink + (stuff.size() - blanks.end()); + sum += stuff.size() - blanks.end(); } } - }); - row("timestamp, then name and text", boost_ns, re2_ns); - row("the same, prefixes only (re2)", boost_ns, re2_prefix_ns); - row("the same, whole matches only (re2)", boost_ns, re2_whole_ns); + return sum; + }; } // --- a group's members' dates ----------------------------------------- - void bench_dates() + TEST_CASE("alregex_bench: group members with a date read from each row", "[alregex_bench][!benchmark]") { - heading("Group members: a date read from each row", "row"); + ll_test::skipUnlessOptimised(); const char* const DATE = "([0-9]{1,2})/([0-9]{1,2})/([0-9]{4})"; const std::string when = "09/23/2026"; // As LLGroupMgr::formatDateString did: compiled for every row. - const double boost_ns = time_per_item(1, [&] { + BENCHMARK("a date's pattern compiled per row, boost") + { + size_t sum = 0; const boost::regex expression(DATE); boost::cmatch result; if (boost::regex_match(when.c_str(), result, expression)) { - g_sink = g_sink + result[3].length(); + sum += result[3].length(); } - }); - static const ALRegex expression(DATE); - const double re2_ns = time_per_item(1, [&] { + return sum; + }; + const ALRegex expression(DATE); + BENCHMARK("a date's pattern compiled once, re2") + { + size_t sum = 0; ALRegexMatch result; if (expression.match(when, &result)) { - g_sink = g_sink + result.length(3); + sum += result.length(3); } - }); - row("compiled per row (boost) vs once (re2)", boost_ns, re2_ns); + return sum; + }; } } -#endif // LL_RELEASE - -int main(int, char**) -{ -#if !defined(LL_RELEASE) - std::printf("Skipped: an unoptimised build has no numbers worth reading\n"); - return 125; -#else - std::printf("alregex_bench: Boost.Regex against ALRegex over RE2"); - bench_urls(); - bench_url_gate(); - bench_rlva(); - bench_lexing(); - bench_highlights(); - bench_chat_log(); - bench_dates(); - std::printf("\n(checksum %zu)\n", static_cast(g_sink)); - return 0; -#endif -} diff --git a/indra/llmath/CMakeLists.txt b/indra/llmath/CMakeLists.txt index 493440ab90..6dddd76515 100644 --- a/indra/llmath/CMakeLists.txt +++ b/indra/llmath/CMakeLists.txt @@ -133,6 +133,9 @@ if(AL_BUILD_TESTS) tests/v4coloru_test.cpp tests/v4math_test.cpp tests/xform_test.cpp + # Nanoseconds per element for the SIMD math the geometry rebuild spends + # its time in, and for the choices the SIMD series makes by a number. + BENCH_SOURCES tests/llsimd_bench.cpp LIBRARIES llmath llcommon ) @@ -154,8 +157,4 @@ if(AL_BUILD_TESTS) endif() endforeach() endif() - - # Nanoseconds per element for the SIMD math the geometry rebuild spends - # its time in, and for the choices the SIMD series makes by a number. - al_add_bench(llsimd PROJECT llmath LIBRARIES llmath llcommon) endif() diff --git a/indra/llmath/tests/llsimd_bench.cpp b/indra/llmath/tests/llsimd_bench.cpp index 984e4c9874..77aad6e494 100644 --- a/indra/llmath/tests/llsimd_bench.cpp +++ b/indra/llmath/tests/llsimd_bench.cpp @@ -23,16 +23,15 @@ * $/LicenseInfo$ */ -// Each row is one kernel over one array, reported as nanoseconds per -// element: the median of five samples, each sample as many passes over the -// array as fit in twenty milliseconds. Two array sizes: four thousand -// elements, which sit in the first-level cache, and four million, which -// stream from memory. A row differs from the row beside it in one thing. +// Each row is one kernel run over two arrays, a benchmark for each: four +// thousand elements, which sit in the first-level cache, and four million, +// which stream from memory. A benchmark is one whole pass over its array, +// named for the elements it covers; Catch2 reports the mean of its samples +// and the interval around it, which divided by that count is nanoseconds +// per element. A row differs from the row beside it in one thing. // // The output is a table, not a verdict; a number is read against the row -// next to it from the same run on the same quiet machine. An unoptimised -// build exits 125, which CTest reads as skipped, since its numbers would -// say nothing. +// next to it from the same run on the same quiet machine. #include "linden_common.h" @@ -46,57 +45,77 @@ #include "alsimd.h" #include "alsimdkernels.inl" -#include +#include "altestbenchmark.h" + #include -#include #include #include #include +#include #include #include namespace { - using clock = std::chrono::steady_clock; - - // Everything a kernel produces feeds this, so nothing is computed for - // nothing. - volatile U32 g_sink = 0; + constexpr size_t CACHED = 4 * 1024; + constexpr size_t STREAMING = 4 * 1024 * 1024; - // The median of five samples of ns per element, each sample enough - // passes to fill twenty milliseconds. - template - double time_per_element(size_t count, F&& pass) + // A count as a benchmark's name gives it: 4,096. + std::string grouped(size_t count) { - pass(); - double samples[5]; - for (double& sample : samples) + std::string digits = std::to_string(count); + for (size_t at = digits.size(); at > 3; at -= 3) { - size_t passes = 0; - const auto start = clock::now(); - clock::duration elapsed{}; - do - { - pass(); - ++passes; - elapsed = clock::now() - start; - } while (elapsed < std::chrono::milliseconds(20)); - const double ns = std::chrono::duration(elapsed).count(); - sample = ns / (double(passes) * double(count)); + digits.insert(at - 3, ","); } - std::sort(samples, samples + 5); - return samples[2]; + return digits; } - constexpr size_t CACHED = 4 * 1024; - constexpr size_t STREAMING = 4 * 1024 * 1024; + // The machine and the build the numbers are of, printed once a run, by + // whichever case comes first. + void describe_machine() + { + static const bool described = [] + { + const char* isa = AL_SIMD_NEON ? "NEON" : AL_SIMD_AVX512 ? "AVX-512" : AL_SIMD_AVX2 ? "AVX2" : AL_SIMD_AVX ? "AVX" : AL_SIMD_SSE4 ? "SSE4.2" : "SSE2"; + std::printf("llsimd_bench on %s\n built for %s, level %d, %s backend, FMA %s\n", + LLProcessorInfo().getCPUBrandName().c_str(), isa, AL_ISA_LEVEL, + AL_SIMD_VEXT ? "vector-extension" : "intrinsic", AL_SIMD_FMA ? "on" : "off"); + std::fflush(stdout); + return true; + }(); + (void)described; + } + // The arrays a kernel runs over, as a kernel sees them: views, of which + // each call makes a copy of its own. A local, its bounds and pointers + // stay in registers through the loop, as they did when each row made + // its arrays where it ran. A kernel run over arrays kept in a static + // reloads them for every element, since any store it makes might have + // moved them, and was 40% slower over the cached arrays for it. struct Vectors { - std::vector a, b, c, out; + std::span a, b, c, out; // four joint.weight values per vector, joints 0 to 7 - std::vector weights; - explicit Vectors(size_t n) : a(n), b(n), c(n), out(n), weights(n) + std::span weights; + // Everything a kernel produces feeds this, so nothing is computed + // for nothing. + U32 checksum() const + { + U32 sum = 0; + for (const LLVector4a& v : out) + { + sum ^= std::bit_cast(v[0]) + std::bit_cast(v[3]); + } + return sum; + } + }; + + // What the views are of. + struct Storage + { + std::vector a, b, c, out, weights; + explicit Storage(size_t n) : a(n), b(n), c(n), out(n), weights(n) { std::mt19937 rng(0x5eed); std::uniform_real_distribution dist(-8.f, 8.f); @@ -109,32 +128,45 @@ namespace weights[i].set(joint(rng), joint(rng), joint(rng), joint(rng)); } } - U32 checksum() const - { - U32 sum = 0; - for (const LLVector4a& v : out) - { - sum ^= std::bit_cast(v[0]) + std::bit_cast(v[3]); - } - return sum; - } + Vectors view() { return { a, b, c, out, weights }; } }; - // One row: the kernel run over the cached and the streaming arrays. - template - void row(const char* name, Kernel&& kernel) - { - Vectors small(CACHED); - Vectors large(STREAMING); - const double cached = time_per_element(CACHED, [&] { kernel(small); g_sink = g_sink + small.checksum(); }); - const double streaming = time_per_element(STREAMING, [&] { kernel(large); g_sink = g_sink + large.checksum(); }); - std::printf(" %-44s %8.3f %8.3f\n", name, cached, streaming); - std::fflush(stdout); + // The arrays every row runs over, made once for every case of a run. A + // kernel writes only `out`, so each row finds the inputs as the first + // did. + struct Arrays + { + Storage cached{CACHED}; + Storage streaming{STREAMING}; + }; + + Arrays& arrays() + { + static Arrays made; + return made; } - void print_header(const char* section) + // One row: the kernel run over the cached and the streaming arrays, a + // pass and its checksum a call. The kernel is copied into each call as + // the arrays' views are, so a matrix it captured is a local too. + template + void row(const std::string& name, Kernel&& kernel) { - std::printf("\n%s\n %-44s %8s %8s\n", section, "ns per element", "cached", "stream"); + Arrays& arrays_ = arrays(); + BENCHMARK(name + ", " + grouped(arrays_.cached.out.size()) + " elements, cached") + { + Vectors v = arrays_.cached.view(); + auto run = kernel; + run(v); + return v.checksum(); + }; + BENCHMARK(name + ", " + grouped(arrays_.streaming.out.size()) + " elements, stream") + { + Vectors v = arrays_.streaming.view(); + auto run = kernel; + run(v); + return v.checksum(); + }; } } @@ -161,10 +193,11 @@ static void copy_128(char* __restrict dst, const char* __restrict src, size_t by } // The memcpy: the tree's aligned copy against a 128-bit loop and the C -// runtime's, per call, at the sizes the volume slabs come in. -static void bench_memcpy() +// runtime's, a call each, at the sizes the volume slabs come in. +TEST_CASE("llsimd_bench: memcpy per call", "[llsimd_bench][!benchmark]") { - std::printf("\nmemcpy, ns per call %10s %10s %10s\n", "aligned16", "128-bit", "memcpy"); + ll_test::skipUnlessOptimised(); + describe_machine(); const size_t sizes[] = {64, 256, 1024, 4096, 16384, 65536, 262144, 1024 * 1024, 4 * 1024 * 1024}; for (size_t size : sizes) { @@ -172,18 +205,22 @@ static void bench_memcpy() char* dst = (char*)ll_aligned_malloc_64(size); std::memset(src, 0x5a, size); std::memset(dst, 0, size); - auto time = [&](auto&& copy) + const std::string bytes = std::to_string(size) + " bytes, "; + BENCHMARK(bytes + "aligned16") { - return time_per_element(1, [&] - { - copy(dst, src, size); - g_sink = g_sink + (U32)dst[size - 1]; - }); + ll_memcpy_nonaliased_aligned_16(dst, src, size); + return dst[size - 1]; + }; + BENCHMARK(bytes + "128-bit loop") + { + copy_128(dst, src, size); + return dst[size - 1]; + }; + BENCHMARK(bytes + "memcpy") + { + std::memcpy(dst, src, size); + return dst[size - 1]; }; - const double ours = time([](char* d, const char* s, size_t n) { ll_memcpy_nonaliased_aligned_16(d, s, n); }); - const double narrow = time(copy_128); - const double crt = time([](char* d, const char* s, size_t n) { std::memcpy(d, s, n); }); - std::printf(" %-30zu %10.1f %10.1f %10.1f\n", size, ours, narrow, crt); ll_aligned_free_64(src); ll_aligned_free_64(dst); } @@ -192,9 +229,10 @@ static void bench_memcpy() // The dot product three ways: the class as it stands, the shuffle form and // the DPPS instruction, so the guard that picks between them is decided by a // number. -static void bench_dot() +TEST_CASE("llsimd_bench: dot3 in all lanes", "[llsimd_bench][!benchmark]") { - print_header("dot3, all lanes"); + ll_test::skipUnlessOptimised(); + describe_machine(); row("LLVector4a::setAllDot3", [](Vectors& v) { for (size_t i = 0; i < v.a.size(); ++i) @@ -232,9 +270,10 @@ static void bench_dot() // A multiply-add as the class does it, as the ops layer does it, and as the // intrinsic does it where there is one: what the fast floating-point // contract already fuses, and what an explicit fused form adds. -static void bench_fma() +TEST_CASE("llsimd_bench: a * b + c", "[llsimd_bench][!benchmark]") { - print_header("a * b + c"); + ll_test::skipUnlessOptimised(); + describe_machine(); row("LLVector4a setMul then add", [](Vectors& v) { for (size_t i = 0; i < v.a.size(); ++i) @@ -265,9 +304,10 @@ static void bench_fma() // The per-element operations the types are rebuilt on, each as the class // does it today beside the ops layer's form. -static void bench_elementwise() +TEST_CASE("llsimd_bench: per-element operations", "[llsimd_bench][!benchmark]") { - print_header("per element"); + ll_test::skipUnlessOptimised(); + describe_machine(); row("LLVector4a::setSelectWithMask", [](Vectors& v) { for (size_t i = 0; i < v.a.size(); ++i) @@ -360,9 +400,10 @@ static void bench_elementwise() // The transform the geometry rebuild spends its time in: a point through // an affine matrix, the matrix way and the ops way. -static void bench_transform() +TEST_CASE("llsimd_bench: affine transform of a point", "[llsimd_bench][!benchmark]") { - print_header("affine transform of a point"); + ll_test::skipUnlessOptimised(); + describe_machine(); LLMatrix4a mat; mat.initAll(LLVector3(1.f, 2.f, 0.5f), LLQuaternion(0.7f, LLVector3(0.f, 0.f, 1.f)), LLVector3(1.f, 2.f, 3.f)); row("LLMatrix4a::affineTransform", [mat](Vectors& v) @@ -406,39 +447,32 @@ static void bench_kernels_at(const char* width) using namespace alsimd::kernels; LLMatrix4a mat; mat.initAll(LLVector3(1.f, 2.f, 0.5f), LLQuaternion(0.7f, LLVector3(0.f, 0.f, 1.f)), LLVector3(1.f, 2.f, 3.f)); - std::string name; - name = std::string("transform_points ") + width; - row(name.c_str(), [mat](Vectors& v) + row(std::string("transform_points ") + width, [mat](Vectors& v) { transform_impl(mat, v.a.data(), v.out.data(), v.a.size(), 0.f); }); - name = std::string("transform_directions ") + width; - row(name.c_str(), [mat](Vectors& v) + row(std::string("transform_directions ") + width, [mat](Vectors& v) { transform_impl(mat, v.a.data(), v.out.data(), v.a.size(), 0.f); }); - name = std::string("transform_texcoords ") + width; - row(name.c_str(), [](Vectors& v) + row(std::string("transform_texcoords ") + width, [](Vectors& v) { const LLVector4a trans(-0.5f), rot0(0.8f, -0.6f, 0.8f, -0.6f), rot1(0.6f, 0.8f, 0.6f, 0.8f), scale(2.f, 2.f, 2.f, 2.f), offset(0.5f); transform_texcoords_impl((const F32*) v.a.data(), (F32*) v.out.data(), v.a.size(), trans, rot0, rot1, scale, offset); }); - name = std::string("extents ") + width; - row(name.c_str(), [](Vectors& v) + row(std::string("extents ") + width, [](Vectors& v) { extents_impl(v.a.data(), v.a.size(), v.out[0], v.out[1]); }); - name = std::string("offset_indices_u16 x8 ") + width; - row(name.c_str(), [](Vectors& v) + row(std::string("offset_indices_u16 x8 ") + width, [](Vectors& v) { // eight indices per vector of the arrays, so the count is per vector const U16* src = (const U16*) v.a.data(); U16* dst = (U16*) v.out.data(); offset_indices_u16_impl(src, dst, v.a.size() * 8, 3); }); - name = std::string("dequantize_u16x3 ") + width; - row(name.c_str(), [](Vectors& v) + row(std::string("dequantize_u16x3 ") + width, [](Vectors& v) { // six bytes per vertex read from the source array, one vector written const size_t n = v.a.size() * 16 / 6 < v.out.size() ? v.a.size() * 16 / 6 : v.out.size(); @@ -446,9 +480,10 @@ static void bench_kernels_at(const char* width) }); } -static void bench_kernels() +TEST_CASE("llsimd_bench: batch kernels", "[llsimd_bench][!benchmark]") { - print_header("batch kernels"); + ll_test::skipUnlessOptimised(); + describe_machine(); bench_kernels_at("128"); #if AL_SIMD_AVX bench_kernels_at("256"); @@ -521,30 +556,44 @@ namespace } }; + // One row: the morph applied over the cached and the streaming + // vertices, a pass and its checksum a call. The calls take the weight + // and its negative in turn, so what they accumulate comes back to where + // it was rather than growing call on call; the work is the same either + // way. template - void morph_row(const char* name, Kernel&& kernel) + void morph_row(const std::string& name, F32 weight, Kernel&& kernel) { Morph small(CACHED / 4); Morph large(STREAMING / 16); - const double cached = time_per_element(small.index.size(), [&] { kernel(small); g_sink = g_sink + small.checksum(); }); - const double streaming = time_per_element(large.index.size(), [&] { kernel(large); g_sink = g_sink + large.checksum(); }); - std::printf(" %-44s %8.3f %8.3f\n", name, cached, streaming); - std::fflush(stdout); + size_t small_calls = 0; + BENCHMARK(name + ", " + grouped(small.index.size()) + " morph vertices, cached") + { + kernel(small, ++small_calls % 2 ? weight : -weight); + return small.checksum(); + }; + size_t large_calls = 0; + BENCHMARK(name + ", " + grouped(large.index.size()) + " morph vertices, stream") + { + kernel(large, ++large_calls % 2 ? weight : -weight); + return large.checksum(); + }; } } -static void bench_morph() +TEST_CASE("llsimd_bench: morph apply", "[llsimd_bench][!benchmark]") { - print_header("morph apply, per morph vertex"); + ll_test::skipUnlessOptimised(); + describe_machine(); const F32 delta_weight = 0.125f; const F32 soften = 0.65f; - morph_row("LLPolyMorphTarget::apply as written", [=](Morph& m) + morph_row("LLPolyMorphTarget::apply as written", delta_weight, [=](Morph& m, F32 weight) { for (size_t i = 0; i < m.index.size(); ++i) { const U32 at = m.index[i]; - const F32 w = delta_weight * m.mask[i]; + const F32 w = weight * m.mask[i]; LLVector4a pos = m.coord_delta[i]; pos.mul(w); @@ -574,13 +623,13 @@ static void bench_morph() } }); - morph_row("alsimd::morph_apply", [=](Morph& m) + morph_row("alsimd::morph_apply", delta_weight, [=](Morph& m, F32 weight) { alsimd::MorphApply apply; apply.count = m.index.size(); apply.index = m.index.data(); apply.mask = m.mask.data(); - apply.weight = delta_weight; + apply.weight = weight; apply.soften = soften; apply.coord_delta = m.coord_delta.data(); apply.normal_delta = m.normal_delta.data(); @@ -595,25 +644,3 @@ static void bench_morph() alsimd::morph_apply(apply); }); } - -int main(int, char**) -{ -#if !defined(LL_RELEASE) - std::printf("Skipped: an unoptimised build has no numbers worth reading\n"); - return 125; -#else - const char* isa = AL_SIMD_NEON ? "NEON" : AL_SIMD_AVX512 ? "AVX-512" : AL_SIMD_AVX2 ? "AVX2" : AL_SIMD_AVX ? "AVX" : AL_SIMD_SSE4 ? "SSE4.2" : "SSE2"; - std::printf("llsimd_bench on %s\n built for %s, level %d, %s backend, FMA %s\n", - LLProcessorInfo().getCPUBrandName().c_str(), isa, AL_ISA_LEVEL, - AL_SIMD_VEXT ? "vector-extension" : "intrinsic", AL_SIMD_FMA ? "on" : "off"); - bench_memcpy(); - bench_dot(); - bench_fma(); - bench_elementwise(); - bench_transform(); - bench_kernels(); - bench_morph(); - std::printf("\n(checksum %u)\n", (unsigned)g_sink); - return 0; -#endif -} diff --git a/indra/llui/CMakeLists.txt b/indra/llui/CMakeLists.txt index 8db20427ce..e084d35920 100644 --- a/indra/llui/CMakeLists.txt +++ b/indra/llui/CMakeLists.txt @@ -706,6 +706,12 @@ if(AL_BUILD_TESTS) tests/llurlregistry_test.cpp tests/llview_test.cpp tests/llxuiparser_test.cpp + # The benchmarks: comparing big scripts, by lines and laid out, and what + # each comparison says of some edits, which each change to how texts are + # compared is measured and judged by; and the code editor over + # 50,000-line scripts, which each change to the edit pipeline, the + # highlighter, the layout, the folds or find is measured by. + BENCH_SOURCES tests/altextdiff_bench.cpp tests/altextengine_bench.cpp SUPPORT llui_test_fakes llui_test_checks llrender_test_checks llcommon_test_checks LIBRARIES llui ${test_libs} DEFINES "LLUI_TEST_APP_DIR=\"${INDRA_SOURCE_DIR}/newview\"" @@ -729,42 +735,12 @@ if(AL_BUILD_TESTS) llui_gl_tests PROJECT llui SOURCES tests/aldiffview_gl_test.cpp tests/alsurface_gl_test.cpp tests/altextview_gl_test.cpp + # The same editor drawn, a frame at a time, into llrender's hidden + # window: what the drawing steps are measured by. + BENCH_SOURCES tests/altextdraw_bench.cpp SUPPORT llui_test_fakes llui_gl_test_checks llrender_test_checks llcommon_test_checks LIBRARIES llui ${test_libs} DEFINES "LLUI_TEST_APP_DIR=\"${INDRA_SOURCE_DIR}/newview\"" GL ) - - # The benchmarks, still executables of their own, link the fakes with - # the library. - set(test_libs llui llui_test_fakes ${test_libs}) - - # The code editor over 50,000-line scripts: what each change to the edit - # pipeline, the highlighter, the layout, the folds or find is measured by. - al_add_bench( - altextengine - PROJECT llui - LIBRARIES ${test_libs} - INCLUDES ${INDRA_SOURCE_DIR}/test - DEFINES "LLUI_TEST_APP_DIR=\"${INDRA_SOURCE_DIR}/newview\"" - ) - # Comparing big scripts, by lines and laid out, and what each comparison - # says of some edits: what each change to how texts are compared is - # measured and judged by. - al_add_bench( - altextdiff - PROJECT llui - LIBRARIES ${test_libs} - INCLUDES ${INDRA_SOURCE_DIR}/test - DEFINES "LLUI_TEST_APP_DIR=\"${INDRA_SOURCE_DIR}/newview\"" - ) - # The same editor drawn, a frame at a time, into llrender's hidden window: - # what the drawing steps are measured by. - al_add_bench( - altextdraw - PROJECT llui - LIBRARIES ${test_libs} - INCLUDES ${INDRA_SOURCE_DIR}/test - DEFINES "LLUI_TEST_APP_DIR=\"${INDRA_SOURCE_DIR}/newview\"" - ) endif() diff --git a/indra/llui/tests/altextdiff_bench.cpp b/indra/llui/tests/altextdiff_bench.cpp index 524be549a5..de18cb60d7 100644 --- a/indra/llui/tests/altextdiff_bench.cpp +++ b/indra/llui/tests/altextdiff_bench.cpp @@ -26,9 +26,9 @@ // What comparing costs over generated LSL scripts of 5,000 and 50,000 // lines, each against itself edited ten times and a thousand: the lines // alone (ALTextDiff), and the comparison laid out as a view shows it -- -// lines, each change's lines paired and their words -- (ALDiffModel). A -// number is milliseconds per operation: the median of five samples, each -// as many passes as fit in fifty milliseconds (and at least one). +// lines, each change's lines paired and their words -- (ALDiffModel). Each +// benchmark is one operation, named for its text's length; Catch2 reports +// the mean of its samples and the interval around it. // // After the times, what each comparison says of some edits a person // makes -- a line changed, a block moved, a function reformatted, lines @@ -37,8 +37,6 @@ // or worse by being lower; they are read against the same rows from // before a change to how texts are compared, to see that it did what it // meant to and nothing else. -// -// An unoptimised build exits 125, which CTest reads as skipped. #include "linden_common.h" @@ -47,47 +45,17 @@ #include "altextdiff.h" #include "albigscript.h" +#include "altestbenchmark.h" #include -#include #include -#include +#include #include #include #include -#ifndef LLUI_TEST_APP_DIR -# define LLUI_TEST_APP_DIR "" -#endif - -#if defined(LL_RELEASE) namespace { - using clock = std::chrono::steady_clock; - - volatile size_t g_sink = 0; - - double ms_per_item(size_t count, const std::function& pass) - { - pass(); - double samples[5]; - for (double& sample : samples) - { - size_t passes = 0; - const auto start = clock::now(); - clock::duration elapsed{}; - do - { - pass(); - ++passes; - elapsed = clock::now() - start; - } while (elapsed < std::chrono::milliseconds(50)); - sample = std::chrono::duration(elapsed).count() / (double(passes) * double(count)); - } - std::sort(samples, samples + 5); - return samples[2]; - } - std::string joined(const std::vector& lines) { std::string out; @@ -185,11 +153,6 @@ namespace return out; } - void row(const char* name, double small, double big) - { - std::printf(" %-54s %10.3f %10.3f\n", name, small, big); - } - // What a comparison says, laid out: its changes, the lines marked on // each side, the lines paired as changed, and the words marked in them. void says(const char* name, const std::string& left, const std::string& right, const ALTextDiff::Options& options = ALTextDiff::Options()) @@ -216,61 +179,42 @@ namespace } std::printf(" %-44s %8d %8d %8d %8d %8d %8d\n", name, model.changeCount(), removed, added, paired, marks, moved); } -} -#endif // LL_RELEASE -int main(int, char**) -{ -#if !defined(LL_RELEASE) - std::printf("Skipped: an unoptimised build has no numbers worth reading\n"); - return 125; -#else - const std::vector small = ALTextDiff::split(ll_test::bigLSL(5000)); - const std::vector big = ALTextDiff::split(ll_test::bigLSL(50000)); - const std::vector small10 = edited(small, 10, 1); - const std::vector small1k = edited(small, 1000, 2); - const std::vector big10 = edited(big, 10, 3); - const std::vector big1k = edited(big, 1000, 4); - const std::string small_t = joined(small); - const std::string big_t = joined(big); - const std::string small10t = joined(small10); - const std::string small1kt = joined(small1k); - const std::string big10t = joined(big10); - const std::string big1kt = joined(big1k); - - std::printf("altextdiff_bench: comparing generated LSL scripts (ms per operation)\n"); - std::printf("\n %-54s %10s %10s\n", "", "5,000", "50,000"); - - std::printf("\nLines (ALTextDiff::lines)\n"); - const auto lines = [&](const std::vector& left, const std::vector& right, const ALTextDiff::Options& options) { - return ms_per_item(1, [&] { g_sink = g_sink + ALTextDiff::lines(left, right, options).size(); }); + // The texts compared, made once for every case of a run: each script, + // edited ten times and a thousand, as lines and as text. + struct Texts + { + std::vector small = ALTextDiff::split(ll_test::bigLSL(5000)); + std::vector big = ALTextDiff::split(ll_test::bigLSL(50000)); + std::vector small10 = edited(small, 10, 1); + std::vector small1k = edited(small, 1000, 2); + std::vector big10 = edited(big, 10, 3); + std::vector big1k = edited(big, 1000, 4); + std::string small_t = joined(small); + std::string big_t = joined(big); + std::string small10t = joined(small10); + std::string small1kt = joined(small1k); + std::string big10t = joined(big10); + std::string big1kt = joined(big1k); }; + + const Texts& texts() { - const ALTextDiff::Options histogram; - row("histogram, ten edits", lines(small, small10, histogram), lines(big, big10, histogram)); - row("histogram, a thousand edits", lines(small, small1k, histogram), lines(big, big1k, histogram)); - ALTextDiff::Options patience; - patience.algorithm = ALTextDiff::Algorithm::Patience; - row("patience, ten edits", lines(small, small10, patience), lines(big, big10, patience)); - row("patience, a thousand edits", lines(small, small1k, patience), lines(big, big1k, patience)); - ALTextDiff::Options minimal; - minimal.algorithm = ALTextDiff::Algorithm::Minimal; - row("minimal, ten edits", lines(small, small10, minimal), lines(big, big10, minimal)); - row("minimal, a thousand edits", lines(small, small1k, minimal), lines(big, big1k, minimal)); - ALTextDiff::Options structural; - structural.algorithm = ALTextDiff::Algorithm::Structural; - row("structural, ten edits", lines(small, small10, structural), lines(big, big10, structural)); - row("structural, a thousand edits", lines(small, small1k, structural), lines(big, big1k, structural)); + static const Texts made; + return made; } - std::printf("\nLaid out (ALDiffModel: lines, pairs, words, rows)\n"); // LSL's grammar, where the source tree has it, for words cut as code. - std::string error; - std::shared_ptr lsl = ALSyntaxGrammar::fromFile(std::string(LLUI_TEST_APP_DIR) + "/app_settings/syntax/lsl.xml", error); - const auto lexer = [&lsl]() { return lsl ? ALDiffLexer::lexerOf(std::make_shared(lsl)) : ALTextDiff::lexer_t(); }; + std::shared_ptr lslGrammar() + { + std::string error; + return ALSyntaxGrammar::fromFile(std::string(LLUI_TEST_APP_DIR) + "/app_settings/syntax/lsl.xml", error); + } + // A model's lexer as a view gives it: saying what it read again, and // told what an edit knows; and a merge's of its own, likewise. - const auto lexing = [&lsl](ALDiffModel& model) { + void lexing(ALDiffModel& model, const std::shared_ptr& lsl) + { const std::shared_ptr compared = lsl ? std::make_shared(lsl) : nullptr; const std::shared_ptr merging = lsl ? std::make_shared(lsl) : nullptr; model.setLexer(compared ? ALDiffLexer::lexerOf(compared) : ALTextDiff::lexer_t(), merging ? ALDiffLexer::lexerOf(merging) : ALTextDiff::lexer_t(), @@ -278,63 +222,51 @@ int main(int, char**) compared ? ALDiffLexer::toldOf(compared) : ALTextDiff::told_t(), merging ? ALDiffLexer::rereadOf(merging) : ALTextDiff::reread_t(), merging ? ALDiffLexer::toldOf(merging) : ALTextDiff::told_t()); + } + + // A live comparison, the right typed in: a character put in a line near + // the middle, and taken out again, each call one keystroke compared + // again. Blanks let go of where asked, which a grammar's strings keep, + // or comments, which a grammar says where they are. + struct Typing + { + bool grammar = false; + ALTextDiff::Algorithm algorithm = ALTextDiff::Algorithm::Histogram; + ALTextDiff::ranges_t ranges; + bool blanks = false; + bool comments = false; }; - const auto laid = [&](const std::string& left, const std::string& right, bool grammar) { - return ms_per_item(1, [&] { - ALDiffModel model; - if (grammar) - { - lexing(model); - } - model.setTexts(left, right); - g_sink = g_sink + static_cast(model.rowCount(ALDiffModel::Layout::Sides)); - }); - }; - row("ten edits", laid(small_t, small10t, false), laid(big_t, big10t, false)); - row("a thousand edits", laid(small_t, small1kt, false), laid(big_t, big1kt, false)); - row("ten edits, words by LSL's grammar", laid(small_t, small10t, true), laid(big_t, big10t, true)); - row("a thousand edits, words by LSL's grammar", laid(small_t, small1kt, true), laid(big_t, big1kt, true)); - // A live comparison, the right typed in: a character put in a line - // near the middle and compared again, then taken out and compared - // again; per keystroke. Blanks let go of where asked, which a grammar's - // strings keep, or comments, which a grammar says where they are. - const auto typed = [&](const std::string& left, const std::string& right, bool grammar = false, - ALTextDiff::Algorithm algorithm = ALTextDiff::Algorithm::Histogram, const ALTextDiff::ranges_t& ranges = {}, - bool blanks = false, bool comments = false) { + + void typed(const char* name, const std::string& left, const std::string& right, const Typing& typing = Typing()) + { ALDiffModel model; - if (grammar) + if (typing.grammar) { - lexing(model); + lexing(model, lslGrammar()); } ALTextDiff::Likeness like; - like.ignoreWhitespace = blanks; - like.ignoreComments = comments; + like.ignoreWhitespace = typing.blanks; + like.ignoreComments = typing.comments; model.setLikeness(like); - model.setAlgorithm(algorithm); - model.setTexts(left, right, ranges); + model.setAlgorithm(typing.algorithm); + model.setTexts(left, right, typing.ranges); const size_t at = right.find('\n', right.size() / 2); std::string with = right; with.insert(at, "x"); - return ms_per_item(2, [&] { - model.setRightText(with); - model.setRightText(right); - g_sink = g_sink + static_cast(model.changeCount()); - }); - }; - row("a keystroke, ten edits", typed(small_t, small10t), typed(big_t, big10t)); - row("a keystroke, a thousand edits", typed(small_t, small1kt), typed(big_t, big1kt)); - row("a keystroke, a thousand edits, LSL's grammar", typed(small_t, small1kt, true), typed(big_t, big1kt, true)); - row("a keystroke, a thousand edits, LSL's, blanks let go", typed(small_t, small1kt, true, ALTextDiff::Algorithm::Histogram, {}, true), - typed(big_t, big1kt, true, ALTextDiff::Algorithm::Histogram, {}, true)); - row("a keystroke, a thousand edits, LSL's, comments let go", typed(small_t, small1kt, true, ALTextDiff::Algorithm::Histogram, {}, false, true), - typed(big_t, big1kt, true, ALTextDiff::Algorithm::Histogram, {}, false, true)); - row("a keystroke, a thousand edits, by structure", typed(small_t, small1kt, true, ALTextDiff::Algorithm::Structural), - typed(big_t, big1kt, true, ALTextDiff::Algorithm::Structural)); + size_t keystrokes = 0; + BENCHMARK(name) + { + model.setRightText(++keystrokes % 2 ? with : right); + return model.changeCount(); + }; + } + // The same keystroke in a merge: the text before both versions the // base, theirs the left with ten edits, ours the right with a thousand. - const auto merged = [&](const std::string& base, const std::string& left, const std::string& right, bool comments) { + void merged(const char* name, const std::string& base, const std::string& left, const std::string& right, bool comments) + { ALDiffModel model; - lexing(model); + lexing(model, lslGrammar()); ALTextDiff::Likeness like; like.ignoreComments = comments; model.setLikeness(like); @@ -343,87 +275,190 @@ int main(int, char**) const size_t at = right.find('\n', right.size() / 2); std::string with = right; with.insert(at, "x"); - return ms_per_item(2, [&] { - model.setRightText(with); - model.setRightText(right); - g_sink = g_sink + static_cast(model.conflictCount()); - }); - }; - row("a keystroke in a merge, LSL's grammar", merged(small_t, small10t, small1kt, false), merged(big_t, big10t, big1kt, false)); - row("a keystroke in a merge, LSL's, comments let go", merged(small_t, small10t, small1kt, true), merged(big_t, big10t, big1kt, true)); + size_t keystrokes = 0; + BENCHMARK(name) + { + model.setRightText(++keystrokes % 2 ? with : right); + return model.conflictCount(); + }; + } + + // Another version on the left, as a slider over a script's saves steps: + // a line apart from the one before. + void stepped(const char* name, const std::string& left, const std::string& right) { - // Another version on the left, as a slider over a script's saves - // steps: a line apart from the one before. - const auto stepped = [&](const std::string& left, const std::string& right) { - ALDiffModel model; - model.setTexts(left, right); - const size_t at = left.find('\n', left.size() / 3); - std::string with = left; - with.insert(at, " // older"); - return ms_per_item(2, [&] { - model.setLeftText(with); - model.setLeftText(left); - g_sink = g_sink + static_cast(model.changeCount()); - }); + ALDiffModel model; + model.setTexts(left, right); + const size_t at = left.find('\n', left.size() / 3); + std::string with = left; + with.insert(at, " // older"); + size_t steps = 0; + BENCHMARK(name) + { + model.setLeftText(++steps % 2 ? with : left); + return model.changeCount(); }; - row("another version on the left, ten edits", stepped(small_t, small10t), stepped(big_t, big10t)); } + + // A script beside what it was converted to: every line written + // otherwise, each anchored to its own, as the converter lines them up. + std::string converted(const std::vector& lines, ALTextDiff::ranges_t& ranges) { - // A script beside what it was converted to: every line written - // otherwise, each anchored to its own, as the converter lines them up. - const auto converted = [](const std::vector& lines, ALTextDiff::ranges_t& ranges) { - std::vector out; - for (size_t n = 0; n < lines.size(); ++n) + std::vector out; + for (size_t n = 0; n < lines.size(); ++n) + { + out.push_back(lines[n] + " -- as SLua"); + ranges.push_back({ static_cast(n), static_cast(n), static_cast(n), static_cast(n) }); + } + return joined(out); + } + + + TEST_CASE("altextdiff_bench: the lines alone by each algorithm", "[altextdiff_bench][!benchmark]") + { + ll_test::skipUnlessOptimised(); + const Texts& t = texts(); + const auto lines = [](const char* name, const std::vector& left, const std::vector& right, + ALTextDiff::Algorithm algorithm) { + ALTextDiff::Options options; + options.algorithm = algorithm; + BENCHMARK(name) { - out.push_back(lines[n] + " -- as SLua"); - ranges.push_back({ static_cast(n), static_cast(n), static_cast(n), static_cast(n) }); - } - return joined(out); + return ALTextDiff::lines(left, right, options).size(); + }; }; + lines("histogram, ten edits, 5,000 lines", t.small, t.small10, ALTextDiff::Algorithm::Histogram); + lines("histogram, ten edits, 50,000 lines", t.big, t.big10, ALTextDiff::Algorithm::Histogram); + lines("histogram, a thousand edits, 5,000 lines", t.small, t.small1k, ALTextDiff::Algorithm::Histogram); + lines("histogram, a thousand edits, 50,000 lines", t.big, t.big1k, ALTextDiff::Algorithm::Histogram); + lines("patience, ten edits, 5,000 lines", t.small, t.small10, ALTextDiff::Algorithm::Patience); + lines("patience, ten edits, 50,000 lines", t.big, t.big10, ALTextDiff::Algorithm::Patience); + lines("patience, a thousand edits, 5,000 lines", t.small, t.small1k, ALTextDiff::Algorithm::Patience); + lines("patience, a thousand edits, 50,000 lines", t.big, t.big1k, ALTextDiff::Algorithm::Patience); + lines("minimal, ten edits, 5,000 lines", t.small, t.small10, ALTextDiff::Algorithm::Minimal); + lines("minimal, ten edits, 50,000 lines", t.big, t.big10, ALTextDiff::Algorithm::Minimal); + lines("minimal, a thousand edits, 5,000 lines", t.small, t.small1k, ALTextDiff::Algorithm::Minimal); + lines("minimal, a thousand edits, 50,000 lines", t.big, t.big1k, ALTextDiff::Algorithm::Minimal); + lines("structural, ten edits, 5,000 lines", t.small, t.small10, ALTextDiff::Algorithm::Structural); + lines("structural, ten edits, 50,000 lines", t.big, t.big10, ALTextDiff::Algorithm::Structural); + lines("structural, a thousand edits, 5,000 lines", t.small, t.small1k, ALTextDiff::Algorithm::Structural); + lines("structural, a thousand edits, 50,000 lines", t.big, t.big1k, ALTextDiff::Algorithm::Structural); + } + + TEST_CASE("altextdiff_bench: laid out and made whole", "[altextdiff_bench][!benchmark]") + { + ll_test::skipUnlessOptimised(); + const Texts& t = texts(); + const std::shared_ptr lsl = lslGrammar(); + const auto laid = [&lsl](const char* name, const std::string& left, const std::string& right, bool grammar) { + BENCHMARK(name) + { + ALDiffModel model; + if (grammar) + { + lexing(model, lsl); + } + model.setTexts(left, right); + return model.rowCount(ALDiffModel::Layout::Sides); + }; + }; + laid("ten edits, 5,000 lines", t.small_t, t.small10t, false); + laid("ten edits, 50,000 lines", t.big_t, t.big10t, false); + laid("a thousand edits, 5,000 lines", t.small_t, t.small1kt, false); + laid("a thousand edits, 50,000 lines", t.big_t, t.big1kt, false); + laid("ten edits, words by LSL's grammar, 5,000 lines", t.small_t, t.small10t, true); + laid("ten edits, words by LSL's grammar, 50,000 lines", t.big_t, t.big10t, true); + laid("a thousand edits, words by LSL's grammar, 5,000 lines", t.small_t, t.small1kt, true); + laid("a thousand edits, words by LSL's grammar, 50,000 lines", t.big_t, t.big1kt, true); + } + + TEST_CASE("altextdiff_bench: laid out with a keystroke compared again", "[altextdiff_bench][!benchmark]") + { + ll_test::skipUnlessOptimised(); + const Texts& t = texts(); + typed("a keystroke, ten edits, 5,000 lines", t.small_t, t.small10t); + typed("a keystroke, ten edits, 50,000 lines", t.big_t, t.big10t); + typed("a keystroke, a thousand edits, 5,000 lines", t.small_t, t.small1kt); + typed("a keystroke, a thousand edits, 50,000 lines", t.big_t, t.big1kt); + Typing grammar; + grammar.grammar = true; + typed("a keystroke, a thousand edits, LSL's grammar, 5,000 lines", t.small_t, t.small1kt, grammar); + typed("a keystroke, a thousand edits, LSL's grammar, 50,000 lines", t.big_t, t.big1kt, grammar); + Typing blanks = grammar; + blanks.blanks = true; + typed("a keystroke, a thousand edits, LSL's, blanks let go, 5,000 lines", t.small_t, t.small1kt, blanks); + typed("a keystroke, a thousand edits, LSL's, blanks let go, 50,000 lines", t.big_t, t.big1kt, blanks); + Typing comments = grammar; + comments.comments = true; + typed("a keystroke, a thousand edits, LSL's, comments let go, 5,000 lines", t.small_t, t.small1kt, comments); + typed("a keystroke, a thousand edits, LSL's, comments let go, 50,000 lines", t.big_t, t.big1kt, comments); + Typing structure = grammar; + structure.algorithm = ALTextDiff::Algorithm::Structural; + typed("a keystroke, a thousand edits, by structure, 5,000 lines", t.small_t, t.small1kt, structure); + typed("a keystroke, a thousand edits, by structure, 50,000 lines", t.big_t, t.big1kt, structure); + merged("a keystroke in a merge, LSL's grammar, 5,000 lines", t.small_t, t.small10t, t.small1kt, false); + merged("a keystroke in a merge, LSL's grammar, 50,000 lines", t.big_t, t.big10t, t.big1kt, false); + merged("a keystroke in a merge, LSL's, comments let go, 5,000 lines", t.small_t, t.small10t, t.small1kt, true); + merged("a keystroke in a merge, LSL's, comments let go, 50,000 lines", t.big_t, t.big10t, t.big1kt, true); + stepped("another version on the left, ten edits, 5,000 lines", t.small_t, t.small10t); + stepped("another version on the left, ten edits, 50,000 lines", t.big_t, t.big10t); + ALTextDiff::ranges_t small_ranges; ALTextDiff::ranges_t big_ranges; - const std::string small_c = converted(small, small_ranges); - const std::string big_c = converted(big, big_ranges); - row("a keystroke, every line converted, anchored", typed(small_t, small_c, false, ALTextDiff::Algorithm::Histogram, small_ranges), - typed(big_t, big_c, false, ALTextDiff::Algorithm::Histogram, big_ranges)); - row("a keystroke, converted, LSL's, blanks let go", typed(small_t, small_c, true, ALTextDiff::Algorithm::Histogram, small_ranges, true), - typed(big_t, big_c, true, ALTextDiff::Algorithm::Histogram, big_ranges, true)); - row("a keystroke, converted, LSL's, comments let go", - typed(small_t, small_c, true, ALTextDiff::Algorithm::Histogram, small_ranges, false, true), - typed(big_t, big_c, true, ALTextDiff::Algorithm::Histogram, big_ranges, false, true)); + const std::string small_c = converted(t.small, small_ranges); + const std::string big_c = converted(t.big, big_ranges); + Typing anchored_small; + anchored_small.ranges = small_ranges; + Typing anchored_big; + anchored_big.ranges = big_ranges; + typed("a keystroke, every line converted, anchored, 5,000 lines", t.small_t, small_c, anchored_small); + typed("a keystroke, every line converted, anchored, 50,000 lines", t.big_t, big_c, anchored_big); + anchored_small.grammar = anchored_big.grammar = true; + anchored_small.blanks = anchored_big.blanks = true; + typed("a keystroke, converted, LSL's, blanks let go, 5,000 lines", t.small_t, small_c, anchored_small); + typed("a keystroke, converted, LSL's, blanks let go, 50,000 lines", t.big_t, big_c, anchored_big); + anchored_small.blanks = anchored_big.blanks = false; + anchored_small.comments = anchored_big.comments = true; + typed("a keystroke, converted, LSL's, comments let go, 5,000 lines", t.small_t, small_c, anchored_small); + typed("a keystroke, converted, LSL's, comments let go, 50,000 lines", t.big_t, big_c, anchored_big); } - std::printf("\nWhat it says (5,000 lines)\n"); - std::printf(" %-44s %8s %8s %8s %8s %8s %8s\n", "", "changes", "out", "in", "paired", "words", "moved"); - ALTextDiff::Options by_grammar; - by_grammar.lexer = lexer(); - says("ten edits", small_t, small10t); - says("a thousand edits", small_t, small1kt); - says("a function moved", small_t, joined(moved(small))); - says("calls and braces reformatted", small_t, joined(reformatted(small, 200, 200))); + TEST_CASE("altextdiff_bench: what each comparison says of some edits", "[altextdiff_bench][!benchmark]") { - // A line taken out of the middle of lines changed: what pairing by - // place gets wrong. - std::vector right = small; - for (size_t i = 300; i < 306; ++i) + ll_test::skipUnlessOptimised(); + const Texts& t = texts(); + std::printf("\nWhat it says (5,000 lines)\n"); + std::printf(" %-44s %8s %8s %8s %8s %8s %8s\n", "", "changes", "out", "in", "paired", "words", "moved"); + ALTextDiff::Options by_grammar; + if (const std::shared_ptr lsl = lslGrammar()) { - right[i] += " // edited"; + by_grammar.lexer = ALDiffLexer::lexerOf(std::make_shared(lsl)); } - right.erase(right.begin() + 302); - says("one taken out among six changed", small_t, joined(right)); - } - says("a thousand edits, LSL's grammar", small_t, small1kt, by_grammar); - for (const auto& [algorithm, name] : { std::make_pair(ALTextDiff::Algorithm::Patience, "patience"), std::make_pair(ALTextDiff::Algorithm::Minimal, "minimal"), - std::make_pair(ALTextDiff::Algorithm::Structural, "structural") }) - { - ALTextDiff::Options by; - by.algorithm = algorithm; - says((std::string("a thousand edits, ") + name).c_str(), small_t, small1kt, by); - says((std::string("a function moved, ") + name).c_str(), small_t, joined(moved(small)), by); - says((std::string("calls and braces reformatted, ") + name).c_str(), small_t, joined(reformatted(small, 200, 200)), by); + says("ten edits", t.small_t, t.small10t); + says("a thousand edits", t.small_t, t.small1kt); + says("a function moved", t.small_t, joined(moved(t.small))); + says("calls and braces reformatted", t.small_t, joined(reformatted(t.small, 200, 200))); + { + // A line taken out of the middle of lines changed: what pairing + // by place gets wrong. + std::vector right = t.small; + for (size_t i = 300; i < 306; ++i) + { + right[i] += " // edited"; + } + right.erase(right.begin() + 302); + says("one taken out among six changed", t.small_t, joined(right)); + } + says("a thousand edits, LSL's grammar", t.small_t, t.small1kt, by_grammar); + for (const auto& [algorithm, name] : { std::make_pair(ALTextDiff::Algorithm::Patience, "patience"), std::make_pair(ALTextDiff::Algorithm::Minimal, "minimal"), + std::make_pair(ALTextDiff::Algorithm::Structural, "structural") }) + { + ALTextDiff::Options by; + by.algorithm = algorithm; + says((std::string("a thousand edits, ") + name).c_str(), t.small_t, t.small1kt, by); + says((std::string("a function moved, ") + name).c_str(), t.small_t, joined(moved(t.small)), by); + says((std::string("calls and braces reformatted, ") + name).c_str(), t.small_t, joined(reformatted(t.small, 200, 200)), by); + } + std::fflush(stdout); } - - std::printf("\n(checksum %zu)\n", static_cast(g_sink)); - return 0; -#endif } diff --git a/indra/llui/tests/altextdraw_bench.cpp b/indra/llui/tests/altextdraw_bench.cpp index 45bcf7c8bb..126c25819e 100644 --- a/indra/llui/tests/altextdraw_bench.cpp +++ b/indra/llui/tests/altextdraw_bench.cpp @@ -25,75 +25,64 @@ // What the script studio's editor costs to draw, a frame at a time, over a // script of 50,000 lines, LSL and SLua side by side: the code editor as the // studio makes it, drawn into llrender's hidden window with the viewer's GL. -// A number is the CPU's milliseconds per frame: the median of five samples, -// each as many frames as fit in fifty milliseconds. What is counted of a -// frame is the editor's draw and the batch flushed, which is everything -// the viewer's thread does for it, the driver's work on each draw call -// included; the GPU is waited for between frames, off the clock, so that -// nothing queues up across them and what it takes to draw is not counted. +// Each benchmark is one frame, named for its language, and its time is the +// CPU's; Catch2 reports the mean of its samples and the interval around it. +// What is counted of a frame is the editor's draw and the batch flushed, +// which is everything the viewer's thread does for it, the driver's work on +// each draw call included; the GPU is waited for between frames, off the +// clock, so that nothing queues up across them and what it takes to draw is +// not counted. // // Nothing here changes between frames, so every row is an idle frame: what // the editor spends redrawing what it drew the frame before. The output is a // table, read against the same row from another build on the same quiet -// machine. An unoptimised build, or one with no GL, exits 125, which CTest -// reads as skipped. +// machine. #include "linden_common.h" #include "llglheaders.h" -#include "../../llrender/tests/llheadlessgl_fixture.h" +#include "aluigl_fixture.h" #include "alcodeeditor.h" #include "alheadlessui_fixture.h" #include "albigscript.h" +#include "altestbenchmark.h" #include "../lluictrlfactory.h" -#include -#include #include -#include #include -#include #include +#include -#if defined(LL_RELEASE) namespace { - using clock = std::chrono::steady_clock; - constexpr int LINES = 50000; - // Milliseconds per call of `frame`, which says how long its own part - // took. - double ms_per_frame(const std::function& frame) + // The scripts, made once for every case of a run. + struct Scripts { - frame(); - double samples[5]; - for (double& sample : samples) - { - size_t frames = 0; - double counted = 0.0; - const auto start = clock::now(); - do - { - counted += frame(); - ++frames; - } while (clock::now() - start < std::chrono::milliseconds(50)); - sample = counted / double(frames); - } - std::sort(samples, samples + 5); - return samples[2]; + std::string lsl = ll_test::bigLSL(LINES); + std::string slua = ll_test::bigSLua(LINES); + }; + + const Scripts& scripts() + { + static const Scripts made; + return made; } struct Subject { - const char* name; - const char* syntax; - std::string text; - ALCodeEditor* editor = nullptr; + const char* name; + const char* syntax; + // The column its numbers were read under, which ends each + // benchmark's name. + const char* language; + const std::string& text; + ALCodeEditor* editor = nullptr; }; ALCodeEditor* makeEditor(const Subject& s) @@ -109,21 +98,62 @@ namespace return editor; } - // The draw and the batch handed over, on the clock; then the GPU - // waited for. - double drawFrame(ALCodeEditor& e) + // Both languages' editors, made for a case in the binary's shared + // context, whose UI puts the fonts' glyphs in its textures, and dying + // when the case ends however it ends. + struct Editors { - const auto start = clock::now(); - e.draw(); - gGL.flush(); - const double ms = std::chrono::duration(clock::now() - start).count(); - glFinish(); - return ms; - } + Subject subjects[2] = { + { "lsl", "lsl", "LSL", scripts().lsl }, + { "slua", "slua", "SLua", scripts().slua }, + }; - void row(const char* name, double lsl, double slua) + Editors() + { + ll_test::uiGL(); + ll_test::installWhiteTexture(); + if (!ll_test::HeadlessUI::get(/*gl_textures=*/true).ok()) + { + SKIP("no UI, since LLUI_TEST_APP_DIR does not point at the source tree"); + } + for (Subject& s : subjects) + { + s.editor = makeEditor(s); + } + } + + ~Editors() + { + for (Subject& s : subjects) + { + s.editor->die(); + } + } + + Subject* begin() { return subjects; } + Subject* end() { return subjects + 2; } + }; + + // A frame of each language's editor, once `setup` has made it what the + // row draws: the draw and the batch handed over, on the clock; then the + // GPU waited for. A frame is one run of a sample, so the wait is between + // every two. + template + void both(const char* name, Editors& editors, Setup&& setup) { - std::printf(" %-52s %10.3f %10.3f\n", name, lsl, slua); + for (Subject& s : editors) + { + ALCodeEditor& e = *s.editor; + setup(e); + BENCHMARK_ADVANCED(std::string(name) + ", " + s.language)(Catch::Benchmark::Chronometer meter) + { + meter.measure([&] { + e.draw(); + gGL.flush(); + }); + glFinish(); + }; + } } // How many draw calls a frame makes: each flush of the batch the @@ -139,30 +169,23 @@ namespace return capture.size(); } - template - void countBoth(const char* name, Subject (&subjects)[2], Setup&& setup) + // A table of counts, under its heading and the languages' columns. + void countHeading(const char* heading) { - size_t counts[2]; - for (int i = 0; i < 2; ++i) - { - ALCodeEditor& e = *subjects[i].editor; - setup(e); - counts[i] = drawCalls(e); - } - std::printf(" %-52s %10zu %10zu\n", name, counts[0], counts[1]); + std::printf("\n%s\n %-52s %10s %10s\n", heading, "", "LSL", "SLua"); } template - void both(const char* name, Subject (&subjects)[2], Setup&& setup) + void countBoth(const char* name, Editors& editors, Setup&& setup) { - double ms[2]; + size_t counts[2]; for (int i = 0; i < 2; ++i) { - ALCodeEditor& e = *subjects[i].editor; + ALCodeEditor& e = *editors.subjects[i].editor; setup(e); - ms[i] = ms_per_frame([&] { return drawFrame(e); }); + counts[i] = drawCalls(e); } - row(name, ms[0], ms[1]); + std::printf(" %-52s %10zu %10zu\n", name, counts[0], counts[1]); } // Every place a word is written, as a whole word or not. @@ -202,105 +225,67 @@ namespace } return out; } -} -#endif // LL_RELEASE -int main(int, char**) -{ -#if !defined(LL_RELEASE) - std::printf("Skipped: an unoptimised build has no numbers worth reading\n"); - return 125; -#else - // The context first: the fonts put their glyphs in its textures. - std::unique_ptr gl; - try - { - gl = std::make_unique(true, true, true, /*needs_render=*/true); - ll_test::installWhiteTexture(); - } - catch (const std::exception& e) - { - std::printf("Skipped: no GL (%s)\n", e.what()); - return 125; - } - ll_test::HeadlessUI& ui = ll_test::HeadlessUI::get(/*gl_textures=*/true); - if (!ui.ok()) + TEST_CASE("altextdraw_bench: an idle frame at the top", "[altextdraw_bench][!benchmark]") { - std::printf("Skipped: no UI, since LLUI_TEST_APP_DIR does not point at the source tree\n"); - return 125; - } - - Subject subjects[2] = { - { "lsl", "lsl", ll_test::bigLSL(LINES), nullptr }, - { "slua", "slua", ll_test::bigSLua(LINES), nullptr }, - }; - for (Subject& s : subjects) - { - s.editor = makeEditor(s); - } - - std::printf("altextdraw_bench: the code editor drawn, 1000 x 800, over generated scripts (ms per frame)\n"); - std::printf("\n %-52s %10s %10s\n", "", "LSL", "SLua"); - - std::printf("\nAn idle frame at the top\n"); - both("plain", subjects, [](ALCodeEditor&) {}); - both("a squiggle under every line", subjects, [](ALCodeEditor& e) { e.setDecorations(squiggleEveryLine(e)); }); - both(" and every \"total\" lit", subjects, [](ALCodeEditor& e) { e.setHighlights(ALCodeEditor::Highlight::Search, placesOf(e, "total")); }); - for (Subject& s : subjects) - { - s.editor->setDecorations({}); - s.editor->clearHighlights(); - } - both("with the map beside the text", subjects, [](ALCodeEditor& e) { e.setScrollMap(true); }); - for (Subject& s : subjects) - { - s.editor->setScrollMap(false); + ll_test::skipUnlessOptimised(); + Editors editors; + both("plain", editors, [](ALCodeEditor&) {}); + both("a squiggle under every line", editors, [](ALCodeEditor& e) { e.setDecorations(squiggleEveryLine(e)); }); + both("a squiggle under every line and every \"total\" lit", editors, + [](ALCodeEditor& e) { e.setHighlights(ALCodeEditor::Highlight::Search, placesOf(e, "total")); }); + for (Subject& s : editors) + { + s.editor->setDecorations({}); + s.editor->clearHighlights(); + } + both("with the map beside the text", editors, [](ALCodeEditor& e) { e.setScrollMap(true); }); } // A line of 20,000 characters near the top, as generated code or a list // written out at length has: scrolled across to its middle, then // wrapped, which makes it every row in sight. - std::printf("\nA line of 20,000 characters near the top\n"); + TEST_CASE("altextdraw_bench: a line of twenty thousand characters near the top", "[altextdraw_bench][!benchmark]") { + ll_test::skipUnlessOptimised(); + Editors editors; std::string long_line; while (long_line.size() < 20000) { long_line += "value = f(x, [y, z]) + "; } - for (Subject& s : subjects) + for (Subject& s : editors) { s.editor->setCaret(ALTextPos(2, 0)); s.editor->insertText(long_line + "\n"); } - both("scrolled across to its middle", subjects, [](ALCodeEditor& e) { e.setScrollX(e.layout().line(2).width * 0.5f); }); - both("wrapped", subjects, [](ALCodeEditor& e) { + both("scrolled across to its middle", editors, [](ALCodeEditor& e) { e.setScrollX(e.layout().line(2).width * 0.5f); }); + both("wrapped", editors, [](ALCodeEditor& e) { e.setScrollX(0.f); e.setWordWrap(true); }); - countBoth(" draw calls", subjects, [](ALCodeEditor&) {}); - for (Subject& s : subjects) - { - s.editor->setWordWrap(false); - s.editor->deleteRange(ALTextRange(ALTextPos(2, 0), ALTextPos(3, 0))); - } - } - std::printf("\nDraw calls a frame\n"); - countBoth("plain", subjects, [](ALCodeEditor&) {}); - countBoth("a squiggle under every line", subjects, [](ALCodeEditor& e) { e.setDecorations(squiggleEveryLine(e)); }); - countBoth("the map beside the text", subjects, [](ALCodeEditor& e) { - e.setDecorations({}); - e.setScrollMap(true); - }); - for (Subject& s : subjects) - { - s.editor->setScrollMap(false); + countHeading("A line of 20,000 characters near the top"); + countBoth("draw calls, wrapped", editors, [](ALCodeEditor&) {}); + std::fflush(stdout); } - for (Subject& s : subjects) + // Each after a frame drawn plain, as an editor in sight has been. + TEST_CASE("altextdraw_bench: draw calls a frame", "[altextdraw_bench][!benchmark]") { - delete s.editor; + ll_test::skipUnlessOptimised(); + Editors editors; + for (Subject& s : editors) + { + drawCalls(*s.editor); + } + countHeading("Draw calls a frame"); + countBoth("plain", editors, [](ALCodeEditor&) {}); + countBoth("a squiggle under every line", editors, [](ALCodeEditor& e) { e.setDecorations(squiggleEveryLine(e)); }); + countBoth("the map beside the text", editors, [](ALCodeEditor& e) { + e.setDecorations({}); + e.setScrollMap(true); + }); + std::fflush(stdout); } - return 0; -#endif } diff --git a/indra/llui/tests/altextengine_bench.cpp b/indra/llui/tests/altextengine_bench.cpp index 2045b3e2fe..a5b5738c87 100644 --- a/indra/llui/tests/altextengine_bench.cpp +++ b/indra/llui/tests/altextengine_bench.cpp @@ -25,16 +25,16 @@ // What the script studio's editor costs over a script of 50,000 lines, LSL // and SLua side by side: the code editor as the studio makes it, over the -// source tree's fonts with no window behind it. A number is milliseconds -// per operation: the median of five samples, each as many passes as fit in -// fifty milliseconds (and at least one). Where an operation leaves the text -// changed, the pass puts it back, and the number is per edit. +// source tree's fonts with no window behind it. Each benchmark is one +// operation, named for its language, or one pass named for what it covers; +// Catch2 reports the mean of its samples and the interval around it. Where +// an operation leaves the text changed, the calls take turns putting it +// back -- a character typed, then taken away -- and each is one edit. // // The output is a table, not a verdict; a number is read against the same // row from another build, on the same quiet machine. Every change to the // edit pipeline, the highlighter, the layout, the folds or find is measured -// here before and after. An unoptimised build exits 125, which CTest reads -// as skipped, since its numbers would say nothing. +// here before and after. #include "linden_common.h" @@ -44,92 +44,54 @@ #include "aloutputview.h" #include "alquickopen.h" #include "../llspellcheckengine.h" +#include "altextfind.h" #include "altextsearch.h" #include "alvimkeymap.h" #include "alheadlessui_fixture.h" +#include "aluistatescope.h" #include "albigscript.h" +#include "altestbenchmark.h" #include "../llfocusmgr.h" #include "../lluictrlfactory.h" #include -#include #include -#include -#include +#include #include +#include -// Only an optimised build measures, and only it has a use for these: an -// unoptimised one would say they are unused. -#if defined(LL_RELEASE) namespace { - using clock = std::chrono::steady_clock; - constexpr int LINES = 50000; - // Everything a row finds feeds this, so nothing is found for nothing. - volatile size_t g_sink = 0; - - // Milliseconds per item: `pass` does `count` of them. - double ms_per_item(size_t count, const std::function& pass) + // The scripts, made once for every case of a run. + struct Scripts { - pass(); - double samples[5]; - for (double& sample : samples) - { - size_t passes = 0; - const auto start = clock::now(); - clock::duration elapsed{}; - do - { - pass(); - ++passes; - elapsed = clock::now() - start; - } while (elapsed < std::chrono::milliseconds(50)); - sample = std::chrono::duration(elapsed).count() / (double(passes) * double(count)); - } - std::sort(samples, samples + 5); - return samples[2]; - } + std::string lsl = ll_test::bigLSL(LINES); + std::string slua = ll_test::bigSLua(LINES); + }; - // As ms_per_item, a pass at a time, but each pass's own setup -- an - // editor emptied before a load -- left out of what is timed. - double ms_per_item_after(const std::function& setup, const std::function& pass) - { - setup(); - pass(); - double samples[5]; - for (double& sample : samples) - { - size_t passes = 0; - clock::duration spent{}; - const auto start = clock::now(); - do - { - setup(); - const auto from = clock::now(); - pass(); - spent += clock::now() - from; - ++passes; - } while (clock::now() - start < std::chrono::milliseconds(50)); - sample = std::chrono::duration(spent).count() / double(passes); - } - std::sort(samples, samples + 5); - return samples[2]; + const Scripts& scripts() + { + static const Scripts made; + return made; } // One language's editor, over its big script. struct Subject { - const char* name; - const char* syntax; - std::string text; - ALCodeEditor* editor = nullptr; + const char* name; + const char* syntax; + // The column its numbers were read under, which ends each + // benchmark's name. + const char* language; + const std::string& text; // What opens a block comment at the top of the text, and what it // opens, closed again. - const char* opener; + const char* opener; + ALCodeEditor* editor = nullptr; }; ALCodeEditor* makeEditor(const Subject& s) @@ -145,15 +107,94 @@ namespace return editor; } + // The headless UI the binary made before its first case: a case skips + // where it could not be. + void requireUI() + { + if (!ll_test::HeadlessUI::get().ok()) + { + SKIP("no UI, since LLUI_TEST_APP_DIR does not point at the source tree"); + } + } + + // The views a case made, dying when it ends however it ends, and focus + // -- which the find bar takes, and the text when it goes -- put back as + // the case found it. + struct Views + { + ll_test::FocusScope focus; + std::vector made; + + ~Views() + { + for (LLView* view : made) + { + view->die(); + } + } + + template + T* keep(T* view) + { + made.push_back(view); + return view; + } + }; + + // The find's worker, which a text this long is looked through on, + // closed and waited for when a case ends: nothing of its own left + // running. + struct WorkerClosed + { + ~WorkerClosed() { ALTextFind::closeWorker(); } + }; + + // Both languages' editors, made for a case. + struct Editors + { + WorkerClosed worker; + Views views; + Subject subjects[2] = { + { "lsl", "lsl", "LSL", scripts().lsl, "/*" }, + { "slua", "slua", "SLua", scripts().slua, "--[[" }, + }; + + Editors() + { + requireUI(); + for (Subject& s : subjects) + { + s.editor = views.keep(makeEditor(s)); + } + } + + Subject* begin() { return subjects; } + Subject* end() { return subjects + 2; } + }; + S32 lastLine(ALCodeEditor& e) { return e.document().lineCount() - 1; } - // A character typed at the top of the text and taken away again, both - // through the undo journal as a person's edits go. + // A character typed at the top of the text, or taken away again: a + // step of the two that leave the text as it was, each through the undo + // journal as a person's edits go. + void editAtTop(ALCodeEditor& e, std::string_view what, size_t step) + { + if (step == 0) + { + e.setCaret(ALTextPos(0, 0)); + e.insertText(what); + } + else + { + e.deleteRange(ALTextRange(ALTextPos(0, 0), ALTextPos(0, static_cast(what.size())))); + } + } + + // Both steps. void editAtTop(ALCodeEditor& e, std::string_view what) { - e.setCaret(ALTextPos(0, 0)); - e.insertText(what); - e.deleteRange(ALTextRange(ALTextPos(0, 0), ALTextPos(0, static_cast(what.size())))); + editAtTop(e, what, 0); + editAtTop(e, what, 1); } // A text with every third line that is a closer alone -- `}`, `end` -- @@ -200,15 +241,10 @@ namespace } } - void row(const char* name, double lsl, double slua, const char* note = "") - { - std::printf(" %-52s %10.3f %10.3f %s\n", name, lsl, slua, note); - } - - // A row with one number, not one for each language. - void rowOne(const char* name, double ms) + // A table of counts, under its heading and the languages' columns. + void countHeading(const char* heading) { - std::printf(" %-52s %10.3f\n", name, ms); + std::printf("\n%s\n %-52s %10s %10s\n", heading, "", "LSL", "SLua"); } void countRow(const char* name, size_t lsl, size_t slua) @@ -216,367 +252,434 @@ namespace std::printf(" %-52s %10zu %10zu\n", name, lsl, slua); } + // A benchmark for each language, named for the row and its column. template - void both(const char* name, Subject (&subjects)[2], size_t count, F&& op) + void both(const char* name, Editors& editors, F&& op) { - double ms[2]; - for (int i = 0; i < 2; ++i) + for (Subject& s : editors) { - ALCodeEditor& e = *subjects[i].editor; - ms[i] = ms_per_item(count, [&] { op(subjects[i], e); }); + ALCodeEditor& e = *s.editor; + BENCHMARK(std::string(name) + ", " + s.language) + { + return op(s, e); + }; } - row(name, ms[0], ms[1]); } - template - void bothAfter(const char* name, Subject (&subjects)[2], S&& setup, F&& op) + // As both, each call a step of a pass of `steps` that leaves the text + // as it found it -- a character typed, then taken away -- taken in + // turn; and the pass finished after, off the clock, so that the next + // row finds the text as this one did. + template + void bothInSteps(const char* name, Editors& editors, size_t steps, F&& op) { - double ms[2]; - for (int i = 0; i < 2; ++i) + for (Subject& s : editors) { - ALCodeEditor& e = *subjects[i].editor; - ms[i] = ms_per_item_after([&] { setup(subjects[i], e); }, [&] { op(subjects[i], e); }); + ALCodeEditor& e = *s.editor; + size_t calls = 0; + BENCHMARK(std::string(name) + ", " + s.language) + { + return op(s, e, calls++ % steps); + }; + while (calls % steps != 0) + { + op(s, e, calls++ % steps); + } } - row(name, ms[0], ms[1]); } -} -#endif // LL_RELEASE -int main(int, char**) -{ -#if !defined(LL_RELEASE) - std::printf("Skipped: an unoptimised build has no numbers worth reading\n"); - return 125; -#else - ll_test::HeadlessUI& ui = ll_test::HeadlessUI::get(); - if (!ui.ok()) - { - std::printf("Skipped: no UI, since LLUI_TEST_APP_DIR does not point at the source tree\n"); - return 125; - } - - Subject subjects[2] = { - { "lsl", "lsl", ll_test::bigLSL(LINES), nullptr, "/*" }, - { "slua", "slua", ll_test::bigSLua(LINES), nullptr, "--[[" }, - }; - for (Subject& s : subjects) + // As both, each call's own setup -- an editor emptied before a load -- + // left off the clock. An operation of milliseconds is one run of a + // sample, so the setup is before every one. + template + void bothAfter(const char* name, Editors& editors, S&& setup, F&& op) { - s.editor = makeEditor(s); + for (Subject& s : editors) + { + ALCodeEditor& e = *s.editor; + BENCHMARK_ADVANCED(std::string(name) + ", " + s.language)(Catch::Benchmark::Chronometer meter) + { + setup(s, e); + meter.measure([&] { op(s, e); }); + }; + } } - std::printf("altextengine_bench: the code editor over generated scripts (ms per operation)\n"); - std::printf("\n %-52s %10s %10s\n", "", "LSL", "SLua"); - countRow("lines", subjects[0].editor->document().lineCount(), subjects[1].editor->document().lineCount()); - countRow("bytes", subjects[0].text.size(), subjects[1].text.size()); - - std::printf("\nLoading and highlighting\n"); - // Into an editor holding nothing, as a script opened is; and over the - // same text again, as one loaded again is, whose lines are made in - // the strings of the lines they replace. - bothAfter("load: setText into an empty text", subjects, [](Subject&, ALCodeEditor& e) { e.setText(std::string_view()); }, - [](Subject& s, ALCodeEditor& e) { e.setText(s.text); }); - both("load: setText over the same text", subjects, 1, [](Subject& s, ALCodeEditor& e) { e.setText(s.text); }); - both("full highlight: every line lexed", subjects, 1, [](Subject&, ALCodeEditor& e) { - e.highlighter().wordsChanged(); - g_sink = g_sink + e.highlighter().tokens(lastLine(e)).size(); - }); - // The whole script inside one comment -- LSL's /* */, SLua's long - // bracket --[==[ ]==] -- lexed: every byte of it inside a span. + TEST_CASE("altextengine_bench: loading and highlighting", "[altextengine_bench][!benchmark]") { + ll_test::skipUnlessOptimised(); + Editors editors; + countHeading("The scripts"); + countRow("lines", editors.subjects[0].editor->document().lineCount(), editors.subjects[1].editor->document().lineCount()); + countRow("bytes", editors.subjects[0].text.size(), editors.subjects[1].text.size()); + std::fflush(stdout); + + // Into an editor holding nothing, as a script opened is; and over the + // same text again, as one loaded again is, whose lines are made in + // the strings of the lines they replace. + bothAfter("load: setText into an empty text", editors, [](Subject&, ALCodeEditor& e) { e.setText(std::string_view()); }, + [](Subject& s, ALCodeEditor& e) { e.setText(s.text); }); + both("load: setText over the same text", editors, [](Subject& s, ALCodeEditor& e) { e.setText(s.text); }); + both("full highlight: every line lexed", editors, [](Subject&, ALCodeEditor& e) { + e.highlighter().wordsChanged(); + return e.highlighter().tokens(lastLine(e)).size(); + }); + // The whole script inside one comment -- LSL's /* */, SLua's long + // bracket --[==[ ]==] -- lexed: every byte of it inside a span. const char* openers[2] = { "/*\n", "--[==[\n" }; const char* closers[2] = { "\n*/", "\n]==]" }; for (int i = 0; i < 2; ++i) { - subjects[i].editor->setText(openers[i] + subjects[i].text + closers[i]); + editors.subjects[i].editor->setText(openers[i] + editors.subjects[i].text + closers[i]); } - both("full highlight: all of it in one comment", subjects, 1, [](Subject&, ALCodeEditor& e) { + both("full highlight: all of it in one comment", editors, [](Subject&, ALCodeEditor& e) { e.highlighter().wordsChanged(); - g_sink = g_sink + e.highlighter().tokens(lastLine(e)).size(); + return e.highlighter().tokens(lastLine(e)).size(); }); - for (Subject& s : subjects) - { - s.editor->setText(s.text); - } } - std::printf("\nAn edit at the top (typed and taken away, through the journal)\n"); - for (Subject& s : subjects) + TEST_CASE("altextengine_bench: an edit at the top typed and taken away", "[altextengine_bench][!benchmark]") { - s.editor->highlighter().tokens(lastLine(*s.editor)); - } - both("the edit alone, per edit", subjects, 2, [](Subject&, ALCodeEditor& e) { editAtTop(e, "x"); }); - both("a line broken at the top and joined, the edit alone", subjects, 1, [](Subject&, ALCodeEditor& e) { - e.setCaret(ALTextPos(0, 0)); - e.insertText("\n"); - e.deleteRange(ALTextRange(ALTextPos(0, 0), ALTextPos(1, 0))); - }); - size_t relexed[2] = {}; - both("the edit, then the highlight to the end, per edit", subjects, 2, [&](Subject& s, ALCodeEditor& e) { - e.setCaret(ALTextPos(0, 0)); - e.insertText("x"); - g_sink = g_sink + e.highlighter().tokens(lastLine(e)).size(); - relexed[&s - subjects] = e.highlighter().lastLexed(); - e.deleteRange(ALTextRange(ALTextPos(0, 0), ALTextPos(0, 1))); - g_sink = g_sink + e.highlighter().tokens(lastLine(e)).size(); - }); - countRow(" lines relexed after it (where relexing stops)", relexed[0], relexed[1]); - both("a block comment opened at the top, then closed", subjects, 2, [&](Subject& s, ALCodeEditor& e) { - e.setCaret(ALTextPos(0, 0)); - e.insertText(s.opener); - g_sink = g_sink + e.highlighter().tokens(lastLine(e)).size(); - relexed[&s - subjects] = e.highlighter().lastLexed(); - e.deleteRange(ALTextRange(ALTextPos(0, 0), ALTextPos(0, static_cast(strlen(s.opener))))); - g_sink = g_sink + e.highlighter().tokens(lastLine(e)).size(); - }); - countRow(" lines relexed (the first comment's end stops it)", relexed[0], relexed[1]); - - // Only the undo timed: a hundred characters typed at the top as a run, - // then taken back in one step. - { - double ms[2]; - for (int i = 0; i < 2; ++i) + ll_test::skipUnlessOptimised(); + Editors editors; + for (Subject& s : editors) { - ALCodeEditor& e = *subjects[i].editor; - double samples[5]; - for (double& sample : samples) + s.editor->highlighter().tokens(lastLine(*s.editor)); + } + bothInSteps("the edit alone, per edit", editors, 2, [](Subject&, ALCodeEditor& e, size_t step) { editAtTop(e, "x", step); }); + both("a line broken at the top and joined, the edit alone", editors, [](Subject&, ALCodeEditor& e) { + e.setCaret(ALTextPos(0, 0)); + e.insertText("\n"); + e.deleteRange(ALTextRange(ALTextPos(0, 0), ALTextPos(1, 0))); + }); + size_t relexed[2] = {}; + size_t comment[2] = {}; + bothInSteps("the edit, then the highlight to the end, per edit", editors, 2, [&](Subject& s, ALCodeEditor& e, size_t step) { + editAtTop(e, "x", step); + const size_t tokens = e.highlighter().tokens(lastLine(e)).size(); + if (step == 0) { - e.setCaret(ALTextPos(0, 0)); - for (int c = 0; c < 100; ++c) - { - e.handleUnicodeCharHere(static_cast('a' + c % 26)); - } - const auto start = clock::now(); - e.undo(); - sample = std::chrono::duration(clock::now() - start).count(); + relexed[&s - editors.subjects] = e.highlighter().lastLexed(); + } + return tokens; + }); + bothInSteps("a block comment opened at the top, then closed", editors, 2, [&](Subject& s, ALCodeEditor& e, size_t step) { + editAtTop(e, s.opener, step); + const size_t tokens = e.highlighter().tokens(lastLine(e)).size(); + if (step == 0) + { + comment[&s - editors.subjects] = e.highlighter().lastLexed(); + } + return tokens; + }); + + // A hundred characters typed at the top as a run, then taken back in + // one step and put back in another, a step a call. Typed again for + // every undo, the run would cost far more than the undo it is for: + // Catch2 warms an operation this quick up by running it tens of + // thousands of times at once. + for (Subject& s : editors) + { + s.editor->setCaret(ALTextPos(0, 0)); + for (int c = 0; c < 100; ++c) + { + s.editor->handleUnicodeCharHere(static_cast('a' + c % 26)); } - std::sort(samples, samples + 5); - ms[i] = samples[2]; } - row("undo of a 100-character typing run", ms[0], ms[1]); + bothInSteps("undo of a 100-character typing run", editors, 2, [](Subject&, ALCodeEditor& e, size_t step) { + if (step == 0) + { + e.undo(); + } + else + { + e.redo(); + } + }); + + countHeading("An edit at the top (typed and taken away, through the journal)"); + countRow("lines relexed after it (where relexing stops)", relexed[0], relexed[1]); + countRow("lines relexed (the first comment's end stops it)", comment[0], comment[1]); + std::fflush(stdout); } // Completion as a word is typed at the end of the script, the list // open: each key narrows it, over the grammar's words, the language's // and the script's own. - std::printf("\nCompletion\n"); - for (Subject& s : subjects) + TEST_CASE("altextengine_bench: completion as a word is typed", "[altextengine_bench][!benchmark]") { - s.editor->setCaret(s.editor->document().end()); - s.editor->insertText("\n"); - } - size_t offered[2] = {}; - both("a word typed and taken back, the list open, per key", subjects, 8, [&](Subject& s, ALCodeEditor& e) { - for (const char c : std::string("tota")) + ll_test::skipUnlessOptimised(); + Editors editors; + for (Subject& s : editors) { - e.handleUnicodeCharHere(static_cast(c)); + s.editor->setCaret(s.editor->document().end()); + s.editor->insertText("\n"); } - offered[&s - subjects] = e.completions().size(); - for (int i = 0; i < 4; ++i) + size_t offered[2] = {}; + bothInSteps("a word typed and taken back, the list open, per key", editors, 8, [&](Subject& s, ALCodeEditor& e, size_t step) { + if (step < 4) + { + e.handleUnicodeCharHere(static_cast("tota"[step])); + if (step == 3) + { + offered[&s - editors.subjects] = e.completions().size(); + } + } + else + { + e.handleKeyHere(KEY_BACKSPACE, MASK_NONE); + } + }); + for (Subject& s : editors) { - e.handleKeyHere(KEY_BACKSPACE, MASK_NONE); + s.editor->closeCompletion(); } - }); - countRow(" offered at four letters", offered[0], offered[1]); - for (Subject& s : subjects) - { - ALCodeEditor& e = *s.editor; - e.closeCompletion(); - e.deleteRange(ALTextRange(e.document().lineEnd(lastLine(e) - 1), e.document().end())); - } - std::printf("\nFinding\n"); - size_t found[2] = {}; - both("regex find-all: every call, \\w+\\s*\\(", subjects, 1, [&](Subject& s, ALCodeEditor& e) { - ALTextSearchOptions options; - options.regex = true; - options.caseSensitive = true; - found[&s - subjects] = ALTextSearch::matches(e.document(), "\\w+\\s*\\(", options).size(); - g_sink = g_sink + found[&s - subjects]; - }); - countRow(" matches", found[0], found[1]); - both("plain find-all: total", subjects, 1, [&](Subject& s, ALCodeEditor& e) { - ALTextSearchOptions options; - found[&s - subjects] = ALTextSearch::matches(e.document(), "total", options).size(); - g_sink = g_sink + found[&s - subjects]; - }); - countRow(" matches", found[0], found[1]); - - size_t replaced[2] = {}; - both("Replace All: total for count, then undone", subjects, 1, [&](Subject& s, ALCodeEditor& e) { - if (!e.findShown()) - { - e.showFind(true); - e.findBar()->setQuery("total"); - e.findBar()->setReplacement("count"); - } - replaced[&s - subjects] = e.replaceAllMatches(); - e.undo(); - }); - countRow(" replaced", replaced[0], replaced[1]); - both("Replace All: (\\w+)\\s*\\( for $1 (, then undone", subjects, 1, [&](Subject& s, ALCodeEditor& e) { - e.findBar()->setQuery("(\\w+)\\s*\\("); - e.findBar()->setReplacement("$1 ("); - e.findBar()->findChild("regex")->handleMouseDown(0, 0, MASK_NONE); - replaced[&s - subjects] = e.replaceAllMatches(); - e.findBar()->findChild("regex")->handleMouseDown(0, 0, MASK_NONE); - e.undo(); - }); - countRow(" replaced", replaced[0], replaced[1]); - for (Subject& s : subjects) - { - s.editor->findBar()->setQuery("total"); + countHeading("Completion"); + countRow("offered at four letters", offered[0], offered[1]); + std::fflush(stdout); } - // A word typed into the bar a key at a time, the bar open over the text. - both("a query typed into the find bar, per key", subjects, 5, [&](Subject&, ALCodeEditor& e) { - for (const char* typed : { "t", "to", "tot", "tota", "total" }) + + TEST_CASE("altextengine_bench: finding and replacing", "[altextengine_bench][!benchmark]") + { + ll_test::skipUnlessOptimised(); + Editors editors; + size_t found[2][2] = {}; + both("regex find-all: every call, \\w+\\s*\\(", editors, [&](Subject& s, ALCodeEditor& e) { + ALTextSearchOptions options; + options.regex = true; + options.caseSensitive = true; + return found[0][&s - editors.subjects] = ALTextSearch::matches(e.document(), "\\w+\\s*\\(", options).size(); + }); + both("plain find-all: total", editors, [&](Subject& s, ALCodeEditor& e) { + ALTextSearchOptions options; + return found[1][&s - editors.subjects] = ALTextSearch::matches(e.document(), "total", options).size(); + }); + + for (Subject& s : editors) { - e.findBar()->setQuery(typed); + s.editor->showFind(true); + s.editor->findBar()->setQuery("total"); + s.editor->findBar()->setReplacement("count"); } - }); - for (Subject& s : subjects) - { - g_sink = g_sink + s.editor->findMatches().size(); + size_t replaced[2][2] = {}; + both("Replace All: total for count, then undone", editors, [&](Subject& s, ALCodeEditor& e) { + replaced[0][&s - editors.subjects] = e.replaceAllMatches(); + e.undo(); + }); + both("Replace All: (\\w+)\\s*\\( for $1 (, then undone", editors, [&](Subject& s, ALCodeEditor& e) { + e.findBar()->setQuery("(\\w+)\\s*\\("); + e.findBar()->setReplacement("$1 ("); + e.findBar()->findChild("regex")->handleMouseDown(0, 0, MASK_NONE); + replaced[1][&s - editors.subjects] = e.replaceAllMatches(); + e.findBar()->findChild("regex")->handleMouseDown(0, 0, MASK_NONE); + e.undo(); + }); + for (Subject& s : editors) + { + s.editor->findBar()->setQuery("total"); + } + // A word typed into the bar a key at a time, the bar open over the + // text, a key a call. + bothInSteps("a query typed into the find bar, per key", editors, 5, [](Subject&, ALCodeEditor& e, size_t step) { + static const char* const typed[] = { "t", "to", "tot", "tota", "total" }; + e.findBar()->setQuery(typed[step]); + }); + + countHeading("Finding"); + countRow("regex find-all: every call, \\w+\\s*\\(: matches", found[0][0], found[0][1]); + countRow("plain find-all: total: matches", found[1][0], found[1][1]); + countRow("Replace All: total for count: replaced", replaced[0][0], replaced[0][1]); + countRow("Replace All: (\\w+)\\s*\\( for $1 (: replaced", replaced[1][0], replaced[1][1]); + std::fflush(stdout); } // What lies over the text slides with each edit: every match found, // a squiggle on every other line. - std::printf("\nLayers over the text\n"); - size_t lit[2] = {}; - for (Subject& s : subjects) - { - lit[&s - subjects] = s.editor->findMatches().size(); - } - both("an edit at the top, every match of \"total\" lit", subjects, 2, [](Subject&, ALCodeEditor& e) { editAtTop(e, "x"); }); - countRow(" matches lit", lit[0], lit[1]); - for (Subject& s : subjects) + TEST_CASE("altextengine_bench: layers over the text sliding with an edit", "[altextengine_bench][!benchmark]") { - s.editor->hideFind(); - std::vector squiggles; - for (S32 line = 0; line < s.editor->document().lineCount(); line += 2) + ll_test::skipUnlessOptimised(); + Editors editors; + size_t lit[2] = {}; + for (Subject& s : editors) { - ALCodeEditor::Decoration d; - d.range = ALTextRange(ALTextPos(line, 0), ALTextPos(line, llmin(4, s.editor->document().lineLength(line)))); - squiggles.push_back(d); + s.editor->showFind(true); + s.editor->findBar()->setQuery("total"); + lit[&s - editors.subjects] = s.editor->findMatches().size(); } - s.editor->setDecorations(std::move(squiggles)); - } - both("an edit at the top, a squiggle on every other line", subjects, 2, [](Subject&, ALCodeEditor& e) { editAtTop(e, "x"); }); - for (Subject& s : subjects) - { - s.editor->setDecorations({}); + bothInSteps("an edit at the top, every match of \"total\" lit", editors, 2, + [](Subject&, ALCodeEditor& e, size_t step) { editAtTop(e, "x", step); }); + for (Subject& s : editors) + { + s.editor->hideFind(); + std::vector squiggles; + for (S32 line = 0; line < s.editor->document().lineCount(); line += 2) + { + ALCodeEditor::Decoration d; + d.range = ALTextRange(ALTextPos(line, 0), ALTextPos(line, llmin(4, s.editor->document().lineLength(line)))); + squiggles.push_back(d); + } + s.editor->setDecorations(std::move(squiggles)); + } + bothInSteps("an edit at the top, a squiggle on every other line", editors, 2, + [](Subject&, ALCodeEditor& e, size_t step) { editAtTop(e, "x", step); }); + + countHeading("Layers over the text"); + countRow("matches lit", lit[0], lit[1]); + std::fflush(stdout); } // The spell check over comments and strings, as the platform's own // checker answers it: every line checked afresh, and Next Misspelling // round a script with none, the most it ever looks at. + TEST_CASE("altextengine_bench: spelling as the platform's checker answers", "[altextengine_bench][!benchmark]") { + ll_test::skipUnlessOptimised(); + Editors editors; + std::unique_ptr engine = LLSpellCheckEngine::create(); const bool real = engine && engine->setLanguage("en_US"); - std::printf("\nSpelling (%s)\n", real ? "the platform's checker, en_US" : "no checker to be had: every word right"); + size_t asked = 0; auto check = [&](const std::string& word) { ++asked; return !real || engine->checkWord(word); }; - for (Subject& s : subjects) + // The checker taken back from the editors however the case ends: it + // asks the engine, which goes before they do. + struct Unchecked + { + Editors& editors; + ~Unchecked() + { + for (Subject& s : editors) + { + s.editor->setSpellCheck(false); + s.editor->setSpellChecker(nullptr); + } + } + } unchecked{ editors }; + for (Subject& s : editors) { s.editor->setSpellChecker(check); s.editor->setSpellCheck(true); } size_t words[2] = {}; - both("every line checked afresh", subjects, 1, [&](Subject& s, ALCodeEditor& e) { + both("every line checked afresh", editors, [&](Subject& s, ALCodeEditor& e) { e.recheckSpelling(); - asked = 0; + asked = 0; + size_t found = 0; for (S32 line = 0; line < e.document().lineCount(); ++line) { - g_sink = g_sink + e.misspellings(line).size(); + found += e.misspellings(line).size(); } - words[&s - subjects] = asked; + words[&s - editors.subjects] = asked; + return found; }); - countRow(" words asked of the checker", words[0], words[1]); - both("Next Misspelling round a script with none, afresh", subjects, 1, [](Subject&, ALCodeEditor& e) { + both("Next Misspelling round a script with none, afresh", editors, [](Subject&, ALCodeEditor& e) { e.recheckSpelling(); - g_sink = g_sink + e.misspellingFrom(ALTextPos(0, 0), true).has_value(); - }); - both("Next Misspelling round it again, checked already", subjects, 1, [](Subject&, ALCodeEditor& e) { - g_sink = g_sink + e.misspellingFrom(ALTextPos(0, 0), true).has_value(); + return e.misspellingFrom(ALTextPos(0, 0), true).has_value(); }); - for (Subject& s : subjects) - { - s.editor->setSpellCheck(false); - s.editor->setSpellChecker(nullptr); - } + both("Next Misspelling round it again, checked already", editors, + [](Subject&, ALCodeEditor& e) { return e.misspellingFrom(ALTextPos(0, 0), true).has_value(); }); + + countHeading(real ? "Spelling (the platform's checker, en_US)" : "Spelling (no checker to be had: every word right)"); + countRow("words asked of the checker", words[0], words[1]); + std::fflush(stdout); } - std::printf("\nEvery line at once\n"); - both("Select All and Tab, then undone", subjects, 1, [](Subject&, ALCodeEditor& e) { - e.perform(ALEditorCommand::SelectAll); - e.handleKeyHere(KEY_TAB, MASK_NONE); - e.undo(); - }); + TEST_CASE("altextengine_bench: every line at once", "[altextengine_bench][!benchmark]") + { + ll_test::skipUnlessOptimised(); + Editors editors; + both("Select All and Tab, then undone", editors, [](Subject&, ALCodeEditor& e) { + e.perform(ALEditorCommand::SelectAll); + e.handleKeyHere(KEY_TAB, MASK_NONE); + e.undo(); + }); + } // What the crash journal writes as typing goes on: a history of // 3,000 steps, a character typed at the top before each write. As LLSD // built step by step, as it was, and as notation from what each step - // was written as the last time. - std::printf("\nThe history written against a crash (3,000 steps)\n"); - for (Subject& s : subjects) + // was written as the last time. The character is taken back after each + // write, off the clock, so that every write is of the same history. + TEST_CASE("altextengine_bench: the history written against a crash", "[altextengine_bench][!benchmark]") { - ALCodeEditor& e = *s.editor; - e.undoJournal().clear(); - for (S32 i = 0; i < 3000; ++i) + ll_test::skipUnlessOptimised(); + Editors editors; + for (Subject& s : editors) { - // Each at a line of its own: a step of its own. - e.setCaret(ALTextPos(i * 7 % lastLine(e), 0)); - e.insertText("x"); + ALCodeEditor& e = *s.editor; + e.undoJournal().clear(); + for (S32 i = 0; i < 3000; ++i) + { + // Each at a line of its own: a step of its own. + e.setCaret(ALTextPos(i * 7 % lastLine(e), 0)); + e.insertText("x"); + } } - } - size_t written[2] = {}; - both("as LLSD, every step built", subjects, 1, [&](Subject& s, ALCodeEditor& e) { - e.setCaret(ALTextPos(0, 0)); - e.insertText("y"); - const LLSD history = e.undoJournal().asLLSD(); - written[&s - subjects] = history["undo"].size(); - g_sink = g_sink + written[&s - subjects]; - }); - both("as notation, from each step's last", subjects, 1, [&](Subject& s, ALCodeEditor& e) { - e.setCaret(ALTextPos(0, 0)); - e.insertText("y"); - g_sink = g_sink + e.undoJournal().asNotation().size(); - }); - countRow(" steps written", written[0], written[1]); - // The scripts as they were: an x at the start of every seventh line - // makes SLua's ends names, and every block after the first so made - // would be measured open to the end of the text. - for (Subject& s : subjects) - { - s.editor->setText(s.text); - s.editor->undoJournal().clear(); + const auto typedThenWritten = [&editors](const char* name, auto&& write) { + for (Subject& s : editors) + { + ALCodeEditor& e = *s.editor; + BENCHMARK_ADVANCED(std::string(name) + ", " + s.language)(Catch::Benchmark::Chronometer meter) + { + // Each y a step of its own, however many a sample + // types, so as many undos take them all away and the + // next sample writes the same history. + meter.measure([&] { + e.undoJournal().breakRun(); + e.setCaret(ALTextPos(0, 0)); + e.insertText("y"); + return write(s, e); + }); + for (int i = 0; i < meter.runs(); ++i) + { + e.undo(); + } + }; + } + }; + size_t written[2] = {}; + typedThenWritten("as LLSD, every step built", [&](Subject& s, ALCodeEditor& e) { + const LLSD history = e.undoJournal().asLLSD(); + return written[&s - editors.subjects] = static_cast(history["undo"].size()); + }); + typedThenWritten("as notation, from each step's last", [](Subject&, ALCodeEditor& e) { return e.undoJournal().asNotation().size(); }); + + countHeading("The history written against a crash (3,000 steps)"); + countRow("steps written", written[0], written[1]); + std::fflush(stdout); } - // A bracket left open at the top: its partner looked for as the caret - // rests beside it, and the depth the rainbow and the indent read at the - // end, each after a character typed and taken away below it. // Where a caret is as it is seen: every line's end counted in columns, // as the trailer and vim's vertical motions ask of one line at a time; // then the same of a line with a combining accent in it, which the // characters are walked for. - std::printf("\nDisplay columns (every line's end)\n"); - both("ASCII lines, per thousand", subjects, static_cast(LINES / 1000), [](Subject&, ALCodeEditor& e) { - const ALTextDocument& d = e.document(); - for (S32 line = 0; line < d.lineCount(); ++line) - { - g_sink = g_sink + static_cast(d.displayColumn(ALTextPos(line, d.lineLength(line)), 4)); - } - }); + TEST_CASE("altextengine_bench: display columns at every line's end", "[altextengine_bench][!benchmark]") { + ll_test::skipUnlessOptimised(); + Editors editors; + both("ASCII lines, all 50,000", editors, [](Subject&, ALCodeEditor& e) { + const ALTextDocument& d = e.document(); + size_t columns = 0; + for (S32 line = 0; line < d.lineCount(); ++line) + { + columns += static_cast(d.displayColumn(ALTextPos(line, d.lineLength(line)), 4)); + } + return columns; + }); const ALTextDocument accented("\tinteger cafe\xCC\x81 = llStringLength(\"cr\xC3\xA8" "me br\xC3\xBB" "l\xC3\xA9" "e\"); // \xE6\x97\xA5\xE6\x9C\xAC"); - rowOne("a line with accents and CJK, per thousand", ms_per_item(1, [&] { + BENCHMARK("a line with accents and CJK, per thousand") + { + size_t columns = 0; for (int i = 0; i < 1000; ++i) { - g_sink = g_sink + static_cast(accented.displayColumn(ALTextPos(0, accented.lineLength(0)), 4)); + columns += static_cast(accented.displayColumn(ALTextPos(0, accented.lineLength(0)), 4)); } - })); + return columns; + }; } // A line of 20,000 characters, as generated code or a list written out @@ -584,67 +687,81 @@ int main(int, char**) // and taken away, the line laid out again after each; and the x of a // column along it with the column back from that x, as the caret and // the mouse ask. - std::printf("\nA line of 20,000 characters\n"); + TEST_CASE("altextengine_bench: a line of twenty thousand characters", "[altextengine_bench][!benchmark]") { + ll_test::skipUnlessOptimised(); + Editors editors; std::string long_line; while (long_line.size() < 20000) { long_line += "value = f(x, [y, z]) + "; } - for (Subject& s : subjects) + for (Subject& s : editors) { s.editor->setCaret(ALTextPos(5, 0)); s.editor->insertText(long_line + "\n"); } - // Each pass another letter somewhere else in the middle, so that the - // line is one the shaper has not seen, as it is after a keystroke. - both("a keystroke in its middle, laid out again", subjects, 2, [](Subject&, ALCodeEditor& e) { - static S32 pass = 0; - ++pass; - const S32 at = 5000 + (pass * 7) % 10000; - const char typed[2] = { static_cast('a' + pass % 26), 0 }; - e.setCaret(ALTextPos(5, at)); - e.insertText(typed); - g_sink = g_sink + e.layout().line(5).glyphs.size(); - e.deleteRange(ALTextRange(ALTextPos(5, at), ALTextPos(5, at + 1))); - g_sink = g_sink + e.layout().line(5).glyphs.size(); - }); - both("a column's x and the column at an x, per thousand", subjects, 1, [](Subject&, ALCodeEditor& e) { - ALTextLayout& laid = e.layout(); + // Each keystroke another letter somewhere else in the middle, so + // that the line is one the shaper has not seen, as it is after a + // keystroke. + S32 pass = 0; + S32 at = 0; + bothInSteps("a keystroke in its middle, laid out again", editors, 2, [&](Subject&, ALCodeEditor& e, size_t step) { + if (step == 0) + { + ++pass; + at = 5000 + (pass * 7) % 10000; + const char typed[2] = { static_cast('a' + pass % 26), 0 }; + e.setCaret(ALTextPos(5, at)); + e.insertText(typed); + } + else + { + e.deleteRange(ALTextRange(ALTextPos(5, at), ALTextPos(5, at + 1))); + } + return e.layout().line(5).glyphs.size(); + }); + both("a column's x and the column at an x, per thousand", editors, [](Subject&, ALCodeEditor& e) { + ALTextLayout& laid = e.layout(); + size_t column = 0; for (S32 i = 0; i < 1000; ++i) { - const S32 column = (i * 7919) % 20000; - g_sink = g_sink + static_cast(laid.columnAt(5, 0, laid.xOf(5, column) + 1.f, true)); + column += static_cast(laid.columnAt(5, 0, laid.xOf(5, (i * 7919) % 20000) + 1.f, true)); } + return column; }); - for (Subject& s : subjects) - { - s.editor->deleteRange(ALTextRange(ALTextPos(5, 0), ALTextPos(6, 0))); - } } // The widest line, of 400 characters, near the top: a character typed // at its end and taken away; and the same with the widest asked after // each, as the horizontal scrollbar asks after every key. - std::printf("\nThe widest line\n"); - for (Subject& s : subjects) - { - s.editor->setCaret(ALTextPos(5, 0)); - s.editor->insertText(std::string(400, 'w') + "\n"); - } - const auto typedAtWidest = [](ALCodeEditor& e, bool ask) { - const S32 end = e.document().lineLength(5); - e.setCaret(ALTextPos(5, end)); - e.insertText("x"); - g_sink = g_sink + (ask ? static_cast(e.layout().contentWidth()) : 0); - e.deleteRange(ALTextRange(ALTextPos(5, end), ALTextPos(5, end + 1))); - g_sink = g_sink + (ask ? static_cast(e.layout().contentWidth()) : 0); - }; - both("a character typed at its end and taken back, per edit", subjects, 2, [&](Subject&, ALCodeEditor& e) { typedAtWidest(e, false); }); - both("the same, the widest asked after each", subjects, 2, [&](Subject&, ALCodeEditor& e) { typedAtWidest(e, true); }); - for (Subject& s : subjects) + TEST_CASE("altextengine_bench: the widest line", "[altextengine_bench][!benchmark]") { - s.editor->deleteRange(ALTextRange(ALTextPos(5, 0), ALTextPos(6, 0))); + ll_test::skipUnlessOptimised(); + Editors editors; + for (Subject& s : editors) + { + s.editor->setCaret(ALTextPos(5, 0)); + s.editor->insertText(std::string(400, 'w') + "\n"); + } + const auto typedAtWidest = [](ALCodeEditor& e, size_t step, bool ask) { + if (step == 0) + { + const S32 end = e.document().lineLength(5); + e.setCaret(ALTextPos(5, end)); + e.insertText("x"); + } + else + { + const S32 end = e.document().lineLength(5) - 1; + e.deleteRange(ALTextRange(ALTextPos(5, end), ALTextPos(5, end + 1))); + } + return ask ? static_cast(e.layout().contentWidth()) : 0; + }; + bothInSteps("a character typed at its end and taken back, per edit", editors, 2, + [&](Subject&, ALCodeEditor& e, size_t step) { return typedAtWidest(e, step, false); }); + bothInSteps("the same, the widest asked after each", editors, 2, + [&](Subject&, ALCodeEditor& e, size_t step) { return typedAtWidest(e, step, true); }); } // Word wrap on: a character typed at the top and taken away, and the @@ -652,249 +769,266 @@ int main(int, char**) // goes into; the width moved by a pixel and back, and the lines in // view laid out again after each, as a window dragged wider does; and // a line's top asked at random, as drawing and the mouse ask. - std::printf("\nWord wrap\n"); - for (Subject& s : subjects) + TEST_CASE("altextengine_bench: word wrap", "[altextengine_bench][!benchmark]") { - s.editor->setWordWrap(true); - for (S32 line = 0; line < 60; ++line) + ll_test::skipUnlessOptimised(); + Editors editors; + for (Subject& s : editors) { - g_sink = g_sink + s.editor->layout().line(line).rows.size(); - } - } - both("a keystroke at the top, the last line's top found", subjects, 2, [](Subject&, ALCodeEditor& e) { - e.setCaret(ALTextPos(0, 0)); - e.insertText("x"); - g_sink = g_sink + static_cast(e.layout().lineTop(lastLine(e))); - e.deleteRange(ALTextRange(ALTextPos(0, 0), ALTextPos(0, 1))); - g_sink = g_sink + static_cast(e.layout().lineTop(lastLine(e))); - }); - both("the width moved by a pixel, 60 lines laid out again", subjects, 2, [](Subject&, ALCodeEditor& e) { - ALTextLayout& laid = e.layout(); - const S32 width = laid.wrapWidth(); - for (const S32 to : { width + 1, width }) - { - laid.setWrapWidth(to); + s.editor->setWordWrap(true); for (S32 line = 0; line < 60; ++line) { - g_sink = g_sink + laid.line(line).rows.size(); + s.editor->layout().line(line); } - g_sink = g_sink + static_cast(laid.lineTop(60)); - } - }); - both("a line's top and the line at a y, per thousand", subjects, 1, [](Subject&, ALCodeEditor& e) { - ALTextLayout& laid = e.layout(); - for (S32 i = 0; i < 1000; ++i) - { - const S32 line = (i * 7919) % LINES; - g_sink = g_sink + static_cast(laid.lineAtY(laid.lineTop(line) + 1)); } - }); - for (Subject& s : subjects) - { - s.editor->setWordWrap(false); + bothInSteps("a keystroke at the top, the last line's top found", editors, 2, [](Subject&, ALCodeEditor& e, size_t step) { + editAtTop(e, "x", step); + return e.layout().lineTop(lastLine(e)); + }); + bothInSteps("the width moved by a pixel, 60 lines laid out again", editors, 2, [](Subject&, ALCodeEditor& e, size_t step) { + ALTextLayout& laid = e.layout(); + laid.setWrapWidth(laid.wrapWidth() + (step == 0 ? 1 : -1)); + size_t rows = 0; + for (S32 line = 0; line < 60; ++line) + { + rows += laid.line(line).rows.size(); + } + return rows + static_cast(laid.lineTop(60)); + }); + both("a line's top and the line at a y, per thousand", editors, [](Subject&, ALCodeEditor& e) { + ALTextLayout& laid = e.layout(); + size_t lines = 0; + for (S32 i = 0; i < 1000; ++i) + { + lines += static_cast(laid.lineAtY(laid.lineTop((i * 7919) % LINES) + 1)); + } + return lines; + }); } - std::printf("\nBrackets (an opener at the top, unmatched)\n"); - for (Subject& s : subjects) + // A bracket left open at the top: its partner looked for as the caret + // rests beside it, and the depth the rainbow and the indent read at the + // end, each after a character typed and taken away below it. + TEST_CASE("altextengine_bench: brackets with an opener at the top unmatched", "[altextengine_bench][!benchmark]") { - s.editor->setCaret(ALTextPos(0, 0)); - s.editor->insertText("("); - } - both("its partner looked for, per look", subjects, 1, [](Subject&, ALCodeEditor& e) { - ALTextPos out; - g_sink = g_sink + e.matchBracketAt(ALTextPos(0, 0), out); - }); - both("an edit on the next line, then looked for again", subjects, 1, [](Subject&, ALCodeEditor& e) { - e.setCaret(ALTextPos(1, 0)); - e.insertText("x"); - e.deleteRange(ALTextRange(ALTextPos(1, 0), ALTextPos(1, 1))); - ALTextPos out; - g_sink = g_sink + e.matchBracketAt(ALTextPos(0, 0), out); - }); - both("an edit at the top, then the depth at the end", subjects, 1, [](Subject&, ALCodeEditor& e) { - e.setCaret(ALTextPos(0, 1)); - e.insertText("x"); - e.deleteRange(ALTextRange(ALTextPos(0, 1), ALTextPos(0, 2))); - g_sink = g_sink + e.bracketDepthAt(e.document().end()); - }); - for (Subject& s : subjects) - { - s.editor->deleteRange(ALTextRange(ALTextPos(0, 0), ALTextPos(0, 1))); - s.editor->undoJournal().clear(); + ll_test::skipUnlessOptimised(); + Editors editors; + for (Subject& s : editors) + { + s.editor->setCaret(ALTextPos(0, 0)); + s.editor->insertText("("); + } + both("its partner looked for, per look", editors, [](Subject&, ALCodeEditor& e) { + ALTextPos out; + return e.matchBracketAt(ALTextPos(0, 0), out); + }); + both("an edit on the next line, then looked for again", editors, [](Subject&, ALCodeEditor& e) { + e.setCaret(ALTextPos(1, 0)); + e.insertText("x"); + e.deleteRange(ALTextRange(ALTextPos(1, 0), ALTextPos(1, 1))); + ALTextPos out; + return e.matchBracketAt(ALTextPos(0, 0), out); + }); + both("an edit at the top, then the depth at the end", editors, [](Subject&, ALCodeEditor& e) { + e.setCaret(ALTextPos(0, 1)); + e.insertText("x"); + e.deleteRange(ALTextRange(ALTextPos(0, 1), ALTextPos(0, 2))); + return e.bracketDepthAt(e.document().end()); + }); } - std::printf("\nFolding\n"); - size_t regions[2] = {}; - both("fold regions rebuilt", subjects, 1, [&](Subject& s, ALCodeEditor& e) { - ALFoldModel model; - regions[&s - subjects] = model.regions(e.document(), e.getTabWidth()).size(); - g_sink = g_sink + regions[&s - subjects]; - }); - countRow(" regions", regions[0], regions[1]); - both("an edit at the top, then the editor's fold regions", subjects, 1, [&](Subject&, ALCodeEditor& e) { - editAtTop(e, "x"); - g_sink = g_sink + e.foldRegions().size(); - }); - const auto brokenAndJoined = [](Subject&, ALCodeEditor& e) { - e.setCaret(ALTextPos(0, 0)); - e.insertText("\n"); - e.deleteRange(ALTextRange(ALTextPos(0, 0), ALTextPos(1, 0))); - g_sink = g_sink + e.foldRegions().size(); - }; - both("a line broken at the top and joined, then the editor's fold regions", subjects, 1, brokenAndJoined); - // A third of the blocks never closed, so that what is open grows down - // the text, and every place the walk keeps is under all of it. - for (Subject& s : subjects) - { - s.editor->setText(withoutClosers(s.text, &s == subjects ? "}" : "end")); - g_sink = g_sink + s.editor->foldRegions().size(); - } - both("a third never closed: the same", subjects, 1, brokenAndJoined); - for (Subject& s : subjects) + TEST_CASE("altextengine_bench: folding", "[altextengine_bench][!benchmark]") { - s.editor->setText(s.text); - s.editor->undoJournal().clear(); - } - - std::printf("\nLaying out every line\n"); - const S32 tab = subjects[0].editor->layout().tabWidth(); - S32 width = 600; - both("unwrapped", subjects, 1, [&](Subject&, ALCodeEditor& e) { - ALTextLayout& layout = e.layout(); - // A tab width either way, so that every pass lays out afresh. - layout.setTabWidth(layout.tabWidth() == tab ? tab + 1 : tab); - for (S32 line = 0; line < layout.lineCount(); ++line) + ll_test::skipUnlessOptimised(); + Editors editors; + size_t regions[2] = {}; + both("fold regions rebuilt", editors, [&](Subject& s, ALCodeEditor& e) { + ALFoldModel model; + return regions[&s - editors.subjects] = model.regions(e.document(), e.getTabWidth()).size(); + }); + both("an edit at the top, then the editor's fold regions", editors, [](Subject&, ALCodeEditor& e) { + editAtTop(e, "x"); + return e.foldRegions().size(); + }); + const auto brokenAndJoined = [](Subject&, ALCodeEditor& e) { + e.setCaret(ALTextPos(0, 0)); + e.insertText("\n"); + e.deleteRange(ALTextRange(ALTextPos(0, 0), ALTextPos(1, 0))); + return e.foldRegions().size(); + }; + both("a line broken at the top and joined, then the editor's fold regions", editors, brokenAndJoined); + // A third of the blocks never closed, so that what is open grows down + // the text, and every place the walk keeps is under all of it. + for (Subject& s : editors) { - g_sink = g_sink + layout.rowCount(line); + s.editor->setText(withoutClosers(s.text, &s == editors.subjects ? "}" : "end")); + s.editor->foldRegions(); } - g_sink = g_sink + layout.totalHeight(); - }); - for (Subject& s : subjects) - { - s.editor->layout().setTabWidth(tab); + both("a third never closed: the same", editors, brokenAndJoined); + + countHeading("Folding"); + countRow("regions", regions[0], regions[1]); + std::fflush(stdout); } - both("wrapped at 600 px", subjects, 1, [&](Subject&, ALCodeEditor& e) { - ALTextLayout& layout = e.layout(); - // One pixel either way, so that every pass lays out afresh. - width = width == 600 ? 601 : 600; - layout.setWrapWidth(width); - for (S32 line = 0; line < layout.lineCount(); ++line) + + TEST_CASE("altextengine_bench: laying out every line", "[altextengine_bench][!benchmark]") + { + ll_test::skipUnlessOptimised(); + Editors editors; + const S32 tab = editors.subjects[0].editor->layout().tabWidth(); + S32 width = 600; + both("unwrapped", editors, [&](Subject&, ALCodeEditor& e) { + ALTextLayout& layout = e.layout(); + // A tab width either way, so that every call lays out afresh. + layout.setTabWidth(layout.tabWidth() == tab ? tab + 1 : tab); + size_t rows = 0; + for (S32 line = 0; line < layout.lineCount(); ++line) + { + rows += static_cast(layout.rowCount(line)); + } + return rows + static_cast(layout.totalHeight()); + }); + for (Subject& s : editors) { - g_sink = g_sink + layout.rowCount(line); + s.editor->layout().setTabWidth(tab); } - g_sink = g_sink + layout.totalHeight(); - }); + both("wrapped at 600 px", editors, [&](Subject&, ALCodeEditor& e) { + ALTextLayout& layout = e.layout(); + // One pixel either way, so that every call lays out afresh. + width = width == 600 ? 601 : 600; + layout.setWrapWidth(width); + size_t rows = 0; + for (S32 line = 0; line < layout.lineCount(); ++line) + { + rows += static_cast(layout.rowCount(line)); + } + return rows + static_cast(layout.totalHeight()); + }); + } // Vim's commands over the lines a :g picks, each line's change one - // edit or all of them one; and text typed again by `.`. - std::printf("\nVim over every line\n"); - ALVimKeymap* vims[2] = {}; - for (Subject& s : subjects) - { - auto keymap = std::make_unique(); - vims[&s - subjects] = keymap.get(); - s.editor->setModalKeymap(std::move(keymap)); - } - both(":g/total/s//count/, then undone", subjects, 1, [&](Subject& s, ALCodeEditor& e) { - vims[&s - subjects]->takeLine(e, ':', "g/total/s//count/", true); - e.undo(); - }); - both(":%s/\\(\\w\\+\\)\\s*(/\\1 (/g, then undone", subjects, 1, [&](Subject& s, ALCodeEditor& e) { - vims[&s - subjects]->takeLine(e, ':', "%s/\\(\\w\\+\\)\\s*(/\\1 (/g", true); - e.undo(); - }); - both(":g/total/>, then undone", subjects, 1, [&](Subject& s, ALCodeEditor& e) { - vims[&s - subjects]->takeLine(e, ':', "g/total/>", true); - e.undo(); - }); - both(":g/total/d, then undone", subjects, 1, [&](Subject& s, ALCodeEditor& e) { - vims[&s - subjects]->takeLine(e, ':', "g/total/d", true); - e.undo(); - }); - for (Subject& s : subjects) - { - s.editor->setCaret(ALTextPos(0, 0)); - typeKeys(*s.editor, "Athe quick brown fox jumps over the lazy dog, the quick brown fox jumps over the lazy dog, and again.\x1b"); - } - both("`.`: a 100-character insert typed again, then undone", subjects, 1, [](Subject&, ALCodeEditor& e) { - typeKeys(e, "."); - e.undo(); - }); - // Stepping by character along that line, as l does. - both("l along a line of 100 characters, per step", subjects, 100, [](Subject&, ALCodeEditor& e) { - typeKeys(e, "0"); - for (int i = 0; i < 99; ++i) - { - typeKeys(e, "l"); - } - }); - // A search, then n through its matches; and a search typed on the - // search line, with the cursor moved along it, as incsearch sees it. - for (Subject& s : subjects) - { - vims[&s - subjects]->takeLine(*s.editor, '/', "total", true); - } - both("n through the matches of \"total\", per n", subjects, 10, [](Subject&, ALCodeEditor& e) { - typeKeys(e, "nnnnnnnnnn"); - }); - both("/total typed, the cursor moved along it, per key", subjects, 15, [](Subject&, ALCodeEditor& e) { - typeKeys(e, "/total"); - for (int i = 0; i < 4; ++i) - { - e.handleKeyHere(KEY_LEFT, MASK_NONE); - } - for (int i = 0; i < 4; ++i) - { - e.handleKeyHere(KEY_RIGHT, MASK_NONE); - } - e.handleKeyHere(KEY_ESCAPE, MASK_NONE); - }); - // A macro of 500 motions played back while a mapping is made: every - // key it feeds is looked at as the start of one. - for (Subject& s : subjects) - { - vims[&s - subjects]->takeLine(*s.editor, ':', "nmap x", true); - s.editor->setCaret(ALTextPos(0, 0)); - typeKeys(*s.editor, "gg"); - std::string recorded = "qa"; - recorded += std::string(250, 'j') + std::string(250, 'k') + "q"; - typeKeys(*s.editor, recorded.c_str()); - } - both("@a: a macro of 500 motions, a mapping made", subjects, 1, [](Subject&, ALCodeEditor& e) { typeKeys(e, "@a"); }); - // The whole text selected by lines, then moved within: every motion - // makes a selection of all of it. - for (Subject& s : subjects) - { - typeKeys(*s.editor, "ggVG"); - } - both("a motion with the whole text selected (ggVG, then k and j)", subjects, 2, [](Subject&, ALCodeEditor& e) { typeKeys(e, "kj"); }); - for (Subject& s : subjects) + // edit or all of them one; and text typed again by `.`. Over every + // line laid out, wrapped at 600 px: the layout as laying out every line + // leaves it, which these rows were first measured over. + TEST_CASE("altextengine_bench: vim over every line", "[altextengine_bench][!benchmark]") { - s.editor->handleKeyHere(KEY_ESCAPE, MASK_NONE); - } - both(":sort, then undone", subjects, 1, [&](Subject& s, ALCodeEditor& e) { - vims[&s - subjects]->takeLine(e, ':', "sort", true); - e.undo(); - }); - both(":sort i, then undone", subjects, 1, [&](Subject& s, ALCodeEditor& e) { - vims[&s - subjects]->takeLine(e, ':', "sort i", true); - e.undo(); - }); - - for (Subject& s : subjects) - { - s.editor->die(); + ll_test::skipUnlessOptimised(); + Editors editors; + ALVimKeymap* vims[2] = {}; + for (Subject& s : editors) + { + ALTextLayout& layout = s.editor->layout(); + layout.setWrapWidth(600); + for (S32 line = 0; line < layout.lineCount(); ++line) + { + layout.rowCount(line); + } + auto keymap = std::make_unique(); + vims[&s - editors.subjects] = keymap.get(); + s.editor->setModalKeymap(std::move(keymap)); + } + both(":g/total/s//count/, then undone", editors, [&](Subject& s, ALCodeEditor& e) { + vims[&s - editors.subjects]->takeLine(e, ':', "g/total/s//count/", true); + e.undo(); + }); + both(":%s/\\(\\w\\+\\)\\s*(/\\1 (/g, then undone", editors, [&](Subject& s, ALCodeEditor& e) { + vims[&s - editors.subjects]->takeLine(e, ':', "%s/\\(\\w\\+\\)\\s*(/\\1 (/g", true); + e.undo(); + }); + both(":g/total/>, then undone", editors, [&](Subject& s, ALCodeEditor& e) { + vims[&s - editors.subjects]->takeLine(e, ':', "g/total/>", true); + e.undo(); + }); + both(":g/total/d, then undone", editors, [&](Subject& s, ALCodeEditor& e) { + vims[&s - editors.subjects]->takeLine(e, ':', "g/total/d", true); + e.undo(); + }); + for (Subject& s : editors) + { + s.editor->setCaret(ALTextPos(0, 0)); + typeKeys(*s.editor, "Athe quick brown fox jumps over the lazy dog, the quick brown fox jumps over the lazy dog, and again.\x1b"); + } + both("`.`: a 100-character insert typed again, then undone", editors, [](Subject&, ALCodeEditor& e) { + typeKeys(e, "."); + e.undo(); + }); + // Stepping by character along that line, as l does: to its start, + // then a step at a time along it. + bothInSteps("l along a line of 100 characters, per step", editors, 100, + [](Subject&, ALCodeEditor& e, size_t step) { typeKeys(e, step == 0 ? "0" : "l"); }); + // A search, then n through its matches; and a search typed on the + // search line, with the cursor moved along it, as incsearch sees it. + for (Subject& s : editors) + { + vims[&s - editors.subjects]->takeLine(*s.editor, '/', "total", true); + } + both("n through the matches of \"total\", per n", editors, [](Subject&, ALCodeEditor& e) { typeKeys(e, "n"); }); + bothInSteps("/total typed, the cursor moved along it, per key", editors, 15, [](Subject&, ALCodeEditor& e, size_t step) { + if (step < 6) + { + const char key[2] = { "/total"[step], 0 }; + typeKeys(e, key); + } + else if (step < 10) + { + e.handleKeyHere(KEY_LEFT, MASK_NONE); + } + else if (step < 14) + { + e.handleKeyHere(KEY_RIGHT, MASK_NONE); + } + else + { + e.handleKeyHere(KEY_ESCAPE, MASK_NONE); + } + }); + // A macro of 500 motions played back while a mapping is made: every + // key it feeds is looked at as the start of one. + for (Subject& s : editors) + { + vims[&s - editors.subjects]->takeLine(*s.editor, ':', "nmap x", true); + s.editor->setCaret(ALTextPos(0, 0)); + typeKeys(*s.editor, "gg"); + std::string recorded = "qa"; + recorded += std::string(250, 'j') + std::string(250, 'k') + "q"; + typeKeys(*s.editor, recorded.c_str()); + } + both("@a: a macro of 500 motions, a mapping made", editors, [](Subject&, ALCodeEditor& e) { typeKeys(e, "@a"); }); + // The whole text selected by lines, then moved within: every motion + // makes a selection of all of it. + for (Subject& s : editors) + { + typeKeys(*s.editor, "ggVG"); + } + bothInSteps("a motion with the whole text selected (ggVG, then k and j)", editors, 2, + [](Subject&, ALCodeEditor& e, size_t step) { typeKeys(e, step == 0 ? "k" : "j"); }); + for (Subject& s : editors) + { + s.editor->handleKeyHere(KEY_ESCAPE, MASK_NONE); + } + both(":sort, then undone", editors, [&](Subject& s, ALCodeEditor& e) { + vims[&s - editors.subjects]->takeLine(e, ':', "sort", true); + e.undo(); + }); + both(":sort i, then undone", editors, [&](Subject& s, ALCodeEditor& e) { + vims[&s - editors.subjects]->takeLine(e, ':', "sort i", true); + e.undo(); + }); } // The Output pane's log, full: four lanes of 500 entries, every other // one an error and every fourth with a URL in it. - std::printf("\nThe Output log (2,000 entries)\n"); + TEST_CASE("altextengine_bench: the Output log of two thousand entries", "[altextengine_bench][!benchmark]") { + ll_test::skipUnlessOptimised(); + requireUI(); + Views views; + ALOutputView::Params p(LLUICtrlFactory::getDefaultParams()); p.name = "output"; p.rect = LLRect(0, 800, 1000, 0); p.capacity = 500; - ALOutputView* log = LLUICtrlFactory::create(p); + ALOutputView* log = views.keep(LLUICtrlFactory::create(p)); log->setFont(LLFontGL::getFontMonospace()); S32 serial = 0; const auto next_one = [&serial]() { @@ -912,28 +1046,31 @@ int main(int, char**) { log->append(next_one()); } - rowOne("a filter typed: the errors alone, then all again", ms_per_item(1, [&] { - log->setFilter([](const ALOutputView::Entry& entry) { return entry.kind == "error"; }); - log->setFilter(nullptr); - g_sink = g_sink + log->document().lineCount(); - })); - rowOne("an entry past the fill, the oldest let go of", ms_per_item(100, [&] { - for (S32 i = 0; i < 100; ++i) - { - log->append(next_one()); - } - g_sink = g_sink + log->document().lineCount(); - })); - log->die(); + BENCHMARK("a filter typed: the errors alone, then all again") + { + log->setFilter([](const ALOutputView::Entry& entry) { return entry.kind == "error"; }); + log->setFilter(nullptr); + return log->document().lineCount(); + }; + BENCHMARK("an entry past the fill, the oldest let go of") + { + log->append(next_one()); + return log->document().lineCount(); + }; } + // Quick Open over the names of a big project, a query typed a letter - // at a time and taken back. - std::printf("\nQuick Open (2,000 candidates)\n"); + // at a time and taken back, a key a call. + TEST_CASE("altextengine_bench: Quick Open over two thousand candidates", "[altextengine_bench][!benchmark]") { + ll_test::skipUnlessOptimised(); + requireUI(); + Views views; + ALQuickOpen::Params p(LLUICtrlFactory::getDefaultParams()); p.name = "quick"; p.rect = LLRect(0, 400, 500, 0); - ALQuickOpen* quick = LLUICtrlFactory::create(p); + ALQuickOpen* quick = views.keep(LLUICtrlFactory::create(p)); std::vector names; for (S32 i = 0; i < 2000; ++i) { @@ -945,22 +1082,22 @@ int main(int, char**) names.push_back(std::move(one)); } quick->setCandidates(std::move(names)); - rowOne("a query typed and taken back, per key", ms_per_item(10, [&] { - const std::string query = "flscst"; - for (size_t n = 1; n <= query.size(); ++n) - { - quick->setQuery(query.substr(0, n)); - } - for (size_t n = query.size() - 1; n > 0; --n) - { - quick->setQuery(query.substr(0, n)); - } - quick->setQuery(std::string()); - g_sink = g_sink + quick->listed().size(); - })); - quick->die(); + // Six letters typed, then six taken back: the query from "f" to + // "flscst", and back through "f" to nothing. + const std::string query = "flscst"; + size_t keys = 0; + const auto key = [&]() { + const size_t step = keys++ % 12; + quick->setQuery(query.substr(0, step < 6 ? step + 1 : 11 - step)); + return quick->listed().size(); + }; + BENCHMARK("a query typed and taken back, per key") + { + return key(); + }; + while (keys % 12 != 0) + { + key(); + } } - std::printf("\n(checksum %zu)\n", static_cast(g_sink)); - return 0; -#endif } diff --git a/indra/test/CMakeLists.txt b/indra/test/CMakeLists.txt index c2044d129f..76ea30cc23 100644 --- a/indra/test/CMakeLists.txt +++ b/indra/test/CMakeLists.txt @@ -50,6 +50,7 @@ target_sources( PUBLIC FILE_SET HEADERS FILES + altestbenchmark.h altestfailure.h altestfatal.h altesthelper.h @@ -79,12 +80,13 @@ set_target_properties(al_test_support PROPERTIES FOLDER "Tests") # run one at a time by the entries below, which check the exit code and the # output CTest sees from the harness: the state guard and what a fixture may # tell it, LOGFAIL's replay, a crash and an uncaught LL_ERRS, a fixture's end -# failing, the skip and no-match codes, the ISA gate, and a listing needing no -# setup. +# failing, the skip and no-match codes, the ISA gate, a listing needing no +# setup, and which runs a benchmark file's cases. al_add_test_binary( test_support_tests PROJECT test SOURCES tests/altestsupport_test.cpp + BENCH_SOURCES tests/altestsupport_bench.cpp LIBRARIES llcommon ) @@ -210,6 +212,21 @@ al_expect_from_test_support( EXIT 125 OUTPUT "does not run x86-64 level 9" ) +# A benchmark file's cases are hidden: the filter a test file's entry runs +# matches none of them, and the one its benchmark entry runs runs them. Few +# samples and no warm-up, since the run, not the number, is checked; Catch2 +# still spends about two seconds measuring its clock first. +al_expect_from_test_support( + benchmark_hidden_from_its_file + ARGS "[#altestsupport_bench]~[.]" + EXIT 2 +) +al_expect_from_test_support( + benchmark_runs_from_its_entry + ARGS "[#altestsupport_bench][!benchmark]|--benchmark-samples|2|--benchmark-warmup-time|1" + EXIT 0 + OUTPUT "a word's length" +) # The ISA level is one no host runs: a listing that set up first, or held to # the gate, would skip rather than list. al_expect_from_test_support( diff --git a/indra/test/altestbenchmark.h b/indra/test/altestbenchmark.h new file mode 100644 index 0000000000..3862bfd3a2 --- /dev/null +++ b/indra/test/altestbenchmark.h @@ -0,0 +1,47 @@ +/** + * @file altestbenchmark.h + * @brief What a benchmark file includes: Catch2's benchmarks, and the skip + * that keeps an unoptimised build's numbers from being read. + * + * $LicenseInfo:firstyear=2026&license=viewerlgpl$ + * Alchemy Viewer Source Code + * Copyright (C) 2026, Rye + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * $/LicenseInfo$ + */ + +#ifndef AL_ALTESTBENCHMARK_H +#define AL_ALTESTBENCHMARK_H + +#include +#include + +namespace ll_test +{ + // A benchmark case's first line. Outside an optimised build the numbers + // say nothing of the code as it ships, so the case skips there; the code + // still compiles in every configuration. A benchmark file's CTest entry + // reads a run whose every case skipped as skipped, as it does a test + // file's. + inline void skipUnlessOptimised() + { +#if !defined(LL_RELEASE) + SKIP("an unoptimised build has no numbers worth reading"); +#endif + } +} + +#endif // AL_ALTESTBENCHMARK_H diff --git a/indra/test/tests/altestsupport_bench.cpp b/indra/test/tests/altestsupport_bench.cpp new file mode 100644 index 0000000000..890b87a2d7 --- /dev/null +++ b/indra/test/tests/altestsupport_bench.cpp @@ -0,0 +1,42 @@ +/** + * @file altestsupport_bench.cpp + * @brief A benchmark file of the harness's own, which CTest entries in + * test/CMakeLists.txt run to check that its file's filter leaves it + * out and its benchmark entry's runs it. + * + * $LicenseInfo:firstyear=2026&license=viewerlgpl$ + * Alchemy Viewer Source Code + * Copyright (C) 2026, Rye + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "altestbenchmark.h" + +#include + +// No skipUnlessOptimised(): the entries that run this check the harness, +// not a number, in every configuration. +TEST_CASE("altestsupport_bench: a benchmark runs where its entry asks for it", "[altestsupport_bench][!benchmark]") +{ + const std::string word = "benchmark"; + BENCHMARK("a word's length") + { + return word.size(); + }; +}