Test framework, stage 4: the benchmarks in Catch2

indra/alscript/CMakeLists.txt

@@ -299,6 +299,9 @@ if(AL_BUILD_TESTS)
       tests/alscriptweight_test.cpp
       tests/alsourcemap_test.cpp
       tests/aluploadheader_test.cpp
+    # The preprocessor, the formatter, the optimizer and one analysis of each
+    # language over big scripts, with the grid's definitions loaded.
+    BENCH_SOURCES tests/alscript_bench.cpp
     SUPPORT alscript_test_checks llcommon_test_checks
     LIBRARIES alscript llcommon ll::slua ll::tailslide
     DEFINES
@@ -315,13 +318,4 @@ if(AL_BUILD_TESTS)
       tests/alscriptlintpass_test.cpp
     RERUN_ENVIRONMENT AL_TEST_LUAU_SOLVER=new
   )
-  # The preprocessor, the formatter, the optimizer and one analysis of each
-  # language over big scripts, with the grid's definitions loaded.
-  al_add_bench(
-    alscript
-    PROJECT alscript
-    LIBRARIES alscript llcommon
-    INCLUDES ${INDRA_SOURCE_DIR}/test
-    DEFINES ${definitions_defines}
-  )
 endif()

indra/cmake/ALAddTestBinary.cmake

@@ -14,7 +14,7 @@ endif()
 #                    [INCLUDES <dir>...] [DEFINES <define>...] [ENVIRONMENT <VAR=value>...]
 #                    [GL] [PYTHON] [ISA_TIER <tier>] [RESOURCE_LOCK <name>...] [LAUNCHER <command>...]
 #                    [RERUN <run> RERUN_SOURCES <test file>... RERUN_ENVIRONMENT <VAR=value>...]
-#                    [COMBINED])
+#                    [BENCH_SOURCES <benchmark file>...] [COMBINED])
 #
 # Builds the test files into one Catch2 executable, <name>, with the harness
 # (al_test_support: the main, the listeners and the matchers) and SUPPORT's
@@ -69,13 +69,24 @@ endif()
 # with AL_TEST_PER_CASE each of their cases as <name>_<run>/<case>. alscript's
 # tests that check SLua run again under Luau's new type solver, as a scripter
 # may choose it.
+#
+# BENCH_SOURCES are benchmark files, built into the binary beside the tests,
+# whose cases are tagged [!benchmark]: hidden, so no test file's entry, the
+# combined run or the per-case entries run them. Each is an entry of its
+# own, bench/<name>/<file stem>, labelled benchmark and manual, which the
+# default run leaves out and BUILD_AND_RUN_BENCHMARKS runs. It runs alone,
+# since anything beside it would move its numbers, and reports to the
+# console and as XML, Testing/benchmarks/<name>_<file stem>.xml, which keeps
+# the numbers where Catch2's JSON reporter would drop them. A benchmark case
+# skips outside an optimised build (altestbenchmark.h), and the entry then
+# reads as skipped.
 function(al_add_test_binary name)
   cmake_parse_arguments(
     PARSE_ARGV 1
     arg
     "GL;PYTHON;COMBINED"
     "PROJECT;ISA_TIER;RERUN"
-    "SOURCES;UNIT;LIBRARIES;SUPPORT;INCLUDES;DEFINES;ENVIRONMENT;RESOURCE_LOCK;LAUNCHER;RERUN_SOURCES;RERUN_ENVIRONMENT"
+    "SOURCES;UNIT;LIBRARIES;SUPPORT;INCLUDES;DEFINES;ENVIRONMENT;RESOURCE_LOCK;LAUNCHER;RERUN_SOURCES;RERUN_ENVIRONMENT;BENCH_SOURCES"
   )
   if(NOT arg_PROJECT)
     message(FATAL_ERROR "al_add_test_binary(${name}): PROJECT is required")
@@ -107,15 +118,20 @@ function(al_add_test_binary name)
   endif()
 
   set(stems)
-  foreach(source IN LISTS arg_SOURCES)
+  set(bench_stems)
+  foreach(source IN LISTS arg_SOURCES arg_BENCH_SOURCES)
     get_filename_component(stem "${source}" NAME_WLE)
-    if(stem IN_LIST stems)
+    if(stem IN_LIST stems OR stem IN_LIST bench_stems)
       message(
         FATAL_ERROR
         "al_add_test_binary(${name}): two test files are named ${stem}; the stem names a file's entry and its tag"
       )
     endif()
-    list(APPEND stems ${stem})
+    if(source IN_LIST arg_BENCH_SOURCES)
+      list(APPEND bench_stems ${stem})
+    else()
+      list(APPEND stems ${stem})
+    endif()
 
     # A file is in one binary at each tier: the variants of an ISA_TIER
     # binary build the same files at the others.
@@ -145,7 +161,7 @@ function(al_add_test_binary name)
   # vcpkg's toolchain hangs an app-local copy step on every executable it
   # sees declared; a test runs from the build tree, as al_add_test explains.
   set(VCPKG_APPLOCAL_DEPS OFF)
-  add_executable(${target} ${arg_SOURCES} ${arg_UNIT})
+  add_executable(${target} ${arg_SOURCES} ${arg_BENCH_SOURCES} ${arg_UNIT})
   target_link_libraries(${target} PRIVATE al::flags al_test_support ${arg_SUPPORT} ${arg_LIBRARIES})
   target_include_directories(${target} PRIVATE ${arg_INCLUDES} ${INDRA_SOURCE_DIR}/test)
   if(arg_UNIT)
@@ -377,5 +393,34 @@ function(al_add_test_binary name)
     endif()
   endif()
 
+  # A benchmark file's run, whichever way the tests are registered. It names
+  # the file's tag and Catch2's benchmark tag together, which chooses the
+  # hidden cases a tag alone would leave out.
+  if(arg_BENCH_SOURCES)
+    set(bench_dir "${CMAKE_BINARY_DIR}/Testing/benchmarks")
+    file(MAKE_DIRECTORY "${bench_dir}")
+    foreach(stem IN LISTS bench_stems)
+      add_test(
+        NAME bench/${target}/${stem}
+        COMMAND
+          ${arg_LAUNCHER} $<TARGET_FILE:${target}> "[#${stem}][!benchmark]" --reporter console
+          --reporter "xml::out=${bench_dir}/${target}_${stem}.xml" ${arguments}
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+      )
+      set_tests_properties(
+        bench/${target}/${stem}
+        PROPERTIES
+          ENVIRONMENT "${environment}"
+          ENVIRONMENT_MODIFICATION "${AL_TEST_ENVIRONMENT}"
+          LABELS "${labels};benchmark;manual"
+          DISABLED ${disabled}
+          SKIP_RETURN_CODE 125
+          RUN_SERIAL TRUE
+          TIMEOUT 3600
+      )
+    endforeach()
+    add_dependencies(BUILD_BENCHMARKS ${target})
+  endif()
+
   add_dependencies(BUILD_TESTS ${target})
 endfunction()

indra/cmake/LLAddBuildTest.cmake

@@ -189,52 +189,3 @@ function(al_add_test name)
 
   add_dependencies(BUILD_TESTS ${target})
 endfunction()
-
-# al_add_bench(<name> PROJECT <project> [SOURCES <file>...] [LIBRARIES <target>...]
-#              [INCLUDES <dir>...] [DEFINES <define>...])
-#
-# Builds tests/<name>_bench.cpp, which brings its own main, into
-# BENCH_<name> and registers it as a test labelled benchmark and manual. The
-# default test run leaves manual out; BUILD_AND_RUN_BENCHMARKS runs it, verbose,
-# since the numbers are the output. A benchmark exits 125, which CTest reads
-# as skipped, from a build whose numbers would say nothing.
-function(al_add_bench name)
-  cmake_parse_arguments(PARSE_ARGV 1 arg "" "PROJECT" "SOURCES;LIBRARIES;INCLUDES;DEFINES")
-  if(NOT arg_PROJECT)
-    message(FATAL_ERROR "al_add_bench(${name}): PROJECT is required")
-  endif()
-  if(arg_UNPARSED_ARGUMENTS)
-    message(FATAL_ERROR "al_add_bench(${name}): unexpected arguments: ${arg_UNPARSED_ARGUMENTS}")
-  endif()
-
-  set(target BENCH_${name})
-  set(VCPKG_APPLOCAL_DEPS OFF)
-  add_executable(${target} tests/${name}_bench.cpp ${arg_SOURCES})
-  target_link_libraries(${target} PRIVATE al::flags ${arg_LIBRARIES})
-  target_include_directories(${target} PRIVATE ${arg_INCLUDES} ${INDRA_SOURCE_DIR}/llmath)
-  target_compile_definitions(${target} PRIVATE ${arg_DEFINES})
-  set_target_properties(${target} PROPERTIES FOLDER "Benchmarks/${arg_PROJECT}")
-  if(WINDOWS)
-    set_target_properties(${target} PROPERTIES AL_SKIP_RELEASE_DEBUG_INFO ON)
-  elseif(DARWIN)
-    set_target_properties(
-      ${target}
-      PROPERTIES XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY "-" BUILD_RPATH "${SHARED_LIB_STAGING_DIR}"
-    )
-  endif()
-
-  add_test(
-    NAME ${target}
-    COMMAND $<TARGET_FILE:${target}>
-    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-  )
-  set_tests_properties(
-    ${target}
-    PROPERTIES
-      ENVIRONMENT_MODIFICATION "${AL_TEST_ENVIRONMENT}"
-      LABELS "benchmark;manual"
-      SKIP_RETURN_CODE 125
-  )
-
-  add_dependencies(BUILD_BENCHMARKS ${target})
-endfunction()

indra/llcommon/CMakeLists.txt

@@ -364,6 +364,9 @@ if(AL_BUILD_TESTS)
       tests/threadsafeschedule_test.cpp
       tests/tuple_test.cpp
       tests/workqueue_test.cpp
+    # The viewer's own patterns under Boost.Regex and under RE2, side by
+    # side, over the text each is run on.
+    BENCH_SOURCES tests/alregex_bench.cpp
     SUPPORT llcommon_test_checks
     LIBRARIES llcommon
   )
@@ -455,8 +458,4 @@ if(AL_BUILD_TESTS)
       PROPERTIES COMPILE_OPTIONS -Wno-uninitialized
     )
   endif()
-
-  # The viewer's own patterns under Boost.Regex and under RE2, side by side,
-  # over the text each is run on.
-  al_add_bench(alregex PROJECT llcommon LIBRARIES llcommon)
 endif()