Test framework, stage 2: Catch2, its harness and converter, and llmath in one binary

.github/workflows/build.yaml

@@ -514,6 +514,39 @@ jobs:
             esac
           fi
 
+      # Each Catch2 binary's run of all its tests in one process, while it is
+      # still combined_pending: the default run leaves those out until they
+      # are known to share a process, and this is where that is learned, on
+      # every row. It reports, and never fails the job. The runs go side by
+      # side, so the step takes about as long as the longest, which CTest
+      # stops at an hour; the step's own limit is past that, so a hang is
+      # recorded as CTest's timeout and reaches the JUnit report.
+      - name: Run pending combined tests
+        if: ${{ !cancelled() && matrix.build_variant == 'Tests' }}
+        continue-on-error: true
+        timeout-minutes: 75
+        shell: bash
+        env:
+          BUILD_DIRECTORY: ${{ steps.configure.outputs.build_directory }}
+        run: |
+          case "$BUILD_CONFIG" in
+            release) cmake_config=Release ;;
+            relwithdebinfo) cmake_config=RelWithDebInfo ;;
+            optdebug) cmake_config=OptDebug ;;
+            *) cmake_config=Debug ;;
+          esac
+          # This run writes LastTest.log over the build's; the build's is kept
+          # under a name the upload below still takes.
+          log="$BUILD_DIRECTORY/Testing/Temporary/LastTest.log"
+          if [[ -f "$log" ]]; then
+            cp "$log" "$BUILD_DIRECTORY/Testing/Temporary/LastTest-build.log"
+          fi
+          jobs="$(nproc 2>/dev/null || sysctl -n hw.logicalcpu)"
+          # CTest reads a relative --output-junit from the test directory.
+          ctest --test-dir "$BUILD_DIRECTORY" -C "$cmake_config" -L combined_pending \
+            --parallel "$jobs" --output-on-failure --no-tests=ignore \
+            --output-junit Testing/combined-pending-results.xml
+
       # What each test did, on a row whose tests failed as much as on one
       # whose tests passed: the JUnit file CTest writes and its own log. The
       # build step's artifact name is unset when a test fails, so this one
@@ -525,6 +558,7 @@ jobs:
           name: "test-results-${{ runner.os }}-${{ matrix.arch }}"
           path: |
             ${{ steps.configure.outputs.build_directory }}/Testing/test-results.xml
+            ${{ steps.configure.outputs.build_directory }}/Testing/combined-pending-results.xml
             ${{ steps.configure.outputs.build_directory }}/Testing/Temporary/LastTest*.log
           if-no-files-found: warn
 

CLAUDE.md

@@ -57,10 +57,12 @@ The viewer executable lands at `build-<OS>-<preset>/newview/<CONFIG>/` (e.g., `S
 Tests are off by default. To enable: add `-DAL_BUILD_TESTS=ON` to the configure command. Then run via CTest:
 
 ```
-ctest --test-dir build-<OS>-<preset> --output-on-failure
+ctest --test-dir build-<OS>-<preset> --output-on-failure -LE manual
 ```
 
-Unit tests live alongside the library they test in `indra/<library>/tests/` directories. The test framework is TUT (Template Unit Test). Test targets are defined by the `LL_ADD_PROJECT_UNIT_TESTS` macro in `indra/cmake/LLAddBuildTest.cmake`. Integration tests are in `indra/integration_tests/`.
+`-LE manual` leaves out the benchmarks and the pending one-process runs of Catch2 binaries (label `combined_pending`), as the build's own test target does.
+
+Unit tests live alongside the library they test in `indra/<library>/tests/` directories. They are moving from TUT (Template Unit Test) to Catch2 a library at a time. TUT test targets are defined by the `LL_ADD_PROJECT_UNIT_TESTS` macro in `indra/cmake/LLAddBuildTest.cmake`; a converted library builds its files into one Catch2 binary with `al_add_test_binary` (`indra/cmake/ALAddTestBinary.cmake`), on the harness in `indra/test/`. Integration tests are in `indra/integration_tests/`.
 
 ## Architecture
 

doc/BUILD.md

@@ -430,10 +430,12 @@ Build, then run with CTest:
 
 ```
 cmake --build <build-dir> --config RelWithDebInfo
-ctest --test-dir <build-dir> --output-on-failure
+ctest --test-dir <build-dir> --output-on-failure -LE manual
 ```
 
-Unit tests live alongside the library they cover in `indra/<library>/tests/`, written against the TUT (Template Unit Test) framework. Integration tests are in `indra/integration_tests/`.
+`-LE manual` leaves out what the build's own test target leaves out: the benchmarks, and each Catch2 binary's run of all its files in one process while that run is still labelled `combined_pending` (`ctest -L combined_pending` runs those).
+
+Unit tests live alongside the library they cover in `indra/<library>/tests/`. They are moving from the TUT (Template Unit Test) framework to Catch2 a library at a time: a converted library builds its test files into one binary (`al_add_test_binary`, `indra/cmake/ALAddTestBinary.cmake`), each file still a CTest entry of its own, on the harness in `indra/test/`. Integration tests are in `indra/integration_tests/`.
 
 ## Editing XUI
 

indra/CMakeLists.txt

@@ -41,6 +41,13 @@ cmake_dependent_option(
   "AL_BUILD_TESTS"
   OFF
 )
+cmake_dependent_option(
+  AL_TEST_PER_CASE
+  "Register a Catch2 binary's test cases one CTest entry each, found when ctest runs, in place of one entry a test file: for finding a failure that comes from order or shared state"
+  OFF
+  "AL_BUILD_TESTS"
+  OFF
+)
 option(AL_BUILD_SHARED_LLCOMMON "Build llcommon as a shared library" OFF)
 option(AL_BUILD_PACKAGE "Add the package target: the CPack archive of the installed tree" ON)
 
@@ -291,6 +298,10 @@ endif()
 # vcpkg feature flags MUST be set before the first project() call below  #
 ##########################################################################
 
+if(AL_BUILD_TESTS)
+  list(APPEND VCPKG_MANIFEST_FEATURES "tests")
+endif()
+
 if(AL_USE_KDU)
   list(APPEND VCPKG_MANIFEST_FEATURES "kdu")
 endif()
@@ -406,16 +417,19 @@ if(AL_BUILD_TESTS)
   include(ProcessorCount)
   ProcessorCount(NUM_CORES)
 
-  # Set parallel level to decrease test runtime. Benchmarks are registered
-  # as tests under their own label and left out of the default run, since
-  # their numbers are their output and a shared parallel run would ruin them.
+  # Set parallel level to decrease test runtime. What is labelled manual runs
+  # only when asked for by its own label: benchmarks, since their numbers are
+  # their output and a shared parallel run would ruin them, and a Catch2
+  # binary's run of all its tests in one process until they are known to
+  # share one (combined_pending; see ALAddTestBinary.cmake). One label, since
+  # these arguments reach a shell, which would take a | in a regex for a pipe.
   # Each run also writes its results as JUnit, which CI keeps.
   set(
     CMAKE_CTEST_ARGUMENTS
     -j${NUM_CORES}
     --output-on-failure
     -LE
-    benchmark
+    manual
     --output-junit
     "${CMAKE_BINARY_DIR}/Testing/test-results.xml"
   )
@@ -477,6 +491,7 @@ endforeach()
 # Add our test macros
 if(AL_BUILD_TESTS)
   include(LLAddBuildTest)
+  include(ALAddTestBinary)
 endif()
 
 # The build system's own files, for the IDE.