Measure benchmark memory usage natively instead of via /usr/bin/time

include/rootbench/MemoryMeasurement.h

@@ -0,0 +1,19 @@
+///\file Utilities to measure the memory usage of child processes.
+#ifndef RB_MEMORYMEASUREMENT_H
+#define RB_MEMORYMEASUREMENT_H
+
+#include <string>
+
+namespace RB {
+/// Runs the given command through the shell, like std::system(), and stores
+/// the maximum resident set size of the child process tree in KiB in
+/// maxRssKiB, as reported by wait4(). This is the same value that
+/// "/usr/bin/time -v" reports as "Maximum resident set size", but without
+/// depending on an external time binary.
+///
+///\returns the exit status of the command as reported by std::system(), or -1
+/// if the child process could not be created.
+int RunCommandMeasuringRss(const std::string &cmd, long &maxRssKiB);
+} // namespace RB
+
+#endif

lib/CMakeLists.txt

@@ -1,4 +1,5 @@
 RB_ADD_LIBRARY(RBSupport
   ErrorHandling.cxx
+  MemoryMeasurement.cxx
 )
 target_include_directories(RBSupport PUBLIC ${PROJECT_BINARY_DIR}/include ${PROJECT_SOURCE_DIR}/include)

lib/MemoryMeasurement.cxx

@@ -0,0 +1,35 @@
+///\file Contains utilities to measure the memory usage of child processes.
+#include "rootbench/MemoryMeasurement.h"
+
+#include <sys/resource.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+namespace RB {
+
+int RunCommandMeasuringRss(const std::string &cmd, long &maxRssKiB)
+{
+   maxRssKiB = 0;
+
+   pid_t pid = fork();
+   if (pid < 0)
+      return -1;
+   if (pid == 0) {
+      execl("/bin/sh", "sh", "-c", cmd.c_str(), static_cast<char *>(nullptr));
+      _exit(127);
+   }
+
+   int status = 0;
+   rusage usage{};
+   if (wait4(pid, &status, 0, &usage) < 0)
+      return -1;
+#ifdef __APPLE__
+   // On macOS, ru_maxrss is in bytes instead of KiB.
+   maxRssKiB = usage.ru_maxrss / 1024;
+#else
+   maxRssKiB = usage.ru_maxrss;
+#endif
+   return status;
+}
+
+} // namespace RB

root/interpreter/InterpreterTest.h

@@ -1,47 +1,38 @@
-#include <iostream>
-#include <cstdlib>
+#include <chrono>
 #include <string>
-#include <cstdlib>
-#include <fstream>
-#include <iostream>
-
-#include <unistd.h>
 
 #include "benchmark/benchmark.h"
 
-#include <chrono>
-
+#include "rootbench/MemoryMeasurement.h"
 #include "rootbench/RBConfig.h"
 
-static int runTutorial(const std::string& dir, const std::string& filename, const std::string& perffile) {
+static int runTutorial(const std::string& dir, const std::string& filename, long& maxRssKiB) {
   std::string rootsys = RB::GetRootSys();
   std::string rootInvocation;
   if (!filename.empty()) {
     std::string fullpath = rootsys + "/" + dir + "/" + filename;
-    // We are writing /usr/bin/time -v output in file to get maximum resident memory for the benchmark
-    rootInvocation = "/usr/bin/time -v -o \"" + perffile + "\" root.exe -l -q -b -n -x \"" + fullpath + "\" -e return ";
+    rootInvocation = "root.exe -l -q -b -n -x \"" + fullpath + "\" -e return ";
   } else {
-    rootInvocation = "/usr/bin/time -v -o \"" + perffile + "\" root.exe -l -q -b ";
+    rootInvocation = "root.exe -l -q -b ";
   }
 
-    return std::system(rootInvocation.c_str());
+  return RB::RunCommandMeasuringRss(rootInvocation, maxRssKiB);
 }
 
 static void TestTutorial(benchmark::State &state, const char *dir, const char *tutorial) {
-  int peakSize = 0;
-  std::string perftutorial ("perfile.txt");
+  long peakSizeKiB = 0;
   for(auto _ : state){
       auto start = std::chrono::high_resolution_clock::now();
-      runTutorial(dir, tutorial,perftutorial);
+      int status = runTutorial(dir, tutorial, peakSizeKiB);
       auto end   = std::chrono::high_resolution_clock::now();
+      if (status != 0) {
+         state.SkipWithError(("failed to run tutorial \"" + std::string(tutorial) + "\"").c_str());
+         return;
+      }
       auto elapsed_seconds =
       std::chrono::duration_cast<std::chrono::duration<double>>(
         end - start);
       state.SetIterationTime(elapsed_seconds.count());
-      std::string memorytutorial = "cat \"" + perftutorial + "\"| grep 'Maximum resident set size' | awk '{print $6}' > tmp_mem_file";
-      int res = std::system(memorytutorial.c_str());
-      (void) res;
-      std::ifstream("tmp_mem_file") >> peakSize;
    }
-   state.counters.insert({{"RSS", peakSize}});
+   state.counters.insert({{"RSS", static_cast<double>(peakSizeKiB)}});
 }

root/pyroot/PyROOTTest.h

@@ -1,35 +1,36 @@
-#include <iostream>
-#include <cstdlib>
+#include <chrono>
 #include <string>
-#include <cstdlib>
-#include <fstream>
-#include <iostream>
-
-#include <unistd.h>
 
 #include "benchmark/benchmark.h"
 
-#include <chrono>
-
+#include "rootbench/MemoryMeasurement.h"
 #include "rootbench/RBConfig.h"
 
-static int runTutorial(const std::string& dir, const std::string& filename) {
+static int runTutorial(const std::string& dir, const std::string& filename, long& maxRssKiB) {
   std::string rootsys = RB::GetRootSys();
   std::string fullpath = rootsys + "/" + dir + "/" + filename;
-  std::string rootInvocation = "root -l -b -q -e 'TPython::Exec(\" exec( open(\"" + fullpath + "\").read())\")'";
-    return std::system(rootInvocation.c_str());
+  // Exit with a non-zero status if the Python tutorial fails, since a failure
+  // of the interpreted code does not propagate into the exit status of root
+  // by itself.
+  std::string rootInvocation =
+      "root -l -b -q -e 'gSystem->Exit(!TPython::Exec(\"exec(open(\\\"" + fullpath + "\\\").read())\"))'";
+  return RB::RunCommandMeasuringRss(rootInvocation, maxRssKiB);
 }
 
 static void TestTutorial(benchmark::State &state, const char *dir, const char *tutorial) {
-  int peakSize = 0;
+  long peakSizeKiB = 0;
   for(auto _ : state){
       auto start = std::chrono::high_resolution_clock::now();
-      runTutorial(dir, tutorial);
+      int status = runTutorial(dir, tutorial, peakSizeKiB);
       auto end   = std::chrono::high_resolution_clock::now();
+      if (status != 0) {
+         state.SkipWithError(("failed to run tutorial \"" + std::string(tutorial) + "\"").c_str());
+         return;
+      }
       auto elapsed_seconds =
       std::chrono::duration_cast<std::chrono::duration<double>>(
         end - start);
       state.SetIterationTime(elapsed_seconds.count());
    }
-   state.counters.insert({{"RSS", peakSize}});
+   state.counters.insert({{"RSS", static_cast<double>(peakSizeKiB)}});
 }

root/pyroot/RunPyROOT.cxx

@@ -2,22 +2,16 @@
 
 #include "benchmark/benchmark.h"
 
-BENCHMARK_CAPTURE(TestTutorial, Test_hsimple_py, "tutorials/pyroot/", "hsimple.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_framework_py, "tutorials/pyroot/", "framework.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_hsum_py, "tutorials/pyroot/", "hsum.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_formula1_py, "tutorials/pyroot/", "formula1.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_fillrandom_py, "tutorials/pyroot/", "fillrandom.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_fit1_py, "tutorials/pyroot/", "fit1.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_h1draw_py, "tutorials/pyroot/", "h1draw.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_graph_py, "tutorials/pyroot/", "graph.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_gerrors_py, "tutorials/pyroot/", "gerrors.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_tornado_py, "tutorials/pyroot/", "tornado.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_surfaces_py, "tutorials/pyroot/", "surfaces.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_zdemo_py, "tutorials/pyroot/", "zdemo.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_geometry_py, "tutorials/pyroot/", "geometry.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_na49view_py, "tutorials/pyroot/", "na49view.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_file_py, "tutorials/pyroot/", "file.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_ntuple1_py, "tutorials/pyroot/", "ntuple1.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
-BENCHMARK_CAPTURE(TestTutorial, Test_rootmarks_py, "tutorials/pyroot/", "rootmarks.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
+// The Python tutorials that no longer exist in ROOT (framework.py, hsum.py,
+// fillrandom.py, h1draw.py, graph.py, gerrors.py, zdemo.py, file.py and
+// rootmarks.py) were dropped from the benchmarks, as well as geometry.py and
+// na49view.py, which cannot run standalone because they depend on a geometry
+// that needs to be created by other macros first.
+BENCHMARK_CAPTURE(TestTutorial, Test_hsimple_py, "tutorials", "hsimple.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
+BENCHMARK_CAPTURE(TestTutorial, Test_formula1_py, "tutorials", "visualisation/graphics/formula1.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
+BENCHMARK_CAPTURE(TestTutorial, Test_fit1_py, "tutorials", "math/fit/fit1.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
+BENCHMARK_CAPTURE(TestTutorial, Test_tornado_py, "tutorials", "visualisation/graphics/tornado.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
+BENCHMARK_CAPTURE(TestTutorial, Test_surfaces_py, "tutorials", "visualisation/graphics/surfaces.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
+BENCHMARK_CAPTURE(TestTutorial, Test_ntuple1_py, "tutorials", "io/tree/ntuple1.py")->Unit(benchmark::kMicrosecond)->UseManualTime();
 
 BENCHMARK_MAIN();