diff --git a/CHANGELIST.md b/CHANGELIST.md index b53db34..c3f546e 100644 --- a/CHANGELIST.md +++ b/CHANGELIST.md @@ -3,6 +3,7 @@ ### 2.0.0 - Updated JUCE to 9.0.3 - Fixed the crash handler so a backtrace is shown again when a plugin crashes, and made it async-signal-safe [#144] +- Added `--junit-xml ` and `--json-results ` to both `pluginval validate` and `pluginval test` to write machine-readable reports for CI (also settable as `JUNIT_XML` / `JSON_RESULTS`). A report that can't be written fails the run. See `docs/Adding pluginval to CI.md` - Added acceptance testing: `pluginval test ` renders a plugin with a known state, input (audio/MIDI) and optional playhead, then compares the output against a checked-in reference WAV, exiting `0` on a match and `1` on a mismatch (with a diff WAV). Use `--record-missing` to record new references. See `docs/Acceptance testing.md` - Restructured the command line into subcommands: `pluginval validate [options] ` (the default), `pluginval run-tests` and `pluginval strictness-help [level]`. The plugin path is now a positional argument of `validate` - **Deprecated:** the flat flags `--validate `, `--run-tests` and `--strictness-help [level]` still work as aliases (with a one-line notice) but will be removed in a future version. `pluginval ` remains a silent shorthand for `validate` diff --git a/CLAUDE.md b/CLAUDE.md index ff21a61..e4a4a20 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -84,6 +84,7 @@ pluginval/ │ ├── PluginvalSettings.h # Unified settings struct + JSON mapping + toPluginTestOptions() │ ├── SettingsParser.cpp/h # CLI/env/config -> merged JSON -> settings; child-process handoff │ ├── SettingsSerializer.cpp/h # JSON load/save + value coercions (comma lists, hex seed) +│ ├── ReportWriter.cpp/h # JUnit XML + JSON report files (shared by validate and test) │ ├── CrashHandler.cpp/h # Crash reporting utilities │ ├── TestUtilities.cpp/h # Helper functions for tests │ ├── RTCheck.h # Real-time safety checking macros @@ -98,7 +99,7 @@ pluginval/ │ │ ├── TestConfig.cpp/h # snake_case JSON config struct (independent of PluginvalSettings) │ │ ├── AcceptanceTest.cpp/h # plugin load + state + input + render; record-or-compare orchestrator │ │ ├── ReferenceComparator.cpp/h # Comparator interface + registry + SampleComparator -│ │ └── TestReporter.cpp/h # text + JSON result, exit code +│ │ └── TestReporter.cpp/h # text + JSON result, exit code, conversion to a report_writer::Report │ └── tests/ # Individual test implementations │ ├── BasicTests.cpp # Core plugin tests (info, state, audio) │ ├── BusTests.cpp # Audio bus configuration tests @@ -316,6 +317,36 @@ spec: `tests/acceptance/Acceptance testing design.md`; end-user guide: multiplexing, automation, extra comparators, child-process isolation) are notes only. +### Report Output (`--junit-xml` / `--json-results`) + +Both `validate` and `test` can write machine-readable report files through one +shared writer, `report_writer` (`source/ReportWriter.cpp/h`). Stdout and the text +log are unaffected. + +- **Model**: a neutral `report_writer::Report` (command, target, properties and a + list of `TestCase`s with a `passed` / `failed` / `error` status, messages, + duration and optional JSON `details`). `toJUnitXml()` builds a single + `` with `juce::XmlElement`; `toJson()` emits snake_case keys with a + `summary` block. `write()` skips unset files and creates parent directories; if it fails, the + run fails (exit `1`). +- **Two adapters**: `toReport()` in `Validator.cpp` converts the + `juce::UnitTestRunner::TestResult` array collected in `runTests()` (one case per + `beginTest`); `acceptance::reporter::toReport()` converts an acceptance + `TestResult` (its existing JSON goes in `details`). +- **Options**: for `validate` they are ordinary settings (`junitXml` / + `jsonResults` in `PluginvalSettings`, resolved to `juce::File`s on + `PluginTests::Options`), so env vars (`JUNIT_XML` / `JSON_RESULTS`), `--config` + and the child-process handoff all work. For `test` they are parsed in + `settings_parser::dispatch()` into `DispatchResult` and carried on + `acceptance::RunOptions`. +- **Incomplete runs**: before a run starts, `makeIncomplete()` writes a + placeholder report with a single `error` case; the real report overwrites it on + completion, so a crash leaves an error report rather than a stale or missing + one. On a timeout the thread that ends the process writes a "timed out" error + report as its last act (`PluginsUnitTestRunner`'s `onTimeout` for validate, the + acceptance `Watchdog` for test), so the report always agrees with the exit code + even if the tests finish in the moment before the process is terminated. + ### Test Framework Tests are self-registering. To find all tests, look for static instances: @@ -467,6 +498,8 @@ Key options (for `validate`): - `--timeout-ms [ms]` - Test timeout (default: 30000, -1 for none) - `--verbose` - Enable verbose logging - `--output-dir [dir]` - Directory for log files +- `--junit-xml [file]` - Write a JUnit XML report (also accepted by `test`) +- `--json-results [file]` - Write a JSON report (also accepted by `test`) - `--sample-rates [list]` - Comma-separated sample rates - `--block-sizes [list]` - Comma-separated block sizes - `--rtcheck [disabled|enabled|relaxed]` - Real-time safety checking diff --git a/CMakeLists.txt b/CMakeLists.txt index e682e9c..d5cd63b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -134,6 +134,7 @@ set(SourceFiles source/MainComponent.h source/PluginTests.h source/PluginvalSettings.h + source/ReportWriter.h source/SettingsParser.h source/SettingsSerializer.h source/TestUtilities.h @@ -151,6 +152,7 @@ set(SourceFiles source/Main.cpp source/MainComponent.cpp source/PluginTests.cpp + source/ReportWriter.cpp source/SettingsParser.cpp source/SettingsSerializer.cpp source/tests/BasicTests.cpp diff --git a/docs/Acceptance testing.md b/docs/Acceptance testing.md index 6352c71..018305f 100644 --- a/docs/Acceptance testing.md +++ b/docs/Acceptance testing.md @@ -89,6 +89,13 @@ Any other option is an error. `RECORD_MISSING=1`. - `--timeout-ms ` - fail (exit `1`) if the run takes longer than this (default `30000`, `-1` to never time out). Also settable as `TIMEOUT_MS`. +- `--junit-xml ` - write a JUnit XML report, for CI systems that display + test results. Also settable as `JUNIT_XML`. +- `--json-results ` - write a JSON report (the same result that is printed + to stdout, wrapped in a summary). Also settable as `JSON_RESULTS`. + +See [Adding pluginval to CI](Adding%20pluginval%20to%20CI.md#machine-readable-reports) +for the report formats. The render runs on a background thread, as a host's audio thread would, leaving the message thread free for the plugin. diff --git a/docs/Adding pluginval to CI.md b/docs/Adding pluginval to CI.md index 3baa06b..43a63e4 100644 --- a/docs/Adding pluginval to CI.md +++ b/docs/Adding pluginval to CI.md @@ -41,3 +41,62 @@ $ ./pluginval validate --output-dir "./bin" "" || exit 1 ```sh > choco install pluginval ``` + +### Machine-readable reports +Both `pluginval validate` and `pluginval test` can write their results to a file, as well as printing them: + + - `--junit-xml ` writes a JUnit XML report, which most CI systems (GitHub Actions, GitLab, Jenkins, Azure Pipelines) can display as per-test results + - `--json-results ` writes the same results as JSON, for scripts and dashboards + +Relative paths are resolved against the working directory and any missing directories are created. Both options can be used together, and can also be set as the `JUNIT_XML` and `JSON_RESULTS` environment variables. The text output and log files are unchanged. If a report can't be written (e.g. the path isn't writable) the run fails with exit code `1`. + +```sh +$ ./pluginval validate --strictness-level 5 --junit-xml "./reports/MyPlugin.xml" --json-results "./reports/MyPlugin.json" "" +``` + +Each test pluginval runs becomes one ``, with a `` holding the failure messages if it failed: + +```xml + + + + + ... + + + +``` + +The JSON report has the same content: + +```json +{ + "pluginval_version": "2.0.0", + "command": "validate", + "target": "/path/to/MyPlugin.vst3", + "start_time": "...", + "duration_ms": 12345, + "properties": { "plugin_name": "MyPlugin", "strictness_level": "5" }, + "summary": { "tests": 20, "passed": 19, "failed": 1, "errors": 0 }, + "tests": [ + { "name": "Open plugin (cold)", "status": "passed", "duration_ms": 150, "messages": [] } + ] +} +``` + +`status` is `passed`, `failed` or `error`. For `pluginval test`, each test also has a `details` object holding the comparison results. + +If pluginval doesn't finish (the plugin crashes it, or the run times out), the report contains a single `error` entry saying so, so a report left over from an earlier run is never mistaken for a pass. + +##### GitHub Actions +```yaml +- name: Validate plugin + run: ./pluginval validate --strictness-level 5 --skip-gui-tests --junit-xml reports/pluginval.xml ./build/MyPlugin.vst3 + +- name: Upload pluginval report + if: always() + uses: actions/upload-artifact@v4 + with: + name: pluginval-report + path: reports/ +``` diff --git a/docs/Command line options.md b/docs/Command line options.md index 4d1bc56..71e2128 100644 --- a/docs/Command line options.md +++ b/docs/Command line options.md @@ -18,6 +18,8 @@ COMMANDS: pass instead. --timeout-ms Fail if the run takes longer (default 30000, -1 = never). + --junit-xml Write a JUnit XML report. + --json-results Write a JSON report. Any other option is an error. The flat flags --validate , --run-tests and --strictness-help [level] @@ -46,6 +48,8 @@ OPTIONS (for the validate command): --output-dir TEXT Directory in which to write the log files. --output-filename TEXT Filename to write logs into. + --junit-xml TEXT File to write a JUnit XML report to. + --json-results TEXT File to write a JSON report to. --disabled-tests TEXT Comma-separated test names, or a path to a file listing them. --random-seed TEXT Random seed (hex 0x.. or int) for replicable test runs. diff --git a/source/CommandLine.cpp b/source/CommandLine.cpp index f317557..9736bd1 100644 --- a/source/CommandLine.cpp +++ b/source/CommandLine.cpp @@ -232,6 +232,14 @@ void performCommandLine (CommandLineValidator& validator, const juce::String& co options.recordMissing = routed.recordMissing; options.timeoutMs = routed.timeoutMs; + const auto toReportFile = [] (const juce::String& path) + { + return path.isEmpty() ? juce::File() : juce::File::getCurrentWorkingDirectory().getChildFile (path); + }; + + options.junitXmlFile = toReportFile (routed.junitXmlPath); + options.jsonResultsFile = toReportFile (routed.jsonResultsPath); + // Runs async so will quit itself when done validator.runAcceptanceTest (juce::File::getCurrentWorkingDirectory().getChildFile (routed.testConfigPath), options); return; diff --git a/source/CommandLineTests.cpp b/source/CommandLineTests.cpp index 70b5d51..bb5fed9 100644 --- a/source/CommandLineTests.cpp +++ b/source/CommandLineTests.cpp @@ -61,6 +61,8 @@ struct CommandLineTests : public juce::UnitTest expect (opts.realtimeCheck == RealtimeCheck::disabled); expect (opts.dataFile == juce::File()); expect (opts.outputDir == juce::File()); + expect (opts.junitXmlFile == juce::File()); + expect (opts.jsonResultsFile == juce::File()); expect (opts.sampleRates == PluginvalSettings::defaultSampleRates()); expect (opts.blockSizes == PluginvalSettings::defaultBlockSizes()); } @@ -260,6 +262,39 @@ struct CommandLineTests : public juce::UnitTest } } + beginTest ("Report options (--junit-xml, --json-results)"); + { + const auto cwd = juce::File::getCurrentWorkingDirectory(); + + // Command line: relative paths resolve against the working directory. + { + const auto settings = parse ("--junit-xml out/report.xml --json-results out/report.json --validate x"); + expectEquals (juce::String (settings.junitXml), juce::String ("out/report.xml")); + expectEquals (juce::String (settings.jsonResults), juce::String ("out/report.json")); + + const auto opts = settings.toPluginTestOptions(); + expect (opts.junitXmlFile == cwd.getChildFile ("out/report.xml")); + expect (opts.jsonResultsFile == cwd.getChildFile ("out/report.json")); + } + + // Environment variables (names derived from the options). + { + const auto settings = parse ("--validate x", envFrom ({ { "JUNIT_XML", "env.xml" }, { "JSON_RESULTS", "env.json" } })); + expectEquals (juce::String (settings.junitXml), juce::String ("env.xml")); + expectEquals (juce::String (settings.jsonResults), juce::String ("env.json")); + } + + // --config keys mirror the field names. + { + juce::TemporaryFile configFile (".json"); + configFile.getFile().replaceWithText (R"({ "junitXml": "cfg.xml", "jsonResults": "cfg.json" })"); + + const auto settings = parse ("--config " + configFile.getFile().getFullPathName().quoted() + " --validate x"); + expectEquals (juce::String (settings.junitXml), juce::String ("cfg.xml")); + expectEquals (juce::String (settings.jsonResults), juce::String ("cfg.json")); + } + } + beginTest ("Repeatable --config merges per key, last wins"); { juce::TemporaryFile baseFile (".json"); @@ -288,6 +323,8 @@ struct CommandLineTests : public juce::UnitTest opts.randomiseTestOrder = true; opts.withGUI = false; opts.outputFilename = "log.txt"; + opts.junitXmlFile = juce::File::getCurrentWorkingDirectory().getChildFile ("report.xml"); + opts.jsonResultsFile = juce::File::getCurrentWorkingDirectory().getChildFile ("report.json"); opts.disabledTests = juce::StringArray ({ "Test A", "Test B" }); opts.sampleRates = { 44100.0, 96000.0 }; opts.blockSizes = { 64, 512 }; @@ -506,6 +543,186 @@ struct CommandLineTests : public juce::UnitTest } } + beginTest ("Subcommand: test report options (--junit-xml, --json-results)"); + { + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test --junit-xml out/r.xml config.json --json-results out/r.json")); + expect (d.error.isEmpty()); + expectEquals (d.junitXmlPath, juce::String ("out/r.xml")); + expectEquals (d.jsonResultsPath, juce::String ("out/r.json")); + expectEquals (d.testConfigPath, juce::String ("config.json")); + } + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test config.json --junit-xml")); + expect (d.error.contains ("--junit-xml")); + } + { + // The following option isn't taken as the file path. + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test config.json --json-results --record-missing")); + expect (d.error.contains ("--json-results")); + } + } + + beginTest ("Report writer: JUnit XML"); + { + report_writer::Report report; + report.command = "validate"; + report.target = "MyPlugin"; + report.properties.push_back ({ "strictness_level", "5" }); + + report_writer::TestCase passed; + passed.name = "Open plugin (cold)"; + passed.durationSeconds = 0.25; + + report_writer::TestCase failed; + failed.name = "Plugin state"; + failed.status = report_writer::TestCase::Outcome::failed; + failed.messages.add ("Expected <1 & \"2\">"); + failed.messages.add ("second message"); + failed.durationSeconds = 1.0; + + report_writer::TestCase errored; + errored.name = "Editor"; + errored.status = report_writer::TestCase::Outcome::error; + errored.messages.add ("couldn't run"); + + report.cases = { passed, failed, errored }; + + const auto xml = juce::parseXML (report_writer::toJUnitXml (report)); + expect (xml != nullptr, "the report must be well-formed XML"); + + if (xml != nullptr) + { + expect (xml->hasTagName ("testsuites")); + expectEquals (xml->getIntAttribute ("tests"), 3); + expectEquals (xml->getIntAttribute ("failures"), 1); + expectEquals (xml->getIntAttribute ("errors"), 1); + expectEquals (xml->getStringAttribute ("time"), juce::String ("1.250")); + + const auto suite = xml->getChildByName ("testsuite"); + expect (suite != nullptr); + + if (suite != nullptr) + { + expectEquals (suite->getStringAttribute ("name"), juce::String ("pluginval validate: MyPlugin")); + expectEquals (suite->getIntAttribute ("tests"), 3); + expectEquals (suite->getNumChildElements(), 4); // properties + 3 testcases + + const auto property = suite->getChildByName ("properties")->getChildByName ("property"); + expectEquals (property->getStringAttribute ("name"), juce::String ("strictness_level")); + expectEquals (property->getStringAttribute ("value"), juce::String ("5")); + + const auto case0 = suite->getChildElement (1); + expectEquals (case0->getStringAttribute ("classname"), juce::String ("pluginval.validate")); + expectEquals (case0->getStringAttribute ("name"), juce::String ("Open plugin (cold)")); + expectEquals (case0->getStringAttribute ("time"), juce::String ("0.250")); + expectEquals (case0->getNumChildElements(), 0); + + // Special characters survive the round trip (i.e. were escaped). + const auto failure = suite->getChildElement (2)->getChildByName ("failure"); + expect (failure != nullptr); + + if (failure != nullptr) + { + expectEquals (failure->getStringAttribute ("message"), juce::String ("Expected <1 & \"2\">")); + expectEquals (failure->getAllSubText(), juce::String ("Expected <1 & \"2\">\nsecond message")); + } + + expect (suite->getChildElement (3)->getChildByName ("error") != nullptr); + expect (suite->getChildElement (3)->getChildByName ("failure") == nullptr); + } + } + } + + beginTest ("Report writer: JSON"); + { + report_writer::Report report; + report.command = "test"; + report.target = "sine-440"; + report.properties.push_back ({ "reference", "/refs/sine-440.wav" }); + + report_writer::TestCase passed; + passed.name = "a"; + passed.durationSeconds = 0.5; + + report_writer::TestCase failed; + failed.name = "b"; + failed.status = report_writer::TestCase::Outcome::failed; + failed.messages.add ("[sample] too different"); + failed.details = { { "outcome", "failed" } }; + + report.cases = { passed, failed }; + + const auto j = report_writer::toJson (report); + expectEquals (juce::String (j["command"].get()), juce::String ("test")); + expectEquals (juce::String (j["target"].get()), juce::String ("sine-440")); + expectEquals (juce::String (j["properties"]["reference"].get()), juce::String ("/refs/sine-440.wav")); + expectEquals (j["duration_ms"].get(), 500); + expectEquals (j["summary"]["tests"].get(), 2); + expectEquals (j["summary"]["passed"].get(), 1); + expectEquals (j["summary"]["failed"].get(), 1); + expectEquals (j["summary"]["errors"].get(), 0); + expectEquals ((int) j["tests"].size(), 2); + expectEquals (juce::String (j["tests"][0]["status"].get()), juce::String ("passed")); + expect (! j["tests"][0].contains ("details")); + expectEquals (juce::String (j["tests"][1]["status"].get()), juce::String ("failed")); + expectEquals (juce::String (j["tests"][1]["messages"][0].get()), juce::String ("[sample] too different")); + expectEquals (juce::String (j["tests"][1]["details"]["outcome"].get()), juce::String ("failed")); + } + + beginTest ("Report writer: incomplete report and file output"); + { + const auto report = report_writer::makeIncomplete ("validate", "MyPlugin", "did not complete"); + const auto j = report_writer::toJson (report); + expectEquals (j["summary"]["tests"].get(), 1); + expectEquals (j["summary"]["errors"].get(), 1); + expectEquals (juce::String (j["tests"][0]["messages"][0].get()), juce::String ("did not complete")); + + // Files are written (creating parent directories); unset files are skipped. + const auto dir = juce::File::getSpecialLocation (juce::File::tempDirectory) + .getChildFile ("pluginval_report_test_" + juce::String::toHexString (juce::Random::getSystemRandom().nextInt())); + const auto xmlFile = dir.getChildFile ("nested/report.xml"); + const auto jsonFile = dir.getChildFile ("nested/report.json"); + + expect (report_writer::write (report, {}, {})); + expect (! dir.exists()); + + expect (report_writer::write (report, xmlFile, jsonFile)); + expect (juce::parseXML (xmlFile) != nullptr); + expectEquals (nlohmann::json::parse (jsonFile.loadFileAsString().toStdString())["summary"]["errors"].get(), 1); + + dir.deleteRecursively(); + } + + beginTest ("Acceptance result to report"); + { + acceptance::TestResult result; + result.name = "sine-440"; + result.outcome = acceptance::TestResult::Outcome::failed; + result.durationSeconds = 2.0; + + acceptance::ComparisonResult comparison; + comparison.passed = false; + comparison.summary = "max abs diff 0.5"; + result.comparisons.push_back ({ "sample", comparison }); + + const auto failed = acceptance::reporter::toReport (result); + expectEquals (failed.command, juce::String ("test")); + expectEquals ((int) failed.cases.size(), 1); + expect (failed.cases[0].status == report_writer::TestCase::Outcome::failed); + expectEquals (failed.cases[0].messages[0], juce::String ("[sample] max abs diff 0.5")); + expectEquals (failed.cases[0].durationSeconds, 2.0); + expectEquals (juce::String (failed.cases[0].details["outcome"].get()), juce::String ("failed")); + + const auto error = acceptance::reporter::toReport (acceptance::TestResult::makeError ("t", "no plugin")); + expect (error.cases[0].status == report_writer::TestCase::Outcome::error); + expectEquals (error.cases[0].messages[0], juce::String ("no plugin")); + + result.outcome = acceptance::TestResult::Outcome::referenceCreated; + result.comparisons.clear(); + expect (acceptance::reporter::toReport (result).cases[0].status == report_writer::TestCase::Outcome::passed); + } + beginTest ("Acceptance TestConfig JSON parsing (snake_case keys, defaults)"); { const auto json = R"({ diff --git a/source/PluginTests.h b/source/PluginTests.h index 560893a..34b1910 100644 --- a/source/PluginTests.h +++ b/source/PluginTests.h @@ -47,6 +47,8 @@ struct PluginTests : public juce::UnitTest juce::File dataFile; /**< juce::File which tests can use to run user provided data. */ juce::File outputDir; /**< Directory in which to write the log files for each test run. */ juce::String outputFilename; /**< Filename to write logs into */ + juce::File junitXmlFile; /**< File to write a JUnit XML report to (none if default). */ + juce::File jsonResultsFile; /**< File to write a JSON report to (none if default). */ juce::StringArray disabledTests; /**< List of disabled tests. */ std::vector sampleRates; /**< List of sample rates. */ std::vector blockSizes; /**< List of block sizes. */ diff --git a/source/PluginvalSettings.h b/source/PluginvalSettings.h index e8a8255..e4afda6 100644 --- a/source/PluginvalSettings.h +++ b/source/PluginvalSettings.h @@ -54,6 +54,8 @@ struct PluginvalSettings std::string dataFile; std::string outputDir; std::string outputFilename; + std::string junitXml; + std::string jsonResults; std::vector disabledTests; std::vector sampleRates; std::vector blockSizes; @@ -83,6 +85,8 @@ struct PluginvalSettings o.dataFile = juce::String (dataFile); o.outputDir = juce::String (outputDir); o.outputFilename = juce::String (outputFilename); + o.junitXmlFile = toReportFile (junitXml); + o.jsonResultsFile = toReportFile (jsonResults); o.disabledTests = toStringArray (disabledTests); o.sampleRates = sampleRates.empty() ? defaultSampleRates() : sampleRates; o.blockSizes = blockSizes.empty() ? defaultBlockSizes() : blockSizes; @@ -106,6 +110,8 @@ struct PluginvalSettings s.dataFile = o.dataFile.getFullPathName().toStdString(); s.outputDir = o.outputDir.getFullPathName().toStdString(); s.outputFilename = o.outputFilename.toStdString(); + s.junitXml = o.junitXmlFile.getFullPathName().toStdString(); + s.jsonResults = o.jsonResultsFile.getFullPathName().toStdString(); s.disabledTests = fromStringArray (o.disabledTests); s.sampleRates = o.sampleRates; s.blockSizes = o.blockSizes; @@ -115,6 +121,15 @@ struct PluginvalSettings } private: + /** Resolves a report path against the working directory ("" means no report). */ + static juce::File toReportFile (const std::string& path) + { + if (path.empty()) + return {}; + + return juce::File::getCurrentWorkingDirectory().getChildFile (juce::String (path)); + } + static juce::StringArray toStringArray (const std::vector& v) { juce::StringArray a; @@ -139,4 +154,5 @@ struct PluginvalSettings NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT (PluginvalSettings, strictnessLevel, randomSeed, timeoutMs, verbose, numRepeats, randomiseTestOrder, skipGuiTests, dataFile, outputDir, outputFilename, - disabledTests, sampleRates, blockSizes, realtimeCheck, validatePath) + junitXml, jsonResults, disabledTests, sampleRates, blockSizes, realtimeCheck, + validatePath) diff --git a/source/ReportWriter.cpp b/source/ReportWriter.cpp new file mode 100644 index 0000000..7d34240 --- /dev/null +++ b/source/ReportWriter.cpp @@ -0,0 +1,217 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "ReportWriter.h" + +#include +#include + +namespace report_writer +{ + +//============================================================================== +namespace +{ + struct Totals + { + int tests = 0, failed = 0, errors = 0; + double seconds = 0.0; + }; + + Totals getTotals (const Report& report) + { + Totals t; + + for (const auto& c : report.cases) + { + ++t.tests; + t.failed += c.status == TestCase::Outcome::failed ? 1 : 0; + t.errors += c.status == TestCase::Outcome::error ? 1 : 0; + t.seconds += c.durationSeconds; + } + + return t; + } + + const char* toString (TestCase::Outcome s) + { + switch (s) + { + case TestCase::Outcome::passed: return "passed"; + case TestCase::Outcome::failed: return "failed"; + case TestCase::Outcome::error: return "error"; + } + + return "error"; + } + + juce::String formatSeconds (double seconds) + { + return juce::String (seconds, 3); + } + + juce::int64 toMilliseconds (double seconds) + { + return (juce::int64) std::llround (seconds * 1000.0); + } +} + +//============================================================================== +juce::String toJUnitXml (const Report& report) +{ + const auto totals = getTotals (report); + + const auto setTotals = [&totals] (juce::XmlElement& e) + { + e.setAttribute ("tests", totals.tests); + e.setAttribute ("failures", totals.failed); + e.setAttribute ("errors", totals.errors); + e.setAttribute ("time", formatSeconds (totals.seconds)); + }; + + // A plugin path makes for an unwieldy suite name, so use just the file name + const auto shortTarget = juce::File::isAbsolutePath (report.target) ? juce::File (report.target).getFileName() + : report.target; + + juce::XmlElement root ("testsuites"); + root.setAttribute ("name", "pluginval"); + setTotals (root); + + auto suite = root.createNewChildElement ("testsuite"); + suite->setAttribute ("name", "pluginval " + report.command + ": " + shortTarget); + setTotals (*suite); + suite->setAttribute ("skipped", 0); + suite->setAttribute ("timestamp", report.startTime.toISO8601 (true)); + + if (! report.properties.empty()) + { + auto properties = suite->createNewChildElement ("properties"); + + for (const auto& [name, value] : report.properties) + { + auto property = properties->createNewChildElement ("property"); + property->setAttribute ("name", name); + property->setAttribute ("value", value); + } + } + + for (const auto& c : report.cases) + { + auto testCase = suite->createNewChildElement ("testcase"); + + // A fixed classname: plugin names contain dots, which JUnit viewers + // treat as package separators + testCase->setAttribute ("classname", "pluginval." + report.command); + testCase->setAttribute ("name", c.name); + testCase->setAttribute ("time", formatSeconds (c.durationSeconds)); + + if (c.status != TestCase::Outcome::passed) + { + auto problem = testCase->createNewChildElement (c.status == TestCase::Outcome::failed ? "failure" : "error"); + problem->setAttribute ("message", c.messages[0].upToFirstOccurrenceOf ("\n", false, false)); + problem->addTextElement (c.messages.joinIntoString ("\n")); + } + } + + return root.toString(); +} + +nlohmann::ordered_json toJson (const Report& report) +{ + const auto totals = getTotals (report); + + nlohmann::ordered_json j; + j["pluginval_version"] = VERSION; + j["command"] = report.command.toStdString(); + j["target"] = report.target.toStdString(); + j["start_time"] = report.startTime.toISO8601 (true).toStdString(); + j["duration_ms"] = toMilliseconds (totals.seconds); + + auto properties = nlohmann::ordered_json::object(); + + for (const auto& [name, value] : report.properties) + properties[name.toStdString()] = value.toStdString(); + + j["properties"] = properties; + + j["summary"] = { { "tests", totals.tests }, + { "passed", totals.tests - totals.failed - totals.errors }, + { "failed", totals.failed }, + { "errors", totals.errors } }; + + auto tests = nlohmann::ordered_json::array(); + + for (const auto& c : report.cases) + { + nlohmann::ordered_json test; + test["name"] = c.name.toStdString(); + test["status"] = toString (c.status); + test["duration_ms"] = toMilliseconds (c.durationSeconds); + + auto messages = nlohmann::ordered_json::array(); + + for (const auto& m : c.messages) + messages.push_back (m.toStdString()); + + test["messages"] = messages; + + if (! c.details.is_null()) + test["details"] = c.details; + + tests.push_back (test); + } + + j["tests"] = tests; + return j; +} + +Report makeIncomplete (const juce::String& command, const juce::String& target, const juce::String& reason) +{ + TestCase c; + c.name = "pluginval " + command; + c.status = TestCase::Outcome::error; + c.messages.add (reason); + + Report report; + report.command = command; + report.target = target; + report.cases.push_back (c); + return report; +} + +//============================================================================== +bool write (const Report& report, const juce::File& junitXml, const juce::File& jsonResults) +{ + const auto writeFile = [] (const juce::File& file, const juce::String& text) + { + if (file.getParentDirectory().createDirectory().wasOk() + && file.replaceWithText (text, false, false, "\n")) + return true; + + std::cerr << "!!! WARNING: Unable to write the report to " << file.getFullPathName() << std::endl; + return false; + }; + + bool ok = true; + + if (junitXml != juce::File()) + ok = writeFile (junitXml, toJUnitXml (report)) && ok; + + if (jsonResults != juce::File()) + ok = writeFile (jsonResults, juce::String::fromUTF8 (toJson (report).dump (2, ' ', false, nlohmann::ordered_json::error_handler_t::replace).c_str()) + "\n") && ok; + + return ok; +} + +} // namespace report_writer diff --git a/source/ReportWriter.h b/source/ReportWriter.h new file mode 100644 index 0000000..86268d7 --- /dev/null +++ b/source/ReportWriter.h @@ -0,0 +1,78 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include +#include + +#include +#include + +//============================================================================== +/** + Machine-readable reports (JUnit XML and JSON) shared by the validate and + test commands. Each command converts its own results into a Report, which is + then written to whichever files were requested (--junit-xml / --json-results). +*/ +namespace report_writer +{ + //============================================================================== + /** One test within a run. */ + struct TestCase + { + enum class Outcome + { + passed, + failed, /**< The test ran and a check failed. */ + error /**< The test couldn't be run to completion. */ + }; + + juce::String name; + Outcome status = Outcome::passed; + juce::StringArray messages; /**< Failure / error text. */ + double durationSeconds = 0.0; + nlohmann::json details; /**< Optional command-specific data (JSON report only). */ + }; + + /** The results of one run of a command. */ + struct Report + { + juce::String command; /**< "validate" or "test". */ + juce::String target; /**< Plugin path/ID, or the acceptance test name. */ + juce::Time startTime = juce::Time::getCurrentTime(); + std::vector> properties; + std::vector cases; + }; + + //============================================================================== + /** Returns the report as a JUnit XML document (a single testsuite). */ + juce::String toJUnitXml (const Report&); + + /** Returns the report as JSON (snake_case keys). */ + nlohmann::ordered_json toJson (const Report&); + + /** A report with a single error case, for a run that didn't finish. + Written before a run starts so a crash or timeout can't leave a stale + (or missing) report behind; a completed run overwrites it. + */ + Report makeIncomplete (const juce::String& command, const juce::String& target, const juce::String& reason); + + /** Writes the report to whichever files are set (a default juce::File is + skipped), creating parent directories. Returns false, with a warning on + stderr, if a file couldn't be written (which the commands treat as a + failed run). + */ + bool write (const Report&, const juce::File& junitXml, const juce::File& jsonResults); +} diff --git a/source/SettingsParser.cpp b/source/SettingsParser.cpp index 45b6135..999a6a6 100644 --- a/source/SettingsParser.cpp +++ b/source/SettingsParser.cpp @@ -135,6 +135,8 @@ R"(Commands: (default: a missing reference fails). --timeout-ms Fail if the run takes longer than this (default 30000, -1 to never time out). + --junit-xml Write a JUnit XML report. + --json-results Write a JSON report. The flat flags --validate , --run-tests and --strictness-help [level] are deprecated aliases for the commands above and will be removed in a future version. @@ -195,6 +197,8 @@ Precedence (lowest to highest): defaults, environment variables, --config, comma app.add_option ("--data-file", s.dataFile, "Path to a data file tests can use to configure themselves."); app.add_option ("--output-dir", s.outputDir, "Directory in which to write the log files."); app.add_option ("--output-filename", s.outputFilename, "Filename to write logs into."); + app.add_option ("--junit-xml", s.junitXml, "File to write a JUnit XML report to."); + app.add_option ("--json-results", s.jsonResults, "File to write a JSON report to."); app.add_option_function ("--disabled-tests", [&s] (const std::string& v) { s.disabledTests = settings_serializer::disabledTestsToList (juce::String (v)); }, @@ -382,6 +386,9 @@ Precedence (lowest to highest): defaults, environment variables, --config, comma if (const auto env = juce::SystemStats::getEnvironmentVariable ("TIMEOUT_MS", {}); env.isNotEmpty()) result.timeoutMs = env.getLargeIntValue(); + result.junitXmlPath = juce::SystemStats::getEnvironmentVariable ("JUNIT_XML", {}); + result.jsonResultsPath = juce::SystemStats::getEnvironmentVariable ("JSON_RESULTS", {}); + // The options, plus the positional acceptance-test config (the first // non-option token after the verb). for (int i = 1; i < tokensIn.size(); ++i) @@ -405,6 +412,18 @@ Precedence (lowest to highest): defaults, environment variables, --config, comma result.timeoutMs = value.getLargeIntValue(); } + else if (token == "--junit-xml" || token == "--json-results") + { + const auto value = i + 1 < tokensIn.size() ? tokensIn.getReference (++i) : juce::String(); + + if (value.isEmpty() || value.startsWith ("--")) + { + result.error = token + " needs a file path"; + break; + } + + (token == "--junit-xml" ? result.junitXmlPath : result.jsonResultsPath) = value; + } else if (token.startsWith ("-")) { result.error = "unknown option for test: " + token; diff --git a/source/SettingsParser.h b/source/SettingsParser.h index 5d5ae2a..934b0c0 100644 --- a/source/SettingsParser.h +++ b/source/SettingsParser.h @@ -83,6 +83,8 @@ namespace settings_parser juce::String testConfigPath; /**< The positional config.json for the test command. */ bool recordMissing = false; /**< test: record a missing reference (and pass) instead of failing. */ juce::int64 timeoutMs = 30000; /**< test: abort the run after this long (-1 to never time out). */ + juce::String junitXmlPath; /**< test: file to write a JUnit XML report to. */ + juce::String jsonResultsPath; /**< test: file to write a JSON report to. */ juce::String error; /**< test: an invalid option / argument, to report instead of running. */ }; diff --git a/source/Validator.cpp b/source/Validator.cpp index e9a6fdf..6e4b262 100644 --- a/source/Validator.cpp +++ b/source/Validator.cpp @@ -16,6 +16,7 @@ #include "PluginTests.h" #include "CrashHandler.h" #include "CommandLine.h" +#include "ReportWriter.h" #include #include #include @@ -24,9 +25,11 @@ struct PluginsUnitTestRunner : public juce::UnitTestRunner, private juce::Thread { - PluginsUnitTestRunner (std::function logCallback, std::unique_ptr logDestination, juce::int64 timeoutInMs) + PluginsUnitTestRunner (std::function logCallback, std::unique_ptr logDestination, juce::int64 timeoutInMs, + std::function timeoutCallback = {}) : Thread ("TimoutThread"), callback (std::move (logCallback)), + onTimeout (std::move (timeoutCallback)), outputStream (std::move (logDestination)), timeoutMs (timeoutInMs) { @@ -73,6 +76,7 @@ struct PluginsUnitTestRunner : public juce::UnitTestRunner, private: std::function callback; + std::function onTimeout; // Called on the timeout thread just before the process is terminated std::unique_ptr outputStream; const juce::int64 timeoutMs = -1; std::atomic timoutTime { -1 }; @@ -95,6 +99,10 @@ struct PluginsUnitTestRunner : public juce::UnitTestRunner, // Give the log a second to flush the message before terminating juce::Thread::sleep (1000); + + if (onTimeout) + onTimeout(); + juce::Process::terminate(); } @@ -186,6 +194,47 @@ static void updateFileNameIfPossible (PluginTests& test, PluginsUnitTestRunner& } +//============================================================================== +/** Converts the results of a validation into a report for --junit-xml / --json-results. */ +static report_writer::Report toReport (PluginTests& test, const juce::Array& results) +{ + const auto options = test.getOptions(); + + report_writer::Report report; + report.command = "validate"; + report.target = test.getFileOrID(); + + if (! results.isEmpty()) + report.startTime = results.getReference (0).startTime; + + if (auto pd = test.getDescriptions().getFirst()) + { + report.properties.push_back ({ "plugin_name", pd->name }); + report.properties.push_back ({ "plugin_version", pd->version }); + report.properties.push_back ({ "plugin_format", pd->pluginFormatName }); + report.properties.push_back ({ "plugin_manufacturer", pd->manufacturerName }); + } + + report.properties.push_back ({ "strictness_level", juce::String (options.strictnessLevel) }); + report.properties.push_back ({ "timeout_ms", juce::String (options.timeoutMs) }); + + if (options.randomSeed != 0) + report.properties.push_back ({ "random_seed", "0x" + juce::String::toHexString (options.randomSeed) }); + + for (const auto& r : results) + { + report_writer::TestCase c; + c.name = r.subcategoryName; + c.status = r.failures > 0 ? report_writer::TestCase::Outcome::failed + : report_writer::TestCase::Outcome::passed; + c.messages = r.messages; + c.durationSeconds = juce::jmax (0.0, (r.endTime - r.startTime).inSeconds()); + report.cases.push_back (c); + } + + return report; +} + //============================================================================== //============================================================================== inline int getNumFailures (juce::Array results); @@ -194,9 +243,24 @@ inline juce::Array runTests (PluginTests& test { const auto options = test.getOptions(); juce::Array results; - PluginsUnitTestRunner testRunner (std::move (callback), createDestinationFileStream (test), options.timeoutMs); + const auto writeIncompleteReport = [options, fileOrID = test.getFileOrID()] (const juce::String& reason) + { + report_writer::write (report_writer::makeIncomplete ("validate", fileOrID, reason), + options.junitXmlFile, options.jsonResultsFile); + }; + + // Written last on a timeout, so the report always agrees with the exit code + // (the tests can finish in the moment between the timeout and the process ending) + PluginsUnitTestRunner testRunner (std::move (callback), createDestinationFileStream (test), options.timeoutMs, + [options, writeIncompleteReport] + { + writeIncompleteReport ("Validation timed out after " + juce::RelativeTime::milliseconds (options.timeoutMs).getDescription()); + }); testRunner.setAssertOnFailure (false); + // Written up front so a crash can't leave a stale (or missing) report behind + writeIncompleteReport ("Validation did not complete (crashed)"); + juce::Array testsToRun; testsToRun.add (&test); testRunner.runTests (testsToRun, options.randomSeed); @@ -205,6 +269,19 @@ inline juce::Array runTests (PluginTests& test results.add (*testRunner.getResult (i)); updateFileNameIfPossible (test, testRunner); + + // A report that was asked for but couldn't be written is a failure, so CI doesn't silently lose it + if (! report_writer::write (toReport (test, results), options.junitXmlFile, options.jsonResultsFile)) + { + juce::UnitTestRunner::TestResult writeFailure ("pluginval", "Write report files"); + writeFailure.failures = 1; + writeFailure.messages.add ("Unable to write the report file(s)"); + writeFailure.endTime = writeFailure.startTime; + results.add (writeFailure); + + testRunner.logMessage ("*** FAILED: Unable to write the report file(s)"); + } + const int failures = getNumFailures (results); if (! failures) diff --git a/source/acceptance/AcceptanceTest.cpp b/source/acceptance/AcceptanceTest.cpp index 26d67a8..100cdfb 100644 --- a/source/acceptance/AcceptanceTest.cpp +++ b/source/acceptance/AcceptanceTest.cpp @@ -593,6 +593,23 @@ TestResult runTest (const TestConfig& config, const RunOptions& options) //============================================================================== int runTestFile (const juce::File& configFile, const RunOptions& options) { + const auto finish = [&options] (const TestResult& result) + { + // A report that was asked for but couldn't be written is a failure + if (! reporter::report (result, options.junitXmlFile, options.jsonResultsFile)) + { + std::cout << "*** FAILED: Unable to write the report file(s)" << std::endl; + return 1; + } + + return reporter::exitCode (result); + }; + + // Written up front so a crash can't leave a stale (or missing) report behind + report_writer::write (report_writer::makeIncomplete ("test", configFile.getFileNameWithoutExtension(), + "Test did not complete (crashed)"), + options.junitXmlFile, options.jsonResultsFile); + std::vector configs; try @@ -601,33 +618,28 @@ int runTestFile (const juce::File& configFile, const RunOptions& options) } catch (const std::exception& e) { - const auto result = TestResult::makeError (configFile.getFileNameWithoutExtension(), e.what()); - reporter::report (result); - return reporter::exitCode (result); + return finish (TestResult::makeError (configFile.getFileNameWithoutExtension(), e.what())); } if (configs.empty()) - { - const auto result = TestResult::makeError (configFile.getFileNameWithoutExtension(), "no test configs in file"); - reporter::report (result); - return reporter::exitCode (result); - } + return finish (TestResult::makeError (configFile.getFileNameWithoutExtension(), "no test configs in file")); // v1 runs the first entry only; the parser already accepts arrays for the // phase-2 multiplexing extension. if (configs.size() > 1) std::cout << "Note: " << configs.size() << " configs found; v1 runs the first only." << std::endl; - const auto result = runTest (configs.front(), options); - reporter::report (result); - return reporter::exitCode (result); + const auto startTime = juce::Time::getMillisecondCounterHiRes(); + auto result = runTest (configs.front(), options); + result.durationSeconds = (juce::Time::getMillisecondCounterHiRes() - startTime) / 1000.0; + return finish (result); } //============================================================================== struct TestRunner::Watchdog : private juce::Thread { - Watchdog (const juce::File& file, juce::int64 timeout) - : Thread ("pluginval test watchdog"), configFile (file), timeoutMs (timeout) + Watchdog (const juce::File& file, const RunOptions& opts) + : Thread ("pluginval test watchdog"), configFile (file), options (opts) { startThread (juce::Thread::Priority::low); } @@ -645,19 +657,20 @@ struct TestRunner::Watchdog : private juce::Thread private: const juce::File configFile; - const juce::int64 timeoutMs; + const RunOptions options; juce::WaitableEvent finished; void run() override { - if (finished.wait ((double) timeoutMs)) + if (finished.wait ((double) options.timeoutMs)) return; // The worker is stuck in a plugin call that can't be interrupted, so // report the failure and end the process. - const auto result = TestResult::makeError (configFile.getFileNameWithoutExtension(), - "timed out after " + juce::RelativeTime::milliseconds (timeoutMs).getDescription()); - reporter::report (result); + auto result = TestResult::makeError (configFile.getFileNameWithoutExtension(), + "timed out after " + juce::RelativeTime::milliseconds (options.timeoutMs).getDescription()); + result.durationSeconds = (double) options.timeoutMs / 1000.0; + reporter::report (result, options.junitXmlFile, options.jsonResultsFile); std::cout << std::flush; std::_Exit (reporter::exitCode (result)); } @@ -673,7 +686,7 @@ TestRunner::TestRunner (const juce::File& file, const RunOptions& opts, std::fun jassert (onComplete); if (options.timeoutMs > 0) - watchdog = std::make_unique (configFile, options.timeoutMs); + watchdog = std::make_unique (configFile, options); startThread(); } diff --git a/source/acceptance/AcceptanceTest.h b/source/acceptance/AcceptanceTest.h index 65ea6be..a8d1d93 100644 --- a/source/acceptance/AcceptanceTest.h +++ b/source/acceptance/AcceptanceTest.h @@ -52,6 +52,8 @@ struct RunOptions { bool recordMissing = false; /**< Record a missing reference and pass, rather than failing. */ juce::int64 timeoutMs = 30000; /**< Abort the whole run after this long. <= 0 never times out. */ + juce::File junitXmlFile; /**< File to write a JUnit XML report to (none if default). */ + juce::File jsonResultsFile; /**< File to write a JSON report to (none if default). */ }; //============================================================================== @@ -60,7 +62,8 @@ struct RunOptions TestResult runTest (const TestConfig&, const RunOptions&); /** Loads the config file (first entry only for v1), runs it, reports the result - to stdout and returns the process exit code (0 success, 1 failure). */ + to stdout (and any report files in the options) and returns the process exit + code (0 success, 1 failure). */ int runTestFile (const juce::File& configFile, const RunOptions&); //============================================================================== diff --git a/source/acceptance/TestReporter.cpp b/source/acceptance/TestReporter.cpp index a95c100..4f48a34 100644 --- a/source/acceptance/TestReporter.cpp +++ b/source/acceptance/TestReporter.cpp @@ -70,7 +70,43 @@ nlohmann::json toJson (const TestResult& r) return j; } -void report (const TestResult& r) +report_writer::Report toReport (const TestResult& r) +{ + report_writer::TestCase c; + c.name = r.name; + c.durationSeconds = r.durationSeconds; + c.details = toJson (r); + + if (r.outcome == TestResult::Outcome::failed) + { + c.status = report_writer::TestCase::Outcome::failed; + + for (const auto& [method, result] : r.comparisons) + if (! result.passed) + c.messages.add ("[" + method + "] " + result.summary); + + if (r.diffFile != juce::File()) + c.messages.add ("diff written to: " + r.diffFile.getFullPathName()); + } + else if (r.outcome == TestResult::Outcome::error) + { + c.status = report_writer::TestCase::Outcome::error; + c.messages.add (r.message); + } + + report_writer::Report report; + report.command = "test"; + report.target = r.name; + report.startTime = juce::Time::getCurrentTime() - juce::RelativeTime::seconds (r.durationSeconds); + + if (r.referenceFile != juce::File()) + report.properties.push_back ({ "reference", r.referenceFile.getFullPathName() }); + + report.cases.push_back (c); + return report; +} + +bool report (const TestResult& r, const juce::File& junitXml, const juce::File& jsonResults) { switch (r.outcome) { @@ -99,6 +135,8 @@ void report (const TestResult& r) // The structured result for machine consumers. std::cout << toJson (r).dump (2) << std::endl; + + return report_writer::write (toReport (r), junitXml, jsonResults); } int exitCode (const TestResult& r) diff --git a/source/acceptance/TestReporter.h b/source/acceptance/TestReporter.h index 10266b7..4939fc3 100644 --- a/source/acceptance/TestReporter.h +++ b/source/acceptance/TestReporter.h @@ -15,6 +15,7 @@ #pragma once #include "ReferenceComparator.h" +#include "../ReportWriter.h" #include #include @@ -46,6 +47,8 @@ struct TestResult juce::File referenceFile; juce::File diffFile; /**< Written only on failure. */ + double durationSeconds = 0.0; /**< How long the test took to run. */ + //============================================================================== static TestResult makeError (const juce::String& name, const juce::String& message) { @@ -63,8 +66,13 @@ namespace reporter /** Structured, machine-readable representation of a result. */ nlohmann::json toJson (const TestResult&); - /** Prints a human-readable summary followed by the JSON to stdout. */ - void report (const TestResult&); + /** The result as a report for --junit-xml / --json-results. */ + report_writer::Report toReport (const TestResult&); + + /** Prints a human-readable summary followed by the JSON to stdout, then + writes the report files that are set (a default juce::File is skipped). + Returns false if a report file couldn't be written. */ + bool report (const TestResult&, const juce::File& junitXml = {}, const juce::File& jsonResults = {}); /** 0 for referenceCreated / passed, 1 for failed / error. */ int exitCode (const TestResult&); diff --git a/tests/acceptance/CMakeLists.txt b/tests/acceptance/CMakeLists.txt index 5d1f350..67e01b9 100644 --- a/tests/acceptance/CMakeLists.txt +++ b/tests/acceptance/CMakeLists.txt @@ -34,6 +34,10 @@ foreach(case ${acceptance_cases}) file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/$/${case}.json" INPUT "${CMAKE_CURRENT_BINARY_DIR}/${case}.json.gen") + # The report options are passed so the JUnit / JSON writers are exercised too. add_test(NAME "pluginval.acceptance.${case}" - COMMAND pluginval test "${CMAKE_CURRENT_BINARY_DIR}/$/${case}.json") + COMMAND pluginval test + --junit-xml "${CMAKE_CURRENT_BINARY_DIR}/$/reports/${case}.junit.xml" + --json-results "${CMAKE_CURRENT_BINARY_DIR}/$/reports/${case}.json" + "${CMAKE_CURRENT_BINARY_DIR}/$/${case}.json") endforeach()