diff --git a/.github/actions/README.md b/.github/actions/README.md new file mode 100644 index 0000000..45264a9 --- /dev/null +++ b/.github/actions/README.md @@ -0,0 +1,92 @@ +# pluginval GitHub Actions + +Two composite actions for using pluginval from another repository's workflows. +Both run on Linux, macOS and Windows runners, download the matching pluginval +release (or use a binary you give them), run everything they are given, write a +results table to the job summary and fail the step if anything failed. + +On Linux the plugin is run under `xvfb-run` when there is no `DISPLAY` (install +`xvfb` on runners that don't have it). + +## validate + +Validates one or more plugins. + +```yaml +- uses: Tracktion/pluginval/.github/actions/validate@v2 + with: + plugins: | + build/MyPlugin_artefacts/Release/VST3/MyPlugin.vst3 + build/MyPlugin_artefacts/Release/AU/MyPlugin.component + strictness-level: 10 + args: --skip-gui-tests +``` + +| Input | Default | | +| --- | --- | --- | +| `plugins` | (required) | The plugins to validate, one per line. Each is a path, a glob or an AU id. A glob that matches nothing fails. | +| `strictness-level` | `5` | The strictness level (1-10). | +| `args` | | Extra options for `pluginval validate`, split on whitespace. | +| `pluginval-version` | `v2.0.0-beta1` | The pluginval release to download: a tag, or `latest`. | +| `pluginval-path` | | An existing pluginval binary to use instead of downloading one. | + +Every `pluginval validate` option can also be set as an environment variable on +the step (e.g. `env: { TIMEOUT_MS: 60000 }`), which is the way to pass values +containing spaces. + +## acceptance + +Runs the [acceptance tests](../../docs/Acceptance%20testing.md) for every config +(`*.json`) found recursively under a directory, optionally against one built +plugin. + +```yaml +- uses: Tracktion/pluginval/.github/actions/acceptance@v2 + with: + config-dir: tests/acceptance + plugin: build/MyPlugin_artefacts/Release/VST3/MyPlugin.vst3 +``` + +| Input | Default | | +| --- | --- | --- | +| `config-dir` | (required) | Directory searched recursively for configs. | +| `plugin` | | Plugin path or AU id to test every config against, instead of the `plugin` in each config. | +| `record-missing` | `false` | Record a missing reference and pass, instead of failing. | +| `timeout-ms` | | Fail a test if it takes longer than this (pluginval's default if empty). | +| `pluginval-version` | `v2.0.0-beta1` | The pluginval release to download: a tag, or `latest`. | +| `pluginval-path` | | An existing pluginval binary to use instead of downloading one. | + +Notes: + +- Every `*.json` under `config-dir` is treated as a config, apart from the + `.wav.json` manifests. Keep other JSON files (and plugin bundles, + which contain their own) out of it. +- A config holding an array of tests is reported as a failure: use one config + per file. +- Relative `plugin` paths resolve against the working directory; the input, + state and reference paths in a config resolve against the config's directory. + +## Outputs + +Both actions set: + +| Output | | +| --- | --- | +| `passed` / `failed` | How many plugins / configs passed and failed. | +| `report-dir` | A directory holding a JUnit XML report for each one. For `acceptance` it also has a `diffs` directory with the `*-diff.wav` files found under `config-dir`. | + +To keep the reports and diffs from a failed run: + +```yaml +- uses: Tracktion/pluginval/.github/actions/acceptance@v2 + id: acceptance + with: + config-dir: tests/acceptance + plugin: build/MyPlugin_artefacts/Release/VST3/MyPlugin.vst3 + +- uses: actions/upload-artifact@v4 + if: always() + with: + name: pluginval-acceptance-${{ runner.os }} + path: ${{ steps.acceptance.outputs.report-dir }} +``` diff --git a/.github/actions/acceptance/acceptance.sh b/.github/actions/acceptance/acceptance.sh new file mode 100644 index 0000000..68dd63d --- /dev/null +++ b/.github/actions/acceptance/acceptance.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Runs "pluginval test" on every config found (recursively) under +# INPUT_CONFIG_DIR, runs them all, and fails if any failed. See action.yml for +# the inputs. + +set -u +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/../common/common.sh" + +resolve_pluginval + +config_dir="${INPUT_CONFIG_DIR:-}" + +if [ ! -d "$config_dir" ]; then + echo "::error::config-dir is not a directory: $config_dir" + exit 1 +fi + +report_dir="${INPUT_REPORT_DIR:-$RUNNER_TEMP/pluginval-reports/acceptance}" +mkdir -p "$report_dir" + +options=() +[ -n "${INPUT_PLUGIN:-}" ] && options+=(--plugin "$INPUT_PLUGIN") +[ "${INPUT_RECORD_MISSING:-false}" = "true" ] && options+=(--record-missing) +[ -n "${INPUT_TIMEOUT_MS:-}" ] && options+=(--timeout-ms "$INPUT_TIMEOUT_MS") + +passed=0 +failed=0 +index=0 +rows="" + +# Every *.json is a config, except the .wav.json manifests that +# pluginval writes next to each reference. +IFS=$'\n' +configs=( $(find "$config_dir" -type f -name '*.json' ! -name '*.wav.json' | sort) ) +unset IFS + +for config in ${configs[@]+"${configs[@]}"}; do + index=$((index + 1)) + name="${config#"$config_dir"/}" + report="$report_dir/$index-$(sanitise "$(basename "$config" .json)").xml" + + echo "::group::pluginval test: $name" + + # pluginval only runs the first entry of an array config, so reject them + # rather than silently skipping tests. + if [ "$(tr -d '[:space:]' < "$config" | head -c 1)" = "[" ]; then + echo "Array configs aren't supported: use one config per file." + code=1 + else + run_pluginval test ${options[@]+"${options[@]}"} --junit-xml "$report" "$config" + code=$? + fi + + echo "::endgroup::" + + if [ $code -eq 0 ]; then + passed=$((passed + 1)) + rows="$rows| \`$name\` | :white_check_mark: passed |"$'\n' + else + failed=$((failed + 1)) + rows="$rows| \`$name\` | :x: failed |"$'\n' + echo "::error::pluginval acceptance test failed: $name" + fi +done + +if [ $index -eq 0 ]; then + echo "::error::No configs (*.json) found under $config_dir" + exit 1 +fi + +# Gather the diff WAVs written next to failing references, so they can be +# uploaded along with the reports. +find "$config_dir" -type f -name '*-diff.wav' | while IFS= read -r diff; do + mkdir -p "$report_dir/diffs" + cp "$diff" "$report_dir/diffs/$(sanitise "${diff#"$config_dir"/}")" +done + +{ + echo "### pluginval acceptance tests" + echo + echo "| Config | Result |" + echo "| --- | --- |" + printf '%s' "$rows" +} >> "$GITHUB_STEP_SUMMARY" + +{ + echo "passed=$passed" + echo "failed=$failed" + echo "report-dir=$report_dir" +} >> "$GITHUB_OUTPUT" + +echo "pluginval acceptance tests: $passed passed, $failed failed" +[ $failed -eq 0 ] diff --git a/.github/actions/acceptance/action.yml b/.github/actions/acceptance/action.yml new file mode 100644 index 0000000..03f2cce --- /dev/null +++ b/.github/actions/acceptance/action.yml @@ -0,0 +1,51 @@ +name: pluginval acceptance tests +description: Runs pluginval's golden-file acceptance tests for every config in a directory. + +inputs: + config-dir: + description: > + Directory searched recursively for test configs (*.json). The + .wav.json manifests are skipped. + required: true + plugin: + description: > + Plugin path or AU id to test every config against, instead of the + "plugin" in each config. + default: '' + record-missing: + description: Record a missing reference and pass, instead of failing. + default: 'false' + timeout-ms: + description: Fail a test if it takes longer than this (pluginval's default if empty). + default: '' + pluginval-version: + description: The pluginval release to download - a tag (e.g. v2.0.0-beta1) or "latest". + default: v2.0.0-beta1 + pluginval-path: + description: Path to an existing pluginval binary to use instead of downloading one. + default: '' + +outputs: + passed: + description: The number of configs that passed. + value: ${{ steps.run.outputs.passed }} + failed: + description: The number of configs that failed. + value: ${{ steps.run.outputs.failed }} + report-dir: + description: Directory holding a JUnit XML report per config, plus diffs/ with any diff WAVs. + value: ${{ steps.run.outputs.report-dir }} + +runs: + using: composite + steps: + - id: run + shell: bash + env: + INPUT_CONFIG_DIR: ${{ inputs.config-dir }} + INPUT_PLUGIN: ${{ inputs.plugin }} + INPUT_RECORD_MISSING: ${{ inputs.record-missing }} + INPUT_TIMEOUT_MS: ${{ inputs.timeout-ms }} + INPUT_PLUGINVAL_VERSION: ${{ inputs.pluginval-version }} + INPUT_PLUGINVAL_PATH: ${{ inputs.pluginval-path }} + run: bash "$GITHUB_ACTION_PATH/acceptance.sh" diff --git a/.github/actions/common/common.sh b/.github/actions/common/common.sh new file mode 100644 index 0000000..0bdccce --- /dev/null +++ b/.github/actions/common/common.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Shared helpers for the pluginval composite actions (sourced, not run). +# Kept compatible with bash 3.2, which is what macOS runners ship. + +# Defaults so the scripts can also be run outside of GitHub Actions. +if [ -z "${RUNNER_OS:-}" ]; then + if [ "$(uname -s)" = "Darwin" ]; then RUNNER_OS=macOS + elif [ "$(uname -s)" = "Linux" ]; then RUNNER_OS=Linux + else RUNNER_OS=Windows + fi +fi + +: "${RUNNER_TEMP:=$(mktemp -d)}" +: "${GITHUB_STEP_SUMMARY:=/dev/null}" +: "${GITHUB_OUTPUT:=/dev/null}" + +# Sets PLUGINVAL_BIN to the pluginval binary: INPUT_PLUGINVAL_PATH if given, +# otherwise the INPUT_PLUGINVAL_VERSION release ("latest" or a tag like v2.0.0-beta1) +# downloaded for this runner's OS. +resolve_pluginval() +{ + if [ -n "${INPUT_PLUGINVAL_PATH:-}" ]; then + PLUGINVAL_BIN="$INPUT_PLUGINVAL_PATH" + else + local version="${INPUT_PLUGINVAL_VERSION:-latest}" + local dir="$RUNNER_TEMP/pluginval-$version" + local url="https://github.com/Tracktion/pluginval/releases/download/$version/pluginval_$RUNNER_OS.zip" + + if [ "$version" = "latest" ]; then + url="https://github.com/Tracktion/pluginval/releases/latest/download/pluginval_$RUNNER_OS.zip" + fi + + case "$RUNNER_OS" in + macOS) PLUGINVAL_BIN="$dir/pluginval.app/Contents/MacOS/pluginval" ;; + Windows) PLUGINVAL_BIN="$dir/pluginval.exe" ;; + *) PLUGINVAL_BIN="$dir/pluginval" ;; + esac + + if [ ! -f "$PLUGINVAL_BIN" ]; then + echo "Downloading $url" + mkdir -p "$dir" + + if ! curl -fsSL --retry 3 "$url" -o "$dir/pluginval.zip"; then + echo "::error::Couldn't download pluginval $version for $RUNNER_OS ($url)" + exit 1 + fi + + unzip -q -o "$dir/pluginval.zip" -d "$dir" + chmod +x "$PLUGINVAL_BIN" + fi + fi + + if [ ! -f "$PLUGINVAL_BIN" ]; then + echo "::error::pluginval binary not found: $PLUGINVAL_BIN" + exit 1 + fi +} + +# Runs pluginval with the given arguments. On Linux without a display it is +# wrapped in xvfb-run, as plugins generally need an X server to load. +run_pluginval() +{ + if [ "$RUNNER_OS" = "Linux" ] && [ -z "${DISPLAY:-}" ] && command -v xvfb-run > /dev/null; then + xvfb-run --auto-servernum "$PLUGINVAL_BIN" "$@" + else + "$PLUGINVAL_BIN" "$@" + fi +} + +# Prints a file-name-safe version of the argument. +sanitise() +{ + printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '_' +} diff --git a/.github/actions/validate/action.yml b/.github/actions/validate/action.yml new file mode 100644 index 0000000..0eaceb6 --- /dev/null +++ b/.github/actions/validate/action.yml @@ -0,0 +1,48 @@ +name: pluginval validate +description: Validates one or more audio plugins with pluginval. + +inputs: + plugins: + description: > + The plugins to validate, one per line. Each is a path, a glob (e.g. + build/*_artefacts/Release/VST3/*.vst3) or an AU id. + required: true + strictness-level: + description: The strictness level (1-10). + default: '5' + args: + description: > + Extra options for "pluginval validate", split on whitespace (e.g. + "--skip-gui-tests --timeout-ms 60000"). Any option can also be set as an + environment variable on the step. + default: '' + pluginval-version: + description: The pluginval release to download - a tag (e.g. v2.0.0-beta1) or "latest". + default: v2.0.0-beta1 + pluginval-path: + description: Path to an existing pluginval binary to use instead of downloading one. + default: '' + +outputs: + passed: + description: The number of plugins that passed. + value: ${{ steps.run.outputs.passed }} + failed: + description: The number of plugins that failed. + value: ${{ steps.run.outputs.failed }} + report-dir: + description: Directory holding a JUnit XML report per plugin. + value: ${{ steps.run.outputs.report-dir }} + +runs: + using: composite + steps: + - id: run + shell: bash + env: + INPUT_PLUGINS: ${{ inputs.plugins }} + INPUT_STRICTNESS_LEVEL: ${{ inputs.strictness-level }} + INPUT_ARGS: ${{ inputs.args }} + INPUT_PLUGINVAL_VERSION: ${{ inputs.pluginval-version }} + INPUT_PLUGINVAL_PATH: ${{ inputs.pluginval-path }} + run: bash "$GITHUB_ACTION_PATH/validate.sh" diff --git a/.github/actions/validate/validate.sh b/.github/actions/validate/validate.sh new file mode 100644 index 0000000..7ad2841 --- /dev/null +++ b/.github/actions/validate/validate.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Validates each plugin in INPUT_PLUGINS (one per line: a path, a glob or an AU +# id), runs them all, and fails if any failed. See action.yml for the inputs. + +set -u +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/../common/common.sh" + +resolve_pluginval + +report_dir="${INPUT_REPORT_DIR:-$RUNNER_TEMP/pluginval-reports/validate}" +mkdir -p "$report_dir" + +passed=0 +failed=0 +index=0 +rows="" + +# One plugin per line (not comma-separated: AU ids contain commas). Globs are +# left to expand; anything that matches nothing is passed through and fails. +IFS=$'\n' +plugins=( ${INPUT_PLUGINS:-} ) +unset IFS + +for plugin in ${plugins[@]+"${plugins[@]}"}; do + plugin="$(printf '%s' "$plugin" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + [ -z "$plugin" ] && continue + + index=$((index + 1)) + report="$report_dir/$index-$(sanitise "$(basename "$plugin")").xml" + + echo "::group::pluginval validate: $plugin" + # INPUT_ARGS is deliberately unquoted so it splits into separate arguments. + run_pluginval validate --strictness-level "${INPUT_STRICTNESS_LEVEL:-5}" --junit-xml "$report" ${INPUT_ARGS:-} "$plugin" + code=$? + echo "::endgroup::" + + if [ $code -eq 0 ]; then + passed=$((passed + 1)) + rows="$rows| \`$plugin\` | :white_check_mark: passed |"$'\n' + else + failed=$((failed + 1)) + rows="$rows| \`$plugin\` | :x: failed |"$'\n' + echo "::error::pluginval validation failed: $plugin" + fi +done + +if [ $index -eq 0 ]; then + echo "::error::No plugins to validate (the 'plugins' input is empty)" + exit 1 +fi + +{ + echo "### pluginval validate" + echo + echo "| Plugin | Result |" + echo "| --- | --- |" + printf '%s' "$rows" +} >> "$GITHUB_STEP_SUMMARY" + +{ + echo "passed=$passed" + echo "failed=$failed" + echo "report-dir=$report_dir" +} >> "$GITHUB_OUTPUT" + +echo "pluginval validate: $passed passed, $failed failed" +[ $failed -eq 0 ] diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index f764864..8109cdf 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -68,3 +68,102 @@ jobs: if: runner.os != 'Linux' working-directory: build run: ctest -C Release --output-on-failure -R pluginval.acceptance + + # The reusable actions in .github/actions, run against the build above + # (pluginval-path) rather than a downloaded release. + - name: Prepare action fixtures + id: fixtures + shell: bash + run: | + case "$RUNNER_OS" in + macOS) echo "pluginval=build/pluginval_artefacts/Release/pluginval.app/Contents/MacOS/pluginval" >> "$GITHUB_OUTPUT" ;; + Windows) echo "pluginval=build/pluginval_artefacts/Release/pluginval.exe" >> "$GITHUB_OUTPUT" ;; + *) echo "pluginval=build/pluginval_artefacts/Release/pluginval" >> "$GITHUB_OUTPUT" ;; + esac + + # Tone generator configs with the plugin removed, so it has to come from + # the action's "plugin" input. One is nested and a reference manifest is + # added, to check the recursion and that manifests aren't run as configs. + generated=build/tests/acceptance/Release + fixtures=build/action-fixtures + mkdir -p $fixtures/good/nested $fixtures/bad $fixtures/missing + sed '/"plugin"/d' $generated/sine-440.json > $fixtures/good/sine-440.json + sed '/"plugin"/d' $generated/square-220.json > $fixtures/good/nested/square-220.json + cp tests/acceptance/refs/sine-440.wav.json $fixtures/good/nested/ + + # A mismatching reference and an (unsupported) array config. + sed -e '/"plugin"/d' -e 's#refs/sine-440.wav#refs/square-220.wav#' $generated/sine-440.json > $fixtures/bad/wrong-ref.json + (echo "["; cat $fixtures/good/sine-440.json; echo "]") > $fixtures/bad/array.json + + # A reference that doesn't exist yet. + sed -e '/"plugin"/d' -e 's#"reference": ".*"#"reference": "recorded.wav"#' $generated/sine-440.json > $fixtures/missing/missing.json + + - name: "Action: validate" + id: validate + uses: ./.github/actions/validate + with: + pluginval-path: ${{ steps.fixtures.outputs.pluginval }} + args: --skip-gui-tests + plugins: build/tests/test_plugins/*/*_artefacts/Release/VST3/*.vst3 + + - name: "Action: validate (missing plugin, expected to fail)" + id: validate-missing + continue-on-error: true + uses: ./.github/actions/validate + with: + pluginval-path: ${{ steps.fixtures.outputs.pluginval }} + args: --skip-gui-tests + plugins: | + build/tests/test_plugins/gain/*_artefacts/Release/VST3/*.vst3 + build/Missing.vst3 + + - name: "Action: acceptance" + id: acceptance + uses: ./.github/actions/acceptance + with: + pluginval-path: ${{ steps.fixtures.outputs.pluginval }} + config-dir: build/action-fixtures/good + plugin: build/tests/test_plugins/tone_generator/pluginval_tone_generator_artefacts/Release/VST3/pluginval Tone Generator.vst3 + + - name: "Action: acceptance (bad configs, expected to fail)" + id: acceptance-bad + continue-on-error: true + uses: ./.github/actions/acceptance + with: + pluginval-path: ${{ steps.fixtures.outputs.pluginval }} + config-dir: build/action-fixtures/bad + plugin: build/tests/test_plugins/tone_generator/pluginval_tone_generator_artefacts/Release/VST3/pluginval Tone Generator.vst3 + + - name: "Action: acceptance (missing reference, expected to fail)" + id: acceptance-missing + continue-on-error: true + uses: ./.github/actions/acceptance + with: + pluginval-path: ${{ steps.fixtures.outputs.pluginval }} + config-dir: build/action-fixtures/missing + plugin: build/tests/test_plugins/tone_generator/pluginval_tone_generator_artefacts/Release/VST3/pluginval Tone Generator.vst3 + + - name: "Action: acceptance (record the missing reference)" + id: acceptance-record + uses: ./.github/actions/acceptance + with: + pluginval-path: ${{ steps.fixtures.outputs.pluginval }} + config-dir: build/action-fixtures/missing + plugin: build/tests/test_plugins/tone_generator/pluginval_tone_generator_artefacts/Release/VST3/pluginval Tone Generator.vst3 + record-missing: true + + - name: Check the action results + shell: bash + run: | + check() { echo "$1: got '$2', expected '$3'"; [ "$2" = "$3" ] || exit 1; } + check "validate passed" "${{ steps.validate.outputs.passed }}" 3 + check "validate failed" "${{ steps.validate.outputs.failed }}" 0 + check "validate-missing outcome" "${{ steps.validate-missing.outcome }}" failure + check "validate-missing passed" "${{ steps.validate-missing.outputs.passed }}" 1 + check "validate-missing failed" "${{ steps.validate-missing.outputs.failed }}" 1 + check "acceptance passed" "${{ steps.acceptance.outputs.passed }}" 2 + check "acceptance failed" "${{ steps.acceptance.outputs.failed }}" 0 + check "acceptance-bad outcome" "${{ steps.acceptance-bad.outcome }}" failure + check "acceptance-bad failed" "${{ steps.acceptance-bad.outputs.failed }}" 2 + check "acceptance-missing outcome" "${{ steps.acceptance-missing.outcome }}" failure + check "acceptance-record passed" "${{ steps.acceptance-record.outputs.passed }}" 1 diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fbb9dcb..48bdff5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -218,6 +218,8 @@ jobs: uses: softprops/action-gh-release@v2 with: body_path: CHANGELIST.md + # Tags like v2.0.0-beta1 are pre-releases, so they don't become "latest" + prerelease: ${{ contains(github.ref_name, '-') }} files: | CHANGELIST.md/* pluginval_Linux.zip/* diff --git a/CHANGELIST.md b/CHANGELIST.md index c3f546e..0ae417d 100644 --- a/CHANGELIST.md +++ b/CHANGELIST.md @@ -5,6 +5,8 @@ - Fixed the crash handler so a backtrace is shown again when a plugin crashes, and made it async-signal-safe [#144] - Added `--junit-xml ` and `--json-results ` to both `pluginval validate` and `pluginval test` to write machine-readable reports for CI (also settable as `JUNIT_XML` / `JSON_RESULTS`). A report that can't be written fails the run. See `docs/Adding pluginval to CI.md` - Added acceptance testing: `pluginval test ` renders a plugin with a known state, input (audio/MIDI) and optional playhead, then compares the output against a checked-in reference WAV, exiting `0` on a match and `1` on a mismatch (with a diff WAV). Use `--record-missing` to record new references. See `docs/Acceptance testing.md` +- Added `--plugin ` to `pluginval test` to run a config against a given plugin instead of the one it names (the config's `plugin` can then be omitted) +- Added two GitHub Actions for use from other repositories: `Tracktion/pluginval/.github/actions/validate` (validate a list of plugins) and `.../acceptance` (run every acceptance config in a directory against a built plugin). See `.github/actions/README.md` - Restructured the command line into subcommands: `pluginval validate [options] ` (the default), `pluginval run-tests` and `pluginval strictness-help [level]`. The plugin path is now a positional argument of `validate` - **Deprecated:** the flat flags `--validate `, `--run-tests` and `--strictness-help [level]` still work as aliases (with a one-line notice) but will be removed in a future version. `pluginval ` remains a silent shorthand for `validate` - Replaced the hand-rolled command-line parser with CLI11 and a single JSON-based settings pipeline diff --git a/CLAUDE.md b/CLAUDE.md index e4a4a20..074a158 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -121,6 +121,9 @@ pluginval/ │ ├── acceptance/ # Acceptance self-tests: configs (*.json.in), inputs/, checked-in refs/ WAVs │ ├── mac_tests/ # macOS-specific tests │ └── windows_tests.bat # Windows test scripts +├── .github/ +│ ├── actions/ # Composite actions for other repos: validate/, acceptance/, common/ (see README.md there) +│ └── workflows/ # build.yaml (release build), acceptance.yml (acceptance self-tests + action dogfooding) ├── docs/ # Documentation │ ├── Adding pluginval to CI.md │ ├── Command line options.md @@ -281,7 +284,9 @@ spec: `tests/acceptance/Acceptance testing design.md`; end-user guide: - **CLI**: `pluginval test `. A new `Command::test` is recognised by `settings_parser::dispatch()` (captures the positional config path into `DispatchResult::testConfigPath`, plus `recordMissing` / `timeoutMs` from - `--record-missing` / `--timeout-ms` or `RECORD_MISSING` / `TIMEOUT_MS`), + `--record-missing` / `--timeout-ms` or `RECORD_MISSING` / `TIMEOUT_MS`, and + `testPluginOverride` from `--plugin`, which replaces the config's `plugin` + in `runTestFile()` - so `plugin` is only required when `--plugin` is absent), `isCommandLine()` and `getFooterText()`; `CommandLine.cpp`'s `performCommandLine()` has a `Command::test` branch that starts an `acceptance::TestRunner`, which renders on a **background thread** (message @@ -523,7 +528,26 @@ add_pluginval_tests(MyPluginTarget ) ``` -### GitHub Actions Example +### GitHub Actions + +Two composite actions for other repositories live in `.github/actions/` +(documented in `.github/actions/README.md`): + +- `validate/` - validates a newline-separated list of plugins / globs / AU ids. +- `acceptance/` - runs `pluginval test` for every `*.json` (recursively, skipping + the `*.wav.json` manifests and rejecting array configs) under a directory, + optionally passing one built plugin via `--plugin`. + +Each `action.yml` only maps its inputs to `INPUT_*` environment variables and +runs its sibling bash script (`validate.sh` / `acceptance.sh`); shared helpers +(downloading the release or using `pluginval-path`, the Linux `xvfb-run` wrapper) +are in `common/common.sh`. The scripts must stay bash 3.2 compatible (macOS +runners) and can be run locally by setting the `INPUT_*` variables. They run +every item, write a job-summary table and `passed` / `failed` / `report-dir` +outputs, and exit non-zero if anything failed. `.github/workflows/acceptance.yml` +dogfoods both against the in-repo build, including the expected-failure cases. + +Without the actions: ```yaml - name: Download pluginval @@ -573,6 +597,11 @@ Run internal tests via CLI: 3. Commit: `git commit -am "Version X.Y.Z"` 4. Tag: `git tag -a vX.Y.Z -m "X.Y.Z release"` 5. Push: `git push --tags` +6. Once the release binaries are published, bump the `pluginval-version` default + in both `.github/actions/*/action.yml` (and the README tables) to the new tag + +A tag with a hyphen (e.g. `v2.0.0-beta1`) is published as a pre-release, so it +doesn't become the "latest" release that the download URLs in the docs point at. ## Common Tasks for AI Assistants diff --git a/docs/Acceptance testing.md b/docs/Acceptance testing.md index 018305f..cc3b10f 100644 --- a/docs/Acceptance testing.md +++ b/docs/Acceptance testing.md @@ -46,7 +46,7 @@ JSON keys are `snake_case`. The most useful fields: | Field | Notes | |---|---| | `name` | Labels the result and gives the default reference path. **Required** unless `reference` is given. | -| `plugin` | Path to the plugin (or an AU identifier). **Required.** | +| `plugin` | Path to the plugin (or an AU identifier). **Required**, unless `--plugin` is given on the command line. | | `input.audio` / `input.midi` | Input files to feed it. Omit both for silence (e.g. instruments driven only by MIDI, or generators). The audio file's sample rate must match `sample_rate`. Each file channel feeds the plugin input channel with the same index: a mono file feeds channel 0 only (it isn't duplicated), and extra channels on either side are silent / ignored. MIDI meta events (tempo, track names) aren't sent to the plugin. | | `state.parameters` | A map of parameter → **normalised** value (`0` to `1`), applied before rendering, **in file order**. A key of digits only is a parameter index; otherwise it matches the format's parameter ID, then the display name (case-insensitively). | | `state.file` | A binary `getStateInformation` blob to restore first (e.g. a captured preset). Applied before `state.parameters`. | @@ -89,6 +89,9 @@ Any other option is an error. `RECORD_MISSING=1`. - `--timeout-ms ` - fail (exit `1`) if the run takes longer than this (default `30000`, `-1` to never time out). Also settable as `TIMEOUT_MS`. +- `--plugin ` - test this plugin path (or AU identifier) instead of the + config's `plugin`, which can then be left out. This lets one set of configs be + run against whichever binary was just built (e.g. both the VST3 and the AU). - `--junit-xml ` - write a JUnit XML report, for CI systems that display test results. Also settable as `JUNIT_XML`. - `--json-results ` - write a JSON report (the same result that is printed @@ -97,6 +100,18 @@ Any other option is an error. See [Adding pluginval to CI](Adding%20pluginval%20to%20CI.md#machine-readable-reports) for the report formats. +On GitHub Actions, the `acceptance` action runs every config in a directory +(recursively) against a built plugin: + +```yaml +- uses: Tracktion/pluginval/.github/actions/acceptance@v2 + with: + config-dir: tests/acceptance + plugin: build/MyPlugin_artefacts/Release/VST3/MyPlugin.vst3 +``` + +See the [actions README](../.github/actions/README.md) for its inputs and outputs. + The render runs on a background thread, as a host's audio thread would, leaving the message thread free for the plugin. diff --git a/docs/Adding pluginval to CI.md b/docs/Adding pluginval to CI.md index 43a63e4..22b1c85 100644 --- a/docs/Adding pluginval to CI.md +++ b/docs/Adding pluginval to CI.md @@ -9,6 +9,21 @@ There a couple of different ways this can be done but the simplest is as follows 4. If validation fails, fail your build 5. If validation passes, retain the generated log files ready to submit to "Verified by pluginval" +### GitHub Actions +On GitHub Actions the simplest route is the `validate` action, which downloads pluginval, validates each plugin and fails the job if any fail: + +```yaml +- uses: Tracktion/pluginval/.github/actions/validate@v2 + with: + plugins: | + build/MyPlugin_artefacts/Release/VST3/MyPlugin.vst3 + build/MyPlugin_artefacts/Release/AU/MyPlugin.component + strictness-level: 5 + args: --skip-gui-tests +``` + +There is also an `acceptance` action for [acceptance tests](Acceptance%20testing.md). See the [actions README](../.github/actions/README.md) for all the inputs and outputs. + ### Examples In these examples the following is assumed: - macOS/Linux, curl is installed diff --git a/docs/Command line options.md b/docs/Command line options.md index 71e2128..90442aa 100644 --- a/docs/Command line options.md +++ b/docs/Command line options.md @@ -20,6 +20,8 @@ COMMANDS: (default 30000, -1 = never). --junit-xml Write a JUnit XML report. --json-results Write a JSON report. + --plugin Test this plugin path or AU id + instead of the config's. Any other option is an error. The flat flags --validate , --run-tests and --strictness-help [level] diff --git a/source/CommandLine.cpp b/source/CommandLine.cpp index 9736bd1..1b92bbf 100644 --- a/source/CommandLine.cpp +++ b/source/CommandLine.cpp @@ -231,6 +231,7 @@ void performCommandLine (CommandLineValidator& validator, const juce::String& co acceptance::RunOptions options; options.recordMissing = routed.recordMissing; options.timeoutMs = routed.timeoutMs; + options.pluginOverride = routed.testPluginOverride.toStdString(); const auto toReportFile = [] (const juce::String& path) { diff --git a/source/CommandLineTests.cpp b/source/CommandLineTests.cpp index bb5fed9..869fe2c 100644 --- a/source/CommandLineTests.cpp +++ b/source/CommandLineTests.cpp @@ -543,6 +543,29 @@ struct CommandLineTests : public juce::UnitTest } } + beginTest ("Subcommand: test plugin override (--plugin)"); + { + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test --plugin build/P.vst3 config.json")); + expect (d.error.isEmpty()); + expectEquals (d.testPluginOverride, juce::String ("build/P.vst3")); + expectEquals (d.testConfigPath, juce::String ("config.json")); + } + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test config.json")); + expect (d.testPluginOverride.isEmpty()); + } + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test config.json --plugin")); + expect (d.error.contains ("--plugin")); + } + { + // The following option isn't taken as the plugin. + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test config.json --plugin --record-missing")); + expect (d.error.contains ("--plugin")); + } + } + beginTest ("Subcommand: test report options (--junit-xml, --json-results)"); { { @@ -840,7 +863,7 @@ struct CommandLineTests : public juce::UnitTest expect (! rejects (R"({ "name": "t", "plugin": "P.vst3" })")); expect (! rejects (R"({ "reference": "r.wav", "plugin": "P.vst3" })"), "reference alone names the test"); - expect (rejects (R"({ "name": "t" })"), "plugin is required"); + expect (! rejects (R"({ "name": "t" })"), "plugin can come from --plugin instead"); expect (rejects (R"({ "plugin": "P.vst3" })"), "name or reference is required"); expect (rejects (R"({ "name": "t", "plugin": "P.vst3", "comparison": {} })"), "empty comparison"); expect (rejects (R"({ "name": "t", "plugin": "P.vst3", "comparison": 1 })"), "non-object comparison"); diff --git a/source/SettingsParser.cpp b/source/SettingsParser.cpp index 999a6a6..d8105eb 100644 --- a/source/SettingsParser.cpp +++ b/source/SettingsParser.cpp @@ -137,6 +137,8 @@ R"(Commands: (default 30000, -1 to never time out). --junit-xml Write a JUnit XML report. --json-results Write a JSON report. + --plugin Test this plugin path or AU id instead + of the one in the config. The flat flags --validate , --run-tests and --strictness-help [level] are deprecated aliases for the commands above and will be removed in a future version. @@ -424,6 +426,18 @@ Precedence (lowest to highest): defaults, environment variables, --config, comma (token == "--junit-xml" ? result.junitXmlPath : result.jsonResultsPath) = value; } + else if (token == "--plugin") + { + const auto value = i + 1 < tokensIn.size() ? tokensIn.getReference (++i) : juce::String(); + + if (value.isEmpty() || value.startsWith ("--")) + { + result.error = "--plugin needs a plugin path or AU id"; + break; + } + + result.testPluginOverride = value; + } else if (token.startsWith ("-")) { result.error = "unknown option for test: " + token; diff --git a/source/SettingsParser.h b/source/SettingsParser.h index 934b0c0..6896050 100644 --- a/source/SettingsParser.h +++ b/source/SettingsParser.h @@ -85,6 +85,7 @@ namespace settings_parser juce::int64 timeoutMs = 30000; /**< test: abort the run after this long (-1 to never time out). */ juce::String junitXmlPath; /**< test: file to write a JUnit XML report to. */ juce::String jsonResultsPath; /**< test: file to write a JSON report to. */ + juce::String testPluginOverride; /**< test: plugin path or AU id to use instead of the config's. */ juce::String error; /**< test: an invalid option / argument, to report instead of running. */ }; diff --git a/source/acceptance/AcceptanceTest.cpp b/source/acceptance/AcceptanceTest.cpp index 100cdfb..9b8e2b2 100644 --- a/source/acceptance/AcceptanceTest.cpp +++ b/source/acceptance/AcceptanceTest.cpp @@ -629,8 +629,16 @@ int runTestFile (const juce::File& configFile, const RunOptions& options) if (configs.size() > 1) std::cout << "Note: " << configs.size() << " configs found; v1 runs the first only." << std::endl; + auto& config = configs.front(); + + if (! options.pluginOverride.empty()) + config.plugin = options.pluginOverride; + + if (config.plugin.empty()) + return finish (TestResult::makeError (config.getName(), "plugin is required (in the config, or via --plugin)")); + const auto startTime = juce::Time::getMillisecondCounterHiRes(); - auto result = runTest (configs.front(), options); + auto result = runTest (config, options); result.durationSeconds = (juce::Time::getMillisecondCounterHiRes() - startTime) / 1000.0; return finish (result); } diff --git a/source/acceptance/AcceptanceTest.h b/source/acceptance/AcceptanceTest.h index a8d1d93..8dcb1f1 100644 --- a/source/acceptance/AcceptanceTest.h +++ b/source/acceptance/AcceptanceTest.h @@ -54,6 +54,7 @@ struct RunOptions juce::int64 timeoutMs = 30000; /**< Abort the whole run after this long. <= 0 never times out. */ juce::File junitXmlFile; /**< File to write a JUnit XML report to (none if default). */ juce::File jsonResultsFile; /**< File to write a JSON report to (none if default). */ + std::string pluginOverride; /**< Plugin path or AU id to test instead of the config's (none if empty). */ }; //============================================================================== diff --git a/source/acceptance/TestConfig.cpp b/source/acceptance/TestConfig.cpp index bdb25d0..3f13495 100644 --- a/source/acceptance/TestConfig.cpp +++ b/source/acceptance/TestConfig.cpp @@ -134,10 +134,8 @@ void from_json (const nlohmann::ordered_json& j, TestConfig& c) c.comparison = *comp; } - // Validation. - if (c.plugin.empty()) - fail ("plugin is required"); - + // Validation. (The plugin may instead come from --plugin, so it's checked + // by the runner once any override has been applied.) if (c.name.empty() && c.reference.empty()) fail ("name or reference is required"); diff --git a/source/acceptance/TestConfig.h b/source/acceptance/TestConfig.h index 556a582..a6b2e01 100644 --- a/source/acceptance/TestConfig.h +++ b/source/acceptance/TestConfig.h @@ -53,7 +53,7 @@ struct TestConfig }; std::string name; /**< Labels results; derives the default reference path. */ - std::string plugin; /**< Plugin path or AU id. Required. */ + std::string plugin; /**< Plugin path or AU id. Required unless --plugin is given. */ std::string inputAudio; /**< input.audio: path to an input audio file, or empty for silence. */ std::string inputMidi; /**< input.midi: path to a .mid file, or empty for none. */ std::string reference; /**< The golden file. Empty -> derived from name. */ diff --git a/tests/acceptance/Acceptance testing design.md b/tests/acceptance/Acceptance testing design.md index 836a805..27ee2da 100644 --- a/tests/acceptance/Acceptance testing design.md +++ b/tests/acceptance/Acceptance testing design.md @@ -140,7 +140,7 @@ member names verbatim). Missing keys still fall back to the defaults. | Field | Type | Default | Notes | |---|---|---|---| | `name` | string | reference's basename | Used to derive the default reference path and to label results. `name` or `reference` is required. | -| `plugin` | string | (required) | Absolute/relative path, or an AU identifier string. Relative paths resolve against the working directory. | +| `plugin` | string | (required) | Absolute/relative path, or an AU identifier string. Relative paths resolve against the working directory. `--plugin ` on the command line overrides it, in which case it may be omitted. | | `input.audio` | string | none | Path to an input audio file (WAV/AIFF/FLAC via `AudioFormatManager`). Used for effects. Its sample rate must equal `sample_rate`. File channel N feeds plugin input channel N; a mono file feeds channel 0 only (deliberately not duplicated). | | `input.midi` | string | none | Path to a `.mid` file (`juce::MidiFile`). Used for instruments. Meta events are skipped. | | `reference` | string | `/.wav` | The golden file. Absent -> record mode; present -> compare mode. |