diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2808ad9..74afe1d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -8,6 +8,12 @@ on: pull_request: paths-ignore: ['**.md', 'docs/**', 'LICENSE'] +# A new push to a pull request cancels the run it supersedes; pushes to main +# each run to the end. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: flake8: runs-on: ubuntu-latest diff --git a/.github/workflows/publish-release-packages.yml b/.github/workflows/publish-release-packages.yml index 14c9840..daf9a42 100644 --- a/.github/workflows/publish-release-packages.yml +++ b/.github/workflows/publish-release-packages.yml @@ -53,7 +53,8 @@ jobs: # numpy: two tests need it (test_cpython_asym_table through the ulab # bridge, test_rig_comparator directly); the packages themselves do not. - - run: python -m pip install numpy + # pytest and pytest-xdist run the suite in parallel, as tests.yml does. + - run: python -m pip install numpy pytest pytest-xdist # --no-deps is load-bearing: without it a floor raised above the pinned # release re-resolves pydevices-audiodsp from the index and every @@ -67,7 +68,10 @@ jobs: # a test. The tests would generate them too; this makes it explicit. - run: python tools/effect_probes/make_probes.py --ensure - - run: python -m unittest discover -s tests -p "test_*.py" + - run: python -m pytest tests -n auto --dist loadscope -p no:cacheprovider + env: + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" - run: python tools/validate_api.py - run: python tests/parity/effects_library_smoke.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ded620d..371e762 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,6 +15,17 @@ on: paths-ignore: ['docs/**', 'README.md', 'ROADMAP.md', 'CHANGELOG.md', 'AGENTS.md', 'LICENSE'] pull_request: paths-ignore: ['docs/**', 'README.md', 'ROADMAP.md', 'CHANGELOG.md', 'AGENTS.md', 'LICENSE'] + # The full matrix on demand, for a branch that should see every Python + # before it merges rather than after. + workflow_dispatch: + +# A new push to a pull request cancels the run it supersedes: only the newest +# commit's result is ever read. Pushes to main are never cancelled or queued +# behind one another (each gets a group of its own), so every commit on main +# keeps its own full-matrix record. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: test: @@ -22,7 +33,11 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python: ["3.11", "3.12", "3.13", "3.14"] + # Every supported Python on main and on a manual run; on a pull + # request only 3.13, the Python the other two platforms run. A failure + # only another Python shows surfaces on the push to main, or sooner + # from a manual run on the branch. + python: ${{ fromJSON(github.event_name == 'pull_request' && '["3.13"]' || '["3.11", "3.12", "3.13", "3.14"]') }} include: # The components are pure Python; the platform variance comes from # audiodsp's extension underneath them, so one Python is enough to @@ -54,16 +69,33 @@ jobs: echo "tag=$tag" >> "$GITHUB_OUTPUT" echo "audiodsp pinned at $tag" - - name: Install audiodsp at the pin + # Building audiodsp's extension from source is the slowest install here, + # and its output depends only on the pin, the platform and the Python. + # Build the wheel once per combination and keep it. + - name: Cache the audiodsp wheel + id: audiodsp-wheel + uses: actions/cache@v6 + with: + path: .audiodsp-wheel + key: audiodsp-wheel-v1-${{ steps.pin.outputs.tag }}-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python }} + + - name: Build the audiodsp wheel at the pin + if: steps.audiodsp-wheel.outputs.cache-hit != 'true' run: >- - python -m pip install + python -m pip wheel --no-deps --wheel-dir .audiodsp-wheel "pydevices-audiodsp @ git+https://github.com/PyDevices/audiodsp@${{ steps.pin.outputs.tag }}" + - name: Install audiodsp at the pin + shell: bash + run: python -m pip install .audiodsp-wheel/pydevices_audiodsp-*.whl + # Editable, so the tests exercise lib/ directly rather than a copy, and # so each package's own pyproject.toml is exercised too. # numpy: two tests need it (test_cpython_asym_table through the ulab # bridge, test_rig_comparator directly); the packages themselves do not. - - run: python -m pip install numpy + # pytest and pytest-xdist run the suite below; the tests are plain + # unittest and need neither. + - run: python -m pip install numpy pytest pytest-xdist # --no-deps is load-bearing: without it a floor raised above the pinned # release re-resolves pydevices-audiodsp from the index and every @@ -77,7 +109,19 @@ jobs: # a test. The tests would generate them too; this makes it explicit. - run: python tools/effect_probes/make_probes.py --ensure - - run: python -m unittest discover -s tests -p "test_*.py" + # The same tests `python -m unittest discover -s tests -p "test_*.py"` + # finds, spread across every core the runner has: serially the suite + # takes over an hour. A test that writes a file writes it into its own + # temporary directory, so no two workers collide. loadscope keeps each + # class on one worker, in unittest's order, with one setUpClass; + # tests/conftest.py starts the slowest class first. --durations names + # the slowest tests, which bound how fast this step can get. + # One BLAS thread per process: every core already runs a test process, + # and numpy would otherwise start a BLAS thread per core in each one. + - run: python -m pytest tests -n auto --dist loadscope -p no:cacheprovider --durations=25 + env: + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" - run: python tools/validate_api.py - run: python tests/parity/effects_library_smoke.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9ceac03 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,24 @@ +"""pytest's view of the suite, for CI's parallel run (pytest-xdist). + +The tests are plain unittest and `python -m unittest discover -s tests` +ignores this file. It changes only the order pytest hands the tests out in, +never which tests run. + +The slowest files go first. The lifecycle matrix runs the whole matrix on +CPython in its setUpClass, and a few reverb and tape delay classes take +minutes each; started last, any of them would finish long after every other +worker had gone idle. With `--dist loadscope` each class stays on one worker, +so a setUpClass runs once. +""" + +#: Files whose tests start first, slowest first. +SLOW_FIRST = ("test_lifecycle_matrix.py", "test_cpython_effects_reverb.py", + "test_cpython_effects_tapedelay.py") + + +def pytest_collection_modifyitems(config, items): + def rank(item): + name = item.path.name + return SLOW_FIRST.index(name) if name in SLOW_FIRST else len(SLOW_FIRST) + + items.sort(key=rank) diff --git a/tests/test_lifecycle_matrix.py b/tests/test_lifecycle_matrix.py index cce01be..1a3c988 100644 --- a/tests/test_lifecycle_matrix.py +++ b/tests/test_lifecycle_matrix.py @@ -174,13 +174,46 @@ def start_native(binary, names, scratch): text=True, cwd=ROOT, env=env) -def run_cpython(name): +def run_cpython(name, **only): lines = [] lifecycle.run_class(rebuilt.module_class(name), emit=lines.append, - quick=not FULL) + quick=not FULL, **only) return lines +def _run_cpython_part(part): + name, rate, ch, patch = part + return run_cpython(name, rates=(rate,), channels=(ch,), patches=[patch]) + + +def run_cpython_all(names): + """{name: lines}, the CPython matrix for each class. + + The CPython matrix is most of this file's time, and no cell depends on + another, so each class's rates, channel counts and patches run side by + side in processes of their own and their lines are joined in run_class's + order. Each part prints its own SUMMARY line, which parse() skips. + LIFECYCLE_JOBS=1 runs every class in this process, one after another. + """ + jobs = int(os.environ.get("LIFECYCLE_JOBS", "0")) or os.cpu_count() or 1 + if jobs <= 1 or not names: + return {name: run_cpython(name) for name in names} + import concurrent.futures + import multiprocessing + parts = [(name, rate, ch, patch) for name in names + for rate in lifecycle.RATES for ch in lifecycle.CHANNELS + for patch in [None] + sorted(rebuilt.module_class(name).PATCHES)] + out = {name: [] for name in names} + # spawn on every platform: a fork of a process already running threads + # (a test runner's, say) can deadlock. + with concurrent.futures.ProcessPoolExecutor( + min(jobs, len(parts)), + mp_context=multiprocessing.get_context("spawn")) as pool: + for part, lines in zip(parts, pool.map(_run_cpython_part, parts)): + out[part[0]] += lines + return out + + class TestMatrix(unittest.TestCase): """Each class's red cells are exactly KNOWN_RED's, and the three interpreters print the same line for every cell (P6).""" @@ -195,9 +228,7 @@ def setUpClass(cls): if binary is not None and cls.names: cls.procs[family] = start_native(binary, cls.names, cls.scratch.name) - cls.cpython = {} - for name in cls.names: - cls.cpython[name] = run_cpython(name) + cls.cpython = run_cpython_all(cls.names) cls.native_lines = {} for family, proc in cls.procs.items(): out, err = proc.communicate()