From 6114c906f62c4018cb92c7f6ad5da70315ae1288 Mon Sep 17 00:00:00 2001 From: Serhii A Date: Wed, 9 Sep 2026 11:50:45 +0200 Subject: [PATCH] Run the test matrix nightly and turn warnings into errors (#1382) * Run the test matrix nightly and turn warnings into errors CI only ran on push and pull request, so breakage that arrives without a commit -- a dependency release, a tz database update, the calendar rolling over -- stayed invisible until someone happened to push. Master saw CI on 41 distinct days in the last 13 months, twice going more than two months without a run, which is how #1286 reached a downstream packager first. A new Nightly workflow calls the existing Build matrix daily at 04:17 UTC and files one tracking issue per outage, since GitHub's own notification goes to a single person. The schedule lives in its own workflow because GitHub disables a workflow that has a schedule after 60 days without repository activity, and a disabled workflow answers none of its triggers: keeping the cron out of main.yml means an inactivity disable can never take pull request runs down with it. pytest.ini turns warnings into errors. The suite emits none today on 3.10 through 3.14, with and without the calendars and langdetect extras, against both current dependencies and the min pins, so the exemption list starts empty. The exception is the tzlocal 0.2 pin in the min environments, whose invalid string escapes become an import-time SyntaxError under an error filter whenever the installer does not byte-compile, so tox ignores that one message for those two environments. The matrix gains a plain 3.10 leg: 3.10 is a supported interpreter that was only ever tested against the oldest permitted dependency versions, since the Python bump in #1296 consumed the bare 3.10 row. CIFuzz uploads SARIF with github/codeql-action/upload-sarif@v4. v2 has been emitting a failure-level annotation on every run, and v3 emits one of its own, so v4 is what actually clears it. Fixes #1374 Co-Authored-By: Claude Opus 5 (1M context) * Address review findings on the nightly and warnings setup Scope the Nightly token to contents: read at the workflow level, which the called matrix inherits; the report job asks for issues: write itself (CodeQL actions/missing-workflow-permissions). Exempt three environmental warnings from the error filter: tzlocal's two UserWarnings on a host with no or stale time zone configuration, which CI never sees because it always sets TZ but which fail most of the suite for a contributor in a bare container or in nixpkgs' sandbox, and pytest's PytestCacheWarning in a read-only checkout, which turned a green run into exit 1. Drop the DeprecationWarning category from the tzlocal 0.2 escape filter, since CPython 3.12+ emits it as a SyntaxWarning, and carry it through PYTEST_ADDOPTS instead of a copied command line so min and min-all keep following [testenv] commands. Ignore pytest-codspeed's measurement-hook RuntimeWarnings in the benchmark env, so a runner glitch cannot turn CodSpeed red. Upload coverage only on push and pull_request, so a dispatched Nightly does not re-upload master's report. Gate the report job on master and on failure or cancelled, key the tracking issue on a dedicated nightly-failure label so retitling it does not defeat the dedupe, and let a transient listing failure fall through to filing rather than aborting. Lint W605 with ruff: pip's byte-compilation absorbs an invalid-escape warning before pytest can see it, so the linter is the only leg that can. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Opus 5 (1M context) --- .github/workflows/cifuzz.yml | 2 +- .github/workflows/main.yml | 11 +++++ .github/workflows/nightly.yml | 79 +++++++++++++++++++++++++++++++++++ CONTRIBUTING.rst | 4 ++ pyproject.toml | 5 +++ pytest.ini | 14 +++++++ tox.ini | 14 ++++++- 7 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/nightly.yml diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index 111913019..c32c5c9b7 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -28,7 +28,7 @@ jobs: path: ./out/artifacts - name: Upload Sarif if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v4 with: # Path to SARIF file relative to the root of the repository sarif_file: cifuzz-sarif/results.sarif diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cb397b116..0b7ef6939 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,6 +6,12 @@ on: pull_request: branches: - master + # Called by the Nightly workflow, which re-runs these same legs daily. The + # schedule lives there and not here because GitHub disables a workflow that + # has a schedule after 60 days without repository activity, and a disabled + # workflow answers none of its triggers, which would take pull request runs + # down with it. + workflow_call: jobs: build: name: Build @@ -20,6 +26,7 @@ jobs: toxenv: min - python-version: "3.10" toxenv: min-all + - python-version: "3.10" - python-version: "3.11" - python-version: "3.12" - python-version: "3.14" @@ -53,6 +60,10 @@ jobs: TZ=${{ matrix.timezone || 'UTC' }} echo "Running tests with timezone: $TZ" TZ=${{ matrix.timezone || 'UTC' }} tox -e ${{ matrix.toxenv || 'py' }} - name: Upload coverage.xml to codecov + # A nightly run, scheduled or dispatched, re-tests a commit that already + # has a coverage report, so uploading again only repeats the Codecov + # notification for that SHA. + if: github.event_name == 'push' || github.event_name == 'pull_request' uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000..2ba448bed --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,79 @@ +name: Nightly +on: + # Dependency, time zone database and calendar drift arrives with no commit + # behind it, and tests that only fail on the 31st of a month need a run on + # every date; a fixed weekday lands on a 31st about once a year. Off the hour, + # which GitHub names as a high-load window, and early in the UTC day so that + # the Pacific/Auckland and Pacific/Fiji legs stay on the same calendar date as + # the rest of the matrix. + schedule: + - cron: "17 4 * * *" + # Runs the same legs on demand, which is also the only way to exercise the + # reporting below before it is needed. GitHub disables this workflow after 60 + # days without repository activity; re-enable it from the Actions tab or with + # `gh workflow enable nightly.yml`, since a disabled workflow answers no + # triggers at all. + workflow_dispatch: + +# The called matrix only checks out and tests; the report job below asks for +# the one extra scope it needs. +permissions: + contents: read + +jobs: + build: + uses: ./.github/workflows/main.yml + secrets: inherit + + report: + name: Report failure + needs: build + # A push or pull request failure is already in front of whoever caused it. + # A failure with no commit behind it has nowhere to land but a single + # notification email to whoever last edited the schedule. A cancelled build + # counts, since a leg that hangs until GitHub kills it is a failure too. + # Master only, so that a dispatch on a work branch can neither file a + # report nor mask the real one. + if: >- + (needs.build.result == 'failure' || needs.build.result == 'cancelled') + && github.ref == 'refs/heads/master' + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: File a tracking issue unless one is already open + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + TITLE: The nightly build is failing + LABEL: nightly-failure + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SHA: ${{ github.sha }} + run: | + set -euo pipefail + # One issue per outage: a daily comment stream would only be one more + # signal to ignore. The label, not the title, identifies the tracking + # issue, so retitling it to record the cause does not defeat the + # check; --force makes creating the label idempotent. A transient + # listing failure must not stop the report, since a duplicate issue + # is cheaper than silence. + gh label create "$LABEL" --force --color d93f0b \ + --description "Filed by the Nightly workflow" + existing="$(gh issue list --state open --label "$LABEL" --limit 1 \ + --json number --jq '.[0].number // empty' || true)" + if [ -n "$existing" ]; then + echo "Already tracked in #$existing." + exit 0 + fi + gh issue create --title "$TITLE" --label "$LABEL" \ + --label "Type: Maintenance" --body-file - <