From 762079b24856c69e339add5179599c4670b76ab9 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 13 Oct 2026 15:56:44 -0400 Subject: [PATCH 1/2] ci(python): a failed KTLO leg exits 0 and reports a warning, so main stays green (#1672) Compat's KTLO legs carried job-level `continue-on-error`, which passes the run but concludes the leg's check run `failure`: `Compat Python 3.7 (KTLO)` turned every commit red. Every job holding a KTLO leg now works one way. * Each step that can fail on a KTLO leg has an `id` and step-level `continue-on-error: ${{ matrix.ktlo == true }}`, checkout included; Compat drops its job-level form. The job concludes `success`. * A step that needs an earlier one's work waits on its `outcome`: Compat and `pypcap-parity` wait on the install as `test` and `integration` do, and each `pypi` step on the one before it, so no KTLO leg builds or tests with the image's `python` after setup-python failed. * A new last step, `KTLO summary`, turns each failed `outcome` into a `::warning` and a job-summary line. A step's own annotations take the job's `ANNOTATION_LEVEL`, `warning` on a KTLO leg; engine-tests' 3.10 cell too. * `pypi`'s checkout and #1634's tag check still block every leg; Release status's KTLO notice stays. * Prose and pins follow; tests/project/test_ktlo_legs_stay_green_1672.py steps through each job leg by leg and runs the scripts. actionlint adds no finding; the touched tests/project modules and tests/test_tier_guard.py pass. --- .github/workflows/create-release.yml | 33 +- .github/workflows/python-compatibility.yml | 59 ++- .github/workflows/unit-tests.yml | 156 +++++- docs/source/contributing/releasing.rst | 4 +- docs/source/contributing/workflows.rst | 75 +-- pyproject.toml | 11 +- .../project/test_ktlo_legs_stay_green_1672.py | 448 ++++++++++++++++++ .../project/test_python_matrix_tiers_1633.py | 111 +++-- tests/project/test_unit_tests_queue.py | 2 +- 9 files changed, 792 insertions(+), 107 deletions(-) create mode 100644 tests/project/test_ktlo_legs_stay_green_1672.py diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 1788b3237..904582598 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -810,7 +810,11 @@ jobs: # and published but never blocking. Each step that can fail on one # carries `continue-on-error: ${{ matrix.ktlo == true }}`, the # step-level form, so ``needs.pypi.result`` cannot see it -- see the - # same choice in unit-tests.yml's ``test`` job. + # same choice in unit-tests.yml's ``test`` job. #1672: each step after + # setup-python waits on the ``outcome`` of the one it needs, so a KTLO + # leg stops at its first failure, where a supported leg stops, rather + # than building with the image's ``python`` or uploading nothing; and + # ``KTLO summary`` reports each failed step as a warning. include: - python-version: "3.10" ktlo: true @@ -852,6 +856,7 @@ jobs: - name: Install and Setup id: install + if: ${{ steps.setup.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} run: | set -x @@ -861,6 +866,7 @@ jobs: - name: Build Package id: build + if: ${{ steps.install.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} env: PCAPKIT_VERSION: ${{ needs.version_check.outputs.PCAPKIT_VERSION }} @@ -872,13 +878,14 @@ jobs: - name: No Source Distribution for Python ${{ matrix.python-version }} id: no_sdist - if: ${{ matrix.python-version != '3.14' }} + if: ${{ matrix.python-version != '3.14' && steps.build.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} run: | rm dist/*.tar.gz - name: Publish to PyPI id: publish + if: ${{ steps.build.outcome == 'success' && steps.no_sdist.outcome != 'failure' }} continue-on-error: ${{ matrix.ktlo == true }} uses: pypa/gh-action-pypi-publish@release/v1 with: @@ -891,6 +898,7 @@ jobs: ## 1.5.0b2 Python 3.12 leg *after* PyPI had already accepted the upload, ## which skipped the GitHub Release step below and lost that wheel. - name: Publish package to TestPyPI + if: ${{ steps.publish.outcome == 'success' }} continue-on-error: true uses: pypa/gh-action-pypi-publish@release/v1 with: @@ -917,6 +925,7 @@ jobs: # ``::error``, where it only warned. ``conda`` runs the same script. - name: Upload package to GitHub Release id: upload + if: ${{ steps.publish.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} shell: bash env: @@ -973,6 +982,26 @@ jobs: echo "ktlo_$(printf '%s' "$LEG" | tr -d '.')=$result" >> "$GITHUB_OUTPUT" echo "Python $LEG (KTLO, non-blocking): $result (step outcomes: $OUTCOMES)" + # #1672: the warning and job-summary line for each failed step, the same + # step as ends every KTLO leg in unit-tests.yml; see its ``test`` job. + - name: KTLO summary + if: ${{ !cancelled() && matrix.ktlo == true }} + continue-on-error: ${{ matrix.ktlo == true }} + shell: bash + env: + LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} + STEPS: ${{ toJSON(steps) }} + run: | + failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") + if [ -z "$failed" ]; then + echo "$LEG: no step failed." + exit 0 + fi + while IFS= read -r id; do + echo "::warning title=${id} failed on ${LEG//,/%2C}::KTLO legs never block (#1633), so this job still succeeds (#1672); see the ${id} step's log." + echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" + done <<< "$failed" + conda: name: Conda deployment (release) on ${{ matrix.os }} with Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} diff --git a/.github/workflows/python-compatibility.yml b/.github/workflows/python-compatibility.yml index 132409845..bc2ee1080 100644 --- a/.github/workflows/python-compatibility.yml +++ b/.github/workflows/python-compatibility.yml @@ -31,14 +31,15 @@ jobs: # # #1633: two tiers. The supported tier, 3.11-3.15, keeps the plain name, # which is what ruleset 23497679 requires one context per version of. The - # KTLO tier -- end-of-life versions that still run, 3.7-3.10 -- is suffixed - # ` (KTLO)`, so no required context can ever match one, and carries - # `continue-on-error`, so a failed leg shows red without failing the run. - # 3.7 is red by design: it cannot import pcapkit (see "Verify package"). 3.6 - # is left out: setup-python builds it only for retired Ubuntu images. + # KTLO tier, end-of-life 3.7-3.10, is suffixed ` (KTLO)`, so no required + # context can match one. Its steps carry `continue-on-error`, as in + # unit-tests.yml's `test` job: job-level, a failed leg's check run concluded + # `failure`, so every commit showed red (#1672). 3.7 fails "Verify package" + # by design. No 3.6: setup-python builds it only for retired Ubuntu images. name: Compat Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} runs-on: ${{ matrix.os || 'ubuntu-latest' }} - continue-on-error: ${{ matrix.ktlo == true }} + env: + ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }} strategy: fail-fast: false matrix: @@ -65,23 +66,37 @@ jobs: ktlo: true steps: - uses: actions/checkout@v7 + id: checkout + continue-on-error: ${{ matrix.ktlo == true }} - uses: actions/setup-python@v7 + id: python + continue-on-error: ${{ matrix.ktlo == true }} with: python-version: ${{ matrix.python-version }} + # The install needs this leg's interpreter, and the check below needs the + # install, as in unit-tests.yml's `test` job: on a KTLO leg a failed step + # no longer stops the job, so each one waits on the `outcome` it needs. - name: Install package + id: install + if: ${{ steps.python.outcome == 'success' }} + continue-on-error: ${{ matrix.ktlo == true }} run: | python -m pip install -U pip setuptools wheel python -m pip install -e . - # One annotation saying why, so a red leg -- 3.7's, today, where the + # One annotation saying why, so a failed leg -- 3.7's, today, where the # editable install succeeds and 41 modules do not compile -- is legible - # from the run summary without opening the log. Each title escapes `,` as - # `%2C`: the runner splits a command's properties on `,`, so the KTLO - # leg's title would otherwise stop at "(KTLO" (actions/runner + # from the run summary without opening the log. ANNOTATION_LEVEL makes it + # a warning on a KTLO leg (#1672). Each title escapes `,` as `%2C`: the + # runner splits a command's properties on `,`, so the KTLO leg's title + # would otherwise stop at "(KTLO" (actions/runner # src/Runner.Common/ActionCommand.cs:100, at 5f0b3ddcc). - name: Verify package + id: verify + if: ${{ steps.install.outcome == 'success' }} + continue-on-error: ${{ matrix.ktlo == true }} shell: bash env: LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} @@ -91,10 +106,30 @@ jobs: cat "$log" count=$(awk '/^\*\*\* Error compiling /{n++} END{print n+0}' "$log") first=$(awk '/^ File "/{sub(/^ /, ""); print; exit}' "$log") - echo "::error title=pcapkit does not compile on ${LEG//,/%2C}::$count modules fail to compile, the first at ${first:-an unreported place}; see this step's log." + echo "::${ANNOTATION_LEVEL} title=pcapkit does not compile on ${LEG//,/%2C}::$count modules fail to compile, the first at ${first:-an unreported place}; see this step's log." exit 1 fi if ! python -c 'import pcapkit; print(pcapkit.__version__)'; then - echo "::error title=pcapkit does not import on ${LEG//,/%2C}::See the traceback in this step's log." + echo "::${ANNOTATION_LEVEL} title=pcapkit does not import on ${LEG//,/%2C}::See the traceback in this step's log." exit 1 fi + + # #1672: the KTLO leg's report -- the same step ends every KTLO matrix + # job; see unit-tests.yml's `test` job for how it works. + - name: KTLO summary + if: ${{ !cancelled() && matrix.ktlo == true }} + continue-on-error: ${{ matrix.ktlo == true }} + shell: bash + env: + LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} + STEPS: ${{ toJSON(steps) }} + run: | + failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") + if [ -z "$failed" ]; then + echo "$LEG: no step failed." + exit 0 + fi + while IFS= read -r id; do + echo "::warning title=${id} failed on ${LEG//,/%2C}::KTLO legs never block (#1633), so this job still succeeds (#1672); see the ${id} step's log." + echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" + done <<< "$failed" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 91e192f72..9aa6c478b 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -52,6 +52,10 @@ jobs: needs: [changes] runs-on: ${{ matrix.os || 'ubuntu-latest' }} timeout-minutes: 45 + # #1672: the level of every annotation a failing step writes -- a warning on + # a KTLO leg, which must not show red, an error on a supported one. + env: + ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }} strategy: fail-fast: false matrix: @@ -76,8 +80,9 @@ jobs: # `needs.test.result`, so a KTLO leg cannot fail the job: each step # that can fail on one carries `continue-on-error: ${{ matrix.ktlo # == true }}` instead -- the step-level form, whose effect on the job - # is documented, as for engine-tests' KTLO cell. A failure still shows - # as a red step and an error annotation. 3.8 and 3.9 need + # is documented, as for engine-tests' KTLO cell. The leg concludes + # success, and the job's last step, `KTLO summary`, reports each + # failed step as a warning (#1672). 3.8 and 3.9 need # ubuntu-24.04: setup-python has no Ubuntu 26.04 build of either, and # `ubuntu-latest` moves to 26.04 from 2026-10-19. # @@ -103,6 +108,8 @@ jobs: steps: - uses: actions/checkout@v7 + id: checkout + continue-on-error: ${{ matrix.ktlo == true }} - uses: actions/setup-python@v7 id: python @@ -147,7 +154,8 @@ jobs: # needs `steps.install.outcome == 'success'`, so it is skipped rather # than failing against a missing package. The install needs its own # interpreter: if setup-python failed, the image's `python` would be - # tested under this leg's name. On a supported leg either fails the job. + # tested under this leg's name. On a supported leg either fails the job; + # on a KTLO leg the annotation is a warning (#1672). - name: Install package and test dependencies id: install if: ${{ steps.python.outcome == 'success' }} @@ -160,7 +168,7 @@ jobs: if ! python -m pip install -e '.[test,DPKT,crypto,NGAP]' 2>&1 | tee "$RUNNER_TEMP/pip-install.log"; then reason=$(sed -n 's/^ERROR: //p' "$RUNNER_TEMP/pip-install.log" | tail -n 1) if [ -z "$reason" ]; then reason="pip failed without an ERROR line; see this step's log"; fi - echo "::error title=pcapkit did not install on ${LEG//,/%2C}::${reason//%/%25}. The rest of this leg is skipped." + echo "::${ANNOTATION_LEVEL} title=pcapkit did not install on ${LEG//,/%2C}::${reason//%/%25}. The rest of this leg is skipped." exit 1 fi @@ -174,6 +182,7 @@ jobs: # the log says what that was -- print it so the number `-n auto` picked # is a measured fact instead of an assumption about runner size. - name: Report available parallelism + id: parallelism if: ${{ steps.install.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} run: | @@ -335,12 +344,46 @@ jobs: path: ${{ runner.temp }}/junit.xml retention-days: 14 + # #1672: the KTLO leg's report, and the last step of every KTLO matrix + # job, in one script. Each step above that can fail on a KTLO leg has an + # `id` and `continue-on-error: ${{ matrix.ktlo == true }}`, so its failure + # leaves the job, and `required-checks`, at success. This step makes the + # failure visible again: one warning and one job-summary line per failed + # step. `toJSON(steps)` holds each step above with an `id`, and its + # `outcome` is the result before `continue-on-error` applied. jq is on + # every Ubuntu runner image. A step masked on every leg, as the timings + # upload is, is not KTLO's to report. Not removable here: the error the + # runner itself records for a failed `run:` step, `Process completed with + # exit code N.`, whatever its `continue-on-error` (actions/runner + # src/Runner.Worker/Handlers/ScriptHandler.cs:354, at 5f0b3ddcc), and an + # action's own error. Neither changes the job's conclusion. + - name: KTLO summary + if: ${{ !cancelled() && matrix.ktlo == true }} + continue-on-error: ${{ matrix.ktlo == true }} + shell: bash + env: + LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} + STEPS: ${{ toJSON(steps) }} + run: | + failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") + if [ -z "$failed" ]; then + echo "$LEG: no step failed." + exit 0 + fi + while IFS= read -r id; do + echo "::warning title=${id} failed on ${LEG//,/%2C}::KTLO legs never block (#1633), so this job still succeeds (#1672); see the ${id} step's log." + echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" + done <<< "$failed" + integration: name: Integration Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }} needs: [changes] runs-on: ${{ matrix.os || 'ubuntu-latest' }} timeout-minutes: 45 + # See the `test` job above. + env: + ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }} strategy: fail-fast: false matrix: @@ -366,6 +409,8 @@ jobs: steps: - uses: actions/checkout@v7 + id: checkout + continue-on-error: ${{ matrix.ktlo == true }} - uses: actions/setup-python@v7 id: python @@ -410,13 +455,14 @@ jobs: if ! python -m pip install -e '.[test,Scapy,DPKT,cli]' 2>&1 | tee "$RUNNER_TEMP/pip-install.log"; then reason=$(sed -n 's/^ERROR: //p' "$RUNNER_TEMP/pip-install.log" | tail -n 1) if [ -z "$reason" ]; then reason="pip failed without an ERROR line; see this step's log"; fi - echo "::error title=pcapkit did not install on ${LEG//,/%2C}::${reason//%/%25}. The rest of this leg is skipped." + echo "::${ANNOTATION_LEVEL} title=pcapkit did not install on ${LEG//,/%2C}::${reason//%/%25}. The rest of this leg is skipped." exit 1 fi # `shell: bash` rather than the default, because it runs with -o pipefail: # without it the `tee` below would swallow the generator's exit status. - name: Regenerate sample captures + id: samples if: ${{ steps.install.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} shell: bash @@ -425,7 +471,7 @@ jobs: # be built, so no test has run yet and nothing is wrong with the code # under test. if ! python examples/generators/make_samples.py 2>&1 | tee "$RUNNER_TEMP/make-samples.log"; then - echo "::error title=Sample fixture generation failed::examples/generators/make_samples.py could not rebuild examples/captures/. This is a fixture-generation failure, not a test failure -- the test suite has not run." + echo "::${ANNOTATION_LEVEL} title=Sample fixture generation failed::examples/generators/make_samples.py could not rebuild examples/captures/. This is a fixture-generation failure, not a test failure -- the test suite has not run." exit 1 fi @@ -443,6 +489,7 @@ jobs: # See the `test` job above for why this step exists. - name: Report available parallelism + id: parallelism if: ${{ steps.install.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} run: | @@ -467,16 +514,17 @@ jobs: # say so. `--dist load` is pinned explicitly; see the `test` job above # for why. - name: Run full test suite + id: suite if: ${{ steps.install.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} shell: bash run: | if ! selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths()))"); then - echo "::error title=Could not compute the fixture-tier selection::tests._tiers.fixture_tier_paths() failed -- see the traceback above. Refusing to fall back to a bare 'pytest -q', which would silently re-run the entire suite instead of failing loudly." + echo "::${ANNOTATION_LEVEL} title=Could not compute the fixture-tier selection::tests._tiers.fixture_tier_paths() failed -- see the traceback above. Refusing to fall back to a bare 'pytest -q', which would silently re-run the entire suite instead of failing loudly." exit 1 fi if [ -z "$selection" ]; then - echo "::error title=Fixture-tier selection is empty::tests._tiers.fixture_tier_paths() returned nothing, which cannot be right -- tests/integration alone should always be part of it. Refusing to run pytest with no arguments." + echo "::${ANNOTATION_LEVEL} title=Fixture-tier selection is empty::tests._tiers.fixture_tier_paths() returned nothing, which cannot be right -- tests/integration alone should always be part of it. Refusing to run pytest with no arguments." exit 1 fi echo "Fixture-dependent selection: $selection" @@ -493,6 +541,25 @@ jobs: path: ${{ runner.temp }}/junit.xml retention-days: 14 + # #1672: see the `test` job's step of the same name. + - name: KTLO summary + if: ${{ !cancelled() && matrix.ktlo == true }} + continue-on-error: ${{ matrix.ktlo == true }} + shell: bash + env: + LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} + STEPS: ${{ toJSON(steps) }} + run: | + failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") + if [ -z "$failed" ]; then + echo "$LEG: no step failed." + exit 0 + fi + while IFS= read -r id; do + echo "::warning title=${id} failed on ${LEG//,/%2C}::KTLO legs never block (#1633), so this job still succeeds (#1672); see the ${id} step's log." + echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" + done <<< "$failed" + # Per-engine coverage for the third-party capture engines (#845): every # (Python version, engine) pair is checked, and a failure -- or a correctly # asserted decline -- names the one engine and interpreter responsible. Each @@ -546,8 +613,8 @@ jobs: # share one step, and any one of them failing fails the job. 3.10 is KTLO: # end-of-life, still tested on every run, never blocking. It is its own step # with step-level `continue-on-error: true` -- the treatment #845 gave the - # then-prerelease 3.15 cell -- so a 3.10 failure shows as a failed step and an - # error annotation while the job, and `required-checks`, which reads its + # then-prerelease 3.15 cell -- so a 3.10 failure is reported as warnings and a + # job-summary row (#1672) while the job, and `required-checks`, which reads its # result, stay green. 3.7-3.9 are not cells here: setup-python has no # Ubuntu 26.04 build of any of them, and this job installs every interpreter # on `ubuntu-latest`; PCAP_CT's extra is also `>= '3.10'`. @@ -824,7 +891,7 @@ jobs: PyPCAPFile) EXTRA=PyPCAPFile; RAW_PKGS=(pypcapfile); IMPORT_NAME=pcapfile ;; PyPCAP) EXTRA=PyPCAP; RAW_PKGS=(pypcap); IMPORT_NAME=pcap ;; PCAP_CT) EXTRA=PCAP_CT; RAW_PKGS=(pcap-ct libpcap); IMPORT_NAME=pcap ;; - *) echo "::error::unknown engine $ENGINE"; exit 1 ;; + *) echo "::${ANNOTATION_LEVEL}::unknown engine $ENGINE"; exit 1 ;; esac set +e @@ -838,11 +905,11 @@ jobs: echo "$output" if [ "$status" -ne 0 ] && [ "$EXPECT" != "not-installable" ]; then - echo "::error title=$ENGINE failed to install on Python $PY::This cell expects '$EXPECT', which requires a successful install. pip exited $status." + echo "::${ANNOTATION_LEVEL} title=$ENGINE failed to install on Python $PY::This cell expects '$EXPECT', which requires a successful install. pip exited $status." exit 1 fi if [ "$status" -eq 0 ] && [ "$EXPECT" = "not-installable" ]; then - echo "::error title=$ENGINE now installs on Python $PY::This cell is recorded as not-installable in this job's expect case (and in pyproject.toml's marker for $ENGINE). It installed cleanly -- update both rather than leaving this green for the wrong reason." + echo "::${ANNOTATION_LEVEL} title=$ENGINE now installs on Python $PY::This cell is recorded as not-installable in this job's expect case (and in pyproject.toml's marker for $ENGINE). It installed cleanly -- update both rather than leaving this green for the wrong reason." exit 1 fi if [ "$status" -ne 0 ]; then @@ -857,7 +924,7 @@ jobs: 'read timed out|connection (reset|refused|aborted)|temporary failure in name resolution|max retries exceeded|could not fetch url|newconnectionerror|readtimeouterror|httpsconnectionpool|retrying \(retry\(|http error (5[0-9][0-9]|429)|status code 429|too many 429 error responses|no matching distribution found' \ && ! echo "$output" | grep -qiE \ 'failed building wheel|error: command .* failed with exit code|subprocess-exited-with-error'; then - echo "::error title=$ENGINE's install failure on Python $PY looks transient%2C not a compile failure::pip exited $status, but the output matches a network/index failure pattern rather than the C-API compile failure this cell records as expected. Re-run the job rather than trusting this as a genuine 'not-installable' result." + echo "::${ANNOTATION_LEVEL} title=$ENGINE's install failure on Python $PY looks transient%2C not a compile failure::pip exited $status, but the output matches a network/index failure pattern rather than the C-API compile failure this cell records as expected. Re-run the job rather than trusting this as a genuine 'not-installable' result." exit 1 fi echo "::notice title=$ENGINE is not installable on Python $PY (expected)::pip exited $status, matching this cell's recorded expectation ('not-installable'), and the output does not match a transient-failure pattern. No tests run for this cell -- this is a deliberate skip, not a pass." @@ -869,7 +936,7 @@ jobs: # still exits 0 and every test would then skip, so require the # module this cell exists to exercise. if ! python -c "import $IMPORT_NAME" 2>/dev/null; then - echo "::error title=$ENGINE installed but $IMPORT_NAME does not import on Python $PY::pip exited 0 but the module this cell exists to exercise is not importable -- a green install that unlocked nothing." + echo "::${ANNOTATION_LEVEL} title=$ENGINE installed but $IMPORT_NAME does not import on Python $PY::pip exited 0 but the module this cell exists to exercise is not importable -- a green install that unlocked nothing." exit 1 fi @@ -883,7 +950,7 @@ jobs: echo "Running: $TEST_PATHS" # shellcheck disable=SC2086 if ! python -m pytest -q -n auto --dist load $TEST_PATHS; then - echo "::error title=$ENGINE tests failed on Python $PY::See the 'Python $PY ($ENGINE)' group in this step's log." + echo "::${ANNOTATION_LEVEL} title=$ENGINE tests failed on Python $PY::See the 'Python $PY ($ENGINE)' group in this step's log." exit 1 fi echo "| $PY | $ENGINE | $EXPECT | passed |" >> "$GITHUB_STEP_SUMMARY" @@ -902,9 +969,12 @@ jobs: python -c "import os; print('cpu_count', os.cpu_count())" # Every cell runs even after one fails, so one red run reports all of - # them; the step fails at the end if any did. + # them; the step fails at the end if any did. ANNOTATION_LEVEL is the + # level of the cell script's annotations (#1672). - name: Run the 3.11-3.15 cells shell: bash + env: + ANNOTATION_LEVEL: error run: | failed=() for py in 3.11 3.12 3.13 3.14 3.15; do @@ -921,16 +991,21 @@ jobs: fi # KTLO, non-blocking (#1633) -- see this job's comment. Runs on every event, - # as the 3.10 cell did while it was supported. + # as the 3.10 cell did while it was supported. #1672: this step is the + # KTLO cell's report, as `KTLO summary` is a KTLO leg's in the matrix jobs. + # A failure is a job-summary row and warnings, the cell's own included, + # and an interpreter the setup step above could not install fails here. - name: Run the 3.10 cell (KTLO, non-blocking) shell: bash continue-on-error: true + env: + ANNOTATION_LEVEL: warning run: | echo "::group::Python 3.10 ($ENGINE)" if ! bash "$RUNNER_TEMP/engine-cell.sh" 3.10; then echo "| 3.10 | $ENGINE | | FAILED (KTLO, non-blocking) |" >> "$GITHUB_STEP_SUMMARY" echo "::endgroup::" - echo "::error title=$ENGINE failed on Python 3.10 (KTLO%2C non-blocking)::3.10 is end-of-life and KTLO per #1633; this does not fail the job." + echo "::warning title=$ENGINE failed on Python 3.10 (KTLO%2C non-blocking)::3.10 is end-of-life and KTLO per #1633; this does not fail the job." exit 1 fi echo "::endgroup::" @@ -980,6 +1055,8 @@ jobs: # or the other's save fails with `Unable to reserve cache`. The supported # leg, so the cache does not depend on a leg that may not get that far. APT_CACHE_SAVE: ${{ matrix.python-version == '3.11' }} + # See the `test` job above. + ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }} strategy: fail-fast: false matrix: @@ -992,8 +1069,11 @@ jobs: steps: - uses: actions/checkout@v7 + id: checkout + continue-on-error: ${{ matrix.ktlo == true }} - uses: actions/setup-python@v7 + id: python continue-on-error: ${{ matrix.ktlo == true }} with: python-version: ${{ matrix.python-version }} @@ -1060,6 +1140,7 @@ jobs: restore-keys: ${{ steps.apt-plan.outputs.restore-prefix }} - name: Seed apt's archive with the cached .deb files + id: apt-seed if: steps.apt-cache.outputs.cache-matched-key != '' continue-on-error: ${{ matrix.ktlo == true }} run: | @@ -1068,6 +1149,7 @@ jobs: echo "Seeded $(find "$RUNNER_TEMP/apt-debs" -name '*.deb' | wc -l) .deb files" - name: Install libpcap headers, a C toolchain, and tshark + id: apt-install continue-on-error: ${{ matrix.ktlo == true }} timeout-minutes: 10 run: | @@ -1119,7 +1201,12 @@ jobs: path: ${{ runner.temp }}/apt-debs/*.deb key: ${{ steps.apt-plan.outputs.key }} + # As in the `test` job (#1672): the install needs this leg's interpreter, + # and the steps after it need the install, so a KTLO leg whose setup or + # install failed stops there instead of testing the image's `python`. - name: Install package, test and generator dependencies + id: install + if: ${{ steps.python.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} run: | python -m pip install -U pip setuptools wheel @@ -1128,11 +1215,13 @@ jobs: # See the `integration` job above for why this step is shaped the way it # is. - name: Regenerate sample captures + id: samples + if: ${{ steps.install.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} shell: bash run: | if ! python examples/generators/make_samples.py 2>&1 | tee "$RUNNER_TEMP/make-samples.log"; then - echo "::error title=Sample fixture generation failed::examples/generators/make_samples.py could not rebuild examples/captures/. This is a fixture-generation failure, not a test failure -- the test suite has not run." + echo "::${ANNOTATION_LEVEL} title=Sample fixture generation failed::examples/generators/make_samples.py could not rebuild examples/captures/. This is a fixture-generation failure, not a test failure -- the test suite has not run." exit 1 fi @@ -1145,6 +1234,8 @@ jobs: # See the `test` job above for why this step exists. - name: Report available parallelism + id: parallelism + if: ${{ steps.install.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} run: | nproc @@ -1153,20 +1244,41 @@ jobs: # See the `integration` job above for why this is asked of # tests/_tiers.py directly rather than spelled out as literal flags. - name: Run full test suite + id: suite + if: ${{ steps.install.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} shell: bash run: | if ! selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths()))"); then - echo "::error title=Could not compute the fixture-tier selection::tests._tiers.fixture_tier_paths() failed -- see the traceback above. Refusing to fall back to a bare 'pytest -q', which would silently re-run the entire suite instead of failing loudly." + echo "::${ANNOTATION_LEVEL} title=Could not compute the fixture-tier selection::tests._tiers.fixture_tier_paths() failed -- see the traceback above. Refusing to fall back to a bare 'pytest -q', which would silently re-run the entire suite instead of failing loudly." exit 1 fi if [ -z "$selection" ]; then - echo "::error title=Fixture-tier selection is empty::tests._tiers.fixture_tier_paths() returned nothing, which cannot be right -- tests/integration alone should always be part of it. Refusing to run pytest with no arguments." + echo "::${ANNOTATION_LEVEL} title=Fixture-tier selection is empty::tests._tiers.fixture_tier_paths() returned nothing, which cannot be right -- tests/integration alone should always be part of it. Refusing to run pytest with no arguments." exit 1 fi echo "Fixture-dependent selection: $selection" python -m pytest -q -n auto --dist load $selection + # #1672: see the `test` job's step of the same name. + - name: KTLO summary + if: ${{ !cancelled() && matrix.ktlo == true }} + continue-on-error: ${{ matrix.ktlo == true }} + shell: bash + env: + LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} + STEPS: ${{ toJSON(steps) }} + run: | + failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") + if [ -z "$failed" ]; then + echo "$LEG: no step failed." + exit 0 + fi + while IFS= read -r id; do + echo "::warning title=${id} failed on ${LEG//,/%2C}::KTLO legs never block (#1633), so this job still succeeds (#1672); see the ${id} step's log." + echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" + done <<< "$failed" + # GitHub issue #981: two test modules # (tests/protocols/application/test_http_unit.py and # tests/test_base_class_contract.py) each pass alone, and pytest -- the diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index 4de9bb7f0..6941b8af4 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -46,7 +46,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:401-415``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1533-1602``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1645-1714``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:434,619,784,997``), so a @@ -56,7 +56,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Two other checks run unconditionally on the same path, though neither is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1393-1414``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1505-1526``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 8300cee68..a2f5a4180 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -274,7 +274,7 @@ and ``project-tests``, and ``required-checks`` (see `Required Status Checks`_ below), gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1381-1387``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1493-1499``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -542,41 +542,41 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:1694`` + - ``unit-tests.yml:1806`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - - ``python-compatibility.yml:39,46`` + - ``python-compatibility.yml:39,47`` * - ``Compat Python 3.12`` - job ``compatibility``, matrix leg ``3.12`` - - ``python-compatibility.yml:39,47`` + - ``python-compatibility.yml:39,48`` * - ``Compat Python 3.13`` - job ``compatibility``, matrix leg ``3.13`` - - ``python-compatibility.yml:39,48`` + - ``python-compatibility.yml:39,49`` * - ``Compat Python 3.14`` - job ``compatibility``, matrix leg ``3.14`` - - ``python-compatibility.yml:39,49`` + - ``python-compatibility.yml:39,50`` * - ``Compat Python 3.15`` - job ``compatibility``, matrix leg ``3.15`` - - ``python-compatibility.yml:39,50`` + - ``python-compatibility.yml:39,51`` That is the list after `#1633 `__, which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a version between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:1694``). ``Compat Python`` is defined by one, the +(``unit-tests.yml:1806``). ``Compat Python`` is defined by one, the ``compatibility`` job in ``python-compatibility.yml`` (``:39``), expanding to -the five required legs at ``:46-50`` and to four KTLO legs at ``:55-65`` whose +the five required legs at ``:47-51`` and to four KTLO legs at ``:56-66`` whose names end in ``(KTLO)``, so no required context matches one. A comment at -``unit-tests.yml:1605`` mentions the string without defining it. +``unit-tests.yml:1717`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the five ``Compat Python 3.11``-``3.15`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1604-1614``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1716-1726``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``changes`` and ``project-tests`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:1693-1747``). Rulesets match check names literally, with no +(``unit-tests.yml:1805-1859``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the four legs as ``skipped`` only on a docs-only pull request, where @@ -585,32 +585,43 @@ instead. The ``Compat`` legs stay required separately because they come from ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1696``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1808``), so it is never produced -- and never expected -- on those paths. Python versions come in two tiers (#1633). The **supported** tier, 3.11-3.15, blocks: a failed leg fails ``Required checks passed`` or its own required ``Compat`` context. The **KTLO** tier -- end-of-life 3.7-3.10, still built and -tested -- never blocks and is never a required context. Each KTLO leg sits in the -same job as the supported ones, so the two mechanisms differ by whether something -aggregates the job: - -* ``python-compatibility.yml`` has no aggregate, so a KTLO leg carries job-level - ``continue-on-error`` (``:41``): it shows red, and the run stays green. -* In ``unit-tests.yml`` ``required-checks`` reads one ``needs..result`` per job, - and GitHub does not document what that reads when a job-level - ``continue-on-error`` leg fails. So every step that can fail on a KTLO leg carries - ``continue-on-error: ${{ matrix.ktlo == true }}`` instead, whose effect is - documented (``unit-tests.yml:74-82``); ``engine-tests`` runs its 3.10 cell as a - step of its own with ``continue-on-error: true``. +tested -- never blocks, is never a required context, and never shows a commit or a +pull request red (#1672). Each KTLO leg shares its job with supported legs, in +``unit-tests.yml``, ``python-compatibility.yml`` and ``create-release.yml``, and +every such job works the same way: + +* Every step that can fail on a KTLO leg has an ``id`` and + ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:78-87``), + unless it is masked on every leg, as the timings uploads are. Its ``outcome`` is + then ``failure`` but its ``conclusion`` ``success``, so the job, its check run + and ``required-checks`` read ``success``. A step that needs an earlier one's work + waits on that step's ``outcome``: a leg that cannot install stops there, and a + ``pypi`` leg stops at its first failure. +* The last step, ``KTLO summary``, runs on KTLO legs only and reports each failed + ``outcome`` as a warning and a job-summary line. A step's own annotations take + ``ANNOTATION_LEVEL``: ``warning`` on a KTLO leg, ``error`` on a supported one. + ``engine-tests``' 3.10 cell, a step of its own with ``continue-on-error: true``, + reports the same way. +* Never job-level ``continue-on-error``: it passes the *run*, but the leg's check + run still concludes ``failure``, which showed every commit red while + ``Compat Python 3.7 (KTLO)`` carried it. + +Two errors remain on a failed KTLO step, neither changing a conclusion: the +runner's ``Process completed with exit code N.`` for a ``run:`` step, and an +action's own. And #1634's tag check still blocks the ``pypi`` KTLO legs: it fails +every leg alike, and masked, the leg would go on to publish. 3.7 can neither install the ``test`` extra nor import the package, so each 3.7 leg -fails one step, with one error annotation saying why: ``test`` and ``integration`` -at the install step, the ``Compat`` leg at ``Verify package``. Only the ``Compat`` -leg shows red, through its job-level ``continue-on-error``; the ``test`` and -``integration`` legs conclude ``success``, their failed step and annotation inside. -They run on ``ubuntu-22.04``, which retires on 2027-04-17 after brownouts from -2027-03-23 (actions/runner-images#14254). A brownout fails a ``test`` or +fails one step, with a warning saying why -- ``test`` and ``integration`` at the +install step, the ``Compat`` leg at ``Verify package`` -- and concludes +``success``. They run on ``ubuntu-22.04``, which retires on 2027-04-17 after +brownouts from 2027-03-23 (actions/runner-images#14254). A brownout fails a ``test`` or ``integration`` leg before any step runs, where no ``continue-on-error`` reaches, and so fails ``Required checks passed``: they must go before then (#1660). 3.6 is in no matrix: setup-python builds it only for retired Ubuntu images. Nor are @@ -628,7 +639,7 @@ Four jobs, all in ``create-release.yml``, declare a deployment * ``github`` -- ``environment: github-release`` (``create-release.yml:431``) * ``tag`` -- ``environment: conda-tag`` (``create-release.yml:617``) * ``pypi`` -- ``environment: pypi`` (``create-release.yml:779``) -* ``conda`` -- ``environment: anaconda`` (``create-release.yml:994``) +* ``conda`` -- ``environment: anaconda`` (``create-release.yml:1023``) No other workflow in scope here declares one -- including ``deploy-pages.yml``, whose ``deploy-pages`` job pushes straight to the diff --git a/pyproject.toml b/pyproject.toml index 7354cb560..a33750627 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,8 +46,9 @@ license = { text="BSD 3-Clause License" } # # Independently, CI has two tiers (#1633). **3.11 through 3.15** are supported and # blocking. **3.7 through 3.10** are KTLO: end-of-life, still built and tested, never -# blocking. Each 3.7 leg fails a step by design: the import fails as above, and the -# ``test`` extra's ``pytest>=8`` needs 3.8. 3.6 is not run at all: setup-python +# blocking, and a failure there is a warning on a green leg (#1672). Each 3.7 leg +# fails a step by design: the import fails as above, and the ``test`` extra's +# ``pytest>=8`` needs 3.8. 3.6 is not run at all: setup-python # builds it only for retired Ubuntu images, and the build needs ``setuptools>=61`` # besides. The ``PCAP_CT`` extra needs 3.10 regardless. requires-python = ">=3.6, <4" @@ -67,9 +68,9 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python :: 3", # The supported tier only (#1633). A version classifier is a claim of - # support, which the KTLO tier (3.7-3.10) is by definition not: it may stay - # red without blocking anything. ``requires-python`` above is what keeps - # those versions installable. + # support, which the KTLO tier (3.7-3.10) is by definition not: its tests + # may fail, as a warning that blocks nothing. ``requires-python`` above is + # what keeps those versions installable. "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", diff --git a/tests/project/test_ktlo_legs_stay_green_1672.py b/tests/project/test_ktlo_legs_stay_green_1672.py new file mode 100644 index 000000000..45e6096c0 --- /dev/null +++ b/tests/project/test_ktlo_legs_stay_green_1672.py @@ -0,0 +1,448 @@ +# -*- coding: utf-8 -*- +"""#1672: a failed KTLO leg concludes ``success`` and reports a warning. + +The owner's ruling: a KTLO failure must not turn ``main`` or a pull request red. +``Compat Python 3.7 (KTLO)`` carried job-level ``continue-on-error``, which keeps +the *run* from failing but still concludes the job's own check run ``failure`` -- +check run 115490362417, the one failing check of 4673cffa9 -- and the KTLO steps +of every other job wrote ``::error`` annotations. + +Every job holding a KTLO leg now works one way: each step that can fail on the +leg carries ``continue-on-error: ${{ matrix.ktlo == true }}`` and an ``id``; a +step that needs an earlier one waits on that step's ``outcome``; the last step, +``KTLO summary``, reports each failed ``outcome`` as a warning and a job-summary +line; and a step's own annotations take the job's ``ANNOTATION_LEVEL``. + +:func:`run_leg` steps through a job the way the runner does -- the implicit +``success()`` GitHub prepends to an ``if:`` without a status function, and a +``continue-on-error`` step's failed ``outcome`` concluding ``success`` -- so each +step's failure is tried on every leg instead of read off the text. Expressions go +through :mod:`tests.project.test_release_gates`'s evaluator, and the scripts that +write annotations are executed. + +""" + +from __future__ import annotations + +import functools +import json +import os +import pathlib +import re +import shutil +import subprocess +import tempfile +import unittest +from typing import TYPE_CHECKING, NamedTuple + +from tests.project.test_python_matrix_tiers_1633 import (ANNOTATION_LEVEL, KTLO_GUARD, KTLO_LEGS, + LEG_37, TITLE_37, is_ktlo, job, legs, + titles, workflow, workflow_commands) +from tests.project.test_release_gates import NULL, STATUS, evaluate_condition, evaluate_expression + +if TYPE_CHECKING: + from typing import Any, Optional + +#: Every job holding a KTLO matrix leg, as (workflow, job). +KTLO_JOBS = tuple((name, job_id) for name, job_id, _ in KTLO_LEGS) +#: The jobs whose steps write annotations of their own on a KTLO leg, so set the level. +LEVELLED = (('unit-tests.yml', 'test'), ('unit-tests.yml', 'integration'), + ('unit-tests.yml', 'pypcap-parity'), ('python-compatibility.yml', 'compatibility')) +#: Steps that stop every leg alike, KTLO or not, by job. #1634's: a leg that cannot +#: show its tree is the tag's must not go on to publish, as +#: ``test_release_tags_the_run_sha_1634.TestPublishersCheckTheirTags`` pins. +BLOCKS_EVERY_LEG = { + ('create-release.yml', 'pypi'): ('Checkout code', "Check the release tags name this run's commit"), +} +#: Steps whose own ``::error`` a KTLO leg cannot reach, with the ``if:`` that makes it +#: so. ``pypi``'s upload is one script with ``conda``'s (#1634), and runs only once PyPI +#: took this ``dist/``, so its "Nothing to upload" needs a supported leg's failure. +UNREACHABLE_ERRORS = { + ('create-release.yml', 'pypi', 'Upload package to GitHub Release'): "${{ steps.publish.outcome == 'success' }}", +} +#: Steps masked on every leg by their own design, ``continue-on-error: true``: a +#: diagnostic upload (#1538), and TestPyPI, which must not gate the real release. +MASKED_ON_EVERY_LEG = { + ('unit-tests.yml', 'test'): ('Upload test timings',), + ('unit-tests.yml', 'integration'): ('Upload test timings',), + ('create-release.yml', 'pypi'): ('Publish package to TestPyPI',), +} +#: ``engine-tests``'s KTLO steps: the 3.10 interpreter, and the 3.10 cell. +ENGINE_KTLO = ('Run the 3.10 cell (KTLO, non-blocking)',) +#: The workflow commands a step a KTLO leg runs may write. +KTLO_COMMANDS = frozenset({'${ANNOTATION_LEVEL}', 'warning', 'notice', 'group', 'endgroup'}) +#: A workflow command a script writes, by its name. +COMMAND = re.compile(r'''(?:echo|printf|print\(f?)\s*["']::([^\s:"']+)''') +#: A status-check function: an ``if:`` holding none gets ``success() &&`` prepended. +STATUS_FUNCTION = re.compile(r'\b(?:success|failure|cancelled|always)\(\)') +#: The outputs a step with an ``id`` is taken to set: ones that let every later step run. +OUTPUTS = { + 'apt-plan': {'key': 'apt-debs-ImageOS-X64-set-0123456789abcdef', 'restore-prefix': 'apt-debs-ImageOS-X64-set-'}, + 'apt-cache': {'cache-matched-key': 'apt-debs-ImageOS-X64-set-0123456789abcdef', 'cache-hit': 'false'}, +} +SUMMARY = 'KTLO summary' +SUMMARY_IF = '${{ !cancelled() && matrix.ktlo == true }}' +SUMMARY_ENV = {'LEG': "Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}", + 'STEPS': '${{ toJSON(steps) }}'} + + +class Step(NamedTuple): + """One step of a :func:`run_leg` run.""" + + label: str + outcome: str + conclusion: str + + +def label(step: 'dict[str, Any]') -> 'str': + return step.get('name') or step.get('uses') + + +def summary_step(spec: 'dict[str, Any]') -> 'dict[str, Any]': + """The job's ``KTLO summary`` step.""" + found = [step for step in spec['steps'] if step.get('name') == SUMMARY] + if len(found) != 1: + raise AssertionError(f'expected one {SUMMARY!r} step, found {len(found)}') + return found[0] + + +def is_engine_ktlo(step: 'dict[str, Any]') -> 'bool': + """Whether ``step`` is one of ``engine-tests``'s, which run only the KTLO interpreter.""" + return label(step) in ENGINE_KTLO or str(step.get('with', {}).get('python-version')) == '3.10' + + +def stringify(value: 'object') -> 'str': + """``value`` as an ``env:`` entry holds it.""" + if value is NULL or value is None: + return '' + if isinstance(value, bool): + return 'true' if value else 'false' + return str(value) + + +@functools.lru_cache(maxsize=None) +def workflow_env(name: 'str') -> 'tuple[tuple[str, Any], ...]': + """``name``'s workflow-level ``env:``, parsed once.""" + return tuple((workflow(name).get('env') or {}).items()) + + +def leg_env(name: 'str', spec: 'dict[str, Any]', leg: 'dict[str, Any]') -> 'dict[str, str]': + """The workflow's and the job's ``env:``, each expression evaluated on ``leg``.""" + merged = dict(workflow_env(name), **(spec.get('env') or {})) + return {key: stringify(evaluate_expression(value, {'matrix': leg}) + if isinstance(value, str) and '${{' in value else value) + for key, value in merged.items()} + + +def continue_on_error(step: 'dict[str, Any]', context: 'dict[str, Any]') -> 'bool': + value = step.get('continue-on-error', False) + return value if isinstance(value, bool) else evaluate_condition(str(value), context) + + +def run_leg(name: 'str', spec: 'dict[str, Any]', leg: 'dict[str, Any]', + failing: 'int' = -1) -> 'tuple[str, list[Step]]': + """The job's conclusion on ``leg``, and each step's, with step ``failing`` failing. + + A step runs when its ``if:`` holds -- with ``success()`` prepended when it calls + no status function, which GitHub documents as the default -- and a failure is + the job's unless the step's ``continue-on-error`` holds on this leg. + + """ + env = leg_env(name, spec, leg) + steps = {} # type: dict[str, dict[str, Any]] + failed = False + record = [] # type: list[Step] + for index, step in enumerate(spec['steps']): + context = {'matrix': leg, 'env': env, 'steps': steps, 'inputs': {}, + STATUS: {'success': not failed, 'failure': failed, 'cancelled': False}} + condition = '' if step.get('if') is None else str(step['if']) + runs = STATUS_FUNCTION.search(condition) is not None or not failed + if runs and condition: + runs = evaluate_condition(condition, context) + if runs: + outcome = 'failure' if index == failing else 'success' + conclusion = 'success' if outcome == 'failure' and continue_on_error(step, context) else outcome + failed = failed or conclusion == 'failure' + else: + outcome = conclusion = 'skipped' + if 'id' in step: + outputs = OUTPUTS.get(step['id'], {}) if outcome == 'success' else {} + steps[step['id']] = {'outcome': outcome, 'conclusion': conclusion, 'outputs': dict(outputs)} + record.append(Step(label(step), outcome, conclusion)) + return ('failure' if failed else 'success'), record + + +def scenarios(name: 'str', spec: 'dict[str, Any]', leg: 'dict[str, Any]') -> 'list[tuple[int, str, list[Step]]]': + """``(failing step, job conclusion, steps)`` for each step that runs on ``leg``.""" + found = [] + for index in range(len(spec['steps'])): + conclusion, record = run_leg(name, spec, leg, failing=index) + if record[index].outcome != 'skipped': + found.append((index, conclusion, record)) + return found + + +def commands(script: 'str') -> 'set[str]': + """The workflow commands ``script`` writes.""" + return set(COMMAND.findall(script)) + + +def bash(script: 'str', env: 'dict[str, str]', cwd: 'pathlib.Path') -> 'subprocess.CompletedProcess[str]': + """``script`` as ``shell: bash`` runs it.""" + return subprocess.run(['bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c', script], + env=dict(os.environ, **env), cwd=cwd, capture_output=True, text=True, + check=False, timeout=60) + + +class TestNoKTLOFailureTurnsAJobRed(unittest.TestCase): + """(a) Whatever step fails on a KTLO leg, the job concludes ``success``.""" + + def test_no_job_with_a_ktlo_leg_masks_it_at_job_level(self) -> None: + """Job-level, the run passes but the leg's check run concludes ``failure``.""" + for name, job_id in KTLO_JOBS + (('unit-tests.yml', 'engine-tests'),): + with self.subTest(workflow=name, job=job_id): + self.assertNotIn('continue-on-error', job(name, job_id)) + + def test_no_step_can_fail_a_ktlo_leg(self) -> None: + for name, job_id in KTLO_JOBS: + spec = job(name, job_id) + blocking = BLOCKS_EVERY_LEG.get((name, job_id), ()) + for leg in filter(is_ktlo, legs(spec)): + runs = scenarios(name, spec, leg) + self.assertGreater(len(runs), 3, f'{job_id} on {leg} runs almost nothing') + for index, conclusion, _ in runs: + step = label(spec['steps'][index]) + with self.subTest(workflow=name, job=job_id, python=leg['python-version'], step=step): + self.assertEqual(conclusion, 'failure' if step in blocking else 'success') + spec = job('unit-tests.yml', 'engine-tests') + for leg in legs(spec): + ktlo = [(index, conclusion) for index, conclusion, _ in scenarios('unit-tests.yml', spec, leg) + if is_engine_ktlo(spec['steps'][index])] + self.assertEqual(len(ktlo), 2, leg) + for index, conclusion in ktlo: + with self.subTest(engine=leg['engine'], step=label(spec['steps'][index])): + self.assertEqual(conclusion, 'success') + + def test_every_failure_it_masks_reaches_the_summary(self) -> None: + """``KTLO summary`` ends each job, runs after any failure, and can see each one. + + It reads ``toJSON(steps)``, and the ``steps`` context holds only steps with + an ``id``. A step masked on every leg, ``continue-on-error: true``, is not + KTLO's to report. + + """ + scripts = set() + for name, job_id in KTLO_JOBS: + spec = job(name, job_id) + summary = summary_step(spec) + with self.subTest(workflow=name, job=job_id): + self.assertIs(spec['steps'][-1], summary) + self.assertEqual(summary.get('if'), SUMMARY_IF) + self.assertEqual(summary.get('continue-on-error'), KTLO_GUARD) + self.assertEqual(summary.get('shell'), 'bash') + self.assertEqual(summary.get('env'), SUMMARY_ENV) + scripts.add(summary['run']) + blocking = BLOCKS_EVERY_LEG.get((name, job_id), ()) + for leg in filter(is_ktlo, legs(spec)): + for index, _, record in scenarios(name, spec, leg): + step = spec['steps'][index] + if step is summary or label(step) in blocking: + continue + with self.subTest(workflow=name, job=job_id, python=leg['python-version'], step=label(step)): + self.assertNotEqual(record[-1].outcome, 'skipped') + if step.get('continue-on-error') is not True: + self.assertIn('id', step) + self.assertEqual(len(scripts), 1, f'the copies of {SUMMARY!r} have drifted apart') + + +class TestAKTLOFailureIsAWarning(unittest.TestCase): + """(b) Nothing a KTLO leg runs writes an error, and the summary writes warnings.""" + + def setUp(self) -> None: + if shutil.which('bash') is None: # pragma: no cover + self.skipTest('bash is needed to run the workflow shell') + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1672-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + + def test_annotation_level_is_a_warning_on_a_ktlo_leg_only(self) -> None: + for name, job_id in LEVELLED: + spec = job(name, job_id) + with self.subTest(workflow=name, job=job_id): + self.assertEqual((spec.get('env') or {}).get('ANNOTATION_LEVEL'), ANNOTATION_LEVEL) + for leg in legs(spec): + with self.subTest(workflow=name, job=job_id, python=leg['python-version']): + self.assertEqual(leg_env(name, spec, leg)['ANNOTATION_LEVEL'], + 'warning' if is_ktlo(leg) else 'error') + + def test_no_step_a_ktlo_leg_runs_writes_an_error(self) -> None: + for name, job_id in KTLO_JOBS: + spec = job(name, job_id) + blocking = BLOCKS_EVERY_LEG.get((name, job_id), ()) + reached = set() # type: set[int] + for leg in filter(is_ktlo, legs(spec)): + for _, _, record in scenarios(name, spec, leg): + reached.update(index for index, step in enumerate(record) if step.outcome != 'skipped') + for index in sorted(reached): + step = spec['steps'][index] + if label(step) in blocking: + continue + unreachable = UNREACHABLE_ERRORS.get((name, job_id, label(step))) + with self.subTest(workflow=name, job=job_id, step=label(step)): + if unreachable is not None: + self.assertEqual(step.get('if'), unreachable) + continue + self.assertLessEqual(commands(step.get('run', '')), KTLO_COMMANDS) + if '${ANNOTATION_LEVEL}' in step.get('run', ''): + self.assertIn((name, job_id), LEVELLED) + + def summary(self, steps: 'dict[str, str]') -> 'tuple[subprocess.CompletedProcess[str], str]': + """``KTLO summary`` on 3.7, given each step's ``outcome``; its result and job summary.""" + if shutil.which('jq') is None: # pragma: no cover + self.skipTest('jq, on every Ubuntu runner image, is needed to run KTLO summary') + script = summary_step(job('unit-tests.yml', 'test'))['run'] + context = {key: {'outputs': {}, 'outcome': outcome, + 'conclusion': 'success' if outcome == 'failure' else outcome} + for key, outcome in steps.items()} + page = self.tmp / 'summary.md' + page.write_text('', encoding='utf-8') + result = bash(script, {'LEG': LEG_37, 'STEPS': json.dumps(context, indent=2), + 'GITHUB_STEP_SUMMARY': str(page)}, self.tmp) + return result, page.read_text(encoding='utf-8') + + def test_the_summary_warns_once_per_failed_step(self) -> None: + def warning(step: 'str') -> 'str': + return (f'::warning title={step} failed on {TITLE_37}::KTLO legs never block (#1633), so this job ' + f"still succeeds (#1672); see the {step} step's log.") + + def line(step: 'str') -> 'str': + return f'- {LEG_37}: step `{step}` failed; non-blocking, so this job still succeeds.\n' + + result, page = self.summary({'checkout': 'success', 'python': 'success', 'install': 'failure', + 'parallelism': 'skipped', 'tests': 'skipped'}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(workflow_commands(result.stdout), [warning('install')]) + self.assertEqual(titles(result.stdout), [f'install failed on {LEG_37}']) + self.assertEqual(page, line('install')) + + result, page = self.summary({'checkout': 'failure', 'python': 'failure', 'install': 'skipped'}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(workflow_commands(result.stdout), [warning('checkout'), warning('python')]) + self.assertEqual(page, line('checkout') + line('python')) + + def test_the_summary_is_quiet_when_nothing_failed(self) -> None: + result, page = self.summary({'checkout': 'success', 'python': 'success', 'tests': 'success'}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(workflow_commands(result.stdout), []) + self.assertEqual(page, '') + self.assertIn(f'{LEG_37}: no step failed.', result.stdout) + + def engine_cell(self, py: 'str', level: 'Optional[str]') -> 'subprocess.CompletedProcess[str]': + """``engine-tests``'s cell script for Scapy on ``py``, whose extra fails to install. + + The step that writes the script is run, then the script, with ``python`` + stubbed to make a venv whose ``python`` fails ``pip install -e .[Scapy]``. + + """ + writer = next(step for step in job('unit-tests.yml', 'engine-tests')['steps'] + if step.get('name') == 'Write the per-cell script') + bin_dir = self.tmp / 'bin' + bin_dir.mkdir(exist_ok=True) + venv_python = self.tmp / 'venv-python' + venv_python.write_text('#!/bin/bash\ncase "$*" in\n' + ' --version) echo "Python $PY" ;;\n' + ' "-m pip install -e .[Scapy]") echo "ERROR: no Scapy"; exit 1 ;;\n' + 'esac\n', encoding='utf-8') + venv_python.chmod(0o755) + interpreter = bin_dir / f'python{py}' + interpreter.write_text('#!/bin/bash\n[ "$1 $2" = "-m venv" ]\nmkdir -p "$3/bin"\n' + f'cp "{venv_python}" "$3/bin/python"\n' + 'printf \'PATH="%s/bin:$PATH"\\n\' "$3" > "$3/bin/activate"\n', encoding='utf-8') + interpreter.chmod(0o755) + env = {'RUNNER_TEMP': str(self.tmp), 'ENGINE': 'Scapy', 'PY': py, + 'GITHUB_STEP_SUMMARY': str(self.tmp / 'summary.md'), + 'PATH': f'{bin_dir}{os.pathsep}{os.environ["PATH"]}'} + written = bash(writer['run'], env, self.tmp) + self.assertEqual(written.returncode, 0, written.stderr) + if level is not None: + env['ANNOTATION_LEVEL'] = level + return subprocess.run(['bash', str(self.tmp / 'engine-cell.sh'), py], + env=dict({k: v for k, v in os.environ.items() if k != 'ANNOTATION_LEVEL'}, **env), + cwd=self.tmp, capture_output=True, text=True, check=False, timeout=60) + + def test_the_engine_ktlo_cell_warns(self) -> None: + """The 3.10 cell's step, and the cell script it runs, write no error; 3.12's does.""" + steps = {label(step): step for step in job('unit-tests.yml', 'engine-tests')['steps']} + ktlo = steps['Run the 3.10 cell (KTLO, non-blocking)'] + self.assertEqual(ktlo.get('env'), {'ANNOTATION_LEVEL': 'warning'}) + self.assertLessEqual(commands(ktlo['run']), {'group', 'endgroup', 'warning'}) + cell = steps['Write the per-cell script']['run'].split("<<'CELL'\n", 1)[1].split('\nCELL\n', 1)[0] + self.assertLessEqual(commands(cell), {'${ANNOTATION_LEVEL}', 'notice'}) + expected = ("::{} title=Scapy failed to install on Python {}::This cell expects 'supported', " + 'which requires a successful install. pip exited 1.') + for py, level in (('3.10', 'warning'), ('3.12', 'error')): + with self.subTest(python=py): + result = self.engine_cell(py, level) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(workflow_commands(result.stdout), [expected.format(level, py)]) + unset = self.engine_cell('3.10', None) + self.assertNotEqual(unset.returncode, 0) + self.assertIn('ANNOTATION_LEVEL: unbound variable', unset.stderr) + + (self.tmp / 'engine-cell.sh').write_text('exit 1\n', encoding='utf-8') + page = self.tmp / 'cells.md' + result = bash(ktlo['run'], {'RUNNER_TEMP': str(self.tmp), 'ENGINE': 'Scapy', + 'GITHUB_STEP_SUMMARY': str(page)}, self.tmp) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(workflow_commands(result.stdout)[-1], + '::warning title=Scapy failed on Python 3.10 (KTLO%2C non-blocking)::3.10 is ' + 'end-of-life and KTLO per #1633; this does not fail the job.') + self.assertEqual(page.read_text(encoding='utf-8'), '| 3.10 | Scapy | | FAILED (KTLO, non-blocking) |\n') + + +class TestASupportedLegStillFails(unittest.TestCase): + """(c) On a supported leg every step fails the job as before, and no summary runs.""" + + def test_every_failure_still_fails_a_supported_leg(self) -> None: + """A step's failure is the job's, unless it is masked on every leg by its own design. + + An ``outcome`` gate only ever names a step whose failure already fails a + supported leg, so there it adds nothing to the default ``success()``. + + """ + for name, job_id in KTLO_JOBS: + spec = job(name, job_id) + summary = spec['steps'].index(summary_step(spec)) + by_id = {step['id']: step for step in spec['steps'] if 'id' in step} + masked = [label(step) for step in spec['steps'] if step.get('continue-on-error') is True] + with self.subTest(workflow=name, job=job_id): + self.assertEqual(masked, list(MASKED_ON_EVERY_LEG.get((name, job_id), ()))) + for leg in (leg for leg in legs(spec) if not is_ktlo(leg)): + clean, record = run_leg(name, spec, leg) + with self.subTest(workflow=name, job=job_id, python=leg['python-version']): + self.assertEqual(clean, 'success') + self.assertEqual(record[summary].outcome, 'skipped') + context = {'matrix': leg, 'env': leg_env(name, spec, leg)} + for index, conclusion, record in scenarios(name, spec, leg): + step = spec['steps'][index] + with self.subTest(workflow=name, job=job_id, python=leg['python-version'], step=label(step)): + self.assertEqual(conclusion, 'success' if step.get('continue-on-error') is True else 'failure') + self.assertEqual(record[summary].outcome, 'skipped') + for gated in re.findall(r'steps\.([\w-]+)\.outcome', str(step.get('if', ''))): + self.assertFalse(continue_on_error(by_id[gated], context), gated) + + def test_a_supported_engine_cell_still_fails_its_job_as_an_error(self) -> None: + spec = job('unit-tests.yml', 'engine-tests') + loop = next(step for step in spec['steps'] if step.get('name') == 'Run the 3.11-3.15 cells') + self.assertEqual(loop.get('env'), {'ANNOTATION_LEVEL': 'error'}) + for leg in legs(spec): + for index, conclusion, _ in scenarios('unit-tests.yml', spec, leg): + step = spec['steps'][index] + if is_engine_ktlo(step): + continue + with self.subTest(engine=leg['engine'], step=label(step)): + self.assertEqual(conclusion, 'success' if step.get('continue-on-error') is True else 'failure') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_python_matrix_tiers_1633.py b/tests/project/test_python_matrix_tiers_1633.py index 4b36d0bfc..6c7316762 100644 --- a/tests/project/test_python_matrix_tiers_1633.py +++ b/tests/project/test_python_matrix_tiers_1633.py @@ -19,15 +19,17 @@ and PyPI's completeness evidence must not count a KTLO wheel, or one failed KTLO leg would leave every later release run retrying it. The release-side checks execute the workflow's own shell, stubbing only ``curl``. The last class reads -the prose that states the tiers back against the matrices. +the prose that states the tiers back against the matrices. #1672 later made +every KTLO leg conclude ``success``, Compat's included, and report a failed step +as a warning; :mod:`tests.project.test_ktlo_legs_stay_green_1672` pins that. 3.7 joined the tier afterwards, at the owner's request, and cannot install or import pcapkit: the ``test`` extra's ``pytest>=8`` needs 3.8, and the sources use -3.8 syntax. So each of its legs fails a step by design -- only Compat's shows red, -its ``continue-on-error`` being job-level -- and :class:`TestPython37IsKTLO` pins -that they fail fast and say why -- the install and verify steps are executed with -``python`` stubbed, and each annotation is parsed as the runner parses it. 3.6 -stays out: setup-python builds it only for retired images. +3.8 syntax. So each of its legs fails a step by design, and +:class:`TestPython37IsKTLO` pins that they fail fast and say why, at warning level +-- the install and verify steps are executed with ``python`` stubbed, and each +annotation is parsed as the runner parses it. 3.6 stays out: setup-python builds +it only for retired images. """ @@ -56,8 +58,10 @@ WORKFLOWS = ROOT / '.github' / 'workflows' SUPPORTED = ('3.11', '3.12', '3.13', '3.14', '3.15') -#: The step-level guard every failure-prone step of an aggregated job carries. +#: The step-level guard every failure-prone step of a KTLO leg's job carries. KTLO_GUARD = '${{ matrix.ktlo == true }}' +#: The job-level level of every annotation a failing step writes (#1672). +ANNOTATION_LEVEL = "${{ matrix.ktlo && 'warning' || 'error' }}" #: Ruleset 23497679's required contexts once the owner applies #1633's edit. REQUIRED = frozenset({'Required checks passed'} | {f'Compat Python {v}' for v in SUPPORTED}) @@ -114,9 +118,19 @@ r'3\.8, 3\.9 and 3\.10 are end-of-life', r"CI's range starts at 3\.8", r'oldest tested leg is ``Python 3\.8', - # Only Compat's 3.7 leg shows red; step-level ``continue-on-error`` leaves - # the ``test`` and ``integration`` legs concluding ``success``. r"3\.7's legs are red by design", + # #1672: no KTLO leg shows red, Compat's included, and a failed KTLO step's + # own annotation is a warning. + r'3\.7 is red by design', + r'a failed leg shows red without failing the run', + r'it shows red, and the run stays green', + r'Only the ``Compat`` leg shows red', + r'only Compat.s shows red', + r'may stay red without blocking', + r'shows as a red step and an error annotation', + r'shows as a failed step and an error annotation', + r'with one error annotation saying why', + r'a KTLO leg carries job-level', ))) #: Where :data:`RETIRED_CLAIMS` is looked for, as (directory, globs), plus root files. #: The changelog is history and is left out. @@ -193,7 +207,8 @@ def test_every_matrix_runs_the_supported_tier_as_blocking(self) -> None: spec = job(name, job_id) versions = sorted({leg['python-version'] for leg in legs(spec) if not is_ktlo(leg)}) self.assertEqual(versions, sorted(blocking)) - self.assertIn(spec.get('continue-on-error', False), (False, KTLO_GUARD)) + # #1672: not even KTLO-only; job-level, a failed leg's check run is red. + self.assertNotIn('continue-on-error', spec) def test_engine_tests_runs_every_supported_cell_in_its_blocking_step(self) -> None: steps = job('unit-tests.yml', 'engine-tests')['steps'] @@ -250,12 +265,10 @@ def test_no_ktlo_leg_can_fail_an_aggregated_job(self) -> None: self.assertNotIn('continue-on-error', spec) for step in spec['steps']: label = step.get('name') or step.get('uses') - if str(step.get('uses', '')).startswith('actions/checkout@'): - continue condition = str(step.get('if', '')) if 'matrix.coverage' in condition or 'APT_CACHE_SAVE' in condition: continue # only ever on a supported leg: the coverage one, the cache saver - if label == "Check the release tags name this run's commit": + if label in ('Checkout code', "Check the release tags name this run's commit"): continue # #1634: blocks every leg alike, pinned in its own test module with self.subTest(workflow=name, job=job_id, step=label): self.assertIn(step.get('continue-on-error'), (True, KTLO_GUARD)) @@ -269,11 +282,6 @@ def test_engine_tests_isolates_the_ktlo_interpreter_and_cell(self) -> None: self.assertIs(cell.get('continue-on-error'), True) self.assertIn('engine-cell.sh" 3.10', cell['run']) - def test_a_compatibility_ktlo_leg_shows_red_without_failing_the_run(self) -> None: - """No aggregate reads this job, so job-level ``continue-on-error`` is the right form.""" - spec = job('python-compatibility.yml', 'compatibility') - self.assertEqual(spec.get('continue-on-error'), KTLO_GUARD) - def test_no_ktlo_leg_is_a_required_context(self) -> None: spec = job('python-compatibility.yml', 'compatibility') template = spec['name'] @@ -420,6 +428,15 @@ def test_no_conda_leg_is_ktlo_and_315_leaves_base_python_alone(self) -> None: LEG_37 = 'Python 3.7 (KTLO, non-blocking)' #: :data:`LEG_37` as a ``title=`` property must carry it: ``,`` escaped (see :func:`runner_parse`). TITLE_37 = 'Python 3.7 (KTLO%2C non-blocking)' +#: The 3.7 leg's matrix entry, and a supported leg's, as :meth:`TestPython37IsKTLO.run_step` takes them. +KTLO_37 = {'python-version': '3.7', 'ktlo': True} +SUPPORTED_312 = {'python-version': '3.12'} +#: Every KTLO leg's install step, as (workflow, job, step): it needs setup-python's ``python`` (#1672). +INSTALLS = ( + LEGACY_37[0], LEGACY_37[1], + ('unit-tests.yml', 'pypcap-parity', 'Install package, test and generator dependencies'), + ('python-compatibility.yml', 'compatibility', 'Install package'), +) def workflow_commands(output: 'str') -> 'list[str]': @@ -472,7 +489,7 @@ def titles(output: 'str') -> 'list[str]': class TestPython37IsKTLO(unittest.TestCase): - """3.7 runs, non-blocking, and stops at the first failure with one annotation.""" + """3.7 runs, non-blocking, and stops at the first failure with one warning.""" def setUp(self) -> None: if shutil.which('bash') is None: # pragma: no cover @@ -482,12 +499,23 @@ def setUp(self) -> None: self.tmp = pathlib.Path(tmp.name) def run_step(self, name: 'str', job_id: 'str', step_name: 'str', stub: 'str', - env: 'dict[str, str]') -> 'subprocess.CompletedProcess[str]': - """Step ``step_name`` under ``shell: bash`` (``-eo pipefail``), ``python`` stubbed.""" - step = next(step for step in job(name, job_id)['steps'] if step.get('name') == step_name) + env: 'dict[str, str]', leg: 'Optional[dict[str, Any]]' = None, + ) -> 'subprocess.CompletedProcess[str]': + """Step ``step_name`` on ``leg`` (3.7's by default) under ``shell: bash``, ``python`` stubbed. + + ``LEG`` and the job's ``ANNOTATION_LEVEL`` get the values their expressions + take on that leg, and ``-eo pipefail`` is what ``shell: bash`` runs. + + """ + leg = KTLO_37 if leg is None else leg + spec = job(name, job_id) + step = next(step for step in spec['steps'] if step.get('name') == step_name) self.assertEqual(step.get('shell'), 'bash', 'pipefail is what keeps `tee` from hiding a failure') self.assertEqual(step['env']['LEG'], "Python ${{ matrix.python-version }}" "${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}") + self.assertEqual(spec['env']['ANNOTATION_LEVEL'], ANNOTATION_LEVEL) + env = dict(env, LEG=f"Python {leg['python-version']}{' (KTLO, non-blocking)' if is_ktlo(leg) else ''}", + ANNOTATION_LEVEL='warning' if is_ktlo(leg) else 'error') bin_dir = self.tmp / 'bin' bin_dir.mkdir(exist_ok=True) python = bin_dir / 'python' @@ -495,7 +523,7 @@ def run_step(self, name: 'str', job_id: 'str', step_name: 'str', stub: 'str', python.chmod(0o755) return subprocess.run(['bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c', step['run']], env=dict(os.environ, PATH=f'{bin_dir}{os.pathsep}{os.environ["PATH"]}', - RUNNER_TEMP=str(self.tmp), LEG=LEG_37, **env), + RUNNER_TEMP=str(self.tmp), **env), capture_output=True, text=True, check=False, timeout=60) def test_37_is_a_ktlo_leg_of_exactly_the_three_test_matrices(self) -> None: @@ -518,10 +546,12 @@ def test_a_leg_that_cannot_install_stops_there(self) -> None: """The install needs setup-python's interpreter, and every later step needs the install. ``outcome``, not ``conclusion``: ``continue-on-error`` turns a failed step's - conclusion into ``success``, so only its outcome still says it failed. + conclusion into ``success``, so only its outcome still says it failed. #1672 + brought Compat and ``pypcap-parity`` in; ``KTLO summary`` is the one step + that has to run after a failed install, to report it. """ - for name, job_id, install in LEGACY_37[:2]: + for name, job_id, install in INSTALLS: steps = job(name, job_id)['steps'] setup = next(step for step in steps if str(step.get('uses', '')).startswith('actions/setup-python@')) index = next(i for i, step in enumerate(steps) if step.get('name') == install) @@ -533,6 +563,8 @@ def test_a_leg_that_cannot_install_stops_there(self) -> None: condition = str(step.get('if', '')) if 'matrix.coverage' in condition: continue # the 3.14 leg only, which a failed install fails anyway + if step.get('name') == 'KTLO summary': + continue with self.subTest(workflow=name, job=job_id, step=step.get('name') or step.get('uses')): self.assertIn("steps.install.outcome == 'success'", condition) @@ -542,7 +574,7 @@ def test_a_failed_install_fails_once_naming_pips_reason(self) -> None: result = self.run_step(name, job_id, install, PIP_STUB, {'PIP_OUTPUT': PIP_37, 'PIP_STATUS': '1'}) self.assertNotEqual(result.returncode, 0) self.assertEqual(workflow_commands(result.stdout + result.stderr), [ - f'::error title=pcapkit did not install on {TITLE_37}::No matching distribution found ' + f'::warning title=pcapkit did not install on {TITLE_37}::No matching distribution found ' 'for pytest>=8; extra == "test". The rest of this leg is skipped.']) self.assertEqual(titles(result.stdout), [f'pcapkit did not install on {LEG_37}']) self.assertEqual((self.tmp / 'pip-install.log').read_text(encoding='utf-8'), PIP_37 + '\n') @@ -555,7 +587,8 @@ def test_an_install_failure_without_an_error_line_is_still_named(self) -> None: {'PIP_OUTPUT': output, 'PIP_STATUS': '1'}) self.assertNotEqual(result.returncode, 0) self.assertEqual(workflow_commands(result.stdout), [ - f'::error title=pcapkit did not install on {TITLE_37}::{reason}. The rest of this leg is skipped.']) + f'::warning title=pcapkit did not install on {TITLE_37}::{reason}. ' + 'The rest of this leg is skipped.']) def test_a_clean_install_passes_without_an_annotation(self) -> None: for name, job_id, install in LEGACY_37[:2]: @@ -568,10 +601,10 @@ def test_a_clean_install_passes_without_an_annotation(self) -> None: def test_the_compatibility_leg_names_its_first_compile_error(self) -> None: cases = ( ({'COMPILE_OUTPUT': COMPILE_37, 'COMPILE_STATUS': '1', 'IMPORT_STATUS': '0'}, - [f'::error title=pcapkit does not compile on {TITLE_37}::2 modules fail to compile, the first at ' + [f'::warning title=pcapkit does not compile on {TITLE_37}::2 modules fail to compile, the first at ' 'File "pcapkit/const/ftp/command.py", line 286; see this step\'s log.']), ({'COMPILE_OUTPUT': '', 'COMPILE_STATUS': '0', 'IMPORT_STATUS': '1'}, - [f"::error title=pcapkit does not import on {TITLE_37}::See the traceback in this step's log."]), + [f"::warning title=pcapkit does not import on {TITLE_37}::See the traceback in this step's log."]), ({'COMPILE_OUTPUT': '', 'COMPILE_STATUS': '0', 'IMPORT_STATUS': '0'}, []), ) # type: tuple[tuple[dict[str, str], list[str]], ...] for env, expected in cases: @@ -585,11 +618,27 @@ def test_the_compatibility_leg_names_its_first_compile_error(self) -> None: if env['COMPILE_STATUS'] != '0': self.assertIn('SyntaxError: invalid syntax', result.stdout) + def test_a_supported_leg_fails_the_same_steps_at_error_level(self) -> None: + """#1672 changed the KTLO level only: on 3.12 each of these still fails as an ``::error``.""" + cases = [(name, job_id, install, PIP_STUB, {'PIP_OUTPUT': PIP_37, 'PIP_STATUS': '1'}, + '::error title=pcapkit did not install on Python 3.12::No matching distribution found ' + 'for pytest>=8; extra == "test". The rest of this leg is skipped.') + for name, job_id, install in LEGACY_37[:2]] + cases.append(('python-compatibility.yml', 'compatibility', 'Verify package', VERIFY_STUB, + {'COMPILE_OUTPUT': '', 'COMPILE_STATUS': '0', 'IMPORT_STATUS': '1'}, + "::error title=pcapkit does not import on Python 3.12::See the traceback in this step's log.")) + for name, job_id, step_name, stub, env, expected in cases: + with self.subTest(workflow=name, job=job_id): + result = self.run_step(name, job_id, step_name, stub, env, leg=SUPPORTED_312) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(workflow_commands(result.stdout), [expected]) + #: The properties an annotation takes; anything else in one is a value cut at a ``,``. ANNOTATION_PROPERTIES = frozenset({'title', 'file', 'line', 'endline', 'col', 'endcolumn'}) -#: An annotation command and its properties, which end at the next ``::``. -ANNOTATION = re.compile(r'::(?:error|warning|notice) (.*?)::') +#: An annotation command and its properties, which end at the next ``::``. The level +#: can also be ``${ANNOTATION_LEVEL}``, a job's per-leg one (#1672). +ANNOTATION = re.compile(r'::(?:error|warning|notice|\$\{ANNOTATION_LEVEL\}) (.*?)::') #: A ``${...}`` expansion: the shell replaces it, so the runner never sees its text. BRACED = re.compile(r'\$\{([^}]*)\}') #: The one braced form that escapes what it expands: ``${NAME//,/%2C}``. diff --git a/tests/project/test_unit_tests_queue.py b/tests/project/test_unit_tests_queue.py index ed04860b9..e2438a786 100644 --- a/tests/project/test_unit_tests_queue.py +++ b/tests/project/test_unit_tests_queue.py @@ -414,7 +414,7 @@ def test_one_failing_cell_fails_the_step_after_the_rest_ran(self) -> None: self.assertEqual(calls, ['3.11', '3.12', '3.13', '3.14', '3.15']) def test_a_failing_ktlo_cell_fails_its_own_step(self) -> None: - """The step is red; only its ``continue-on-error`` keeps the job green.""" + """The step fails; its ``continue-on-error`` keeps the job green, its annotation is a warning (#1672).""" status, calls = self.run_loop('Run the 3.10 cell (KTLO, non-blocking)', fail='3.10') self.assertNotEqual(status, 0) self.assertEqual(calls, ['3.10']) From 88d6bbc9d0959aed8f0d0fcff1fa31c34a4f5c46 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 13 Oct 2026 16:25:50 -0400 Subject: [PATCH 2/2] docs(releasing): renumber the create-release.yml citations the KTLO steps moved (#1672) The first commit grew pypi by 29 lines, so nine citations below it named the wrong line: conda's needs (four times as `:997`), both upload steps and their release tags, and release_status's App-bot notice. * releasing.rst and workflows.rst cite the lines the prose describes: `:1026`, `:933`/`:1283`, `:926-963`/`:1278-1313`, `:1385-1387`. * TestDocCitationsResolve checks create-release.yml citations as it checks unit-tests.yml's, with each range's job, comma lists, and the bare `:N` form these two pages use; BARE_ELSEWHERE names the five bare citations of other files. The new test fails on the first commit's docs (11 failures) and passes here, as do the 1672, 1633, 1634, release-gates and unit-tests-queue modules. --- docs/source/contributing/releasing.rst | 10 +- docs/source/contributing/workflows.rst | 4 +- tests/project/test_unit_tests_queue.py | 129 ++++++++++++++++++++++++- 3 files changed, 131 insertions(+), 12 deletions(-) diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index 6941b8af4..ae243875b 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -49,7 +49,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1645-1714``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every -publishing job requires it to have succeeded (``:434,619,784,997``), so a +publishing job requires it to have succeeded (``:434,619,784,1026``), so a citation left behind fails the gate well before any approval is requested. Either run the script, or move both fields by hand in the same commit. @@ -78,7 +78,7 @@ neither is a workaround for the other: of *that* tree (``:237``). Every downstream tag and release name is built from that output -- ``v${{ needs.version_check.outputs.PCAPKIT_VERSION }}`` is ``github``'s ``tag_name`` at ``:530`` and the release ``pypi`` and - ``conda`` upload to at ``:924`` and ``:1254``, always from + ``conda`` upload to at ``:933`` and ``:1283``, always from ``PCAPKIT_VERSION``, never from ``github.ref_name`` -- so the version actually released is whatever ``__version__`` says on the tagged commit, not the name of the tag that was pushed. The two agree only if the tag @@ -134,7 +134,7 @@ the default branch, and ``tag`` pulled ``main`` before tagging. with an ``::error`` instead of publishing a second tree under that version; a partial release is finished by re-running the run that tagged it. * ``pypi`` and ``conda`` attach their files with ``gh release upload`` - (``:918-954``, ``:1249-1284``), not ``action-gh-release``, which calls Update + (``:926-963``, ``:1278-1313``), not ``action-gh-release``, which calls Update a release before every upload (3.0.2's ``src/github.ts:631-683``). A file the release already has is left alone (`#1640 `__): the action @@ -248,7 +248,7 @@ the release test gate, ask for a second approval and race the first run's uploads from another concurrency group, so ``version_check`` skips a ``push`` whose ``github.actor_id`` is ``vars.RELEASE_APP_BOT_ID`` (``:210-212``), and ``release_status`` reports it with ``::notice title=Nothing to release`` -(``:1356-1358``). A tag pushed by a person runs as before. Whether a tag made +(``:1385-1387``). A tag pushed by a person runs as before. Whether a tag made through the Releases API fires ``push``, and with the bot as actor, is unmeasured; if not, the guard never matches and the cost is one approval to reject. @@ -392,7 +392,7 @@ removed on `#887 `__. What still makes one approval mean *the whole release* is the ``needs:`` graph: ``tag`` and ``pypi`` both depend on ``github`` (``.github/workflows/create-release.yml:619,784``), and ``conda`` depends on -``tag`` and ``github`` (``:997``) -- so nothing downstream of ``github`` can +``tag`` and ``github`` (``:1026``) -- so nothing downstream of ``github`` can start before it is approved, and rejecting it leaves nothing tagged and nothing published. Before this change ``tag`` depended on ``version_check`` alone, so removing its reviewer without moving this dependency would have let diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index a2f5a4180..661eee48a 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -311,7 +311,7 @@ every job past it is gated the same way: ``github`` needs ``[ unit-tests, version_check ]`` (``:434``), ``tag`` needs ``[ github, unit-tests, version_check ]`` (``:619``), ``pypi`` needs ``[ github, unit-tests, version_check ]`` (``:784``), and ``conda`` needs ``[ tag, github, -unit-tests, version_check ]`` (``:997``). +unit-tests, version_check ]`` (``:1026``). .. mermaid:: @@ -337,7 +337,7 @@ unit-tests, version_check ]`` (``:997``). This is a transitive reduction, as in :doc:`releasing`: ``version_check`` and ``unit-tests`` are also direct ``needs:`` of ``tag`` and ``pypi`` -(``create-release.yml:619,784``) and of ``conda`` (``:997``), but ``github`` +(``create-release.yml:619,784``) and of ``conda`` (``:1026``), but ``github`` already implies those edges, since it depends on both itself. Drawing them would add lines without changing which job can start before which, or which job a skip in ``version_check`` reaches. diff --git a/tests/project/test_unit_tests_queue.py b/tests/project/test_unit_tests_queue.py index e2438a786..b1dd187d1 100644 --- a/tests/project/test_unit_tests_queue.py +++ b/tests/project/test_unit_tests_queue.py @@ -570,15 +570,134 @@ def test_the_scan_is_not_blind(self) -> None: } +#: Every ``create-release.yml`` citation in these documents, as (the job its lines +#: sit in, ``None`` above ``jobs:``; text the first line must contain; text the +#: last line must contain). Lines repeat across jobs -- ``tag`` and ``pypi`` share +#: their ``needs:``, ``pypi`` and ``conda`` their upload step -- so the job is +#: checked too. A list such as ``:619,784`` is one citation per number. +RELEASE_CITATIONS: 'dict[str, list[tuple[Optional[str], str, Optional[str]]]]' = { + 'docs/source/contributing/releasing.rst': [ + (None, 'push:', '- completed'), + ('version_check', 'if: >-', 'vars.RELEASE_APP_BOT_ID) }}'), + ('version_check', '- name: Checkout code', 'fetch-depth: 0'), + ('version_check', 'print(pcapkit.__version__)', None), + ('unit-tests', ' unit-tests:', 'gate-only: true'), + ('github', 'needs: [ unit-tests, version_check ]', None), + ('github', "grep -qi 'unreleased'", 'title=Changelog entry is undated'), + ('github', "- name: Mint the release App's token", 'permission-workflows: write'), + ('github', 'tag_name: "v${{ needs.version_check.outputs.PCAPKIT_VERSION }}"', None), + ('github', "target_commitish: ${{ github.event_name == 'workflow_run' && github.sha", None), + ('github', 'token: ${{ steps.app_token.outputs.token || secrets.GITHUB_TOKEN }}', None), + ('github', '- name: Explain a refused release tag', 'title=Could not create v'), + ('tag', 'needs: [ github, unit-tests, version_check ]', None), + ('tag', '- name: Create Tag', 'fi'), + ('tag', '- name: Reset Build Number', 'exit 1'), + ('pypi', 'environment: pypi', None), + ('pypi', 'id-token: write', None), + ('pypi', 'needs: [ github, unit-tests, version_check ]', None), + ('pypi', 'PCAPKIT_RELEASE_TAG: v${{ needs.version_check.outputs.PCAPKIT_VERSION }}', None), + ('pypi', '- name: Upload package to GitHub Release', 'done <<< "$PCAPKIT_ASSETS"'), + ('conda', 'needs: [ tag, github, unit-tests, version_check ]', None), + ('conda', 'PCAPKIT_RELEASE_TAG: v${{ needs.version_check.outputs.PCAPKIT_VERSION }}', None), + ('conda', '- name: Upload package to GitHub Release', 'done <<< "$PCAPKIT_ASSETS"'), + ('release_status', '[ -n "$PCAPKIT_RELEASE_APP_BOT_ID" ]', 'title=Nothing to release'), + ], + 'docs/source/contributing/workflows.rst': [ + (None, 'workflow_run:', '- completed'), + ('version_check', 'if: >-', 'vars.RELEASE_APP_BOT_ID) }}'), + ('unit-tests', 'needs: [ version_check ]', None), + ('unit-tests', 'uses: ./.github/workflows/unit-tests.yml', None), + ('github', 'environment: github-release', None), + ('github', 'needs: [ unit-tests, version_check ]', None), + ('tag', 'environment: conda-tag', None), + ('tag', 'needs: [ github, unit-tests, version_check ]', None), + ('pypi', 'environment: pypi', None), + ('pypi', 'needs: [ github, unit-tests, version_check ]', None), + ('conda', 'environment: anaconda', None), + ('conda', 'needs: [ tag, github, unit-tests, version_check ]', None), + ], +} + +#: A document that names ``create-release.yml`` also cites it bare, as ``:N``, +#: and each bare citation there counts as one of its citations unless listed +#: here with the file it does cite. A bare ``:997`` is what outlived the line it +#: named when #1672 first moved ``conda`` down 29 lines. +BARE_ELSEWHERE: 'dict[str, dict[str, str]]' = { + 'docs/source/contributing/releasing.rst': { + '494-514': 'tests/project/test_bump_version.py', + '631-647': "softprops/action-gh-release 3.0.2's src/github.ts", + }, + 'docs/source/contributing/workflows.rst': { + '39': '.github/workflows/python-compatibility.yml', + '47-51': '.github/workflows/python-compatibility.yml', + '56-66': '.github/workflows/python-compatibility.yml', + }, +} + +#: A line-number citation: ``N``, ``N-M``, or several joined by commas. +SPANS = r'(\d+(?:-\d+)?(?:,\d+(?:-\d+)?)*)' + + +def documents() -> 'list[str]': + """Every page under ``docs/`` and every top-level Markdown file.""" + return sorted({str(p.relative_to(ROOT)) for p in (ROOT / 'docs').rglob('*.rst')} + | {str(p.relative_to(ROOT)) for p in ROOT.glob('*.md')}) + + +def jobs_by_line(lines: 'list[str]') -> 'list[Optional[str]]': + """The job each workflow line sits in, ``None`` above ``jobs:``.""" + owner: 'list[Optional[str]]' = [] + current: 'Optional[str]' = None + seen = False + for line in lines: + seen = seen or line.startswith('jobs:') + match = re.match(r' ([\w-]+):\s*$', line) if seen else None + current = match.group(1) if match else current + owner.append(current) + return owner + + class TestDocCitationsResolve(unittest.TestCase): - """The prose's ``unit-tests.yml`` line numbers point at what it says they do.""" + """The prose's ``unit-tests.yml`` and ``create-release.yml`` line numbers land where it says.""" + + def test_every_create_release_citation_lands_on_its_anchor(self) -> None: + """Each citation, bare ones included, lands in its anchor's job and on its lines.""" + release = ROOT / '.github' / 'workflows' / 'create-release.yml' + lines = release.read_text(encoding='utf-8').splitlines() + owner = jobs_by_line(lines) + for document in documents(): + text = (ROOT / document).read_text(encoding='utf-8') + groups = re.findall(rf'create-release\.yml:{SPANS}', text) + if 'create-release.yml' in text: + bare = re.findall(rf'``:{SPANS}``', text) + elsewhere = BARE_ELSEWHERE.get(document, {}) + with self.subTest(document=document, table='BARE_ELSEWHERE'): + self.assertFalse(set(elsewhere) - set(bare), + f'{document} no longer cites these bare') + groups += [group for group in bare if group not in elsewhere] + anchors = RELEASE_CITATIONS.get(document, []) + unmatched = list(anchors) + for first, last in ((int(a), int(b or a)) for group in groups + for a, b in re.findall(r'(\d+)(?:-(\d+))?', group)): + with self.subTest(document=document, citation=f'{first}-{last}'): + self.assertLessEqual(last, len(lines)) + hit = next((a for a in anchors if owner[first - 1] == owner[last - 1] == a[0] + and a[1] in lines[first - 1] + and (a[2] is None or a[2] in lines[last - 1])), None) + self.assertIsNotNone( + hit, f'{document} cites create-release.yml:{first}-{last}, which is ' + f'{lines[first - 1].strip()!r} .. {lines[last - 1].strip()!r} in ' + f'{owner[first - 1]!r}; a bare citation of another file goes in ' + f'BARE_ELSEWHERE') + if hit in unmatched: + unmatched.remove(hit) + with self.subTest(document=document): + self.assertFalse(unmatched, f'{document}: no citation lands on {unmatched}') def test_every_citation_lands_on_its_anchor(self) -> None: - """Each citation's range starts and ends on the line its anchor names.""" + """Each ``unit-tests.yml`` citation's range starts and ends on the line its anchor names.""" lines = WORKFLOW.read_text(encoding='utf-8').splitlines() - documents = sorted({str(p.relative_to(ROOT)) for p in (ROOT / 'docs').rglob('*.rst')} - | {str(p.relative_to(ROOT)) for p in ROOT.glob('*.md')}) - for document in documents: + for document in documents(): text = (ROOT / document).read_text(encoding='utf-8') cited = [(int(m.group(1)), int(m.group(2) or m.group(1))) for m in re.finditer(r'unit-tests\.yml:(\d+)(?:-(\d+))?', text)]