diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index 9f49b37738..9ada8b8bd7 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -34,6 +34,11 @@ concurrency: group: pages-${{ github.event.workflow_run.head_sha || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# #2030: one Discussions category holds every report with no pull request to +# comment on, one thread per job. +env: + REPORT_CATEGORY: CI reports + jobs: # #1967: the test gate on the schedule and the `verdict` on a `workflow_run`, # shared with Conda Update and Vendor Update; cron-head.yml says when each @@ -185,3 +190,64 @@ jobs: clean: true # Automatically remove deleted files from the deploy branch # Read back by the step above; keep the two in step. commit-message: Publish the documentation of ${{ github.sha }} + + # #2030: the run's report, the gate's, the build's and the publish's results, + # posted to the thread `deploy-pages: documentation build report`. As + # cron-vendor.yml's `discussion` job: a `workflow_run` follows a push to + # main, so it posts as a push run, only when it failed and once more on the + # next green run, and the schedule posts every run. It runs where + # `deploy-pages` ran or `head`'s gate failed, so never on a pull request's + # run or a verdict for another commit, + # which skip `deploy-pages` deliberately. It reads no artifact, and checks + # out the script alone. The job and its step are `continue-on-error`; + # util/discussion_report.py writes a warning rather than failing. + discussion: + name: Post the Pages report + needs: [ head, deploy-pages, publish-pages ] + if: >- + ${{ always() && github.event_name != 'pull_request' && (needs.deploy-pages.result != 'skipped' || + needs.head.result == 'failure' || needs.head.result == 'cancelled') }} + runs-on: ubuntu-latest + timeout-minutes: 10 + continue-on-error: true + permissions: + contents: read + discussions: write + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: | + util/discussion_report.py + sparse-checkout-cone-mode: false + persist-credentials: false + + # A skipped `deploy-pages` reports the gate's result; a skipped + # `publish-pages` after a green build, the skip. + - name: Post the report to Discussions + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + MODE: ${{ github.event_name == 'workflow_run' && 'push' || 'scheduled' }} + HEAD_RESULT: ${{ needs.head.result }} + BUILD_RESULT: ${{ needs.deploy-pages.result }} + PUBLISH_RESULT: ${{ needs.publish-pages.result }} + run: | + if [ "$BUILD_RESULT" = skipped ]; then + status="$HEAD_RESULT" + elif [ "$BUILD_RESULT" != success ]; then + status="$BUILD_RESULT" + else + status="$PUBLISH_RESULT" + fi + mkdir -p report + { + echo '### GitHub Pages' + echo + echo "- Gate: $HEAD_RESULT" + echo "- Build: $BUILD_RESULT" + echo "- Publish: $PUBLISH_RESULT" + } > report/pages-report.md + python3 util/discussion_report.py --category "$REPORT_CATEGORY" \ + --title 'deploy-pages: documentation build report' --report report/pages-report.md \ + --mode "$MODE" --status "$status" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b96c6dbbc1..4d40fd8001 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -56,11 +56,18 @@ concurrency: group: lint-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# #2030: one Discussions category holds every report with no pull request to +# comment on, one thread per job. +env: + REPORT_CATEGORY: CI reports + jobs: lint: name: Lint runs-on: ubuntu-latest timeout-minutes: 30 + outputs: + status: ${{ steps.run-report.outputs.status }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -97,9 +104,11 @@ jobs: # run` prefix, so this and a developer's `make pylint` cannot drift apart. # `continue-on-error: true` lets every tool run whatever another found; # `set +e` and `status=` keep the summary lines running under the default - # `bash -e`. + # `bash -e`. Each block goes to lint-summary.md too, for the run report + # (#2030). - name: bandit + id: bandit continue-on-error: true run: | set +e @@ -112,11 +121,12 @@ jobs: echo '```' sed -n '/^Run metrics:/,$p' bandit.log || echo 'no run metrics in output' echo '```' - } >> "$GITHUB_STEP_SUMMARY" + } | tee -a "$RUNNER_TEMP/lint-summary.md" >> "$GITHUB_STEP_SUMMARY" exit "$status" - name: mypy + id: mypy continue-on-error: true run: | set +e @@ -129,7 +139,7 @@ jobs: echo '```' tail -n 1 mypy.log echo '```' - } >> "$GITHUB_STEP_SUMMARY" + } | tee -a "$RUNNER_TEMP/lint-summary.md" >> "$GITHUB_STEP_SUMMARY" exit "$status" @@ -141,6 +151,7 @@ jobs: # code's real floor, 3.11 (`enum.StrEnum`, `enum.show_flag_values`). # Raising either belongs in its own change. - name: vermin + id: vermin continue-on-error: true run: | set +e @@ -154,13 +165,14 @@ jobs: grep -E '^(Minimum required versions|Incompatible versions|Target versions not met):' vermin.log \ || echo 'no verdict in output' echo '```' - } >> "$GITHUB_STEP_SUMMARY" + } | tee -a "$RUNNER_TEMP/lint-summary.md" >> "$GITHUB_STEP_SUMMARY" exit "$status" # Last of the advisory steps because it is by far the slowest, so the # quick verdicts reach the summary first. - name: pylint + id: pylint continue-on-error: true run: | set +e @@ -174,7 +186,7 @@ jobs: printf 'messages by class (C/E/R/W):\n' grep -oE ': [A-Z][0-9]{4}:' pylint.log | grep -oE '[A-Z]' | sort | uniq -c || echo 'none' echo '```' - } >> "$GITHUB_STEP_SUMMARY" + } | tee -a "$RUNNER_TEMP/lint-summary.md" >> "$GITHUB_STEP_SUMMARY" exit "$status" @@ -183,8 +195,8 @@ jobs: # is safe because #2023, #2037 and #2038 fixed every finding it reported. # It runs last, though it needs no Python, so that a finding skips none of # the advisory steps above, and on `!cancelled()`, so that a failed Python - # setup does not skip it. Lint is not among ruleset 23497679's required - # checks, so a red Lint does not hold a merge. + # setup does not skip it. `Lint` is one of ruleset 23497679's required + # checks (#1967), so a red Lint holds a merge. # # The image's shellcheck is not pinned: a runner image that adds a check # can flag unchanged code and turn Lint red on an unrelated pull request. @@ -199,6 +211,7 @@ jobs: # yet (rhysd/actionlint#657, fixed by rhysd/actionlint#654). Drop the # pattern once a release with that fix is pinned. - name: actionlint + id: actionlint if: ${{ !cancelled() }} timeout-minutes: 5 env: @@ -228,6 +241,100 @@ jobs: printf 'findings by rule:\n%s\n' "${rules:-none}" printf 'shellcheck findings by code:\n%s\n' "${shellcheck:-none}" echo '```' - } >> "$GITHUB_STEP_SUMMARY" + } | tee -a "$RUNNER_TEMP/lint-summary.md" >> "$GITHUB_STEP_SUMMARY" exit "$status" + + # #2030: the report the `discussion` job posts, on the schedule and on + # dispatch, never on a pull request, whose run has checks of its own: + # each tool's outcome, then the blocks the steps above wrote to the + # summary. `status=warning` where an advisory step failed. Both steps + # are `continue-on-error`, so a report never fails the run. + - name: Write the run report + id: run-report + if: ${{ !cancelled() && github.event_name != 'pull_request' }} + continue-on-error: true + env: + BANDIT: ${{ steps.bandit.outcome }} + MYPY: ${{ steps.mypy.outcome }} + VERMIN: ${{ steps.vermin.outcome }} + PYLINT: ${{ steps.pylint.outcome }} + ACTIONLINT: ${{ steps.actionlint.outcome }} + run: | + status=success + for outcome in "$BANDIT" "$MYPY" "$VERMIN" "$PYLINT"; do + if [ "$outcome" = failure ]; then + status=warning + fi + done + report="$RUNNER_TEMP/lint-report.md" + { + echo '### Lint' + echo + echo "- bandit, advisory: ${BANDIT:-not run}" + echo "- mypy, advisory: ${MYPY:-not run}" + echo "- vermin, advisory: ${VERMIN:-not run}" + echo "- pylint, advisory: ${PYLINT:-not run}" + echo "- actionlint, blocking: ${ACTIONLINT:-not run}" + if [ -s "$RUNNER_TEMP/lint-summary.md" ]; then + echo + cat "$RUNNER_TEMP/lint-summary.md" + fi + } > "$report" + echo "status=$status" >> "$GITHUB_OUTPUT" + cat "$report" + + - name: Upload the run report + if: ${{ !cancelled() && github.event_name != 'pull_request' }} + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: lint-report + path: ${{ runner.temp }}/lint-report.md + retention-days: 14 + + # #2030: posts the report `lint` uploaded to the thread `lint: weekly lint + # report`, every run, as a scheduled report: Lint has no push trigger, and a + # dispatch is a deliberate report, as in ci-timing.yml. Never on a pull + # request. A green job whose advisory steps failed reports `warning`. The job + # and its steps are `continue-on-error`; util/discussion_report.py writes a + # warning rather than failing. + discussion: + name: Post the lint report + needs: [ lint ] + if: ${{ always() && github.event_name != 'pull_request' && needs.lint.result != 'skipped' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + continue-on-error: true + permissions: + contents: read + discussions: write + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # A run that never wrote a report uploaded none; the script then warns. + - name: Download the report + continue-on-error: true + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: lint-report + path: report + + - name: Post the report to Discussions + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + LINT_RESULT: ${{ needs.lint.result }} + LINT_STATUS: ${{ needs.lint.outputs.status }} + run: | + if [ "$LINT_RESULT" = success ]; then + status="${LINT_STATUS:-success}" + else + status="$LINT_RESULT" + fi + python3 util/discussion_report.py --category "$REPORT_CATEGORY" \ + --title 'lint: weekly lint report' --report report/lint-report.md \ + --mode scheduled --status "$status" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 07a1bf5a95..1c27bbd748 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -689,6 +689,9 @@ grepping every workflow file for its name: * - Required context - Emitted by - Where + * - ``Lint`` + - job ``lint``, required since `#1967 `__ + - ``lint.yml`` * - ``Required checks passed`` - job ``required-checks`` - ``unit-tests.yml`` @@ -1221,8 +1224,9 @@ does; each has tests under :file:`tests/project/` that run it. ``gate``, at ``error``. * - :file:`util/discussion_report.py` - ``Post the report to Discussions`` in the ``discussion`` job of - ``ci-timing.yml``, ``cron-vendor.yml``, ``cron-conda.yml`` and - ``cron-python-versions.yml`` (see `CI Reports`_). + ``ci-timing.yml``, ``cron-vendor.yml``, ``cron-conda.yml``, + ``cron-python-versions.yml``, ``lint.yml`` and ``deploy-pages.yml`` (see + `CI Reports`_). * - :file:`util/apt_packages.py` - The apt steps of ``unit-tests.yml``'s ``engine-tests`` and ``pypcap-parity`` (and so its KTLO twin): the plan, seed, install and collect steps, as @@ -1306,6 +1310,20 @@ hand-edit note, for a failed audit of a changed tree, or for an unset because ``pr`` waits for the owner's approval of ``github-release``, and the pull request it opens is its own record. +``lint.yml`` has no push trigger, so it posts as ``scheduled`` to ``lint: weekly +lint report``, on ``schedule`` and ``workflow_dispatch``, never on a pull request, +whose run has checks of its own. Its report lists each tool step's outcome, then +the blocks those steps wrote to the job summary, which each also appends to a file +the report reads. It reports ``warning`` where an advisory step failed, as each +does while its findings stand, and ``failure`` where actionlint failed the job. +``deploy-pages.yml`` posts to ``deploy-pages: documentation build report`` by +``cron-vendor.yml``'s rules: ``push`` on its ``workflow_run``, ``scheduled`` on its +``schedule``, and never on a pull request or a verdict for another commit. Its +report is the results of ``head``, ``deploy-pages`` and ``publish-pages``, so it +uploads no artifact, and its ``discussion`` job checks out the script alone. The +``--status`` is the publish's result after a green build, the build's where the +build did not pass, and the gate's where the gate skipped the build. + The category is the workflow-level ``REPORT_CATEGORY``. The API cannot create one, so it is made by hand, in Settings, Discussions. @@ -1389,8 +1407,9 @@ actionlint runs the image's shellcheck over each ``run:`` block. Unlike Lint's other steps, this one blocks: a finding fails the job, as the owner ruled on #1967 once #2023, #2037 and #2038 had fixed every finding it reported. It runs last and on ``!cancelled()``, so a finding skips none of the advisory steps and a failed -Python setup does not skip it. Lint is not a required check of ruleset 23497679, so -a red Lint does not hold a merge. The image's shellcheck is not pinned, so a runner +Python setup does not skip it. ``Lint`` is a required check of ruleset 23497679 +(`#1967 `__), so a red Lint +holds a merge. The image's shellcheck is not pinned, so a runner image that adds a check can turn Lint red on unchanged code; the fix is to the script it flags. The step's run summary counts the findings by actionlint rule and by shellcheck code. diff --git a/tests/project/test_actionlint_in_lint_1967.py b/tests/project/test_actionlint_in_lint_1967.py index 59753e5532..bc7ca75fdc 100644 --- a/tests/project/test_actionlint_in_lint_1967.py +++ b/tests/project/test_actionlint_in_lint_1967.py @@ -56,7 +56,10 @@ ), } #: The ``if:`` values under which a step still runs after an earlier step failed. -RUNS_AFTER_A_FAILURE = frozenset({'${{ !cancelled() }}', '${{ always() }}', '!cancelled()', 'always()'}) +#: The last is the run report's (#2030), which a pull request skips whatever +#: actionlint found. +RUNS_AFTER_A_FAILURE = frozenset({'${{ !cancelled() }}', '${{ always() }}', '!cancelled()', 'always()', + "${{ !cancelled() && github.event_name != 'pull_request' }}"}) #: An ``-ignore`` argument in the step's script, quoted either way. IGNORE = re.compile(r'''-ignore\s+(?P['"])(?P.*?)(?P=quote)''') diff --git a/tests/project/test_lint_pages_reports_2030.py b/tests/project/test_lint_pages_reports_2030.py new file mode 100644 index 0000000000..d40ace2837 --- /dev/null +++ b/tests/project/test_lint_pages_reports_2030.py @@ -0,0 +1,311 @@ +# -*- coding: utf-8 -*- +"""#2030: the ``discussion`` jobs of :file:`lint.yml` and :file:`deploy-pages.yml`. + +Each posts its workflow's run report to that job's thread in the ``CI reports`` +category with :file:`util/discussion_report.py`, which +:mod:`tests.project.test_discussion_report_2030` tests on its own. + +* :class:`TestTheWorkflowsPost` pins both workflows: the ``discussion`` job + alone holds ``discussions: write``, every other job's permissions as they + were, first-party actions only, the command each posts with, the mode each + event maps to, Lint's report steps and their guards, and when each job runs: + never on a pull request, and for GitHub Pages never on a verdict for another + commit. +* :class:`TestTheLintReport` runs Lint's ``Write the run report`` as the runner + runs it. +* :class:`TestThePostSteps` runs each ``Post the report to Discussions`` with a + stand-in ``python3``: the status it passes, and the report it hands over. + +""" + +from __future__ import annotations + +import json +import os +import pathlib +import shutil +import subprocess # nosec: B404 +import tempfile +import unittest +from typing import TYPE_CHECKING + +from tests._workflows import WORKFLOWS, load, pinned +from tests.project.test_cron_reports_2030 import _ReportStep, launcher, step +from tests.project.test_discussion_report_2030 import (BASH, CATEGORY, FAKE_PYTHON, FIRST_PARTY, + POSTER, RUN, evaluate) +from tests.project.test_release_gates import STATUS, job_blocks, job_runs + +if TYPE_CHECKING: + from typing import Any, Optional + +LINT = 'lint.yml' +PAGES = 'deploy-pages.yml' +POST = 'Post the report to Discussions' +#: Each workflow's thread, and the report file the post step reads. +TITLES = {LINT: ('lint: weekly lint report', 'lint-report.md'), + PAGES: ('deploy-pages: documentation build report', 'pages-report.md')} +#: Every job's own ``permissions:``, and the top-level block, as before #2030. +PERMISSIONS = { + LINT: {None: {'contents': 'read'}, 'lint': None}, + PAGES: {None: {'contents': 'read'}, 'head': {'contents': 'read'}, 'deploy-pages': None, + 'publish-pages': {'actions': 'read', 'contents': 'write'}}, +} +#: Each ``discussion`` job's ``needs:`` and ``if:``. +NEEDS = {LINT: ['lint'], PAGES: ['head', 'deploy-pages', 'publish-pages']} +IFS = { + LINT: "${{ always() && github.event_name != 'pull_request' && needs.lint.result != 'skipped' }}", + PAGES: "${{ always() && github.event_name != 'pull_request' && (needs.deploy-pages.result != 'skipped' || " + "needs.head.result == 'failure' || needs.head.result == 'cancelled') }}", +} +#: The ``--mode`` and ``--status`` each workflow passes, and the step's ``env:`` +#: beside ``GH_TOKEN``. +ARGS = { + LINT: ('--mode scheduled --status "$status"', { + 'LINT_RESULT': '${{ needs.lint.result }}', + 'LINT_STATUS': '${{ needs.lint.outputs.status }}', + }), + PAGES: ('--mode "$MODE" --status "$status"', { + 'MODE': "${{ github.event_name == 'workflow_run' && 'push' || 'scheduled' }}", + 'HEAD_RESULT': '${{ needs.head.result }}', + 'BUILD_RESULT': '${{ needs.deploy-pages.result }}', + 'PUBLISH_RESULT': '${{ needs.publish-pages.result }}', + }), +} +#: Lint's tool steps, each with the ``id`` the report reads its outcome by. +TOOLS = ('bandit', 'mypy', 'vermin', 'pylint', 'actionlint') +#: The guard on Lint's report steps. +REPORTS = "${{ !cancelled() && github.event_name != 'pull_request' }}" + + +class TestTheWorkflowsPost(unittest.TestCase): + """The ``discussion`` job of each workflow, and every other job left as it was.""" + + def test_only_the_posting_job_may_write_discussions(self) -> None: + for name, permissions in PERMISSIONS.items(): + spec = load(name) + with self.subTest(workflow=name): + self.assertEqual(spec['permissions'], permissions[None]) + self.assertEqual(sorted(spec['jobs']), sorted([*filter(None, permissions), 'discussion'])) + self.assertEqual(spec['jobs']['discussion']['permissions'], POSTER) + for job_id, expected in permissions.items(): + if job_id is None: + continue + with self.subTest(job=job_id): + self.assertEqual(spec['jobs'][job_id].get('permissions'), expected) + self.assertNotIn('discussion', json.dumps(spec['jobs'][job_id])) + + def test_the_posting_job_runs_first_party_actions_only(self) -> None: + for name in TITLES: + job = load(name)['jobs']['discussion'] + self.assertNotIn('environment', job) + for entry in job['steps']: + if 'uses' not in entry: + continue + with self.subTest(workflow=name, step=entry.get('name') or entry['uses']): + self.assertTrue(any(pinned(entry['uses'], action) for action in FIRST_PARTY), entry['uses']) + if pinned(entry['uses'], 'actions/checkout'): + self.assertIs(entry['with']['persist-credentials'], False) + + def test_each_workflow_posts_its_report_with_the_script(self) -> None: + for name, (title, file) in TITLES.items(): + spec = load(name) + job = spec['jobs']['discussion'] + with self.subTest(workflow=name): + self.assertEqual(spec['env'], {'REPORT_CATEGORY': CATEGORY}) + self.assertEqual(job['needs'], NEEDS[name]) + self.assertEqual(' '.join(job['if'].split()), IFS[name]) + self.assertIs(job['continue-on-error'], True) + self.assertEqual(job['runs-on'], 'ubuntu-latest') + self.assertTrue(title.startswith(name[:-len('.yml')] + ': '), title) + post = step(name, 'discussion', POST) + self.assertIs(post['continue-on-error'], True) + args, env = ARGS[name] + self.assertEqual(post['env'], {'GH_TOKEN': '${{ github.token }}', **env}) + command = ' '.join(post['run'][post['run'].index(RUN):].replace('\\\n', ' ').split()) + self.assertEqual(command, f'{RUN} --category "$REPORT_CATEGORY" --title \'{title}\' ' + f'--report report/{file} {args}') + + def test_no_other_job_runs_the_script(self) -> None: + for name in TITLES: + for job_id, job in load(name)['jobs'].items(): + if job_id == 'discussion': + continue + with self.subTest(workflow=name, job=job_id): + self.assertNotIn('discussion_report', json.dumps(job)) + + def test_lint_hands_its_report_on(self) -> None: + """Lint's report steps come last, skip a pull request, and never fail the job.""" + spec = load(LINT) + lint = spec['jobs']['lint'] + self.assertEqual(lint['outputs'], {'status': '${{ steps.run-report.outputs.status }}'}) + steps = lint['steps'] + self.assertEqual([entry.get('name') for entry in steps][-2:], ['Write the run report', 'Upload the run report']) + self.assertEqual(steps[-2]['id'], 'run-report') + for entry in steps[-2:]: + self.assertEqual(entry['if'], REPORTS) + self.assertIs(entry['continue-on-error'], True) + self.assertEqual(steps[-2]['env'], {tool.upper(): f'${{{{ steps.{tool}.outcome }}}}' for tool in TOOLS}) + for tool in TOOLS: + with self.subTest(tool=tool): + entry = step(LINT, 'lint', tool) + self.assertEqual(entry['id'], tool) + self.assertIn('| tee -a "$RUNNER_TEMP/lint-summary.md" >> "$GITHUB_STEP_SUMMARY"', entry['run']) + upload = steps[-1] + self.assertTrue(pinned(upload['uses'], 'actions/upload-artifact')) + self.assertEqual(upload['with']['name'], 'lint-report') + self.assertEqual(upload['with']['path'], '${{ runner.temp }}/lint-report.md') + [download] = [entry for entry in spec['jobs']['discussion']['steps'] + if pinned(entry.get('uses'), 'actions/download-artifact')] + self.assertEqual(download['with'], {'name': 'lint-report', 'path': 'report'}) + self.assertIs(download['continue-on-error'], True) + + def test_pages_reads_no_artifact_and_checks_out_the_script_alone(self) -> None: + job = load(PAGES)['jobs']['discussion'] + uses = [entry['uses'] for entry in job['steps'] if 'uses' in entry] + self.assertEqual(len(uses), 1) + self.assertTrue(pinned(uses[0], 'actions/checkout')) + self.assertEqual(job['steps'][0]['with'], {'sparse-checkout': 'util/discussion_report.py\n', + 'sparse-checkout-cone-mode': False, + 'persist-credentials': False}) + + def test_the_mode_follows_the_event(self) -> None: + """Lint has no push trigger; GitHub Pages' ``workflow_run`` follows a push, so ``push``.""" + self.assertEqual(sorted(load(LINT)['on']), ['pull_request', 'schedule', 'workflow_dispatch']) + self.assertEqual(sorted(load(PAGES)['on']), ['pull_request', 'schedule', 'workflow_run']) + expression = step(PAGES, 'discussion', POST)['env']['MODE'] + for event, mode in (('schedule', 'scheduled'), ('workflow_run', 'push')): + with self.subTest(event=event): + self.assertEqual(evaluate(expression, {'github': {'event_name': event}}), mode) + + def test_lint_posts_on_the_schedule_and_dispatch_alone(self) -> None: + jobs = job_blocks((WORKFLOWS / LINT).read_text(encoding='utf-8')) + for event, result, expected in (('schedule', 'success', True), ('schedule', 'failure', True), + ('workflow_dispatch', 'success', True), ('schedule', 'skipped', False), + ('pull_request', 'success', False), ('pull_request', 'failure', False)): + context = {STATUS: {'cancelled': False}, 'github': {'event_name': event, 'event': {}}, + 'needs': {'lint': {'result': result, 'outputs': {}}}} # type: dict[str, Any] + with self.subTest(event=event, lint=result): + self.assertEqual(job_runs(jobs, 'discussion', context), expected) + + def test_pages_posts_on_a_build_or_a_failed_gate_but_not_a_deliberate_skip(self) -> None: + """As cron-vendor's: a pull request's run, or a verdict for another commit, posts nothing.""" + jobs = job_blocks((WORKFLOWS / PAGES).read_text(encoding='utf-8')) + # `(event, head, deploy-pages, whether discussion runs)`. + rows = ( + ('schedule', 'success', 'success', True), + ('schedule', 'success', 'failure', True), + ('schedule', 'failure', 'skipped', True), + ('schedule', 'cancelled', 'skipped', True), + ('workflow_run', 'success', 'success', True), + ('workflow_run', 'failure', 'skipped', True), + ('workflow_run', 'success', 'skipped', False), + ('pull_request', 'skipped', 'success', False), + ('pull_request', 'skipped', 'failure', False), + ) + for event, head, built, expected in rows: + needs = {'head': {'result': head}, 'deploy-pages': {'result': built}, + 'publish-pages': {'result': 'skipped'}} + context = {STATUS: {'cancelled': False}, 'github': {'event_name': event, 'event': {}}, + 'needs': needs} # type: dict[str, Any] + with self.subTest(event=event, head=head, deploy_pages=built): + self.assertEqual(job_runs(jobs, 'discussion', context), expected) + + +class TestTheLintReport(_ReportStep): + """``lint``'s report: each tool's outcome, then the summary blocks.""" + + workflow, job, file = LINT, 'lint', 'lint-report.md' + #: A run where every advisory tool found something, as each does today. + FOUND = {'BANDIT': 'failure', 'MYPY': 'failure', 'VERMIN': 'failure', 'PYLINT': 'failure', + 'ACTIONLINT': 'success'} + + def lines(self, **env: 'str') -> 'str': + return ('### Lint\n\n' + f"- bandit, advisory: {env.get('BANDIT', 'not run')}\n" + f"- mypy, advisory: {env.get('MYPY', 'not run')}\n" + f"- vermin, advisory: {env.get('VERMIN', 'not run')}\n" + f"- pylint, advisory: {env.get('PYLINT', 'not run')}\n" + f"- actionlint, blocking: {env.get('ACTIONLINT', 'not run')}\n") + + def test_an_advisory_finding_warns_and_the_blocks_follow(self) -> None: + blocks = '### mypy\n```\nFound 3 errors in 2 files\n```\n' + (self.temp / 'lint-summary.md').write_text(blocks, encoding='utf-8') + self.assertEqual(self.report(**self.FOUND), (self.lines(**self.FOUND) + '\n' + blocks, 'warning')) + + def test_a_clean_run(self) -> None: + clean = dict.fromkeys(self.FOUND, 'success') + self.assertEqual(self.report(**clean), (self.lines(**clean), 'success')) + + def test_a_blocking_finding_alone_does_not_warn(self) -> None: + """actionlint fails the job, which the post step reads instead.""" + env = {**dict.fromkeys(self.FOUND, 'success'), 'ACTIONLINT': 'failure'} + self.assertEqual(self.report(**env), (self.lines(**env), 'success')) + + def test_a_job_that_stopped_before_the_tools(self) -> None: + self.assertEqual(self.report(), (self.lines(), 'success')) + + +@unittest.skipIf(not BASH, 'needs bash on PATH') +class TestThePostSteps(unittest.TestCase): + """Each ``Post the report to Discussions``, run with a stand-in ``python3``.""" + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-2030-post-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.root = pathlib.Path(tmp.name) + self.bin = self.root / 'bin' + self.bin.mkdir() + (self.bin / 'fake_python.py').write_text(FAKE_PYTHON, encoding='utf-8') + launcher(self.bin, 'python3', 'fake_python.py') + self.log = self.root / 'python.json' + + def run_step(self, name: 'str', artifact: 'Optional[str]' = None, **env: 'str') -> 'tuple[str, str]': + """The ``--status`` the step passed, and the report it handed over.""" + title, file = TITLES[name] + work = self.root / 'work' + shutil.rmtree(work, ignore_errors=True) + (work / 'report').mkdir(parents=True) + if artifact is not None: + (work / 'report' / file).write_text(artifact, encoding='utf-8') + script = self.root / 'step.sh' + script.write_text(step(name, 'discussion', POST)['run'], encoding='utf-8') + env = dict(env, PATH=f'{self.bin}{os.pathsep}{os.environ.get("PATH", "")}', FAKE_PYTHON_LOG=str(self.log), + REPORT_CATEGORY=CATEGORY) + result = subprocess.run([str(BASH), '-e', str(script)], cwd=work, env=env, # nosec: B603 + capture_output=True, text=True, check=False, timeout=60) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + logged = json.loads(self.log.read_text(encoding='utf-8')) + args = logged['args'] + self.assertEqual(args[:9], ['util/discussion_report.py', '--category', CATEGORY, '--title', title, + '--report', f'report/{file}', '--mode', env.get('MODE', 'scheduled')]) + self.assertEqual(args[9], '--status') + return args[10], logged['report'] + + def test_the_lint_status(self) -> None: + for result, status, expected in (('success', 'warning', 'warning'), ('success', 'success', 'success'), + ('success', '', 'success'), ('failure', 'warning', 'failure'), + ('cancelled', '', 'cancelled')): + with self.subTest(result=result, status=status): + self.assertEqual(self.run_step(LINT, 'the week\n', LINT_RESULT=result, LINT_STATUS=status), + (expected, 'the week\n')) + + def test_the_pages_status_and_report(self) -> None: + # `(head, deploy-pages, publish-pages, --status)`. + for head, built, published, expected in ( + ('success', 'success', 'success', 'success'), + ('success', 'success', 'failure', 'failure'), + ('success', 'success', 'cancelled', 'cancelled'), + ('success', 'failure', 'skipped', 'failure'), + ('success', 'cancelled', 'skipped', 'cancelled'), + ('failure', 'skipped', 'skipped', 'failure'), + ('cancelled', 'skipped', 'skipped', 'cancelled'), + ): + for mode in ('push', 'scheduled'): + with self.subTest(head=head, deploy_pages=built, publish_pages=published, mode=mode): + self.assertEqual(self.run_step(PAGES, MODE=mode, HEAD_RESULT=head, BUILD_RESULT=built, + PUBLISH_RESULT=published), (expected, ( + f'### GitHub Pages\n\n- Gate: {head}\n- Build: {built}\n- Publish: {published}\n'))) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_workflow_hardening_1968.py b/tests/project/test_workflow_hardening_1968.py index c0f7532ec7..1a0816c14d 100644 --- a/tests/project/test_workflow_hardening_1968.py +++ b/tests/project/test_workflow_hardening_1968.py @@ -255,14 +255,18 @@ def test_a_pull_request_builds_the_docs_with_read_only_contents(self) -> None: self.assertEqual(job_runs(jobs, 'publish-pages', context), expected) # Only the job that runs no project code holds the write token, and - # only it pushes. + # only it pushes. The run report's job (#2030) may write discussions + # alone. + self.assertEqual(spec['jobs']['discussion']['permissions'], {'contents': 'read', 'discussions': 'write'}) for job_id, job in spec['jobs'].items(): uses = [str(entry.get('uses', '')) for entry in job.get('steps') or []] with self.subTest(job=job_id): pushes = any(entry.startswith('JamesIves/github-pages-deploy-action@') for entry in uses) self.assertEqual(pushes, job_id == 'publish-pages') - writes = 'write' in (job.get('permissions') or spec['permissions']).values() - self.assertEqual(writes, job_id == 'publish-pages') + scopes = dict(job.get('permissions') or spec['permissions']) + if job_id == 'discussion': + del scopes['discussions'] + self.assertEqual('write' in scopes.values(), job_id == 'publish-pages') self.assertFalse([entry for entry in publish['steps'] if 'pip install' in str(entry.get('run', ''))]) def test_the_deploy_waits_on_the_freshness_check(self) -> None: diff --git a/tests/project/test_workflow_loader_1967_unit.py b/tests/project/test_workflow_loader_1967_unit.py index 1e50cfb12c..8f8a23ea2b 100644 --- a/tests/project/test_workflow_loader_1967_unit.py +++ b/tests/project/test_workflow_loader_1967_unit.py @@ -122,7 +122,7 @@ def test_doctored_copy(self) -> None: self.assertEqual(self.parser.call_count, 1) path.write_text(text + '\nenv:\n DOCTORED: "1"\n', encoding='utf-8') self.assertEqual(_workflows.load(path)['env'], {'DOCTORED': '1'}) - self.assertNotIn('env', _workflows.load(OTHER)) + self.assertNotIn('DOCTORED', _workflows.load(OTHER).get('env') or {}) class TestOnIsTheTriggerKey(LoaderTestCase):