diff --git a/.github/workflows/cron-conda.yml b/.github/workflows/cron-conda.yml index 994fd94a88..690d235975 100644 --- a/.github/workflows/cron-conda.yml +++ b/.github/workflows/cron-conda.yml @@ -406,6 +406,12 @@ jobs: needs.conda-update.outputs.CONDA_CHANGED == 'true' permissions: contents: write + # #2060: the Windows leg builds and smoke-tests only, and never blocks. Each + # step that can fail on it has an `id`, and a `continue-on-error` true on + # `windows-latest` alone, so a failure leaves the leg at success, and + # `Windows summary` reports it as a warning, as the KTLO legs do (#1672). + # No job-level `continue-on-error`, which would still turn the leg's check + # red (#1672), so a lost runner fails the job. strategy: fail-fast: false matrix: @@ -415,25 +421,37 @@ jobs: - macos-latest - macos-26-intel - ubuntu-latest - # No Windows leg since 2023, and no Windows steps (#1967): adding the leg - # back means writing its steps, as test_conda_no_windows_steps_1967 pins. + # #2060: builds and smoke-tests win-64, and publishes nothing until + # its runs pass. Every upload step below is gated off it, so the + # `ANACONDA_TOKEN` this job's `environment:` holds reaches no step on it. + - windows-latest # #1633: the supported tier, from .github/python-versions.toml through # `conda-update` (#1660); create-release.yml's ``conda`` job says why # there is no KTLO 3.10 leg. python-version: ${{ fromJSON(needs.conda-update.outputs.PYTHON_CONDA) }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # No persisted token: nothing here runs `git` with credentials, and the + # Windows leg builds a package nobody has published yet (#2060). + - id: checkout + continue-on-error: ${{ matrix.os == 'windows-latest' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: conda-${{ needs.conda-update.outputs.PCAPKIT_VERSION }}+${{ needs.conda-update.outputs.PCAPKIT_BUILD }} fetch-depth: 0 + persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - id: setup-python + continue-on-error: ${{ matrix.os == 'windows-latest' }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} # #1854: pcapkit is not installed here; see create-release.yml's step of - # the same name. + # the same name. `shell: bash` here and below, as Windows runs `pwsh`. - name: Install and Setup + id: install + continue-on-error: ${{ matrix.os == 'windows-latest' }} + shell: bash run: | set -x @@ -441,8 +459,12 @@ jobs: # #1827: pip evaluates conda/requirements.txt's platform markers here, # on this leg's own runner, so Intel macOS fetches its own cryptography - # pin; the recipe's build.sh installs whatever this downloaded. + # pin; the recipe's build.sh, or bld.bat on Windows, installs whatever + # this downloaded. - name: Setup Conda Distribution + id: download + continue-on-error: ${{ matrix.os == 'windows-latest' }} + shell: bash run: | set -x @@ -451,6 +473,8 @@ jobs: # See create-release.yml's step of the same name for the 3.15 exception. - name: Conda environment creation and activation + id: miniconda + continue-on-error: ${{ matrix.os == 'windows-latest' }} uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 env: PCAPKIT_VERSION: ${{ needs.conda-update.outputs.PCAPKIT_VERSION }} @@ -467,13 +491,19 @@ jobs: conda-remove-defaults: false show-channel-urls: true + # A login shell, as for the build below: on Windows only that one has the + # `conda` setup-miniconda's `conda init` defines. - name: Install conda build tools + id: conda-build-tools + continue-on-error: ${{ matrix.os == 'windows-latest' }} + shell: bash -el {0} run: | conda install -n base --override-channels -c conda-forge -y conda-build conda build --version - name: Build conda packages id: conda_build + continue-on-error: ${{ matrix.os == 'windows-latest' }} shell: bash -el {0} env: PCAPKIT_VERSION: ${{ needs.conda-update.outputs.PCAPKIT_VERSION }} @@ -502,6 +532,9 @@ jobs: ubuntu-latest) target_platform="linux-64" ;; + windows-latest) + target_platform="win-64" + ;; *) target_platform="" ;; @@ -540,17 +573,27 @@ jobs: # site-packages on the path. #1822: it also fails unless the package and # every native binary in it are for this runner's platform. # setup-python's interpreter, not conda's: it is the matrix version the - # package was built for, which conda's base is not on 3.15. + # package was built for, which conda's base is not on 3.15. On Windows it + # is `python.exe` at the top of `pythonLocation`, not under `bin/` (#2060). - name: Smoke-test the vendored packages + id: smoke + if: steps.conda_build.outcome == 'success' + continue-on-error: ${{ matrix.os == 'windows-latest' }} + shell: bash env: PCAPKIT_CONDA_PACKAGE: ${{ steps.conda_build.outputs.host_package }} run: | - "${pythonLocation}/bin/python" util/conda_vendor.py --smoke "$PCAPKIT_CONDA_PACKAGE" + python="${pythonLocation}/bin/python" + if [ "$RUNNER_OS" = Windows ]; then + python="$(cygpath -u "$pythonLocation")/python.exe" + fi + "$python" util/conda_vendor.py --smoke "$PCAPKIT_CONDA_PACKAGE" # #2044: not on macos-26-intel, whose leg "Upload conda packages to - # Anaconda (Intel)" uploads instead. + # Anaconda (Intel)" uploads instead. #2060: nor on windows-latest, which + # publishes nothing until a run has shown its package works. - name: Upload conda packages to Anaconda - if: ${{ matrix.os != 'macos-26-intel' }} + if: ${{ matrix.os != 'macos-26-intel' && matrix.os != 'windows-latest' }} uses: anaconda/actions/upload-package@a6f488910b4a1e4f3fe01736241ca1b92b4b355a # v0.3.1 with: token: ${{ secrets.ANACONDA_TOKEN }} @@ -663,6 +706,18 @@ jobs: tag: "v${{ needs.conda-update.outputs.PCAPKIT_VERSION }}" token: "${{ secrets.GITHUB_TOKEN }}" + # #2060: the Windows leg's last step, as `KTLO summary` is a KTLO leg's + # (#1672): one warning per failed step, from each `outcome`. setup-python's + # `python`, or the image's where that failed. + - name: Windows summary + if: ${{ !cancelled() && matrix.os == 'windows-latest' }} + continue-on-error: true + shell: bash + env: + LEG: conda-dist on ${{ matrix.os }} with Python ${{ matrix.python-version }} (unpublished, non-blocking) + STEPS: ${{ toJSON(steps) }} + run: python util/leg_summary.py conda-windows + # #2030: posts `conda-update`'s report with the tag and upload results, and a # short one of its own where `head` failed or was cancelled, as cron-vendor.yml's # `discussion` job does, whose comment gives the rules. It waits for diff --git a/conda/pypcapkit/bld.bat b/conda/pypcapkit/bld.bat index d5a4303ae6..5033d027b4 100644 --- a/conda/pypcapkit/bld.bat +++ b/conda/pypcapkit/bld.bat @@ -1,10 +1,21 @@ @ECHO OFF +SETLOCAL EnableDelayedExpansion -REM "%PYTHON%" -m pip install conda\wheels\* --target pcapkit/_extern -vv -FOR /F "Tokens=*" %A IN ('DIR /B conda\wheels\') DO @( - "%PYTHON%" -m pip install conda\wheels\%A --target pcapkit\_extern -vv +REM build.sh's two calls, for Windows (#2060). cmd does not expand the glob in +REM build.sh's `pip install conda/wheels/*`, so collect what Setup Conda +REM Distribution downloaded and install it in one call, letting pip resolve the +REM files together. In a batch file a FOR variable takes two percent signs; the +REM loop this replaces had one, a syntax error, and likely why no win-64 package +REM was ever built. +SET WHEELS= +FOR %%W IN (conda\wheels\*) DO SET WHEELS=!WHEELS! "%%W" +IF NOT DEFINED WHEELS ( + ECHO bld.bat: conda\wheels holds nothing to install 1>&2 + EXIT 1 ) -"%PYTHON%" -m pip install . -vv +"%PYTHON%" -m pip install !WHEELS! --target pcapkit\_extern -vv +IF ERRORLEVEL 1 EXIT 1 +"%PYTHON%" -m pip install . -vv IF ERRORLEVEL 1 EXIT 1 diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 187dc1f09a..02c88ba4f5 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -1548,6 +1548,15 @@ it, so the scheduled jobs reading them name it too * Conda Update's ``conda-dist`` -- ``environment: anaconda`` * Vendor Update's ``vendor-update`` -- ``environment: conda-tag`` +``conda-dist`` also has a ``windows-latest`` leg, which builds and smoke-tests a +``win-64`` package and publishes nothing +(`#2060 `__). Every upload +step is gated off it, and those are the only steps that read ``ANACONDA_TOKEN``, +so no step on that leg receives the secret. +:file:`tests/project/test_conda_windows_leg_2060.py` checks both. Each step that +can fail there continues on error, and ``Windows summary`` reports each failure +as a warning, as ``KTLO summary`` does for a KTLO leg, so the leg stays green. + A required reviewer on ``conda-tag`` or ``anaconda`` would therefore hold every scheduled bump for an approval, besides the release-graph reason :doc:`releasing` gives for keeping them without one. Each run of these jobs diff --git a/tests/project/python_versions_fixtures/versions-manifest.json b/tests/project/python_versions_fixtures/versions-manifest.json index 0f7aa692bf..ee9985ea2a 100644 --- a/tests/project/python_versions_fixtures/versions-manifest.json +++ b/tests/project/python_versions_fixtures/versions-manifest.json @@ -30,6 +30,11 @@ "arch": "x64", "platform": "linux", "platform_version": "26.04" + }, + { + "filename": "python-3.15.0-win32-x64.zip", + "arch": "x64", + "platform": "win32" } ] }, @@ -64,6 +69,11 @@ "arch": "x64", "platform": "linux", "platform_version": "26.04" + }, + { + "filename": "python-3.14.8-win32-x64.zip", + "arch": "x64", + "platform": "win32" } ] }, @@ -110,6 +120,11 @@ "arch": "x64", "platform": "linux", "platform_version": "26.04" + }, + { + "filename": "python-3.13.16-win32-x64.zip", + "arch": "x64", + "platform": "win32" } ] }, @@ -168,6 +183,11 @@ "arch": "x64", "platform": "linux", "platform_version": "26.04" + }, + { + "filename": "python-3.12.15-win32-x64.zip", + "arch": "x64", + "platform": "win32" } ] }, @@ -238,6 +258,11 @@ "arch": "x64", "platform": "linux", "platform_version": "26.04" + }, + { + "filename": "python-3.11.17-win32-x64.zip", + "arch": "x64", + "platform": "win32" } ] }, diff --git a/tests/project/test_conda_cryptography_pin_1827.py b/tests/project/test_conda_cryptography_pin_1827.py index 9985cd5211..97c8069920 100644 --- a/tests/project/test_conda_cryptography_pin_1827.py +++ b/tests/project/test_conda_cryptography_pin_1827.py @@ -183,11 +183,11 @@ def test_a_dropped_or_moved_platform_pin_fails(self) -> None: def test_an_override_no_build_can_use_fails(self) -> None: lines = written() - pins = pinned(('cryptography', 'osx-64', '48.0.1'), ('cryptography', 'win-64', '1.0'), + pins = pinned(('cryptography', 'osx-64', '48.0.1'), ('cryptography', 'linux-aarch64', '1.0'), ('gone', 'osx-64', '1.0')) with mock.patch.object(vendor(), 'PLATFORM_PINS', pins): problems = self.problems('\n'.join(lines)) - self.assertEqual(problems, ['override: cryptography is pinned for win-64, which no conda leg builds', + self.assertEqual(problems, ['override: cryptography is pinned for linux-aarch64, which no conda leg builds', 'override: gone is pinned for osx-64, and no conda build vendors it']) def test_the_writer_refuses_an_override_no_build_can_use(self) -> None: diff --git a/tests/project/test_conda_intel_cli_upload_2044.py b/tests/project/test_conda_intel_cli_upload_2044.py index 7b8de586a6..a20b72c991 100644 --- a/tests/project/test_conda_intel_cli_upload_2044.py +++ b/tests/project/test_conda_intel_cli_upload_2044.py @@ -119,8 +119,9 @@ def test_the_intel_leg_runs_the_cli_step_only(self) -> None: self.assertTrue(runs(step(name, job_id, CLI_STEP), INTEL)) def test_every_other_leg_runs_the_action_only(self) -> None: + # #2060: but the Windows leg, which runs neither; test_conda_windows_leg_2060 pins that. for name, job_id in JOBS: - others = [os for os in legs(name, job_id) if os != INTEL] + others = [os for os in legs(name, job_id) if os not in (INTEL, 'windows-latest')] with self.subTest(workflow=name, job=job_id): self.assertTrue(others, 'no other leg to check') for os in others: diff --git a/tests/project/test_conda_native_platforms_1822.py b/tests/project/test_conda_native_platforms_1822.py index 14db045e07..232557e5c0 100644 --- a/tests/project/test_conda_native_platforms_1822.py +++ b/tests/project/test_conda_native_platforms_1822.py @@ -4,11 +4,12 @@ * Both conda jobs map each runner label of their ``os`` axis to the conda platform of that runner's own architecture, as GitHub's runner-images table gives it (the snapshot in :file:`python_versions_fixtures`), and between them - publish exactly :data:`conda_vendor.PLATFORMS`. + build exactly :data:`conda_vendor.PLATFORMS`; win-64 only in Conda Update, + which does not publish it yet (#2060). * Neither job runs ``conda convert``. Run as the job runs it, with a stand-in ``conda``, a leg publishes the one package conda built, and fails before any upload when that is not the platform its label maps to. -* ``--smoke`` reads each native binary's ELF or Mach-O header and fails on one +* ``--smoke`` reads each native binary's ELF, Mach-O or PE header and fails on one that is not for the package's platform, and on a package that is not for the runner's own. @@ -65,8 +66,11 @@ #: Intel, per https://docs.github.com/en/actions/reference/runners/github-hosted-runners #: as read on 2026-10-16; the test below checks the snapshot of the image table too. LEGS = {'macos-latest': 'osx-arm64', 'macos-26-intel': 'osx-64', 'ubuntu-latest': 'linux-64'} +#: Each job's legs: Conda Update's add #2060's unpublished ``windows-latest``. +JOB_LEGS = {('cron-conda.yml', 'conda-dist'): dict(LEGS, **{'windows-latest': 'win-64'}), + ('create-release.yml', 'conda'): LEGS} #: ``platform.machine()``, as the runner-images table names the architecture. -ARCH = {'x86_64': 'x64', 'arm64': 'arm64'} +ARCH = {'x86_64': 'x64', 'arm64': 'arm64', 'AMD64': 'x64'} _MODULES = {} # type: dict[str, Any] @@ -117,7 +121,7 @@ def test_each_leg_publishes_its_runners_own_platform(self) -> None: self.assertTrue(blocks, f'{name} {job_id} maps no runner label') for block in blocks: with self.subTest(workflow=name, job=job_id): - self.assertEqual({label: block.get(label) for label in labels}, LEGS) + self.assertEqual({label: block.get(label) for label in labels}, JOB_LEGS[(name, job_id)]) for label in labels: sys_platform, _, machine = vendor.PLATFORMS[block[label]] with self.subTest(workflow=name, job=job_id, label=label): @@ -127,8 +131,9 @@ def test_each_leg_publishes_its_runners_own_platform(self) -> None: def test_the_legs_are_the_vendored_platforms(self) -> None: """What conda/requirements.txt is derived for is what the legs build.""" - self.assertEqual(sorted(load('conda_vendor').PLATFORMS), sorted(LEGS.values())) - self.assertEqual(sorted(load('conda_vendor').BINARIES), sorted(LEGS.values())) + built = sorted({platform for legs in JOB_LEGS.values() for platform in legs.values()}) + self.assertEqual(sorted(load('conda_vendor').PLATFORMS), built) + self.assertEqual(sorted(load('conda_vendor').BINARIES), built) #: Stands in for ``conda``: ``build`` writes an empty package under @@ -191,7 +196,7 @@ def build(self, name: 'str', job_id: 'str', label: 'str', built: 'str') -> 'tupl def test_a_leg_publishes_only_the_package_conda_built(self) -> None: for name, job_id in JOBS: - for label, platform in LEGS.items(): + for label, platform in JOB_LEGS[(name, job_id)].items(): with self.subTest(workflow=name, label=label): status, log, packages = self.build(name, job_id, label, platform) self.assertEqual(status, 0, log) @@ -219,6 +224,13 @@ def macho(cpu: 'int') -> 'bytes': return b'\xcf\xfa\xed\xfe' + cpu.to_bytes(4, 'little') + bytes(4) + (8).to_bytes(4, 'little') + bytes(16) +def pe(machine: 'int', offset: 'int' = 0x80) -> 'bytes': + """A PE image's header for COFF ``Machine``, its ``PE\\0\\0`` at ``offset`` (#2060).""" + head = bytearray(b'MZ\x90\x00' + bytes(offset - 4)) + head[0x3c:0x40] = offset.to_bytes(4, 'little') + return bytes(head) + b'PE\0\0' + machine.to_bytes(2, 'little') + bytes(18) + + def fat(*cpus: 'int') -> 'bytes': """A universal Mach-O's header with a slice per ``cputype``.""" return b'\xca\xfe\xba\xbe' + len(cpus).to_bytes(4, 'big') + b''.join( @@ -235,12 +247,16 @@ def fat(*cpus: 'int') -> 'bytes': 'macho-arm64': (macho(ARM64), ('Mach-O', {'arm64'})), 'universal2': (fat(X86_64, ARM64), ('Mach-O', {'x86_64', 'arm64'})), 'fat-arm64': (fat(ARM64), ('Mach-O', {'arm64'})), + 'pe-x86_64': (pe(0x8664), ('PE', {'x86_64'})), + 'pe-arm64': (pe(0xaa64), ('PE', {'arm64'})), + 'pe-i386': (pe(0x014c), ('PE', {'i386'})), } #: The stubs a package for each platform may carry. ACCEPTS = { 'linux-64': {'elf-x86_64'}, 'osx-64': {'macho-x86_64', 'universal2'}, 'osx-arm64': {'macho-arm64', 'universal2', 'fat-arm64'}, + 'win-64': {'pe-x86_64'}, } @@ -289,8 +305,8 @@ def test_other_files_are_passed_over_and_an_unreadable_library_is_not(self) -> N (root / 'pkg' / 'libffi.so.8').write_text('not a library', encoding='utf-8') count, problems = vendor.foreign(root, 'linux-64') self.assertEqual(count, 0) - self.assertEqual(problems, ['pkg/_native.pyd: neither ELF nor Mach-O, so its machine is unknown', - 'pkg/libffi.so.8: neither ELF nor Mach-O, so its machine is unknown']) + self.assertEqual(problems, ['pkg/_native.pyd: not ELF, Mach-O or PE, so its machine is unknown', + 'pkg/libffi.so.8: not ELF, Mach-O or PE, so its machine is unknown']) def test_noarch_carries_no_binary(self) -> None: vendor = load('conda_vendor') @@ -303,7 +319,7 @@ def test_the_runner_is_read_from_its_system_and_machine(self) -> None: vendor = load('conda_vendor') for here, subdir in ((('Linux', 'x86_64'), 'linux-64'), (('Darwin', 'x86_64'), 'osx-64'), (('Darwin', 'arm64'), 'osx-arm64'), (('Linux', 'aarch64'), None), - (('Windows', 'AMD64'), None)): + (('Windows', 'AMD64'), 'win-64'), (('Windows', 'ARM64'), None)): with self.subTest(here=here), mock.patch.object(vendor.platform, 'system', return_value=here[0]), \ mock.patch.object(vendor.platform, 'machine', return_value=here[1]): self.assertEqual(vendor.runner_platform(), subdir) diff --git a/tests/project/test_conda_no_windows_steps_1967.py b/tests/project/test_conda_no_windows_steps_1967.py index a4d6ac6245..23dc16fa0d 100644 --- a/tests/project/test_conda_no_windows_steps_1967.py +++ b/tests/project/test_conda_no_windows_steps_1967.py @@ -7,12 +7,16 @@ (Windows)``, the ``matrix.os != 'windows-latest'`` guards on their twins, and a ``windows-latest`` arm in each ``case "${{ matrix.os }}"``: code no run reached. -* :class:`TestNoDeadRunnerCode` pins that no step's ``if:`` names a runner label - the job's matrix lacks, no ``case`` arm maps one, and, with no Windows leg, no - step is Windows-only. So adding a Windows leg back means adding its steps back - deliberately, rather than reviving ones nobody has run. +* :class:`TestNoDeadRunnerCode` pins that no step's ``if:`` or + ``continue-on-error`` names a runner label the job's matrix lacks, no ``case`` + arm maps one, and, with no Windows leg, no step is Windows-only. So adding a + Windows leg back means adding its steps back deliberately, rather than + reviving ones nobody has run. #2060 did so for ``conda-dist``, whose Windows + steps are therefore live; ``conda`` still has no Windows leg. * :class:`TestTheCheckSeesDeadCode` feeds :func:`dead` the jobs as they stood - before, rebuilt from the current ones, so the pin cannot pass by reading nothing. + before, rebuilt from the current ones, so the pin cannot pass by reading + nothing, and ``conda-dist`` without its Windows leg, whose steps must then read + as dead. """ @@ -53,9 +57,10 @@ def dead(spec: 'dict[str, Any]') -> 'list[str]': for step in spec['steps']: name = str(step.get('name') or step.get('uses') or step.get('id')) condition = str(step.get('if', '')) - for label in GUARD.findall(condition): - if label not in labels: - found.append(f'{name}: `if:` compares matrix.os with {label}, which is no leg') + for key in ('if', 'continue-on-error'): + for label in GUARD.findall(str(step.get(key, ''))): + if label not in labels: + found.append(f'{name}: `{key}:` compares matrix.os with {label}, which is no leg') if not windows_leg and (WINDOWS_ONLY.search(condition) or WINDOWS.search(str(step.get('name', '')))): found.append(f'{name}: Windows-only, and no leg is Windows') for block in CASE.findall(str(step.get('run', ''))): @@ -74,12 +79,28 @@ def test_no_step_or_arm_names_a_missing_leg(self) -> None: self.assertEqual(dead(_workflows.job(name, job_id)), []) +def without_windows(step: 'dict[str, Any]') -> 'dict[str, Any]': + """``step`` without #2060's Windows guard, ``case`` arm or ``if:`` clause.""" + step = {key: value for key, value in step.items() + if not (key == 'continue-on-error' and WINDOWS.search(str(value)))} + if 'if' in step: + step['if'] = str(step['if']).replace(" && matrix.os != 'windows-latest'", '') + if 'run' in step: + step['run'] = re.sub(r'\n\s*windows-latest\)\n[^\n]*\n\s*;;', '', str(step['run'])) + return step + + class TestTheCheckSeesDeadCode(unittest.TestCase): """:func:`dead` reports each kind of dead code the jobs held before #1967.""" def test_it_reports_a_windows_step_guard_and_arm(self) -> None: for name, job_id in JOBS: spec = copy.deepcopy(_workflows.job(name, job_id)) + # #2060: conda-dist as it stood before its Windows leg came back. + spec['strategy']['matrix']['os'] = [os for os in spec['strategy']['matrix']['os'] + if not WINDOWS.search(os)] + spec['steps'] = [without_windows(step) for step in spec['steps'] + if not WINDOWS.search(str(step.get('name', '')))] install = next(step for step in spec['steps'] if step.get('name') == 'Install and Setup') install['if'] = "${{ matrix.os != 'windows-latest' }}" spec['steps'].append({'name': 'Install and Setup (Windows)', @@ -102,6 +123,18 @@ def test_it_reports_a_windows_step_guard_and_arm(self) -> None: 'Map the leg: `case` arm windows-latest, which is no leg', ]) + def test_dropping_conda_dists_windows_leg_leaves_its_steps_dead(self) -> None: + """#2060's steps are live only while the leg is there.""" + spec = copy.deepcopy(_workflows.job('cron-conda.yml', 'conda-dist')) + spec['strategy']['matrix']['os'].remove('windows-latest') + found = dead(spec) + for line in ('Upload conda packages to Anaconda: `if:` compares matrix.os with windows-latest, which is no leg', + 'Install and Setup: `continue-on-error:` compares matrix.os with windows-latest, which is no leg', + 'Windows summary: Windows-only, and no leg is Windows', + 'Build conda packages: `case` arm windows-latest, which is no leg'): + with self.subTest(line=line): + self.assertIn(line, found) + def test_a_windows_leg_allows_its_own_steps(self) -> None: spec = {'strategy': {'matrix': {'os': ['ubuntu-latest', 'windows-latest']}}, 'steps': [{'name': 'Setup (Windows)', 'if': "${{ matrix.os == 'windows-latest' }}", diff --git a/tests/project/test_conda_platform_pin_refresh_1851.py b/tests/project/test_conda_platform_pin_refresh_1851.py index 44d730beb3..f232b9c43f 100644 --- a/tests/project/test_conda_platform_pin_refresh_1851.py +++ b/tests/project/test_conda_platform_pin_refresh_1851.py @@ -362,7 +362,7 @@ def test_a_pin_never_passes_the_installed_release(self) -> None: self.assertEqual(self.refresh(recorded(**{'51.0.0': INTEL_AGAIN}), installed='50.0.2'), (table(), [])) def test_a_pin_on_a_platform_no_leg_builds_is_left_alone(self) -> None: - pins = {'cryptography': {'win-64': vendor().Pin('1.0', TODAY, 'a stand-in')}} + pins = {'cryptography': {'linux-aarch64': vendor().Pin('1.0', TODAY, 'a stand-in')}} result, notes = self.refresh(recorded(**{'51.0.0': INTEL_AGAIN}), pins=pins) self.assertEqual(result, pins) self.assertEqual([level for level, _ in notes], ['warning']) diff --git a/tests/project/test_conda_vendor_1814.py b/tests/project/test_conda_vendor_1814.py index 3a79776725..15772b663c 100644 --- a/tests/project/test_conda_vendor_1814.py +++ b/tests/project/test_conda_vendor_1814.py @@ -206,7 +206,7 @@ def lacking(name: 'str') -> 'list[str]': # Conda Update's newest Python. with self.assertRaises(module.NotInstalled) as elsewhere: module.derive([Requirement('core')], module.environments(['3.1', '3.2']), lacking) - self.assertIn('shim is needed on Python 3.1 on darwin, linux (core requires shim; ' + self.assertIn('shim is needed on Python 3.1 on darwin, linux, win32 (core requires shim; ' 'python_version < "3.2")', str(elsewhere.exception)) self.assertIn(f'and Python {here} on {sys.platform}, which runs this, does not have it: ' 'its marker leaves it out here', str(elsewhere.exception)) diff --git a/tests/project/test_conda_windows_leg_2060.py b/tests/project/test_conda_windows_leg_2060.py new file mode 100644 index 0000000000..a6e0015f58 --- /dev/null +++ b/tests/project/test_conda_windows_leg_2060.py @@ -0,0 +1,346 @@ +# -*- coding: utf-8 -*- +"""#2060, phase 1: Conda Update builds and smoke-tests win-64, and publishes none of it. + +``conda-dist`` in :file:`.github/workflows/cron-conda.yml` gained a +``windows-latest`` leg. ``create-release.yml``'s ``conda`` job has none yet; +phase 2 adds it, with the uploads. + +* :class:`TestTheWindowsLegPublishesNothing`: every step reading a secret or + publishing is evaluated on ``windows-latest`` with + :func:`tests.project.test_release_gates.evaluate_condition` and must not run. + So no step on that leg receives ``ANACONDA_TOKEN``, which the job's + ``environment: anaconda`` holds. +* :class:`TestTheWindowsLegNeverBlocks`: each step that runs there has an ``id`` + and a ``continue-on-error`` true on that leg alone, and ``Windows summary``, + the last step, reports their failures as warnings, as ``KTLO summary`` does (#1672). +* :class:`TestBldBat`: ``conda/pypcapkit/bld.bat`` does what ``build.sh`` does: + one install of every downloaded file, then the project, each checked. Its + previous ``FOR`` loop wrote ``%A``, a syntax error in a batch file. +* :class:`TestPortableExecutable` and :class:`TestTheWindowsLayout`: + :file:`util/conda_vendor.py` reads a ``.pyd``'s PE machine, and ``--smoke`` + takes a Windows package's ``Lib/site-packages``, whose path names no Python. + +Nothing here runs on Windows; what a real Conda Update run shows is the rest. + +""" + +from __future__ import annotations + +import bz2 +import copy +import io +import json +import os +import pathlib +import re +import subprocess # nosec: B404 +import sys +import tarfile +import tempfile +import unittest +from typing import TYPE_CHECKING +from unittest import mock + +from tests import _workflows +from tests.project.test_conda_native_platforms_1822 import DISTRIBUTION, SHIM, elf, load, pe +from tests.project.test_release_gates import PUBLISHING_MARKERS, STATUS, evaluate_condition + +if TYPE_CHECKING: + from typing import Any + +#: Test tier (#1958): reads the workflow and recipe files, and drives the script offline. +__tier__ = 'LATEST' + +ROOT = pathlib.Path(__file__).resolve().parents[2] +RECIPE = ROOT / 'conda' / 'pypcapkit' +JOB = ('cron-conda.yml', 'conda-dist') +WINDOWS = 'windows-latest' +SUMMARY = 'Windows summary' +#: The steps that read ``ANACONDA_TOKEN``: the uploads, and nothing else. +ANACONDA_UPLOADS = ['Upload conda packages to Anaconda', 'Upload conda packages to Anaconda (Intel)'] +#: What publishes besides :data:`PUBLISHING_MARKERS`: the GitHub Release upload. +MORE_MARKERS = ('ncipollo/release-action',) + + +def spec() -> 'dict[str, Any]': + return _workflows.job(*JOB) + + +def context(os_label: 'str') -> 'dict[str, object]': + """A leg on ``os_label`` where every earlier step succeeded and the release exists.""" + steps = {step['id']: {'outcome': 'success', 'conclusion': 'success', + 'outputs': {'published': 'true'}} + for step in spec()['steps'] if 'id' in step} + return {'matrix': {'os': os_label}, 'steps': steps, + STATUS: {'cancelled': False, 'success': True, 'failure': False}} + + +def runs(step: 'dict[str, Any]', os_label: 'str') -> 'bool': + return evaluate_condition(str(step.get('if', 'true')), context(os_label)) + + +def sensitive(step: 'dict[str, Any]') -> 'bool': + """Whether ``step`` reads a secret or publishes anything.""" + text = repr(step) + return 'secrets.' in text or any(marker in text for marker in (*PUBLISHING_MARKERS, *MORE_MARKERS)) + + +def windows_publishers(job: 'dict[str, Any]') -> 'list[str]': + """Each step of ``job`` that reads a secret or publishes, and would run on the Windows leg.""" + return [str(step.get('name') or step.get('uses')) for step in job['steps'] + if sensitive(step) and runs(step, WINDOWS)] + + +class TestTheWindowsLegPublishesNothing(unittest.TestCase): + """No step that publishes, or reads a secret, runs on ``windows-latest``.""" + + def test_the_leg_exists_in_conda_update_alone(self) -> None: + self.assertIn(WINDOWS, spec()['strategy']['matrix']['os']) + # Phase 2's: the release job gets the leg with its uploads. + self.assertNotIn(WINDOWS, _workflows.job('create-release.yml', 'conda')['strategy']['matrix']['os']) + + def test_no_secret_or_upload_reaches_the_windows_leg(self) -> None: + self.assertEqual(windows_publishers(spec()), []) + + def test_only_the_uploads_read_the_anaconda_token(self) -> None: + readers = [step.get('name') for step in spec()['steps'] if 'secrets.ANACONDA_TOKEN' in repr(step)] + self.assertEqual(readers, ANACONDA_UPLOADS) + self.assertNotIn('ANACONDA_TOKEN', repr({key: value for key, value in spec().items() if key != 'steps'})) + + def test_every_other_leg_still_uploads(self) -> None: + """So the pin cannot pass by gating every upload off every leg.""" + steps = [step for step in spec()['steps'] if step.get('name') in ANACONDA_UPLOADS] + for os_label in spec()['strategy']['matrix']['os']: + if os_label == WINDOWS: + continue + with self.subTest(os=os_label): + self.assertEqual(sum(runs(step, os_label) for step in steps), 1) + + def test_the_check_sees_an_ungated_upload(self) -> None: + job = copy.deepcopy(spec()) + upload = next(step for step in job['steps'] if step.get('name') == ANACONDA_UPLOADS[0]) + upload['if'] = upload['if'].replace(f" && matrix.os != '{WINDOWS}'", '') + self.assertEqual(windows_publishers(job), [ANACONDA_UPLOADS[0]]) + + +class TestTheWindowsLegNeverBlocks(unittest.TestCase): + """Each step that can fail on the leg continues, and the summary reports it.""" + + def test_each_step_it_runs_continues_on_error_there_alone(self) -> None: + steps = spec()['steps'] + self.assertEqual(steps[-1].get('name'), SUMMARY) + others = [os_label for os_label in spec()['strategy']['matrix']['os'] if os_label != WINDOWS] + for step in steps[:-1]: + if not runs(step, WINDOWS): + continue + name = step.get('name') or step.get('uses') + with self.subTest(step=name): + self.assertIn('id', step) + guard = str(step.get('continue-on-error')) + self.assertTrue(evaluate_condition(guard, context(WINDOWS))) + for os_label in others: + self.assertFalse(evaluate_condition(guard, context(os_label)), os_label) + + def test_the_summary_runs_on_the_leg_alone_and_last(self) -> None: + summary = spec()['steps'][-1] + self.assertIs(summary.get('continue-on-error'), True) + self.assertEqual(summary.get('shell'), 'bash') + self.assertEqual(summary['env']['STEPS'], '${{ toJSON(steps) }}') + self.assertEqual(summary['run'], 'python util/leg_summary.py conda-windows') + for os_label in spec()['strategy']['matrix']['os']: + with self.subTest(os=os_label): + self.assertEqual(runs(summary, os_label), os_label == WINDOWS) + cancelled = dict(context(WINDOWS), **{STATUS: {'cancelled': True, 'success': False, 'failure': False}}) + self.assertFalse(evaluate_condition(summary['if'], cancelled)) + + def test_the_summary_warns_per_failed_step(self) -> None: + with tempfile.TemporaryDirectory() as temp: + page = pathlib.Path(temp) / 'summary.md' + result = subprocess.run( # nosec: B603 + [sys.executable, str(ROOT / 'util' / 'leg_summary.py'), 'conda-windows'], + env=dict(os.environ, LEG='conda-dist on windows-latest', GITHUB_STEP_SUMMARY=str(page), + STEPS=json.dumps({'checkout': {'outcome': 'success'}, 'conda_build': {'outcome': 'failure'}})), + capture_output=True, text=True, check=False, timeout=60) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), [ + '::warning title=conda_build failed on conda-dist on windows-latest::The win-64 conda leg is ' + "unpublished and never blocks (#2060), so this job still succeeds; see the conda_build step's log."]) + + def test_the_bash_steps_name_their_shell(self) -> None: + """A ``run:`` step's default shell on Windows is ``pwsh``.""" + for step in spec()['steps']: + if 'run' in step and runs(step, WINDOWS): + with self.subTest(step=step.get('name')): + self.assertTrue(str(step.get('shell', '')).startswith('bash')) + + def test_the_build_step_maps_the_leg_to_win_64(self) -> None: + build = next(step for step in spec()['steps'] if step.get('id') == 'conda_build') + self.assertRegex(build['run'], r'\n\s*windows-latest\)\n\s*target_platform="win-64"\n') + + def test_the_smoke_test_runs_python_exe_on_windows(self) -> None: + smoke = next(step for step in spec()['steps'] if step.get('id') == 'smoke') + self.assertIn('if [ "$RUNNER_OS" = Windows ]; then\n python="$(cygpath -u "$pythonLocation")/python.exe"\n', + smoke['run']) + + +#: A FOR loop's variable, with the percent signs before it. +FOR_VARIABLE = re.compile(r'^\s*FOR\s+(?:/\w+\s+(?:"[^"]*"\s+)?)*(%+)(\w)\s', re.IGNORECASE | re.MULTILINE) + + +def code(text: 'str') -> 'list[str]': + """A batch file's lines but blank and ``REM`` ones.""" + return [line.strip() for line in text.splitlines() + if line.strip() and not re.match(r'(?i)@?REM\b', line.strip())] + + +def pip_flags(line: 'str') -> 'list[str]': + """What follows the targets of a ``pip install`` line, slashes for backslashes.""" + words = re.split(r'\bpip\s+install\s+', line, maxsplit=1)[1].split() + while words and not words[0].startswith('-'): + words.pop(0) + return [word.replace('\\', '/') for word in words] + + +class TestBldBat(unittest.TestCase): + """``bld.bat`` is ``build.sh`` for cmd.""" + + def read(self, text: 'str') -> 'list[str]': + """What is wrong with ``text`` as ``bld.bat``.""" + problems = [] + lines = code(text) + for percents, name in FOR_VARIABLE.findall('\n'.join(lines)): + if percents != '%%': + problems.append(f'FOR variable {percents}{name}: a batch file needs %%{name}') + for line in lines: + if re.search(rf'(? None: + self.assertEqual(self.read((RECIPE / 'bld.bat').read_text(encoding='utf-8')), []) + + def test_the_check_sees_the_loop_2060_replaced(self) -> None: + old = ('@ECHO OFF\n\n' + 'FOR /F "Tokens=*" %A IN (\'DIR /B conda\\wheels\\\') DO @(\n' + ' "%PYTHON%" -m pip install conda\\wheels\\%A --target pcapkit\\_extern -vv\n' + ')\n\n' + '"%PYTHON%" -m pip install . -vv\n\n' + 'IF ERRORLEVEL 1 EXIT 1\n') + problems = self.read(old) + self.assertIn('FOR variable %A: a batch file needs %%A', problems) + self.assertIn('"%PYTHON%" -m pip install conda\\wheels\\%A --target pcapkit\\_extern -vv: ' + 'not followed by IF ERRORLEVEL 1 EXIT 1', problems) + self.assertIn('"%PYTHON%" -m pip install conda\\wheels\\%A --target pcapkit\\_extern -vv: ' + 'inside a FOR loop, so pip installs one file at a time', problems) + + +class TestPortableExecutable(unittest.TestCase): + """:func:`conda_vendor.native` reads a PE image's COFF ``Machine``.""" + + def test_it_reads_each_machine(self) -> None: + vendor = load('conda_vendor') + for machine, name in ((0x8664, 'x86_64'), (0xaa64, 'arm64'), (0x014c, 'i386'), (0x1234, 'Machine 0x1234')): + with self.subTest(machine=machine): + self.assertEqual(vendor.native(pe(machine)), ('PE', frozenset((name,)))) + self.assertEqual(vendor.native(pe(0x8664, offset=0x200)), ('PE', frozenset(('x86_64',)))) + + def test_an_mz_header_without_a_pe_one_is_unknown(self) -> None: + vendor = load('conda_vendor') + broken = bytearray(pe(0x8664)) + broken[0x80:0x84] = b'NE\0\0' + for head in (b'MZ\x90\x00' + bytes(60), bytes(broken), pe(0x8664)[:0x3f], pe(0x8664)[:0x84]): + with self.subTest(head=len(head)): + self.assertIsNone(vendor.native(head)) + + def test_win_64_takes_an_x86_64_pyd_alone(self) -> None: + vendor = load('conda_vendor') + for head, problem in ((pe(0x8664), None), (pe(0xaa64), 'PE arm64, not PE x86_64 as win-64 needs'), + (elf(62), 'ELF x86_64, not PE x86_64 as win-64 needs')): + with self.subTest(problem=problem), tempfile.TemporaryDirectory() as temp: + path = pathlib.Path(temp) / 'Lib' / 'site-packages' / '_cffi_backend.cp314-win_amd64.pyd' + path.parent.mkdir(parents=True) + path.write_bytes(head) + self.assertEqual(vendor.foreign(pathlib.Path(temp), 'win-64'), (1, [] if problem is None else [ + f'Lib/site-packages/_cffi_backend.cp314-win_amd64.pyd: {problem}'])) + + def test_win_64_environments_are_nt(self) -> None: + vendor = load('conda_vendor') + envs = vendor.environments(['3.14']) + self.assertEqual({(env['sys_platform'], env['os_name']) for env in envs}, + {('linux', 'posix'), ('darwin', 'posix'), ('win32', 'nt')}) + self.assertRegex('cryptography-50.0.2-cp311-abi3-win_amd64.whl', vendor.WHEEL_PLATFORMS['win-64']) + + +class TestTheWindowsLayout(unittest.TestCase): + """``--smoke`` on a package with ``Lib/site-packages``, as conda builds on Windows.""" + + PYTHON = f'{sys.version_info.major}.{sys.version_info.minor}' + + def test_the_python_is_read_from_the_build_string(self) -> None: + vendor = load('conda_vendor') + site = pathlib.Path('root') / 'Lib' / 'site-packages' + for build, built in (('py314h1a2b3c4_0', 'python3.14'), ('py39_0', 'python3.9'), ('py310_2', 'python3.10'), + (None, "Lib/site-packages with build string None"), + ('h1a2b3c4_0', "Lib/site-packages with build string 'h1a2b3c4_0'")): + with self.subTest(build=build): + self.assertEqual(vendor.built_for(site, {} if build is None else {'build': build}), built) + posix = pathlib.Path('root') / 'lib' / 'python3.14' / 'site-packages' + self.assertEqual(vendor.built_for(posix, {'build': 'py39_0'}), 'python3.14') + + def smoke(self, build: 'str') -> 'tuple[int, str]': + vendor = load('conda_vendor') + files = dict({'pcapkit/__init__.py': SHIM.encode()}, **DISTRIBUTION) + files['pcapkit/_extern/native/_speedups.pyd'] = pe(0x8664) + files['pcapkit/_extern/native-1.0.dist-info/RECORD'] = ( + b'native/__init__.py,,\nnative/_speedups.pyd,,\nnative-1.0.dist-info/METADATA,,\n' + b'native-1.0.dist-info/RECORD,,\n') + members = {f'Lib/site-packages/{name}': data for name, data in files.items()} + members['info/index.json'] = json.dumps({'name': 'pypcapkit', 'subdir': 'win-64', 'build': build}).encode() + with tempfile.TemporaryDirectory() as temp: + package = pathlib.Path(temp) / 'pypcapkit-0-py_0.tar.bz2' + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode='w') as archive: + for name, data in members.items(): + info = tarfile.TarInfo(name) + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + package.write_bytes(bz2.compress(buffer.getvalue())) + out, err = io.StringIO(), io.StringIO() + with mock.patch.object(vendor, 'runner_platform', return_value='win-64'), \ + mock.patch('sys.stdout', out), mock.patch('sys.stderr', err): + status = vendor.smoke(package) + return status, out.getvalue() + err.getvalue() + + def test_a_windows_package_for_this_python_passes(self) -> None: + status, log = self.smoke(f'py{self.PYTHON.replace(".", "")}h0_0') + self.assertEqual(status, 0, log) + self.assertIn('win-64: 1 native binaries read, none failing', log) + + def test_a_windows_package_for_another_python_fails(self) -> None: + status, log = self.smoke('py27_0') + self.assertEqual(status, 1, log) + self.assertIn('is built for python2.7, and this is ', log) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_ktlo_legs_stay_green_1672.py b/tests/project/test_ktlo_legs_stay_green_1672.py index 8f69ceb872..9fd8a8570d 100644 --- a/tests/project/test_ktlo_legs_stay_green_1672.py +++ b/tests/project/test_ktlo_legs_stay_green_1672.py @@ -134,6 +134,8 @@ #: Why each summary's warning says the job still succeeds; the scripts differ in nothing else. KTLO_REASON = 'KTLO legs never block (#1633), so this job still succeeds (#1672)' EXPERIMENTAL_REASON = 'Experimental legs never block (#1660), so this job still succeeds' +#: #2060: cron-conda.yml's unpublished Windows leg, pinned in test_conda_windows_leg_2060.py. +CONDA_WINDOWS_REASON = 'The win-64 conda leg is unpublished and never blocks (#2060), so this job still succeeds' #: Each summary's command (#1967): the image's interpreter, not the leg's, which may #: have failed to set up. IMAGE_PYTHON = '/usr/bin/python3' @@ -688,7 +690,8 @@ def test_every_failure_it_masks_reaches_the_experimental_summary(self) -> None: # One script with the KTLO summary's, so a fix to either reaches both (#1967). self.assertEqual(self.summary['run'], EXPERIMENTAL_SUMMARY_RUN) self.assertEqual(summary_step(job('unit-tests.yml', 'test'))['run'], SUMMARY_RUN) - self.assertEqual(REASONS, {'ktlo': KTLO_REASON, 'experimental': EXPERIMENTAL_REASON}) + self.assertEqual(REASONS, {'ktlo': KTLO_REASON, 'experimental': EXPERIMENTAL_REASON, + 'conda-windows': CONDA_WINDOWS_REASON}) def test_each_step_names_the_leg_by_its_tier(self) -> None: """``LEG`` in an annotation's title: ``(experimental, non-blocking)``.""" diff --git a/tests/project/test_python_versions_1660.py b/tests/project/test_python_versions_1660.py index f940e6bea2..d0fd956c94 100644 --- a/tests/project/test_python_versions_1660.py +++ b/tests/project/test_python_versions_1660.py @@ -234,6 +234,8 @@ def setups(spec: 'dict[str, Any]') -> 'list[dict[str, Any]]': #: The conda jobs' ``os`` axis since #1822, which added the Intel macOS runner. CONDA_JOBS = (('create-release.yml', 'conda'), ('cron-conda.yml', 'conda-dist')) CONDA_OS = ['macos-latest', 'macos-26-intel', 'ubuntu-latest'] +#: The extra leg each conda job has beyond :data:`CONDA_OS`: #2060's unpublished Windows one. +CONDA_EXTRA_OS = {('create-release.yml', 'conda'): [], ('cron-conda.yml', 'conda-dist'): ['windows-latest']} MAIN_MATRICES = { ('unit-tests.yml', 'test'): { 'python-version': SUPPORTED_AT_MAIN, @@ -589,7 +591,7 @@ def test_every_matrix_expands_to_mains_legs(self) -> None: leg['coverage'] = True if (name, job_id) in CONDA_JOBS: # #1822: an Intel macOS leg per version, so osx-64 is built natively. - expected = legs(dict(spec, os=CONDA_OS)) + expected = legs(dict(spec, os=CONDA_OS + CONDA_EXTRA_OS[(name, job_id)])) expected = with_pypy(name, job_id, expected) if (name, job_id) in KTLO_TWINS: # #1738: the KTLO legs, in MAIN's order, are the twin job's. diff --git a/tests/project/test_python_versions_update_1660.py b/tests/project/test_python_versions_update_1660.py index 155d57f899..3840f73941 100644 --- a/tests/project/test_python_versions_update_1660.py +++ b/tests/project/test_python_versions_update_1660.py @@ -386,7 +386,8 @@ def test_the_labels_come_from_the_workflows(self) -> None: """``matrix.os || 'ubuntu-latest'``; the conda jobs' ``os`` axis; where ``latest`` is installed.""" found = context(snapshot()).labels self.assertEqual(found, {'default': ['ubuntu-latest'], - 'supported': ['macos-26-intel', 'macos-latest', 'ubuntu-latest'], + # #2060: Conda Update's windows-latest leg. + 'supported': ['macos-26-intel', 'macos-latest', 'ubuntu-latest', 'windows-latest'], 'latest': ['macos-latest', 'ubuntu-latest']}) def test_ubuntu_latest_counts_as_its_image_and_every_newer_one(self) -> None: @@ -396,6 +397,8 @@ def test_ubuntu_latest_counts_as_its_image_and_every_newer_one(self) -> None: [module.Target('linux', '24.04', 'x64'), module.Target('linux', '26.04', 'x64')]) self.assertEqual(module.label_targets('ubuntu-24.04', images), [module.Target('linux', '24.04', 'x64')]) self.assertEqual(module.label_targets('macos-latest', images), [module.Target('darwin', None, 'arm64')]) + # #2060: the Windows Server image; the Arm64 Windows 11 one is no newer x64 image. + self.assertEqual(module.label_targets('windows-latest', images), [module.Target('win32', None, 'x64')]) moved = module.parse_images(move_latest(snapshot(), '26.04')['runner-images']) self.assertEqual(module.label_targets('ubuntu-latest', moved), [module.Target('linux', '26.04', 'x64')]) with self.assertRaisesRegex(ValueError, 'does not list'): diff --git a/tests/project/test_release_conda_leg_smoke_1855.py b/tests/project/test_release_conda_leg_smoke_1855.py index c50f176350..aaec698440 100644 --- a/tests/project/test_release_conda_leg_smoke_1855.py +++ b/tests/project/test_release_conda_leg_smoke_1855.py @@ -68,15 +68,24 @@ def test_the_smoke_test_runs_after_the_build_and_before_every_upload(self) -> No self.assertLess(smoke[0], min(uploads)) def test_it_is_conda_updates_step(self) -> None: - """The same interpreter, package and gate: setup-python's, ``host_package``, none.""" + """The same interpreter and package: setup-python's, ``host_package``. + + #2060: Conda Update's step also finds that interpreter on its Windows + leg, and carries that leg's ``id``, gate and ``continue-on-error``. The + release job has no Windows leg yet, so its step is the POSIX branch alone. + + """ cron = steps(*CRON) release = steps(*RELEASE) expected = cron[smoke_steps(cron)[0]] self.assertEqual(expected['env'], {'PCAPKIT_CONDA_PACKAGE': '${{ steps.conda_build.outputs.host_package }}'}) - self.assertIn('"${pythonLocation}/bin/python" util/conda_vendor.py --smoke', expected['run']) + self.assertIn('python="${pythonLocation}/bin/python"\n', expected['run']) + self.assertIn('"$python" util/conda_vendor.py --smoke "$PCAPKIT_CONDA_PACKAGE"', expected['run']) found = smoke_steps(release) self.assertEqual(len(found), 1, 'the release leg runs no smoke test') - self.assertEqual(release[found[0]], expected) + self.assertEqual(release[found[0]], { + 'name': expected['name'], 'env': expected['env'], + 'run': '"${pythonLocation}/bin/python" util/conda_vendor.py --smoke "$PCAPKIT_CONDA_PACKAGE"\n'}) @unittest.skipIf(yaml is None, 'PyYAML, from the test extra, is not installed') diff --git a/tests/project/test_release_tags_the_run_sha_1634.py b/tests/project/test_release_tags_the_run_sha_1634.py index 55883f2a7b..a043596407 100644 --- a/tests/project/test_release_tags_the_run_sha_1634.py +++ b/tests/project/test_release_tags_the_run_sha_1634.py @@ -1570,9 +1570,9 @@ def test_the_check_reads_the_release_the_action_updates(self) -> None: self.assertNotIn('${{', check['run']) def test_the_anaconda_upload_is_not_gated_on_it(self) -> None: - # #2044: gated on the leg alone. + # #2044: gated on the leg alone; #2060: Windows publishes nothing. self.assertEqual(self.step('conda-dist', 'Upload conda packages to Anaconda').get('if'), - "${{ matrix.os != 'macos-26-intel' }}") + "${{ matrix.os != 'macos-26-intel' && matrix.os != 'windows-latest' }}") self.assertEqual(self.step('conda-dist', 'Upload conda packages to Anaconda (Intel)').get('if'), "${{ matrix.os == 'macos-26-intel' }}") diff --git a/util/conda_vendor.py b/util/conda_vendor.py index 3e64275cee..f243954a48 100644 --- a/util/conda_vendor.py +++ b/util/conda_vendor.py @@ -99,15 +99,17 @@ #: The extras the conda distribution carries, besides ``dependencies``. EXTRAS = ('all',) #: The platforms the conda workflows build on -- ``cron-conda.yml`` and -#: ``create-release.yml`` -- by the conda subdir each leg publishes, as +#: ``create-release.yml`` -- by the conda subdir each leg builds, as #: ``sys.platform``, ``platform.system()`` and ``platform.machine()`` read on its -#: runner: ``ubuntu-latest``, the Intel ``macos-26-intel`` and the arm64 -#: ``macos-latest``. Each leg publishes only its own runner's platform (#1822). -#: They have no Windows leg (#1967), so the recipe's ``bld.bat`` goes unused. +#: runner: ``ubuntu-latest``, the Intel ``macos-26-intel``, the arm64 +#: ``macos-latest`` and ``windows-latest``. Each leg builds only its own runner's +#: platform (#1822). The Windows leg, ``cron-conda.yml``'s alone, builds and +#: smoke-tests with the recipe's ``bld.bat`` and publishes nothing yet (#2060). PLATFORMS = { 'linux-64': ('linux', 'Linux', 'x86_64'), 'osx-64': ('darwin', 'Darwin', 'x86_64'), 'osx-arm64': ('darwin', 'Darwin', 'arm64'), + 'win-64': ('win32', 'Windows', 'AMD64'), } #: The file :data:`PLATFORM_PINS` is read from, and ``--refresh-pins`` rewrites (#1827, #1851). PINS = ROOT / 'conda' / 'platform-pins.toml' @@ -116,23 +118,27 @@ REVIEW_HORIZON = datetime.timedelta(weeks=13) #: The wheel platform tags a build on each of :data:`PLATFORMS` installs, besides #: ``any``, for :func:`installs` (#1851): glibc's manylinux on linux-64, and on -#: macOS the runner's own architecture or universal2, for any macOS version. A -#: tag left out here only leaves a pin where it is. +#: macOS the runner's own architecture or universal2, for any macOS version, and +#: ``win_amd64`` on win-64. A tag left out here only leaves a pin where it is. WHEEL_PLATFORMS = { 'linux-64': re.compile(r'manylinux(?:1|2010|2014|_\d+_\d+)_x86_64'), 'osx-64': re.compile(r'macosx_\d+_\d+_(?:x86_64|universal2)'), 'osx-arm64': re.compile(r'macosx_\d+_\d+_(?:arm64|universal2)'), + 'win-64': re.compile(r'win_amd64'), } #: Where ``--refresh-pins`` reads a package's releases: PyPI's JSON API. PYPI = 'https://pypi.org/pypi/{}/json' #: The object format and the machine of every native binary a package for each #: of :data:`PLATFORMS` may carry (#1822). A universal Mach-O passes when one of #: its slices is the machine. -BINARIES = {'linux-64': ('ELF', 'x86_64'), 'osx-64': ('Mach-O', 'x86_64'), 'osx-arm64': ('Mach-O', 'arm64')} -#: What ELF's ``e_machine`` and Mach-O's ``cputype`` say, as :data:`BINARIES` spells it. +BINARIES = {'linux-64': ('ELF', 'x86_64'), 'osx-64': ('Mach-O', 'x86_64'), 'osx-arm64': ('Mach-O', 'arm64'), + 'win-64': ('PE', 'x86_64')} +#: What ELF's ``e_machine``, Mach-O's ``cputype`` and PE's ``Machine`` say, as +#: :data:`BINARIES` spells it. ELF_MACHINES = {3: 'i386', 20: 'ppc', 21: 'ppc64', 40: 'arm', 62: 'x86_64', 183: 'arm64', 243: 'riscv'} MACHO_CPUS = {7: 'i386', 0x01000007: 'x86_64', 12: 'arm', 0x0100000c: 'arm64', 0x0200000c: 'arm64_32', 18: 'ppc', 0x01000012: 'ppc64'} +PE_MACHINES = {0x014c: 'i386', 0x01c4: 'arm', 0x8664: 'x86_64', 0xaa64: 'arm64'} #: File names a native extension or library goes by, so one whose header this #: cannot read is reported rather than passed over. NATIVE_SUFFIXES = ('.so', '.dylib', '.pyd', '.dll') @@ -245,7 +251,7 @@ def environments(pythons: 'Iterable[str]') -> 'list[dict[str, str]]': 'python_version': python, 'python_full_version': f'{python}.0', 'implementation_name': 'cpython', 'platform_python_implementation': 'CPython', 'implementation_version': f'{python}.0', 'sys_platform': sys_platform, - 'platform_system': system, 'os_name': 'posix', + 'platform_system': system, 'os_name': 'nt' if sys_platform == 'win32' else 'posix', 'platform_machine': machine, 'platform_release': '', 'platform_version': '', }) return found @@ -834,6 +840,14 @@ def refresh_pins() -> 'int': def native(head: 'bytes') -> 'Optional[tuple[str, frozenset[str]]]': """The object format and machines of a binary starting with ``head``; :data:`None` for any other file.""" magic = head[:4] + # #2060: a PE image, as a ``.pyd`` or ``.dll`` is: the DOS header's + # ``e_lfanew``, at 0x3c, points at ``PE\0\0`` and the COFF ``Machine`` after it. + if magic[:2] == b'MZ' and len(head) >= 0x40: + offset = int.from_bytes(head[0x3c:0x40], 'little') + if head[offset:offset + 4] == b'PE\0\0' and len(head) >= offset + 6: + machine = int.from_bytes(head[offset + 4:offset + 6], 'little') + return 'PE', frozenset((PE_MACHINES.get(machine, f'Machine {machine:#06x}'),)) + return None if magic == b'\x7fELF' and len(head) >= 20 and head[5] in (1, 2): order = 'little' if head[5] == 1 else 'big' # type: Literal['little', 'big'] machine = int.from_bytes(head[18:20], order) @@ -859,7 +873,7 @@ def foreign(root: 'pathlib.Path', subdir: 'str') -> 'tuple[int, list[str]]': """How many native binaries ``root`` holds, and each one that does not run on ``subdir``. ``noarch`` may hold none; a platform outside :data:`BINARIES` fails every one. - A file named as a native library whose header is neither ELF nor Mach-O is + A file named as a native library whose header is not ELF, Mach-O or PE is reported too, since this cannot vouch for it. """ @@ -874,7 +888,7 @@ def foreign(root: 'pathlib.Path', subdir: 'str') -> 'tuple[int, list[str]]': name = path.relative_to(root).as_posix() if found is None: if path.name.endswith(NATIVE_SUFFIXES) or '.so.' in path.name: - problems.append(f'{name}: neither ELF nor Mach-O, so its machine is unknown') + problems.append(f'{name}: not ELF, Mach-O or PE, so its machine is unknown') continue count += 1 kind, machines = found @@ -892,6 +906,22 @@ def runner_platform() -> 'Optional[str]': None) +def built_for(site: 'pathlib.Path', index: 'dict[str, Any]') -> 'str': + """The ``pythonX.Y`` a package's ``site-packages`` at ``site`` is for, or what it is if not that. + + A POSIX package names it in the path, ``lib/pythonX.Y/site-packages``. A + Windows one has ``Lib/site-packages`` for every version (#2060), so it is read + from the ``pyXY`` that starts the build string in ``info/index.json``. + + """ + if site.parent.name != 'Lib': + return site.parent.name + match = re.match(r'py(\d)(\d+)(?![0-9])', str(index.get('build', ''))) + if match is None: + return f'Lib/site-packages with build string {index.get("build")!r}' + return f'python{match.group(1)}.{match.group(2)}' + + def smoke(package: 'pathlib.Path') -> 'int': """Unpack a built ``.tar.bz2`` conda package, check its platform and binaries, and run :data:`PROBE`.""" with tempfile.TemporaryDirectory() as temp: @@ -902,10 +932,6 @@ def smoke(package: 'pathlib.Path') -> 'int': print(f'{package}: expected one site-packages/pcapkit, found {len(inits)}', file=sys.stderr) return 1 site = inits[0].parents[1] - built = site.parent.name - if built != f'python{sys.version_info.major}.{sys.version_info.minor}': - print(f'{package} is built for {built}, and this is {sys.version}', file=sys.stderr) - return 1 try: index = json.loads((pathlib.Path(temp) / 'info' / 'index.json').read_text(encoding='utf-8')) subdir = str(index['subdir']) @@ -913,6 +939,10 @@ def smoke(package: 'pathlib.Path') -> 'int': print(f'{package}: info/index.json names no subdir ({error!r}), so its platform is unknown', file=sys.stderr) return 1 + built = built_for(site, index) + if built != f'python{sys.version_info.major}.{sys.version_info.minor}': + print(f'{package} is built for {built}, and this is {sys.version}', file=sys.stderr) + return 1 runner = runner_platform() if subdir not in ('noarch', runner): print(f'{package} is filed under {subdir}, and this runner is ' diff --git a/util/leg_summary.py b/util/leg_summary.py index 4e4db21ab8..995ec877a9 100644 --- a/util/leg_summary.py +++ b/util/leg_summary.py @@ -11,6 +11,10 @@ /usr/bin/python3 util/leg_summary.py ktlo # or: experimental +:file:`.github/workflows/cron-conda.yml`'s ``Windows summary`` runs it as +``python util/leg_summary.py conda-windows``, since a Windows image has no +``/usr/bin/python3`` (#2060). + with ``LEG``, the leg's display name, and ``STEPS``, ``${{ toJSON(steps) }}``, in the environment. ``STEPS`` holds each earlier step with an ``id``, and its ``outcome`` is the step's result before ``continue-on-error`` applied. For each @@ -47,6 +51,7 @@ REASONS = { 'ktlo': 'KTLO legs never block (#1633), so this job still succeeds (#1672)', 'experimental': 'Experimental legs never block (#1660), so this job still succeeds', + 'conda-windows': 'The win-64 conda leg is unpublished and never blocks (#2060), so this job still succeeds', } diff --git a/util/update_python_versions.py b/util/update_python_versions.py index e7e96a8ba8..eb32df60c6 100644 --- a/util/update_python_versions.py +++ b/util/update_python_versions.py @@ -236,9 +236,12 @@ class Target(NamedTuple): arch: str -#: An image's family and version, at the start of the table's Image cell. -IMAGE = re.compile(r'(Ubuntu|macOS)\s+(\d+(?:\.\d+)?)\b') -PLATFORM = {'ubuntu': 'linux', 'macos': 'darwin'} +#: An image's family and version, at the start of the table's Image cell: +#: ``Windows Server 2025`` reads as ``windows`` 2025, for the conda job's +#: ``windows-latest`` leg (#2060). +IMAGE = re.compile(r'(Ubuntu|macOS|Windows)(?:\s+Server)?\s+(\d+(?:\.\d+)?)\b') +#: setup-python's manifest ``platform`` for each family. +PLATFORM = {'ubuntu': 'linux', 'macos': 'darwin', 'windows': 'win32'} def numeric(version: 'str') -> 'tuple[int, ...]': @@ -276,7 +279,7 @@ def markdown_tables(text: 'str') -> 'list[list[list[Any]]]': def parse_images(text: 'str') -> 'list[Image]': - """The Ubuntu and macOS rows of the runner-images README's image table. + """The Ubuntu, macOS and Windows rows of the runner-images README's image table. The labels are the code spans of the YAML Label cell, read from its tokens, so a span holding an escaped pipe reads as GitHub shows it, without the