From 71756901719d0e7c025ff0e39e0782ee4519e545 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Thu, 22 Oct 2026 15:12:29 -0400 Subject: [PATCH] ci(conda): upload the macos-26-intel legs with the anaconda CLI (#2044) anaconda/actions/upload-package v0.3.1 installs its CLI from anaconda.sh/releases/stable/v0.2.1/ana-darwin-x86_64, which returns 404, so "Upload conda packages to Anaconda" fails on every Intel leg. In cron-conda.yml's conda-dist and create-release.yml's conda, that step now skips macos-26-intel, and "Upload conda packages to Anaconda (Intel)" runs there instead: it installs conda-forge's anaconda-client into base and makes the action's own `anaconda --token ... upload --user ...` call, with the same channel, labels and files, plus `--skip-existing`. Every other leg keeps the action. create-release.yml's per-leg check still gates both. releasing.rst states the Intel exception. test_release_gates.py's PUBLISHING_MARKERS recognises the CLI call as a route to anaconda.org. tests/project/test_conda_intel_cli_upload_2044.py evaluates both steps' `if:` per leg and runs the CLI step with a stand-in conda and anaconda; test_release_tags_the_run_sha_1634.py's pins of the action step's `if:` follow. --- .github/workflows/create-release.yml | 54 ++++- .github/workflows/cron-conda.yml | 45 ++++ docs/source/contributing/releasing.rst | 8 +- .../test_conda_intel_cli_upload_2044.py | 217 ++++++++++++++++++ tests/project/test_release_gates.py | 5 +- .../test_release_tags_the_run_sha_1634.py | 11 +- 6 files changed, 334 insertions(+), 6 deletions(-) create mode 100644 tests/project/test_conda_intel_cli_upload_2044.py diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 2893068015..0c724da0cf 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -72,7 +72,8 @@ concurrency: # exists, ``pypi`` PyPI's file count, and ``conda`` Anaconda's aggregate count # (a cost saver) plus, per leg, its own distribution -- the correctness fix, # since ``anaconda/actions/upload-package`` has no ``skip-existing``, only -# ``force:``, never set here. +# ``force:``, never set here. The ``macos-26-intel`` leg uploads with the +# anaconda-client CLI and ``--skip-existing`` instead (#2044). # # GitHub skips a job whose ``needs:`` skipped before evaluating its ``if:``, # unless the ``if:`` calls a status-check function, and ``github`` skips on @@ -1214,8 +1215,10 @@ jobs: run: | "${pythonLocation}/bin/python" util/conda_vendor.py --smoke "$PCAPKIT_CONDA_PACKAGE" + # #2044: not on macos-26-intel, whose leg "Upload conda packages to + # Anaconda (Intel)" uploads instead. - name: Upload conda packages to Anaconda - if: ${{ steps.check_leg.outputs.PCAPKIT_LEG_EXISTS != 'true' }} + if: ${{ matrix.os != 'macos-26-intel' && steps.check_leg.outputs.PCAPKIT_LEG_EXISTS != 'true' }} uses: anaconda/actions/upload-package@a6f488910b4a1e4f3fe01736241ca1b92b4b355a # v0.3.1 with: token: ${{ secrets.ANACONDA_TOKEN }} @@ -1223,6 +1226,53 @@ jobs: packages: ${{ steps.conda_build.outputs.packages }} labels: ${{ needs.version_check.outputs.PCAPKIT_CONDA_LABEL }} + # #2044: upload-package v0.3.1 installs its CLI from + # anaconda.sh/releases/stable/v0.2.1/ana-darwin-x86_64, which returns 404, + # so the Intel leg runs the call the action makes -- `anaconda --token + # upload --user`, with the same channel, labels and files and the new CLI + # parser -- from conda-forge's anaconda-client in base. One difference: + # `--skip-existing`, so a file anaconda.org already has is skipped where + # the action, with no `force:`, fails on it. "Check whether this leg is + # already on Anaconda" still gates both. + - name: Upload conda packages to Anaconda (Intel) + if: ${{ matrix.os == 'macos-26-intel' && steps.check_leg.outputs.PCAPKIT_LEG_EXISTS != 'true' }} + shell: bash + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + ANACONDA_CLI_FORCE_NEW: "1" + PCAPKIT_CONDA_CHANNEL: jarryshaw + PCAPKIT_CONDA_PACKAGES: ${{ steps.conda_build.outputs.packages }} + PCAPKIT_CONDA_LABELS: ${{ needs.version_check.outputs.PCAPKIT_CONDA_LABEL }} + run: | + set -euo pipefail + + conda install -n base --override-channels -c conda-forge -y anaconda-client + + # The action's handling of its `labels:` and `packages:` inputs: a + # comma-separated list, each trimmed, and the build step's + # newline-separated paths. + labels=() + IFS=',' read -ra names <<< "$PCAPKIT_CONDA_LABELS" + for name in ${names[@]+"${names[@]}"}; do + name="$(echo "$name" | xargs)" + if [ -n "$name" ]; then + labels+=(--label "$name") + fi + done + packages=() + while IFS= read -r path; do + if [ -n "$path" ]; then + packages+=("$path") + fi + done <<< "$PCAPKIT_CONDA_PACKAGES" + if [ "${#packages[@]}" -eq 0 ]; then + echo "::error::No conda packages to upload" + exit 1 + fi + + "${CONDA}/bin/anaconda" --token "$ANACONDA_API_TOKEN" upload \ + --user "$PCAPKIT_CONDA_CHANNEL" --skip-existing ${labels[@]+"${labels[@]}"} "${packages[@]}" + # ``pypi``'s upload script, for the same reasons (#1634, #1640). Not # gated on ``check_leg`` as the Anaconda upload is: a leg whose Anaconda # upload landed and whose GitHub upload then failed would get no GitHub diff --git a/.github/workflows/cron-conda.yml b/.github/workflows/cron-conda.yml index 5566495110..b5b579d69e 100644 --- a/.github/workflows/cron-conda.yml +++ b/.github/workflows/cron-conda.yml @@ -490,7 +490,10 @@ jobs: run: | "${pythonLocation}/bin/python" util/conda_vendor.py --smoke "$PCAPKIT_CONDA_PACKAGE" + # #2044: not on macos-26-intel, whose leg "Upload conda packages to + # Anaconda (Intel)" uploads instead. - name: Upload conda packages to Anaconda + if: ${{ matrix.os != 'macos-26-intel' }} uses: anaconda/actions/upload-package@a6f488910b4a1e4f3fe01736241ca1b92b4b355a # v0.3.1 with: token: ${{ secrets.ANACONDA_TOKEN }} @@ -499,6 +502,48 @@ jobs: labels: ${{ needs.conda-update.outputs.PCAPKIT_CONDA_LABEL }} force: true + # #2044: see create-release.yml's step of the same name. One difference + # from the action here: `--skip-existing` keeps a file anaconda.org + # already has, where the action's `force: true` replaces it. + - name: Upload conda packages to Anaconda (Intel) + if: ${{ matrix.os == 'macos-26-intel' }} + shell: bash + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + ANACONDA_CLI_FORCE_NEW: "1" + PCAPKIT_CONDA_CHANNEL: jarryshaw + PCAPKIT_CONDA_PACKAGES: ${{ steps.conda_build.outputs.packages }} + PCAPKIT_CONDA_LABELS: ${{ needs.conda-update.outputs.PCAPKIT_CONDA_LABEL }} + run: | + set -euo pipefail + + conda install -n base --override-channels -c conda-forge -y anaconda-client + + # The action's handling of its `labels:` and `packages:` inputs: a + # comma-separated list, each trimmed, and the build step's + # newline-separated paths. + labels=() + IFS=',' read -ra names <<< "$PCAPKIT_CONDA_LABELS" + for name in ${names[@]+"${names[@]}"}; do + name="$(echo "$name" | xargs)" + if [ -n "$name" ]; then + labels+=(--label "$name") + fi + done + packages=() + while IFS= read -r path; do + if [ -n "$path" ]; then + packages+=("$path") + fi + done <<< "$PCAPKIT_CONDA_PACKAGES" + if [ "${#packages[@]}" -eq 0 ]; then + echo "::error::No conda packages to upload" + exit 1 + fi + + "${CONDA}/bin/anaconda" --token "$ANACONDA_API_TOKEN" upload \ + --user "$PCAPKIT_CONDA_CHANNEL" --skip-existing ${labels[@]+"${labels[@]}"} "${packages[@]}" + # #1634: the upload below never makes the release. ncipollo/release-action # v1.21.0 creates a missing one, and its tag, at ``commit`` # (src/Action.ts:72-92), so a bump made while Create Release waits on diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index 1e7caccdae..c45e3b312e 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -351,7 +351,13 @@ checks *is* the one it is about to create: is already there -- this second check is the actual correctness fix, because ``anaconda/actions/upload-package`` has no ``skip-existing`` equivalent, only ``force:`` (never set), so re-uploading a file it already has fails outright - rather than being absorbed. + rather than being absorbed. The ``macos-26-intel`` leg is the exception: the + action's CLI download returns 404 there, so its "Upload conda packages to + Anaconda (Intel)" step runs ``anaconda upload --skip-existing`` instead + (`#2044 `__), which skips + such a file. Conda Update's Intel leg runs the same step, so it keeps a file + anaconda.org already has, where the action's ``force: true`` on its other legs + replaces it. These evidence checks all run inside ``version_check``, curling PyPI's and Anaconda's public, unauthenticated JSON APIs; the per-leg check runs inside diff --git a/tests/project/test_conda_intel_cli_upload_2044.py b/tests/project/test_conda_intel_cli_upload_2044.py new file mode 100644 index 0000000000..7b8de586a6 --- /dev/null +++ b/tests/project/test_conda_intel_cli_upload_2044.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- +"""#2044: the conda jobs' ``macos-26-intel`` legs upload with the anaconda-client CLI. + +``anaconda/actions/upload-package`` v0.3.1 installs its CLI from +``anaconda.sh/releases/stable/v0.2.1/ana-darwin-x86_64``, which returns 404, so +the step "Upload conda packages to Anaconda" failed on every Intel leg of +``conda-dist`` in :file:`.github/workflows/cron-conda.yml` and would on ``conda`` +in :file:`.github/workflows/create-release.yml`. That step now skips the Intel +leg, and "Upload conda packages to Anaconda (Intel)" runs there alone, making +the action's own ``anaconda --token … upload --user …`` call from conda-forge's +anaconda-client, with ``--skip-existing``. + +Each step's ``if:`` is evaluated per matrix leg with +:func:`tests.project.test_release_gates.evaluate_condition`. The CLI step's +script is run with a stand-in ``conda`` and ``anaconda``, and the arguments it +passes are compared with those :func:`action_arguments` builds as the action's +"Upload to anaconda.org" step does. + +""" + +from __future__ import annotations + +import os +import pathlib +import stat +import subprocess # nosec: B404 +import tempfile +import textwrap +import unittest +from typing import TYPE_CHECKING + +from tests import _workflows +from tests._support import scale_timeout +from tests.project.test_conda_native_platforms_1822 import BASH +from tests.project.test_release_gates import evaluate_condition + +if TYPE_CHECKING: + from typing import Any + +#: Test tier (#1958): reads the workflow files, a question about the tree. +__tier__ = 'LATEST' + +#: The conda jobs, as (workflow, job). +JOBS = (('cron-conda.yml', 'conda-dist'), ('create-release.yml', 'conda')) +ACTION_STEP = 'Upload conda packages to Anaconda' +CLI_STEP = 'Upload conda packages to Anaconda (Intel)' +ACTION = 'anaconda/actions/upload-package' +#: The leg whose upload-package CLI download returns 404. +INTEL = 'macos-26-intel' +SECRET = '${{ secrets.ANACONDA_TOKEN }}' +#: The CLI step's ``env:`` key for each action input it carries. +CARRIED = {'token': 'ANACONDA_API_TOKEN', 'channel': 'PCAPKIT_CONDA_CHANNEL', + 'packages': 'PCAPKIT_CONDA_PACKAGES', 'labels': 'PCAPKIT_CONDA_LABELS'} + +#: A stand-in ``conda`` and ``anaconda``, each appending its arguments, one per +#: line, to ``$FAKE_LOG``, then a blank line. +RECORDER = textwrap.dedent('''\ + #!/usr/bin/env bash + set -eu + { echo "$(basename "$0")"; printf '%s\\n' "$@"; echo; } >> "$FAKE_LOG" +''') + + +def step(name: 'str', job_id: 'str', step_name: 'str') -> 'dict[str, Any]': + """The step named ``step_name`` of job ``job_id`` in workflow ``name``.""" + found = [entry for entry in _workflows.steps(name, job_id) if entry.get('name') == step_name] + if len(found) != 1: + raise AssertionError(f'{name}: {job_id} has {len(found)} steps named {step_name!r}, not one') + return found[0] + + +def index(name: 'str', job_id: 'str', step_name: 'str') -> 'int': + """The position of the step named ``step_name`` in job ``job_id``.""" + return [entry.get('name') for entry in _workflows.steps(name, job_id)].index(step_name) + + +def legs(name: 'str', job_id: 'str') -> 'list[str]': + """The ``os`` values of job ``job_id``'s matrix in workflow ``name``.""" + return list(_workflows.job(name, job_id)['strategy']['matrix']['os']) + + +def runs(entry: 'dict[str, Any]', os: 'str', published: 'str' = 'false') -> 'bool': + """Whether ``entry`` runs on leg ``os``, with ``check_leg`` having found ``published``.""" + context = {'matrix': {'os': os}, + 'steps': {'check_leg': {'outputs': {'PCAPKIT_LEG_EXISTS': published}}}} # type: dict[str, object] + return evaluate_condition(str(entry.get('if', 'true')), context) + + +def action_arguments(inputs: 'dict[str, str]', token: 'str', packages: 'str', labels: 'str') -> 'list[str]': + """The arguments upload-package v0.3.1 passes ``anaconda`` for ``inputs``. + + Its "Upload to anaconda.org" step at commit + a6f488910b4a1e4f3fe01736241ca1b92b4b355a: ``--token``, ``upload``, + ``--user`` with the channel, ``--force`` when ``force`` is ``'true'``, a + ``--label`` per comma-separated label, trimmed, then the packages + word-split. + """ + flags = ['--force'] if str(inputs.get('force', 'false')) == 'true' else [] + for label in labels.split(','): + if label.strip(): + flags += ['--label', label.strip()] + return ['--token', token, 'upload', '--user', str(inputs['channel']), *flags, *packages.split()] + + +class TestOnlyTheIntelLegUsesTheCLI(unittest.TestCase): + """The action on every leg but ``macos-26-intel``, the CLI step there alone.""" + + def test_the_action_step_is_still_the_action(self) -> None: + for name, job_id in JOBS: + with self.subTest(workflow=name, job=job_id): + self.assertTrue(_workflows.pinned(step(name, job_id, ACTION_STEP).get('uses'), ACTION)) + self.assertNotIn('uses', step(name, job_id, CLI_STEP)) + + def test_the_intel_leg_runs_the_cli_step_only(self) -> None: + for name, job_id in JOBS: + with self.subTest(workflow=name, job=job_id): + self.assertIn(INTEL, legs(name, job_id)) + self.assertFalse(runs(step(name, job_id, ACTION_STEP), INTEL)) + self.assertTrue(runs(step(name, job_id, CLI_STEP), INTEL)) + + def test_every_other_leg_runs_the_action_only(self) -> None: + for name, job_id in JOBS: + others = [os for os in legs(name, job_id) if os != INTEL] + with self.subTest(workflow=name, job=job_id): + self.assertTrue(others, 'no other leg to check') + for os in others: + with self.subTest(workflow=name, job=job_id, os=os): + self.assertTrue(runs(step(name, job_id, ACTION_STEP), os)) + self.assertFalse(runs(step(name, job_id, CLI_STEP), os)) + + def test_a_published_release_leg_runs_neither(self) -> None: + """#888's per-leg check still gates both release uploads.""" + for os in legs('create-release.yml', 'conda'): + with self.subTest(os=os): + self.assertFalse(runs(step('create-release.yml', 'conda', ACTION_STEP), os, published='true')) + self.assertFalse(runs(step('create-release.yml', 'conda', CLI_STEP), os, published='true')) + + def test_the_cli_step_follows_the_action_step(self) -> None: + """So it, too, runs after the smoke test and before the GitHub upload.""" + for name, job_id in JOBS: + with self.subTest(workflow=name, job=job_id): + self.assertEqual(index(name, job_id, CLI_STEP), index(name, job_id, ACTION_STEP) + 1) + + +class TestTheCLIStepMatchesTheAction(unittest.TestCase): + """The CLI step uploads what the action would, with ``--skip-existing``.""" + + def test_the_token_comes_through_env(self) -> None: + for name, job_id in JOBS: + entry = step(name, job_id, CLI_STEP) + with self.subTest(workflow=name, job=job_id): + self.assertEqual(entry['env']['ANACONDA_API_TOKEN'], SECRET) + self.assertNotIn('${{', entry['run']) + self.assertIn('--token "$ANACONDA_API_TOKEN"', entry['run']) + + def test_it_carries_the_actions_inputs(self) -> None: + for name, job_id in JOBS: + given = step(name, job_id, ACTION_STEP)['with'] + env = step(name, job_id, CLI_STEP)['env'] + for key, variable in CARRIED.items(): + with self.subTest(workflow=name, job=job_id, input=key): + self.assertEqual(env[variable], given[key]) + + def test_it_runs_the_actions_call_with_skip_existing(self) -> None: + if not BASH: + self.skipTest('needs bash on PATH') + packages = [f'/build/out/osx-64/pypcapkit-1.0-py314_{build}.tar.bz2' for build in (0, 1)] + for name, job_id in JOBS: + given = step(name, job_id, ACTION_STEP)['with'] + entry = step(name, job_id, CLI_STEP) + self.assertEqual(entry.get('shell'), 'bash') + for labels in ('main', ' main , dev ', ''): + with self.subTest(workflow=name, job=job_id, labels=labels): + calls = self.run_script(entry, packages='\n'.join(packages), labels=labels) + self.assertEqual(calls[0], ['conda', 'install', '-n', 'base', '--override-channels', + '-c', 'conda-forge', '-y', 'anaconda-client']) + expected = action_arguments(given, 'TOKEN', '\n'.join(packages), labels) + expected = [argument for argument in expected if argument != '--force'] + expected.insert(expected.index('--user') + 2, '--skip-existing') + self.assertEqual(calls[1:], [['anaconda', *expected]]) + + def test_no_packages_uploads_nothing_and_fails(self) -> None: + """Where the action would run ``anaconda upload`` with no file.""" + if not BASH: + self.skipTest('needs bash on PATH') + for name, job_id in JOBS: + with self.subTest(workflow=name, job=job_id): + calls = self.run_script(step(name, job_id, CLI_STEP), packages='', labels='main', status=1) + self.assertEqual([call[0] for call in calls], ['conda']) + + def run_script(self, entry: 'dict[str, Any]', packages: 'str', labels: 'str', + status: 'int' = 0) -> 'list[list[str]]': + """Each call ``entry``'s script makes to ``conda`` and ``anaconda``, as argument lists. + + The script has to exit with ``status``. + """ + with tempfile.TemporaryDirectory() as temp: + scratch = pathlib.Path(temp) + for tool in (scratch / 'bin' / 'conda', scratch / 'base' / 'bin' / 'anaconda'): + tool.parent.mkdir(parents=True, exist_ok=True) + tool.write_text(RECORDER, encoding='utf-8') + tool.chmod(tool.stat().st_mode | stat.S_IXUSR) + log = scratch / 'log' + log.write_text('', encoding='utf-8') + env = dict(os.environ, PATH=f'{scratch / "bin"}{os.pathsep}{os.environ["PATH"]}', + CONDA=str(scratch / 'base'), FAKE_LOG=str(log), ANACONDA_API_TOKEN='TOKEN', + ANACONDA_CLI_FORCE_NEW='1', PCAPKIT_CONDA_CHANNEL=str(entry['env']['PCAPKIT_CONDA_CHANNEL']), + PCAPKIT_CONDA_PACKAGES=packages, PCAPKIT_CONDA_LABELS=labels) + result = subprocess.run([str(BASH), '-e', '-c', entry['run']], cwd=temp, env=env, # nosec: B603 + capture_output=True, text=True, check=False, timeout=scale_timeout(60)) + self.assertEqual(result.returncode, status, result.stdout + result.stderr) + blocks = log.read_text(encoding='utf-8').split('\n\n') + return [block.splitlines() for block in blocks if block.strip()] + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_release_gates.py b/tests/project/test_release_gates.py index 0af90b7dbe..433eb5381d 100644 --- a/tests/project/test_release_gates.py +++ b/tests/project/test_release_gates.py @@ -274,6 +274,9 @@ PUBLISHING_MARKERS = { 'pypa/gh-action-pypi-publish': 'uploads a distribution to PyPI or TestPyPI', 'anaconda/actions/upload-package': 'uploads a package to anaconda.org', + # #2044: the ``macos-26-intel`` leg's ``anaconda … upload``, which calls the + # anaconda-client binary by path; a bare ``anaconda`` on ``PATH`` is not matched. + '/bin/anaconda"': 'uploads a package to anaconda.org with the anaconda-client CLI', 'softprops/action-gh-release': 'creates a public GitHub Release, and the tag it names', # ``pypi`` and ``conda`` attach their files this way since #1634. 'gh release upload': 'attaches a file to a public GitHub Release', @@ -4075,7 +4078,7 @@ def test_cron_conda_really_does_publish(self) -> None: self.assertIn('ncipollo/release-action', block) self.assertEqual( sorted(publishing_reasons(block)), - ['uploads a package to anaconda.org'], + ['uploads a package to anaconda.org', 'uploads a package to anaconda.org with the anaconda-client CLI'], 'the publishing markers no longer recognise what `conda-dist` does; ' 'see PUBLISHING_MARKERS, which is an allowlist and says nothing about ' 'a route it does not name', diff --git a/tests/project/test_release_tags_the_run_sha_1634.py b/tests/project/test_release_tags_the_run_sha_1634.py index ef4953b300..55883f2a7b 100644 --- a/tests/project/test_release_tags_the_run_sha_1634.py +++ b/tests/project/test_release_tags_the_run_sha_1634.py @@ -626,8 +626,11 @@ def test_conda_uploads_whatever_check_leg_found(self) -> None: already has is what stops the replacement #1640 measured. """ self.assertEqual(self.step('conda', UPLOAD_STEP).get('if'), "${{ matrix.os == 'ubuntu-latest' }}") + # #2044: each Anaconda upload also names its legs. self.assertEqual(self.step('conda', 'Upload conda packages to Anaconda').get('if'), - "${{ steps.check_leg.outputs.PCAPKIT_LEG_EXISTS != 'true' }}") + "${{ matrix.os != 'macos-26-intel' && steps.check_leg.outputs.PCAPKIT_LEG_EXISTS != 'true' }}") + self.assertEqual(self.step('conda', 'Upload conda packages to Anaconda (Intel)').get('if'), + "${{ matrix.os == 'macos-26-intel' && steps.check_leg.outputs.PCAPKIT_LEG_EXISTS != 'true' }}") class TestEveryScriptIsHandedTheRunSha(WorkflowMixin, unittest.TestCase): @@ -1567,7 +1570,11 @@ def test_the_check_reads_the_release_the_action_updates(self) -> None: self.assertNotIn('${{', check['run']) def test_the_anaconda_upload_is_not_gated_on_it(self) -> None: - self.assertNotIn('if', self.step('conda-dist', 'Upload conda packages to Anaconda')) + # #2044: gated on the leg alone. + self.assertEqual(self.step('conda-dist', 'Upload conda packages to Anaconda').get('if'), + "${{ matrix.os != 'macos-26-intel' }}") + self.assertEqual(self.step('conda-dist', 'Upload conda packages to Anaconda (Intel)').get('if'), + "${{ matrix.os == 'macos-26-intel' }}") def test_an_update_targets_the_default_branch(self) -> None: """``github.ref_name`` is the default branch on this workflow's triggers.