diff --git a/.github/workflows/_docker-image.yml b/.github/workflows/_docker-image.yml new file mode 100644 index 00000000000..d40e5c24add --- /dev/null +++ b/.github/workflows/_docker-image.yml @@ -0,0 +1,142 @@ +name: Resolve CI docker image + +# OSDC jobs run inside a container the runner pod pulls before any step of the +# job executes, so unlike linux_job_v2 there is no in-job step that can resolve +# `ci-image:` to an ECR reference or wait for docker-builds to push it. +# Callers depend on this workflow instead and spell the image out as +# 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:-. + +on: + workflow_call: + outputs: + ci-docker-hash: + description: | + Tag suffix shared by every executorch CI image built from this commit, + i.e. `git rev-parse HEAD:.ci/docker`. Matches the tag docker-builds + pushes and the one calculate-docker-image derives for linux_job_v2. + value: ${{ jobs.resolve.outputs.ci-docker-hash }} + +permissions: + contents: read + +jobs: + resolve: + name: resolve + runs-on: ubuntu-latest + # docker-builds gives itself 240 minutes, so the wait below has to be able + # to outlast a full rebuild of the slowest image. + timeout-minutes: 250 + permissions: + contents: read + actions: read + outputs: + ci-docker-hash: ${{ steps.hash.outputs.ci-docker-hash }} + + steps: + - name: Checkout ExecuTorch + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + + - name: Compute the docker tag + id: hash + run: | + set -eu + echo "ci-docker-hash=$(git rev-parse HEAD:.ci/docker)" >> "$GITHUB_OUTPUT" + + # A commit that leaves .ci/docker alone reuses an image that is already in + # ECR, so only a commit that changes the tree has to wait for a rebuild. + - name: Check whether this commit rebuilds the images + id: rebuild + env: + GH_TOKEN: ${{ github.token }} + run: | + set -eu + + REPO="${{ github.repository }}" + BASE="" + + case "${{ github.event_name }}" in + pull_request|pull_request_target) + BASE="${{ github.event.pull_request.base.sha }}" + HEAD="${{ github.event.pull_request.head.sha }}" + ;; + push) + BASE="${{ github.event.before }}" + HEAD="${{ github.sha }}" + ;; + esac + + ZERO_SHA="0000000000000000000000000000000000000000" + if [ -z "$BASE" ] || [ "$BASE" = "$ZERO_SHA" ]; then + # No diff base (workflow_dispatch, schedule, tag or branch creation). + # docker-builds is not path-filtered on those events either, so + # whatever is in ECR for this tag is what the job gets. + echo "No diff base for event '${{ github.event_name }}'; not waiting" + echo "rebuild=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if gh api "repos/$REPO/compare/$BASE...$HEAD" --paginate \ + --jq '.files[]?.filename' | grep -q '^\.ci/docker/'; then + echo "rebuild=true" >> "$GITHUB_OUTPUT" + else + echo "rebuild=false" >> "$GITHUB_OUTPUT" + fi + + - name: Wait for docker-builds to push the images + if: steps.rebuild.outputs.rebuild == 'true' + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + run: | + set -eu + + REPO="${{ github.repository }}" + QUERY="repos/$REPO/actions/workflows/docker-builds.yml/runs?head_sha=$HEAD_SHA&per_page=1" + + # docker-builds has no pull_request trigger, so on a PR it only runs + # once the ciflow/docker label tags this commit. Give that tag a few + # minutes to show up, then say so, rather than blocking for hours on a + # build nobody asked for. + missing=0 + + # 240 x 60s outlasts docker-builds' own 240 minute job timeout. + for _ in $(seq 1 240); do + RUN=$(gh api "$QUERY" \ + --jq '(.workflow_runs[0] // {}) | "\(.status // "") \(.conclusion // "")"' \ + 2>/dev/null || echo " ") + STATUS="${RUN%% *}" + CONCLUSION="${RUN##* }" + + case "$STATUS" in + completed) + if [ "$CONCLUSION" = "success" ]; then + echo "docker-builds succeeded for $HEAD_SHA" + exit 0 + fi + echo "::error::docker-builds concluded '$CONCLUSION' for $HEAD_SHA." \ + "The CI images for this commit were never pushed, so every" \ + "job that needs one would fail pulling its container." + exit 1 + ;; + "") + missing=$((missing + 1)) + if [ "$missing" -gt 5 ]; then + echo "::error::This commit changes .ci/docker, so it needs images" \ + "that do not exist yet, but no docker-builds run was found for" \ + "$HEAD_SHA. Add the ciflow/docker label to build them." + exit 1 + fi + echo "No docker-builds run for $HEAD_SHA yet; waiting" + ;; + *) + missing=0 + echo "docker-builds is $STATUS for $HEAD_SHA; waiting" + ;; + esac + sleep 60 + done + + echo "::error::Timed out waiting for docker-builds on $HEAD_SHA" + exit 1 diff --git a/.github/workflows/_llm_server.yml b/.github/workflows/_llm_server.yml index e1ef5a30db9..255cf282f8d 100644 --- a/.github/workflows/_llm_server.yml +++ b/.github/workflows/_llm_server.yml @@ -4,20 +4,30 @@ on: workflow_call: inputs: docker-image: - description: Docker image to use for Linux tests. + description: | + ECR repository and tag prefix of the CI image, e.g. + ci-image:executorch-ubuntu-22.04-clang12. The registry and the + .ci/docker hash are appended by this workflow. required: false type: string default: ci-image:executorch-ubuntu-22.04-clang12 jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + linux: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: - runner: linux.2xlarge - docker-image: ${{ inputs.docker-image }} + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ inputs.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: recursive ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 60 diff --git a/.github/workflows/_test_arduino_library.yml b/.github/workflows/_test_arduino_library.yml index a3e73958463..9656b4f958a 100644 --- a/.github/workflows/_test_arduino_library.yml +++ b/.github/workflows/_test_arduino_library.yml @@ -14,15 +14,22 @@ on: default: 90 jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + run: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: job-name: arduino-library - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: ${{ inputs.timeout }} diff --git a/.github/workflows/_test_backend.yml b/.github/workflows/_test_backend.yml index 18c70a31531..5355a286cb1 100644 --- a/.github/workflows/_test_backend.yml +++ b/.github/workflows/_test_backend.yml @@ -40,15 +40,25 @@ on: description: 'Runner type for Linux jobs' required: false type: string - default: linux.4xlarge.memory + default: mt-l-x86iavx512-16-128 docker-image: - description: 'Docker image for Linux jobs' + description: | + ECR repository and tag prefix of the CI image, e.g. + ci-image:executorch-ubuntu-22.04-clang12. The registry and the + .ci/docker hash are appended by this workflow. required: false type: string default: ci-image:executorch-ubuntu-22.04-clang12 jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + test-backend-linux: + needs: docker-image if: ${{ inputs.run-linux }} strategy: fail-fast: false @@ -57,11 +67,14 @@ jobs: suite: [models, operators] exclude: ${{ fromJSON(inputs.exclude) }} - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read with: ref: ${{ inputs.ref }} runner: ${{ inputs.runner-linux }} - docker-image: ${{ inputs.docker-image }} + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ inputs.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: recursive timeout: ${{ inputs.timeout }} upload-artifact: test-report-${{ inputs.backend }}-${{ matrix.flow }}-${{ matrix.suite }} diff --git a/.github/workflows/_test_cadence.yml b/.github/workflows/_test_cadence.yml index 2e98d21db1c..e8b6008b0b8 100644 --- a/.github/workflows/_test_cadence.yml +++ b/.github/workflows/_test_cadence.yml @@ -8,7 +8,10 @@ on: workflow_call: inputs: docker-image: - description: 'Docker image to use' + description: | + ECR repository and tag prefix of the CI image, e.g. + ci-image:executorch-ubuntu-22.04-clang12. The registry and the + .ci/docker hash are appended by this workflow. required: false type: string default: ci-image:executorch-ubuntu-22.04-clang12 @@ -16,7 +19,7 @@ on: description: 'Runner type' required: false type: string - default: linux.8xlarge.memory + default: mt-l-x86iavx512-32-256 ref: description: 'Git ref to checkout' required: false @@ -29,12 +32,22 @@ on: default: 90 jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + test-aot: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read with: job-name: test-aot runner: ${{ inputs.runner }} - docker-image: ${{ inputs.docker-image }} + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ inputs.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: recursive ref: ${{ inputs.ref }} timeout: ${{ inputs.timeout }} @@ -50,11 +63,15 @@ jobs: python -m pytest backends/cadence/aot/tests/ -v -n auto --reruns 2 --reruns-delay 1 test-ops: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read with: job-name: test-ops runner: ${{ inputs.runner }} - docker-image: ${{ inputs.docker-image }} + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ inputs.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: recursive ref: ${{ inputs.ref }} timeout: ${{ inputs.timeout }} diff --git a/.github/workflows/_test_cortex_m_e2e.yml b/.github/workflows/_test_cortex_m_e2e.yml index 0510b017723..f25fb09c4ce 100644 --- a/.github/workflows/_test_cortex_m_e2e.yml +++ b/.github/workflows/_test_cortex_m_e2e.yml @@ -23,8 +23,18 @@ on: default: 120 jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + run: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read strategy: matrix: model: ${{ fromJSON(inputs.models) }} @@ -32,8 +42,8 @@ jobs: fail-fast: false with: job-name: ${{ matrix.model }}-${{ matrix.target }} - runner: linux.2xlarge.memory - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: ${{ inputs.timeout }} diff --git a/.github/workflows/_test_cortex_m_ops.yml b/.github/workflows/_test_cortex_m_ops.yml index a9e2e6180c3..61fbce65398 100644 --- a/.github/workflows/_test_cortex_m_ops.yml +++ b/.github/workflows/_test_cortex_m_ops.yml @@ -18,16 +18,26 @@ on: default: 120 jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + run: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read strategy: matrix: target: ${{ fromJSON(inputs.targets) }} fail-fast: false with: job-name: cortex-m-ops-${{ matrix.target }} - runner: linux.2xlarge.memory - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: ${{ inputs.timeout }} diff --git a/.github/workflows/_test_riscv.yml b/.github/workflows/_test_riscv.yml index 223a146e3d8..a70b977d4e8 100644 --- a/.github/workflows/_test_riscv.yml +++ b/.github/workflows/_test_riscv.yml @@ -37,11 +37,21 @@ on: type: string jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + run: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-24.04-gcc14 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-24.04-gcc14-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: ${{ inputs.timeout }} diff --git a/.github/workflows/_unittest.yml b/.github/workflows/_unittest.yml index 4216d3aaa27..3c2398d8d57 100644 --- a/.github/workflows/_unittest.yml +++ b/.github/workflows/_unittest.yml @@ -6,7 +6,10 @@ on: docker-image: required: true type: string - description: Name of the docker image to use. + description: | + ECR repository and tag prefix of the CI image, e.g. + ci-image:executorch-ubuntu-22.04-clang12. The registry and the + .ci/docker hash are appended by this workflow. build-mode: required: true type: string @@ -30,14 +33,21 @@ on: default: 90 jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + linux: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: - runner: linux.2xlarge.memory - docker-image: ${{ inputs.docker-image }} + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ inputs.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: ${{ inputs.linux-timeout }} diff --git a/.github/workflows/android-release-artifacts.yml b/.github/workflows/android-release-artifacts.yml index 0cf179109ed..456d6a4772c 100644 --- a/.github/workflows/android-release-artifacts.yml +++ b/.github/workflows/android-release-artifacts.yml @@ -40,6 +40,12 @@ on: - cron: 0 10 * * * jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + check-if-aar-exists: name: check-if-aar-exists runs-on: ubuntu-22.04 @@ -71,16 +77,16 @@ jobs: build-aar: name: build-aar - needs: check-if-aar-exists + needs: [docker-image, check-if-aar-exists] if: ${{ !github.event.pull_request.head.repo.fork && needs.check-if-aar-exists.outputs.should-skip != 'true' }} - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main secrets: inherit permissions: id-token: write contents: read with: secrets-env: EXECUTORCH_MAVEN_SIGNING_KEYID EXECUTORCH_MAVEN_SIGNING_PASSWORD EXECUTORCH_MAVEN_CENTRAL_PASSWORD EXECUTORCH_MAVEN_CENTRAL_USERNAME EXECUTORCH_MAVEN_SIGNING_GPG_KEY_CONTENTS - docker-image: ci-image:executorch-ubuntu-22.04-clang12-android + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-android-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.sha }} timeout: 90 diff --git a/.github/workflows/build-cadence-runner.yml b/.github/workflows/build-cadence-runner.yml index c12343f48e3..99a8225ab56 100644 --- a/.github/workflows/build-cadence-runner.yml +++ b/.github/workflows/build-cadence-runner.yml @@ -19,6 +19,12 @@ concurrency: cancel-in-progress: true jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + # Same-repo PRs run on pull_request, which reads the PR's own workflow AND code # -- so CI changes, new test jobs, code, and tests are all validated pre-merge. # Fork PRs can't get credentials (OIDC) on pull_request, so Meta-exported forks @@ -26,19 +32,20 @@ jobs: # run condition is inlined per job (GitHub Actions has no YAML anchors and env # is unavailable in job-level if), so keep the copies in sync. cpu-build: + needs: docker-image if: >- github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && contains(github.event.pull_request.labels.*.name, 'CLA Signed') && contains(github.event.pull_request.labels.*.name, 'meta-exported')) - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: job-name: build - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: recursive ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.head.sha || github.sha }} timeout: 90 diff --git a/.github/workflows/build-cmsis-pack.yml b/.github/workflows/build-cmsis-pack.yml index c9c40d670f1..110f4becb1a 100644 --- a/.github/workflows/build-cmsis-pack.yml +++ b/.github/workflows/build-cmsis-pack.yml @@ -44,15 +44,22 @@ concurrency: cancel-in-progress: true jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + build-cmsis-pack: + needs: docker-image name: build-cmsis-pack - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 60 diff --git a/.github/workflows/build-presets.yml b/.github/workflows/build-presets.yml index 89d36cd6b0a..9fb8882fc36 100644 --- a/.github/workflows/build-presets.yml +++ b/.github/workflows/build-presets.yml @@ -13,6 +13,12 @@ concurrency: cancel-in-progress: true jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + apple: uses: pytorch/test-infra/.github/workflows/macos_job.yml@main strategy: @@ -34,7 +40,11 @@ jobs: ${CONDA_RUN} cmake --build cmake-out -j$(( $(sysctl -n hw.ncpu) - 1 )) zephyr: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read strategy: fail-fast: false matrix: @@ -42,8 +52,8 @@ jobs: with: job-name: build ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-zephyr-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-zephyr-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: recursive timeout: 90 script: | @@ -73,26 +83,30 @@ jobs: cmake --preset ${{ matrix.preset }} cmake --build cmake-out -j$(( $(nproc) - 1 )) linux: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read strategy: fail-fast: false matrix: preset: [linux, pybind, llm] - runner: [linux.2xlarge, linux.arm64.2xlarge] + runner: [mt-l-x86iavx512-8-64, mt-l-arm64g4-16-62] docker-image: [executorch-ubuntu-22.04-clang12, executorch-ubuntu-22.04-gcc11-aarch64] # Excluding specific runner + docker image combinations that don't make sense: - # - Excluding the ARM64 gcc image on the x86 runner (linux.2xlarge) - # - Excluding the x86 clang image on the ARM64 runner (linux.arm64.2xlarge) + # - Excluding the ARM64 gcc image on the x86 runner + # - Excluding the x86 clang image on the ARM64 runner exclude: - - runner: linux.2xlarge + - runner: mt-l-x86iavx512-8-64 docker-image: executorch-ubuntu-22.04-gcc11-aarch64 - - runner: linux.arm64.2xlarge + - runner: mt-l-arm64g4-16-62 docker-image: executorch-ubuntu-22.04-clang12 with: job-name: build ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} runner: ${{ matrix.runner }} - docker-image: ci-image:${{ matrix.docker-image }} + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:${{ matrix.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: recursive timeout: 90 script: | diff --git a/.github/workflows/cuda-perf.yml b/.github/workflows/cuda-perf.yml index 0adf3f87d32..662c84914f2 100644 --- a/.github/workflows/cuda-perf.yml +++ b/.github/workflows/cuda-perf.yml @@ -138,7 +138,6 @@ jobs: runner: ${{ contains(matrix.model, 'Qwen3.5-35B-A3B') && 'mt-l-x86iavx512-11-125-a100' || 'mt-l-x86aavx2-29-113-a10g' }} gpu-arch-type: cuda gpu-arch-version: "13.0" - use-custom-docker-registry: false submodules: recursive upload-artifact: model-${{ matrix.model_safe }}-${{ matrix.quant }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} @@ -218,7 +217,6 @@ jobs: runner: ${{ contains(matrix.model, 'Qwen3.5-35B-A3B') && 'mt-l-x86iavx512-11-125-a100' || 'mt-l-x86aavx2-29-113-a10g' }} gpu-arch-type: cuda gpu-arch-version: "13.0" - use-custom-docker-registry: false submodules: recursive download-artifact: model-${{ matrix.model_safe }}-${{ matrix.quant }} upload-artifact: results-${{ matrix.model_safe }}-${{ matrix.quant }} diff --git a/.github/workflows/cuda-windows.yml b/.github/workflows/cuda-windows.yml index 19c8deef651..451830d185e 100644 --- a/.github/workflows/cuda-windows.yml +++ b/.github/workflows/cuda-windows.yml @@ -27,6 +27,12 @@ permissions: contents: read jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + changed-files: name: Get changed files uses: ./.github/workflows/_get-changed-files.yml @@ -44,7 +50,7 @@ jobs: # filter so push commits that don't touch CUDA-relevant paths skip # this job on non-sampled commits. See _ci-run-decision.yml for # the sampling policy. - needs: [changed-files, run-decision] + needs: [docker-image, changed-files, run-decision] if: | (github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request') && ( @@ -54,7 +60,7 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda-windows.yml') || needs.run-decision.outputs.is-full-run == 'true' ) - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -89,10 +95,10 @@ jobs: with: timeout: 90 secrets-env: EXECUTORCH_HF_TOKEN - runner: linux.g5.4xlarge.nvidia.gpu + runner: mt-l-x86aavx2-29-113-a10g gpu-arch-type: cuda gpu-arch-version: "13.0" - docker-image: ci-image:executorch-ubuntu-22.04-cuda-windows + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-cuda-windows-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: recursive upload-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index b490e28c95f..40395d1b583 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -65,16 +65,15 @@ jobs: cuda-version: ["12.6", "13.0"] name: test-executorch-cuda-build-${{ matrix.cuda-version }} - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: timeout: 90 - runner: linux.g5.4xlarge.nvidia.gpu + runner: mt-l-x86aavx2-29-113-a10g gpu-arch-type: cuda gpu-arch-version: ${{ matrix.cuda-version }} - use-custom-docker-registry: false submodules: recursive ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | @@ -124,16 +123,15 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e.sh') || needs.run-decision.outputs.is-full-run == 'true' - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: timeout: 90 - runner: linux.g5.4xlarge.nvidia.gpu + runner: mt-l-x86aavx2-29-113-a10g gpu-arch-type: cuda gpu-arch-version: "13.0" - use-custom-docker-registry: false submodules: recursive ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | @@ -168,16 +166,15 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e.sh') || needs.run-decision.outputs.is-full-run == 'true' - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: timeout: 90 - runner: linux.g5.4xlarge.nvidia.gpu + runner: mt-l-x86aavx2-29-113-a10g gpu-arch-type: cuda gpu-arch-version: "13.0" - use-custom-docker-registry: false submodules: recursive ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | @@ -225,16 +222,15 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e.sh') || needs.run-decision.outputs.is-full-run == 'true' - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: timeout: 90 - runner: linux.g5.4xlarge.nvidia.gpu + runner: mt-l-x86aavx2-29-113-a10g gpu-arch-type: cuda gpu-arch-version: "13.0" - use-custom-docker-registry: false submodules: recursive ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | @@ -382,7 +378,6 @@ jobs: runner: ${{ (matrix.model.name == 'Qwen3.5-35B-A3B-HQQ-INT4' || matrix.model.name == 'gemma-4-31B-it-GGUF') && 'mt-l-x86iavx512-11-125-a100' || 'mt-l-x86aavx2-29-113-a10g' }} gpu-arch-type: cuda gpu-arch-version: "13.0" - use-custom-docker-registry: false submodules: recursive upload-artifact: ${{ matrix.model.repo }}-${{ matrix.model.name }}-cuda-${{ matrix.quant }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} @@ -457,7 +452,6 @@ jobs: runner: mt-l-x86iavx512-11-125-a100 gpu-arch-type: cuda gpu-arch-version: "13.0" - use-custom-docker-registry: false submodules: recursive ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | @@ -511,7 +505,7 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e.sh') || needs.run-decision.outputs.is-full-run == 'true' ) - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -533,10 +527,9 @@ jobs: timeout: 120 secrets-env: EXECUTORCH_HF_TOKEN download-artifact: ${{ matrix.artifact }} - runner: linux.g5.4xlarge.nvidia.gpu + runner: mt-l-x86aavx2-29-113-a10g gpu-arch-type: cuda gpu-arch-version: "13.0" - use-custom-docker-registry: false submodules: recursive ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 0921d78f821..7ae9dc7d430 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -18,8 +18,15 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + build: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -29,8 +36,8 @@ jobs: - build-tool: buck2 with: job-name: Build doc - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12-android + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-android-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' repository: pytorch/executorch upload-artifact: docs @@ -98,15 +105,16 @@ jobs: permissions: id-token: write contents: write - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main secrets: inherit with: repository: pytorch/executorch download-artifact: docs ref: gh-pages - # Publish on the host so actions/checkout's persisted credentials are - # available to git push. - run-with-docker: false + # v2 needed run-with-docker: false here, because the script ran through a + # docker exec that could not see the credentials checkout persisted on the + # host. v3 checks out and runs the script in the same container, so the + # git config the push relies on is already there. timeout: 90 script: | set -euo pipefail diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 922ed95ab8d..3a93948b728 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -16,6 +16,12 @@ concurrency: cancel-in-progress: true jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + update-pytorch-commit-hash: runs-on: ubuntu-latest environment: ${{ (github.event_name == 'schedule') && 'update-commit-hash' || '' }} @@ -50,8 +56,9 @@ jobs: timeout: 180 test-static-hf-llm-qnn-linux: + needs: docker-image name: test-static-hf-llm-qnn-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -60,8 +67,8 @@ jobs: task: [smollm2_135m] fail-fast: false with: - runner: linux.24xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-94-192 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 900 diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml index 01bff087124..e93ee546b78 100644 --- a/.github/workflows/periodic.yml +++ b/.github/workflows/periodic.yml @@ -21,6 +21,12 @@ concurrency: permissions: read-all jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + gather-models: runs-on: ubuntu-22.04 outputs: @@ -42,17 +48,17 @@ jobs: test-models-linux: name: test-models-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read - needs: gather-models + needs: [docker-image, gather-models] strategy: matrix: ${{ fromJSON(needs.gather-models.outputs.models) }} fail-fast: false with: runner: ${{ matrix.runner }} - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: ${{ matrix.timeout }} diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 0d76e991d79..67480815fa6 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -15,6 +15,12 @@ concurrency: cancel-in-progress: true jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + # Emits the list of changed files for the current PR or push commit. # On PR: PR diff. On push: diff against `github.event.before`. # On events without a diff base (workflow_dispatch, tag creation, @@ -35,8 +41,9 @@ jobs: uses: ./.github/workflows/_ci-run-decision.yml test-qnn-wheel-packages-linux: + needs: docker-image name: test-qnn-wheel-packages-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -45,8 +52,8 @@ jobs: matrix: python-version: [ "3.10", "3.11", "3.12", "3.13" ] with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 180 @@ -79,7 +86,7 @@ jobs: contents: read test-minimal-wheel-linux: - needs: changed-files + needs: [docker-image, changed-files] if: | github.event_name != 'pull_request' || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_minimal_wheel.sh') || @@ -92,13 +99,13 @@ jobs: contains(needs.changed-files.outputs.changed-files, 'setup.py') || contains(needs.changed-files.outputs.changed-files, 'tools/cmake/') name: test-minimal-wheel-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 @@ -109,16 +116,17 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_minimal_wheel.sh test-setup-linux-gcc: + needs: docker-image name: test-setup-linux-gcc - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-gcc11 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-gcc11-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -134,8 +142,9 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "add" "${BUILD_TOOL}" "portable" test-models-linux-basic: + needs: docker-image name: test-models-linux-basic - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -144,23 +153,23 @@ jobs: model: [mv3, vit] backend: [portable, xnnpack-quantization-delegation] build-tool: [cmake, buck2] - runner: [linux.2xlarge, linux.arm64.2xlarge] + runner: [mt-l-x86iavx512-8-64, mt-l-arm64g4-16-62] docker-image: [executorch-ubuntu-22.04-clang12, executorch-ubuntu-22.04-gcc11-aarch64] # Excluding specific runner + docker image combinations that don't make sense: - # - Excluding the ARM64 gcc image on the x86 runner (linux.2xlarge) - # - Excluding the x86 clang image on the ARM64 runner (linux.arm64.2xlarge) + # - Excluding the ARM64 gcc image on the x86 runner + # - Excluding the x86 clang image on the ARM64 runner exclude: - - runner: linux.2xlarge + - runner: mt-l-x86iavx512-8-64 docker-image: executorch-ubuntu-22.04-gcc11-aarch64 - - runner: linux.arm64.2xlarge + - runner: mt-l-arm64g4-16-62 docker-image: executorch-ubuntu-22.04-clang12 # TODO: Need to figure out why buck2 doesnt work on Graviton instances. - - runner: linux.arm64.2xlarge + - runner: mt-l-arm64g4-16-62 build-tool: buck2 fail-fast: false with: runner: ${{ matrix.runner }} - docker-image: ci-image:${{ matrix.docker-image }} + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:${{ matrix.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -178,8 +187,9 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" test-models-linux: + needs: docker-image name: test-models-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -187,33 +197,33 @@ jobs: matrix: model: [linear, add, add_mul, ic3, mv2, resnet18, resnet50, mobilebert, emformer_transcribe] backend: [portable, xnnpack-quantization-delegation] - runner: [linux.2xlarge] + runner: [mt-l-x86iavx512-8-64] include: - model: ic4 backend: portable - runner: linux.4xlarge.memory + runner: mt-l-x86iavx512-16-128 - model: ic4 backend: xnnpack-quantization-delegation - runner: linux.4xlarge.memory + runner: mt-l-x86iavx512-16-128 - model: emformer_join backend: portable - runner: linux.4xlarge.memory + runner: mt-l-x86iavx512-16-128 - model: emformer_join backend: xnnpack-quantization-delegation - runner: linux.4xlarge.memory + runner: mt-l-x86iavx512-16-128 - model: phi_4_mini backend: portable - runner: linux.4xlarge.memory + runner: mt-l-x86iavx512-16-128 - model: llama3_2_vision_encoder backend: portable - runner: linux.4xlarge.memory + runner: mt-l-x86iavx512-16-128 - model: w2l backend: portable - runner: linux.4xlarge.memory + runner: mt-l-x86iavx512-16-128 fail-fast: false with: runner: ${{ matrix.runner }} - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -231,16 +241,17 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" test-parakeet-xnnpack-linux: + needs: docker-image name: test-parakeet-xnnpack-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.4xlarge.memory - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-16-128 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 @@ -264,16 +275,17 @@ jobs: echo "::endgroup::" test-voxtral-realtime-xnnpack-linux: + needs: docker-image name: test-voxtral-realtime-xnnpack-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.4xlarge.memory - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-16-128 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 @@ -300,9 +312,10 @@ jobs: echo "::endgroup::" test-llama-runner-linux: + needs: docker-image # Test Both linux x86 and linux aarch64 name: test-llama-runner-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -310,25 +323,25 @@ jobs: matrix: dtype: [fp32] mode: [xnnpack+custom+qe,xnnpack+custom+quantize_kv,xnnpack+quantize_kv] - runner: [linux.2xlarge, linux.arm64.2xlarge] + runner: [mt-l-x86iavx512-8-64, mt-l-arm64g4-16-62] docker-image: [executorch-ubuntu-22.04-clang12, executorch-ubuntu-22.04-gcc11-aarch64] include: - dtype: bf16 mode: custom - runner: linux.2xlarge + runner: mt-l-x86iavx512-8-64 docker-image: executorch-ubuntu-22.04-clang12 # Excluding specific runner + docker image combinations that don't make sense: - # - Excluding the ARM64 gcc image on the x86 runner (linux.2xlarge) - # - Excluding the x86 clang image on the ARM64 runner (linux.arm64.2xlarge) + # - Excluding the ARM64 gcc image on the x86 runner + # - Excluding the x86 clang image on the ARM64 runner exclude: - - runner: linux.2xlarge + - runner: mt-l-x86iavx512-8-64 docker-image: executorch-ubuntu-22.04-gcc11-aarch64 - - runner: linux.arm64.2xlarge + - runner: mt-l-arm64g4-16-62 docker-image: executorch-ubuntu-22.04-clang12 fail-fast: false with: runner: ${{ matrix.runner }} - docker-image: ci-image:${{ matrix.docker-image }} + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:${{ matrix.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 900 @@ -351,16 +364,17 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -dtype "${DTYPE}" -mode "${MODE}" -upload "${ARTIFACTS_DIR_NAME}" test-llama-runner-linux-android: + needs: docker-image name: test-llama-runner-linux-android - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12-android + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-android-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -376,16 +390,17 @@ jobs: bash .ci/scripts/build_llama_android.sh "${BUILD_TOOL}" test-custom-ops-linux: + needs: docker-image name: test-custom-ops-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -400,16 +415,17 @@ jobs: PYTHON_EXECUTABLE=python bash examples/portable/custom_ops/test_custom_ops.sh "${BUILD_TOOL}" test-selective-build-linux: + needs: docker-image name: test-selective-build-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -424,9 +440,10 @@ jobs: PYTHON_EXECUTABLE=python bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}" test-multimodal-linux: + needs: docker-image if: ${{ !github.event.pull_request.head.repo.fork }} name: test-multimodal-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -437,8 +454,8 @@ jobs: model: ["gemma3-4b"] # llava gives segfault so not covering. with: secrets-env: EXECUTORCH_HF_TOKEN - runner: linux.24xlarge.memory - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-94-768 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -463,16 +480,17 @@ jobs: echo "::endgroup::" test-moshi-linux: + needs: docker-image name: test-moshi-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -495,16 +513,17 @@ jobs: python -m unittest examples.models.moshi.mimi.test_mimi test-quantized-aot-lib-linux: + needs: docker-image name: test-quantized-aot-lib-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -518,16 +537,17 @@ jobs: PYTHON_EXECUTABLE=python bash examples/xnnpack/quantization/test_quantize.sh "${BUILD_TOOL}" mv2 test-binary-size-linux-gcc: + needs: docker-image name: test-binary-size-linux-gcc - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-gcc9-nopytorch + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-gcc9-nopytorch-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -561,16 +581,17 @@ jobs: fi test-binary-size-linux: + needs: docker-image name: test-binary-size-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -605,8 +626,9 @@ jobs: fi test-arm-cortex-m-size-test: + needs: docker-image name: test-arm-cortex-m-size-test - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -615,8 +637,8 @@ jobs: os: [bare_metal, zephyr-preset] fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -698,14 +720,15 @@ jobs: fi test-mcu-cortex-m-backend: + needs: docker-image name: test-mcu-cortex-m-backend - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: - runner: linux.2xlarge.memory - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 @@ -765,16 +788,17 @@ jobs: docker-image: ci-image:executorch-ubuntu-22.04-clang12 test-qnn-buck-build-linux: + needs: docker-image name: test-qnn-buck-build-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -799,8 +823,9 @@ jobs: buck2 build //backends/qualcomm/... test-arm-backend-no-driver: + needs: docker-image name: test-arm-backend-no-driver - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -813,8 +838,8 @@ jobs: - test_arm_backend: test_run_tosa fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -834,14 +859,15 @@ jobs: backends/arm/test/test_arm_backend.sh "${ARM_TEST}" test-arm-backend-public-api-backward-compatibility: + needs: docker-image name: test-arm-backend-public-api-backward-compatibility - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: - runner: linux.2xlarge.memory - docker-image: ci-image:executorch-ubuntu-24.04-arm-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-24.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 @@ -861,8 +887,9 @@ jobs: python backends/arm/test/public_api_bc/run_public_api_bc_scenarios.py test-llama-runner-qnn-linux: + needs: docker-image name: test-llama-runner-qnn-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -873,8 +900,8 @@ jobs: mode: [qnn] fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 900 @@ -900,8 +927,9 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -mode "${MODE}" -dtype "${DTYPE}" -pt2e_quantize "${PT2E_QUANTIZE}" test-static-llama-qnn-linux: + needs: docker-image name: test-static-llama-qnn-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -910,8 +938,8 @@ jobs: task: [stories_110m, stories_260k_bc] fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 180 @@ -934,8 +962,9 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_qnn_static_llm.sh ${{ matrix.task }} test-sqnr-static-llm-qnn-linux: + needs: docker-image name: test-sqnr-static-llm-qnn-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -944,8 +973,8 @@ jobs: task: [smollm2_135m] fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 180 @@ -968,8 +997,9 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_qnn_static_llm.sh ${{ matrix.task }} sqnr test-qnn-models-linux: + needs: docker-image name: test-qnn-models-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -978,8 +1008,8 @@ jobs: model: [mv2, mv3, dl3] fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 180 @@ -993,14 +1023,15 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh ${{ matrix.model }} "cmake" "qnn" test-qnn-direct-build-linux: + needs: docker-image name: test-qnn-direct-build-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 30 @@ -1023,19 +1054,20 @@ jobs: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 run-linux: true - runner-linux: linux.2xlarge + runner-linux: mt-l-x86iavx512-8-64 test-qnn-passes-linux: + needs: docker-image name: test-qnn-passes-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 30 @@ -1060,16 +1092,17 @@ jobs: pytest -xvs backends/qualcomm/tests/test_passes.py test-qnn-delegate-linux: + needs: docker-image name: test-qnn-delegate-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -1098,16 +1131,17 @@ jobs: -k "TestQNNFloatingPointOperator or TestQNNQuantizedOperator" test-phi-3-mini-runner-linux: + needs: docker-image name: test-phi-3-mini-runner-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.24xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-94-192 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -1128,16 +1162,17 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_phi_3_mini.sh Release test-qnn-python-imports-linux: + needs: docker-image name: test-qnn-python-imports-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 15 @@ -1176,16 +1211,17 @@ jobs: --module-prefix executorch.examples.qualcomm test-eval_llama-wikitext-linux: + needs: docker-image name: test-eval_llama-wikitext-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.24xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-94-192 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -1205,7 +1241,7 @@ jobs: # TODO(larryliu0820): Fix this issue before reenabling it: https://gist.github.com/larryliu0820/7377ecd0d79dbc06076cec8d9f2b85d2 # test-eval_llama-mmlu-linux: # name: test-eval_llama-mmlu-linux - # uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + # uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main # permissions: # id-token: write # contents: read @@ -1231,16 +1267,17 @@ jobs: # PYTHON_EXECUTABLE=python bash .ci/scripts/test_eval_llama_mmlu.sh test-llama_runner_eager-linux: + needs: docker-image name: test-llama_runner_eager-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.24xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-94-192 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -1258,16 +1295,17 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama_runner_eager.sh test-lora-linux: + needs: docker-image name: test-lora-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.24xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-94-192 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -1285,16 +1323,17 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_lora.sh test-lora-multimethod-linux: + needs: docker-image name: test-lora-multimethod-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.24xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-94-192 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -1312,16 +1351,17 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_lora_multimethod.sh test-mediatek-models-linux: + needs: docker-image name: test-mediatek-models-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.24xlarge - docker-image: ci-image:executorch-ubuntu-22.04-mediatek-sdk + runner: mt-l-x86iavx512-94-192 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-mediatek-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -1339,16 +1379,17 @@ jobs: # placeholder for mediatek to add more tests test-openvino-linux: + needs: docker-image name: test-openvino-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-gcc11 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-gcc11-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -1361,16 +1402,17 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_openvino.sh test-build-wasm-linux: + needs: docker-image name: test-build-wasm-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -1389,8 +1431,9 @@ jobs: PYTHON_EXECUTABLE=python bash examples/wasm/test_build_wasm.sh unittest-wasm-bindings: + needs: docker-image name: unittest-wasm-bindings - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -1399,8 +1442,8 @@ jobs: enable-etdump: ['', '--enable-etdump'] fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -1435,13 +1478,14 @@ jobs: pnpm test unittest-nxp-neutron: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 150 @@ -1479,18 +1523,19 @@ jobs: bash backends/nxp/run_unittests.sh test-samsung-quantmodels-linux: + needs: docker-image name: test-samsung-quantmodels-linux # Skip this job if the pull request is from a fork (secrets are not available) if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request' - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read secrets: inherit with: secrets-env: SAMSUNG_AI_LITECORE_KEY - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12-android + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-android-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 180 @@ -1517,18 +1562,19 @@ jobs: done test-samsung-models-linux: + needs: docker-image name: test-samsung-models-linux # Skip this job if the pull request is from a fork (secrets are not available) if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request' - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read secrets: inherit with: secrets-env: SAMSUNG_AI_LITECORE_KEY - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12-android + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-android-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 360 @@ -1557,14 +1603,15 @@ jobs: python -m unittest discover -s backends/samsung/test/models -p "test_*.py" test-vulkan-models-linux: + needs: docker-image name: test-vulkan-models-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -1598,14 +1645,15 @@ jobs: done test-vulkan-operators-linux: + needs: docker-image name: test-vulkan-operators-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -1690,14 +1738,15 @@ jobs: echo "::endgroup::" nxp-build-test: + needs: docker-image name: nxp-build-test - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 diff --git a/.github/workflows/test-backend-arm.yml b/.github/workflows/test-backend-arm.yml index d71696ee096..7d844748dac 100644 --- a/.github/workflows/test-backend-arm.yml +++ b/.github/workflows/test-backend-arm.yml @@ -19,6 +19,9 @@ concurrency: jobs: test-arm: uses: ./.github/workflows/_test_backend.yml + permissions: + id-token: write + contents: read with: backend: arm flows: >- @@ -35,6 +38,9 @@ jobs: test-arm-vgf: uses: ./.github/workflows/_test_backend.yml + permissions: + id-token: write + contents: read with: backend: arm-vgf flows: >- diff --git a/.github/workflows/test-backend-coreml.yml b/.github/workflows/test-backend-coreml.yml index 86844ffb559..fca10b619ce 100644 --- a/.github/workflows/test-backend-coreml.yml +++ b/.github/workflows/test-backend-coreml.yml @@ -36,6 +36,9 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.github/workflows/test-backend-coreml.yml') || contains(needs.changed-files.outputs.changed-files, '.github/workflows/_test_backend.yml') uses: ./.github/workflows/_test_backend.yml + permissions: + id-token: write + contents: read with: backend: coreml # The heavier coreml_static_int8 macOS matrix saturates the shared diff --git a/.github/workflows/test-backend-cortex-m.yml b/.github/workflows/test-backend-cortex-m.yml index a3a4e39c9e6..4fd88de9678 100644 --- a/.github/workflows/test-backend-cortex-m.yml +++ b/.github/workflows/test-backend-cortex-m.yml @@ -49,6 +49,9 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.github/workflows/test-backend-cortex-m.yml') || contains(needs.changed-files.outputs.changed-files, '.github/workflows/_test_backend.yml') uses: ./.github/workflows/_test_backend.yml + permissions: + id-token: write + contents: read with: backend: cortex_m flows: '["cortex_m"]' diff --git a/.github/workflows/test-backend-nxp.yml b/.github/workflows/test-backend-nxp.yml index fed7ab7d19b..0db7d7efeac 100644 --- a/.github/workflows/test-backend-nxp.yml +++ b/.github/workflows/test-backend-nxp.yml @@ -39,6 +39,9 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.github/workflows/test-backend-nxp.yml') || contains(needs.changed-files.outputs.changed-files, '.github/workflows/_test_backend.yml') uses: ./.github/workflows/_test_backend.yml + permissions: + id-token: write + contents: read with: backend: nxp flows: '["nxp_neutron_imxrt700_int8_ptq"]' diff --git a/.github/workflows/test-backend-openvino.yml b/.github/workflows/test-backend-openvino.yml index aeeb01e3eb1..cff7f8ba6e9 100644 --- a/.github/workflows/test-backend-openvino.yml +++ b/.github/workflows/test-backend-openvino.yml @@ -21,6 +21,9 @@ concurrency: jobs: test-openvino: uses: ./.github/workflows/_test_backend.yml + permissions: + id-token: write + contents: read with: backend: openvino flows: '["openvino"]' diff --git a/.github/workflows/test-backend-qnn.yml b/.github/workflows/test-backend-qnn.yml index 939b7c36aee..51207d139b3 100644 --- a/.github/workflows/test-backend-qnn.yml +++ b/.github/workflows/test-backend-qnn.yml @@ -19,6 +19,9 @@ concurrency: jobs: test-qnn: uses: ./.github/workflows/_test_backend.yml + permissions: + id-token: write + contents: read with: backend: qnn flows: >- @@ -28,4 +31,4 @@ jobs: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 run-linux: true - runner-linux: linux.8xlarge.memory + runner-linux: mt-l-x86iavx512-32-256 diff --git a/.github/workflows/test-backend-vulkan.yml b/.github/workflows/test-backend-vulkan.yml index 80ac3ee73a1..d3c9829dfe8 100644 --- a/.github/workflows/test-backend-vulkan.yml +++ b/.github/workflows/test-backend-vulkan.yml @@ -21,6 +21,9 @@ jobs: # runners. Runs on every PR and nightly. test-vulkan: uses: ./.github/workflows/_test_backend.yml + permissions: + id-token: write + contents: read with: backend: vulkan flows: >- diff --git a/.github/workflows/test-backend-webgpu.yml b/.github/workflows/test-backend-webgpu.yml index 99dae3ee76d..42e958855ac 100644 --- a/.github/workflows/test-backend-webgpu.yml +++ b/.github/workflows/test-backend-webgpu.yml @@ -19,6 +19,9 @@ concurrency: jobs: test-webgpu: uses: ./.github/workflows/_test_backend.yml + permissions: + id-token: write + contents: read with: backend: webgpu flows: '["webgpu"]' diff --git a/.github/workflows/test-backend-xnnpack.yml b/.github/workflows/test-backend-xnnpack.yml index e9f43608b16..c5abb15e837 100644 --- a/.github/workflows/test-backend-xnnpack.yml +++ b/.github/workflows/test-backend-xnnpack.yml @@ -19,6 +19,9 @@ concurrency: jobs: test-xnnpack: uses: ./.github/workflows/_test_backend.yml + permissions: + id-token: write + contents: read with: backend: xnnpack flows: >- diff --git a/.github/workflows/test-pico2-build.yml b/.github/workflows/test-pico2-build.yml index 860cf5381c6..7219e4172c8 100644 --- a/.github/workflows/test-pico2-build.yml +++ b/.github/workflows/test-pico2-build.yml @@ -18,15 +18,22 @@ concurrency: cancel-in-progress: true jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + test-pico2-fp32-build: + needs: docker-image name: test-pico2-fp32-build - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: - runner: linux.2xlarge.memory - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 @@ -89,14 +96,15 @@ jobs: echo "PASS: FP32 firmware fits in Pico2 memory (SRAM: ${SRAM_USED}/532480, Flash: ${FLASH_USED}/4194304)" test-pico2-cmsis-build: + needs: docker-image name: test-pico2-cmsis-nn-build - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: - runner: linux.2xlarge.memory - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 diff --git a/.github/workflows/test-webgpu-native.yml b/.github/workflows/test-webgpu-native.yml index 123545ffd38..f2e1b9f3ac8 100644 --- a/.github/workflows/test-webgpu-native.yml +++ b/.github/workflows/test-webgpu-native.yml @@ -29,12 +29,22 @@ concurrency: cancel-in-progress: true jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + test-webgpu-native: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + needs: docker-image + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read with: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - runner: linux.4xlarge.memory - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-16-128 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: recursive # The test script alone runs about 60 minutes, and the checkout plus the # Dawn and SwiftShader source builds in front of it have been measured diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 5eb597eed62..e7cb0fdfc4f 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -19,6 +19,12 @@ concurrency: cancel-in-progress: true jobs: + # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the + # container before any step runs, so nothing in the job can resolve it. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + # Emits the list of changed files for the current PR or push commit. # On PR: PR diff. On push: diff against `github.event.before`. # On events without a diff base (workflow_dispatch, tag creation, @@ -81,8 +87,12 @@ jobs: PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" test-arm-backend-zephyr: + needs: docker-image name: test-arm-backend-zephyr - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main + permissions: + id-token: write + contents: read strategy: matrix: include: @@ -93,8 +103,8 @@ jobs: - { readme: zephyr/samples/mv2-ethosu/README.md, target: ethos-u85 } fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-zephyr-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-zephyr-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 @@ -119,8 +129,9 @@ jobs: --zephyr-samples-readme-path "${{ matrix.readme }}" test-models-linux-aarch64: + needs: docker-image name: test-models-linux-aarch64 - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -128,30 +139,30 @@ jobs: matrix: model: [linear, add, add_mul, ic3, ic4, mv2, mv3, resnet18, resnet50, vit, w2l, mobilebert, emformer_join, emformer_transcribe] backend: [portable, xnnpack-quantization-delegation] - runner: [linux.arm64.2xlarge] + runner: [mt-l-arm64g4-16-62] include: - model: lstm backend: portable - runner: linux.arm64.2xlarge + runner: mt-l-arm64g4-16-62 - model: mul backend: portable - runner: linux.arm64.2xlarge + runner: mt-l-arm64g4-16-62 - model: softmax backend: portable - runner: linux.arm64.2xlarge + runner: mt-l-arm64g4-16-62 - model: phi_4_mini backend: portable - runner: linux.arm64.m7g.4xlarge + runner: mt-l-arm64g4-16-62 - model: qwen2_5_1_5b backend: portable - runner: linux.arm64.2xlarge + runner: mt-l-arm64g4-16-62 - model: llama3_2_vision_encoder backend: portable - runner: linux.arm64.2xlarge + runner: mt-l-arm64g4-16-62 fail-fast: false with: runner: ${{ matrix.runner }} - docker-image: ci-image:executorch-ubuntu-22.04-gcc11-aarch64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-gcc11-aarch64-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -223,8 +234,9 @@ jobs: PYTHON_EXECUTABLE=python ${CONDA_RUN} bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}" test-demo-backend-delegation: + needs: docker-image name: test-demo-backend-delegation - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -235,8 +247,8 @@ jobs: - build-tool: cmake fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | @@ -250,8 +262,9 @@ jobs: PYTHON_EXECUTABLE=python bash examples/portable/scripts/test_demo_backend_delegation.sh "${BUILD_TOOL}" test-arm-backend-ethos-u: + needs: docker-image name: test-arm-backend-ethos-u - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -272,8 +285,8 @@ jobs: - test_arm_backend: test_deit_e2e_ethos_u fail-fast: false with: - runner: linux.2xlarge.memory - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 @@ -288,8 +301,10 @@ jobs: .ci/scripts/setup-arm-baremetal-tools.sh # Increase number of files user can monitor to bypass buck failures. - # Hopefully this is high enough for this setup. - sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024 + # Hopefully this is high enough for this setup. This is a node-level + # setting an unprivileged OSDC pod cannot change, so it is best effort. + sudo sysctl fs.inotify.max_user_watches=1048576 2>/dev/null \ + || echo "Could not raise fs.inotify.max_user_watches" ARM_TEST=${{ matrix.test_arm_backend }} @@ -302,8 +317,9 @@ jobs: backends/arm/test/test_arm_backend.sh "${ARM_TEST}" test-arm-backend-vkml: + needs: docker-image name: test-arm-backend-vkml - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -316,8 +332,8 @@ jobs: - test_arm_backend: test_smaller_stories_llama_vkml fail-fast: false with: - runner: linux.2xlarge.memory - docker-image: ci-image:executorch-ubuntu-24.04-arm-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-24.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 120 @@ -331,8 +347,10 @@ jobs: .ci/scripts/setup-arm-baremetal-tools.sh --disable-ethos-u-deps --enable-mlsdk-deps --install-mlsdk-deps-with-pip # Increase number of files user can monitor to bypass buck failures. - # Hopefully this is high enough for this setup. - sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024 + # Hopefully this is high enough for this setup. This is a node-level + # setting an unprivileged OSDC pod cannot change, so it is best effort. + sudo sysctl fs.inotify.max_user_watches=1048576 2>/dev/null \ + || echo "Could not raise fs.inotify.max_user_watches" ARM_TEST=${{ matrix.test_arm_backend }} @@ -431,9 +449,10 @@ jobs: ${CONDA_RUN} sh .ci/scripts/test_llama_torchao_lowbit.sh test-llama-runner-linux: + needs: docker-image # Test Both linux x86 and linux aarch64 name: test-llama-runner-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -441,33 +460,33 @@ jobs: matrix: dtype: [fp32] mode: [portable, xnnpack+custom] - runner: [linux.2xlarge, linux.arm64.2xlarge] + runner: [mt-l-x86iavx512-8-64, mt-l-arm64g4-16-62] docker-image: [executorch-ubuntu-22.04-clang12, executorch-ubuntu-22.04-gcc11-aarch64] include: - dtype: bf16 mode: portable - runner: linux.2xlarge + runner: mt-l-x86iavx512-8-64 docker-image: executorch-ubuntu-22.04-clang12 - dtype: bf16 mode: portable - runner: linux.arm64.2xlarge + runner: mt-l-arm64g4-16-62 docker-image: executorch-ubuntu-22.04-gcc11-aarch64 - dtype: bf16 mode: custom - runner: linux.arm64.2xlarge + runner: mt-l-arm64g4-16-62 docker-image: executorch-ubuntu-22.04-gcc11-aarch64 # Excluding specific runner + docker image combinations that don't make sense: - # - Excluding the ARM64 gcc image on the x86 runner (linux.2xlarge) - # - Excluding the x86 clang image on the ARM64 runner (linux.arm64.2xlarge) + # - Excluding the ARM64 gcc image on the x86 runner + # - Excluding the x86 clang image on the ARM64 runner exclude: - - runner: linux.2xlarge + - runner: mt-l-x86iavx512-8-64 docker-image: executorch-ubuntu-22.04-gcc11-aarch64 - - runner: linux.arm64.2xlarge + - runner: mt-l-arm64g4-16-62 docker-image: executorch-ubuntu-22.04-clang12 fail-fast: false with: runner: ${{ matrix.runner }} - docker-image: ci-image:${{ matrix.docker-image }} + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:${{ matrix.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 900 @@ -543,34 +562,35 @@ jobs: PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_llama.sh -model stories110M -build_tool cmake -dtype "${DTYPE}" -mode "${MODE}" test-torchao-huggingface-checkpoints: + needs: docker-image name: test-torchao-huggingface-checkpoints - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read strategy: matrix: model: [qwen3_4b, phi_4_mini, lfm2_5_1_2b] - runner: [linux.2xlarge] + runner: [mt-l-x86iavx512-8-64] docker-image: [executorch-ubuntu-22.04-clang12] backend: [xnnpack] include: - model: qwen3_4b - runner: linux.arm64.2xlarge + runner: mt-l-arm64g4-16-62 docker-image: executorch-ubuntu-22.04-gcc11-aarch64 backend: torchao - model: phi_4_mini - runner: linux.arm64.2xlarge + runner: mt-l-arm64g4-16-62 docker-image: executorch-ubuntu-22.04-gcc11-aarch64 backend: torchao - model: lfm2_5_1_2b - runner: linux.arm64.2xlarge + runner: mt-l-arm64g4-16-62 docker-image: executorch-ubuntu-22.04-gcc11-aarch64 backend: torchao fail-fast: false with: runner: ${{ matrix.runner }} - docker-image: ci-image:${{ matrix.docker-image }} + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:${{ matrix.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 900 @@ -637,8 +657,9 @@ jobs: echo "::endgroup::" test-qnn-model: + needs: docker-image name: test-qnn-model - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -648,8 +669,8 @@ jobs: model: [dl3, mv3, mv2, ic4, ic3, vit, mb, w2l, conv_former] fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 900 @@ -663,8 +684,9 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh ${{ matrix.model }} "cmake" "qnn" test-qnn-optimum-model: + needs: docker-image name: test-qnn-optimum-model - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -674,8 +696,8 @@ jobs: model: [cvt, dit, efficientnet, focalnet, mobilevit_v1, mobilevit_v2, pvt, swin, albert, bert, distilbert, roberta] # eurobert requires transfomer >= 4.48.0, skip for now fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 900 @@ -778,10 +800,11 @@ jobs: done test-huggingface-transformers-xnnpack: + needs: docker-image # NB: Don't run this on fork PRs because they won't have access to the secret and would fail anyway if: ${{ !github.event.pull_request.head.repo.fork }} name: test-huggingface-transformers-xnnpack - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -801,8 +824,8 @@ jobs: fail-fast: false with: secrets-env: EXECUTORCH_HF_TOKEN - runner: linux.2xlarge.memory - docker-image: ci-image:executorch-ubuntu-22.04-clang12 + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 90 @@ -943,8 +966,9 @@ jobs: ${CONDA_RUN} python .ci/scripts/test_huggingface_optimum_model.py --model ${MODEL} --recipe ${RECIPE} ${QUANTIZE} test-llama-runner-qnn-linux: + needs: docker-image name: test-llama-runner-qnn-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read @@ -955,8 +979,8 @@ jobs: mode: [qnn] fail-fast: false with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + runner: mt-l-x86iavx512-8-64 + docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-qnn-sdk-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: 'recursive' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout: 900 diff --git a/.github/workflows/vulkan.yml b/.github/workflows/vulkan.yml index 031673d6ded..dd1875abf94 100644 --- a/.github/workflows/vulkan.yml +++ b/.github/workflows/vulkan.yml @@ -50,16 +50,15 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/scripts/setup-vulkan-linux-deps.sh') || contains(needs.changed-files.outputs.changed-files, '.github/workflows/vulkan.yml') || needs.run-decision.outputs.is-full-run == 'true' - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main permissions: id-token: write contents: read with: timeout: 120 - runner: linux.g5.4xlarge.nvidia.gpu + runner: mt-l-x86aavx2-29-113-a10g gpu-arch-type: cuda gpu-arch-version: "12.6" - use-custom-docker-registry: false submodules: recursive ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: |