Move Linux CI jobs from linux_job_v2 to linux_job_v3 (OSDC)

.github/workflows/_docker-image.yml

@@ -0,0 +1,142 @@
+name: Resolve CI docker image
+
+# OSDC jobs run inside a container the runner pod pulls before any step of the
+# job executes, so unlike linux_job_v2 there is no in-job step that can resolve
+# `ci-image:<name>` to an ECR reference or wait for docker-builds to push it.
+# Callers depend on this workflow instead and spell the image out as
+# 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:<name>-<hash>.
+
+on:
+  workflow_call:
+    outputs:
+      ci-docker-hash:
+        description: |
+          Tag suffix shared by every executorch CI image built from this commit,
+          i.e. `git rev-parse HEAD:.ci/docker`. Matches the tag docker-builds
+          pushes and the one calculate-docker-image derives for linux_job_v2.
+        value: ${{ jobs.resolve.outputs.ci-docker-hash }}
+
+permissions:
+  contents: read
+
+jobs:
+  resolve:
+    name: resolve
+    runs-on: ubuntu-latest
+    # docker-builds gives itself 240 minutes, so the wait below has to be able
+    # to outlast a full rebuild of the slowest image.
+    timeout-minutes: 250
+    permissions:
+      contents: read
+      actions: read
+    outputs:
+      ci-docker-hash: ${{ steps.hash.outputs.ci-docker-hash }}
+
+    steps:
+      - name: Checkout ExecuTorch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+
+      - name: Compute the docker tag
+        id: hash
+        run: |
+          set -eu
+          echo "ci-docker-hash=$(git rev-parse HEAD:.ci/docker)" >> "$GITHUB_OUTPUT"
+
+      # A commit that leaves .ci/docker alone reuses an image that is already in
+      # ECR, so only a commit that changes the tree has to wait for a rebuild.
+      - name: Check whether this commit rebuilds the images
+        id: rebuild
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -eu
+
+          REPO="${{ github.repository }}"
+          BASE=""
+
+          case "${{ github.event_name }}" in
+            pull_request|pull_request_target)
+              BASE="${{ github.event.pull_request.base.sha }}"
+              HEAD="${{ github.event.pull_request.head.sha }}"
+              ;;
+            push)
+              BASE="${{ github.event.before }}"
+              HEAD="${{ github.sha }}"
+              ;;
+          esac
+
+          ZERO_SHA="0000000000000000000000000000000000000000"
+          if [ -z "$BASE" ] || [ "$BASE" = "$ZERO_SHA" ]; then
+            # No diff base (workflow_dispatch, schedule, tag or branch creation).
+            # docker-builds is not path-filtered on those events either, so
+            # whatever is in ECR for this tag is what the job gets.
+            echo "No diff base for event '${{ github.event_name }}'; not waiting"
+            echo "rebuild=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          if gh api "repos/$REPO/compare/$BASE...$HEAD" --paginate \
+               --jq '.files[]?.filename' | grep -q '^\.ci/docker/'; then
+            echo "rebuild=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "rebuild=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Wait for docker-builds to push the images
+        if: steps.rebuild.outputs.rebuild == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+        run: |
+          set -eu
+
+          REPO="${{ github.repository }}"
+          QUERY="repos/$REPO/actions/workflows/docker-builds.yml/runs?head_sha=$HEAD_SHA&per_page=1"
+
+          # docker-builds has no pull_request trigger, so on a PR it only runs
+          # once the ciflow/docker label tags this commit. Give that tag a few
+          # minutes to show up, then say so, rather than blocking for hours on a
+          # build nobody asked for.
+          missing=0
+
+          # 240 x 60s outlasts docker-builds' own 240 minute job timeout.
+          for _ in $(seq 1 240); do
+            RUN=$(gh api "$QUERY" \
+              --jq '(.workflow_runs[0] // {}) | "\(.status // "") \(.conclusion // "")"' \
+              2>/dev/null || echo " ")
+            STATUS="${RUN%% *}"
+            CONCLUSION="${RUN##* }"
+
+            case "$STATUS" in
+              completed)
+                if [ "$CONCLUSION" = "success" ]; then
+                  echo "docker-builds succeeded for $HEAD_SHA"
+                  exit 0
+                fi
+                echo "::error::docker-builds concluded '$CONCLUSION' for $HEAD_SHA." \
+                     "The CI images for this commit were never pushed, so every" \
+                     "job that needs one would fail pulling its container."
+                exit 1
+                ;;
+              "")
+                missing=$((missing + 1))
+                if [ "$missing" -gt 5 ]; then
+                  echo "::error::This commit changes .ci/docker, so it needs images" \
+                       "that do not exist yet, but no docker-builds run was found for" \
+                       "$HEAD_SHA. Add the ciflow/docker label to build them."
+                  exit 1
+                fi
+                echo "No docker-builds run for $HEAD_SHA yet; waiting"
+                ;;
+              *)
+                missing=0
+                echo "docker-builds is $STATUS for $HEAD_SHA; waiting"
+                ;;
+            esac
+            sleep 60
+          done
+
+          echo "::error::Timed out waiting for docker-builds on $HEAD_SHA"
+          exit 1

.github/workflows/_llm_server.yml

@@ -4,20 +4,30 @@ on:
   workflow_call:
     inputs:
       docker-image:
-        description: Docker image to use for Linux tests.
+        description: |
+          ECR repository and tag prefix of the CI image, e.g.
+          ci-image:executorch-ubuntu-22.04-clang12. The registry and the
+          .ci/docker hash are appended by this workflow.
         required: false
         type: string
         default: ci-image:executorch-ubuntu-22.04-clang12
 
 jobs:
+  # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the
+  # container before any step runs, so nothing in the job can resolve it.
+  docker-image:
+    name: Resolve CI docker image
+    uses: ./.github/workflows/_docker-image.yml
+
   linux:
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    needs: docker-image
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
     permissions:
       id-token: write
       contents: read
     with:
-      runner: linux.2xlarge
-      docker-image: ${{ inputs.docker-image }}
+      runner: mt-l-x86iavx512-8-64
+      docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ inputs.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }}
       submodules: recursive
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 60

.github/workflows/_test_arduino_library.yml

@@ -14,15 +14,22 @@ on:
         default: 90
 
 jobs:
+  # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the
+  # container before any step runs, so nothing in the job can resolve it.
+  docker-image:
+    name: Resolve CI docker image
+    uses: ./.github/workflows/_docker-image.yml
+
   run:
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    needs: docker-image
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
     permissions:
       id-token: write
       contents: read
     with:
       job-name: arduino-library
-      runner: linux.2xlarge
-      docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk
+      runner: mt-l-x86iavx512-8-64
+      docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }}
       submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: ${{ inputs.timeout }}

.github/workflows/_test_backend.yml

@@ -40,15 +40,25 @@ on:
         description: 'Runner type for Linux jobs'
         required: false
         type: string
-        default: linux.4xlarge.memory
+        default: mt-l-x86iavx512-16-128
       docker-image:
-        description: 'Docker image for Linux jobs'
+        description: |
+          ECR repository and tag prefix of the CI image, e.g.
+          ci-image:executorch-ubuntu-22.04-clang12. The registry and the
+          .ci/docker hash are appended by this workflow.
         required: false
         type: string
         default: ci-image:executorch-ubuntu-22.04-clang12
 
 jobs:
+  # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the
+  # container before any step runs, so nothing in the job can resolve it.
+  docker-image:
+    name: Resolve CI docker image
+    uses: ./.github/workflows/_docker-image.yml
+
   test-backend-linux:
+    needs: docker-image
     if: ${{ inputs.run-linux }}
     strategy:
       fail-fast: false
@@ -57,11 +67,14 @@ jobs:
         suite: [models, operators]
         exclude: ${{ fromJSON(inputs.exclude) }}
 
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
+    permissions:
+      id-token: write
+      contents: read
     with:
       ref: ${{ inputs.ref }}
       runner: ${{ inputs.runner-linux }}
-      docker-image: ${{ inputs.docker-image }}
+      docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ inputs.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }}
       submodules: recursive
       timeout: ${{ inputs.timeout }}
       upload-artifact: test-report-${{ inputs.backend }}-${{ matrix.flow }}-${{ matrix.suite }}

.github/workflows/_test_cadence.yml

@@ -8,15 +8,18 @@ on:
   workflow_call:
     inputs:
       docker-image:
-        description: 'Docker image to use'
+        description: |
+          ECR repository and tag prefix of the CI image, e.g.
+          ci-image:executorch-ubuntu-22.04-clang12. The registry and the
+          .ci/docker hash are appended by this workflow.
         required: false
         type: string
         default: ci-image:executorch-ubuntu-22.04-clang12
       runner:
         description: 'Runner type'
         required: false
         type: string
-        default: linux.8xlarge.memory
+        default: mt-l-x86iavx512-32-256
       ref:
         description: 'Git ref to checkout'
         required: false
@@ -29,12 +32,22 @@ on:
         default: 90
 
 jobs:
+  # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the
+  # container before any step runs, so nothing in the job can resolve it.
+  docker-image:
+    name: Resolve CI docker image
+    uses: ./.github/workflows/_docker-image.yml
+
   test-aot:
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    needs: docker-image
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
+    permissions:
+      id-token: write
+      contents: read
     with:
       job-name: test-aot
       runner: ${{ inputs.runner }}
-      docker-image: ${{ inputs.docker-image }}
+      docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ inputs.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }}
       submodules: recursive
       ref: ${{ inputs.ref }}
       timeout: ${{ inputs.timeout }}
@@ -50,11 +63,15 @@ jobs:
         python -m pytest backends/cadence/aot/tests/ -v -n auto --reruns 2 --reruns-delay 1
 
   test-ops:
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    needs: docker-image
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
+    permissions:
+      id-token: write
+      contents: read
     with:
       job-name: test-ops
       runner: ${{ inputs.runner }}
-      docker-image: ${{ inputs.docker-image }}
+      docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ inputs.docker-image }}-${{ needs.docker-image.outputs.ci-docker-hash }}
       submodules: recursive
       ref: ${{ inputs.ref }}
       timeout: ${{ inputs.timeout }}

.github/workflows/_test_cortex_m_e2e.yml

@@ -23,17 +23,27 @@ on:
         default: 120
 
 jobs:
+  # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the
+  # container before any step runs, so nothing in the job can resolve it.
+  docker-image:
+    name: Resolve CI docker image
+    uses: ./.github/workflows/_docker-image.yml
+
   run:
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    needs: docker-image
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
+    permissions:
+      id-token: write
+      contents: read
     strategy:
       matrix:
         model: ${{ fromJSON(inputs.models) }}
         target: ${{ fromJSON(inputs.targets) }}
       fail-fast: false
     with:
       job-name: ${{ matrix.model }}-${{ matrix.target }}
-      runner: linux.2xlarge.memory
-      docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk
+      runner: mt-l-x86iavx512-8-64
+      docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }}
       submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: ${{ inputs.timeout }}

.github/workflows/_test_cortex_m_ops.yml

@@ -18,16 +18,26 @@ on:
         default: 120
 
 jobs:
+  # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the
+  # container before any step runs, so nothing in the job can resolve it.
+  docker-image:
+    name: Resolve CI docker image
+    uses: ./.github/workflows/_docker-image.yml
+
   run:
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    needs: docker-image
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
+    permissions:
+      id-token: write
+      contents: read
     strategy:
       matrix:
         target: ${{ fromJSON(inputs.targets) }}
       fail-fast: false
     with:
       job-name: cortex-m-ops-${{ matrix.target }}
-      runner: linux.2xlarge.memory
-      docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk
+      runner: mt-l-x86iavx512-8-64
+      docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-arm-sdk-${{ needs.docker-image.outputs.ci-docker-hash }}
       submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: ${{ inputs.timeout }}

.github/workflows/_test_riscv.yml

@@ -37,11 +37,21 @@ on:
         type: string
 
 jobs:
+  # linux_job_v3 needs a fully qualified image: the OSDC runner pod pulls the
+  # container before any step runs, so nothing in the job can resolve it.
+  docker-image:
+    name: Resolve CI docker image
+    uses: ./.github/workflows/_docker-image.yml
+
   run:
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    needs: docker-image
+    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
+    permissions:
+      id-token: write
+      contents: read
     with:
-      runner: linux.2xlarge
-      docker-image: ci-image:executorch-ubuntu-24.04-gcc14
+      runner: mt-l-x86iavx512-8-64
+      docker-image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-24.04-gcc14-${{ needs.docker-image.outputs.ci-docker-hash }}
       submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: ${{ inputs.timeout }}