From fac7a6965bf52b691e632697034d6233153aeca1 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Aug 2026 11:52:55 -0700 Subject: [PATCH 1/7] Move docker-builds to OSDC runners OSDC runner pods have no docker daemon, so the ci-image builds move to the in-cluster BuildKit pool. build.sh switches to `docker buildx build --push` when REMOTE_BUILDKIT is set and is otherwise unchanged, and the workflow drives it through test-infra's docker-build-remote-buildkit action, which registers the per-architecture remote builder and rides out a cold builder pool. The tag is now computed in the workflow rather than by calculate-docker-image, but it is the same string that action produced for `ci-image:` consumers (executorch/ci-image:-$(git rev-parse HEAD:.ci/docker)), so jobs still on linux_job_v2 keep resolving their images while the v3 migration lands separately. The orchestrator runner only streams the .ci/docker context to BuildKit and waits, so it drops to the smallest label of the matching architecture, and executorch-ubuntu-22.04-cuda-windows moves off the GPU runner it never used. Authored with Claude Code. --- .ci/docker/build.sh | 11 +++- .github/workflows/docker-builds.yml | 83 +++++++++++++++++------------ 2 files changed, 59 insertions(+), 35 deletions(-) diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index 76af829bd03..672bb8f9590 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -116,7 +116,16 @@ fi # Copy requirements-lintrunner.txt from root to here cp ../../requirements-lintrunner.txt ./ -docker build \ +# OSDC runners have no docker daemon, so the build runs on the in-cluster +# BuildKit pool via a remote buildx builder. That builder has nowhere to load +# an image into, so the result has to go straight to the registry. +if [[ -n "${REMOTE_BUILDKIT:-}" ]]; then + BUILD_CMD=(docker buildx build --push) +else + BUILD_CMD=(docker build) +fi + +"${BUILD_CMD[@]}" \ --no-cache \ --progress=plain \ --build-arg "OS_VERSION=${OS_VERSION}" \ diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index d1689fdcdae..e941ae94052 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -24,6 +24,12 @@ concurrency: env: AWS_DEFAULT_REGION: us-east-1 + ECR_REGISTRY: 308535385114.dkr.ecr.us-east-1.amazonaws.com + ECR_REPOSITORY: executorch/ci-image + +permissions: + id-token: write + contents: read jobs: docker-build: @@ -31,7 +37,12 @@ jobs: strategy: fail-fast: false matrix: - runner: [linux.4xlarge] + # BuildKit runs out of cluster on the OSDC BuildKit pool; this runner is + # only an orchestrator that streams the (tiny) .ci/docker context and + # waits, so the smallest label of the right architecture is enough. The + # architecture still has to match, because the helper picks the + # per-architecture buildkitd address off `uname -m`. + runner: [mt-l-x86iavx512-8-64] docker-image-name: [ executorch-ubuntu-22.04-gcc11, executorch-ubuntu-22.04-gcc9-nopytorch, @@ -45,49 +56,53 @@ jobs: executorch-ubuntu-22.04-clang12-android, executorch-ubuntu-24.04-gcc14, executorch-ubuntu-26.04-gcc14, + # Built on x86: this image only cross-compiles for CUDA on Windows, + # so building it never needed the GPU runner it used to run on. + executorch-ubuntu-22.04-cuda-windows, ] include: - docker-image-name: executorch-ubuntu-22.04-gcc11-aarch64 - runner: linux.arm64.2xlarge + runner: mt-l-arm64g2-6-25 - docker-image-name: executorch-ubuntu-22.04-gcc11-aarch64-android - runner: linux.arm64.2xlarge + runner: mt-l-arm64g2-6-25 - docker-image-name: executorch-ubuntu-22.04-gcc11-aarch64-arm-sdk - runner: linux.arm64.2xlarge - - docker-image-name: executorch-ubuntu-22.04-cuda-windows - runner: linux.g5.4xlarge.nvidia.gpu + runner: mt-l-arm64g2-6-25 - runs-on: [self-hosted, "${{ matrix.runner }}"] - env: - DOCKER_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }} + runs-on: ${{ matrix.runner }} + container: + image: ghcr.io/actions/actions-runner:latest steps: - - name: Clean workspace - shell: bash - run: | - echo "${GITHUB_WORKSPACE}" - sudo rm -rf "${GITHUB_WORKSPACE}" - mkdir "${GITHUB_WORKSPACE}" + - name: Checkout ExecuTorch + uses: actions/checkout@v4 - - name: Setup SSH (Click me for login details) - uses: pytorch/test-infra/.github/actions/setup-ssh@main + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 with: - github-secret: ${{ secrets.GITHUB_TOKEN }} + role-to-assume: arn:aws:iam::308535385114:role/arc + aws-region: us-east-1 + role-duration-seconds: 18000 - - name: Checkout Executorch - uses: actions/checkout@v3 + # buildx forwards the client's registry auth to the remote builder, so the + # push at the end of build.sh authenticates with what this step writes. + - name: Login to ECR + uses: aws-actions/amazon-ecr-login@v2 + with: + registries: "308535385114" - - name: Setup Linux - uses: pytorch/test-infra/.github/actions/setup-linux@main + - name: Compute the image tag + id: tag + shell: bash + run: | + set -eux + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + # Same tag that calculate-docker-image derives for consumers of + # `ci-image:`, so jobs still on linux_job_v2 keep resolving. + DOCKER_TAG=$(git rev-parse HEAD:.ci/docker) + echo "docker-image=${ECR_REGISTRY}/${ECR_REPOSITORY}:${{ matrix.docker-image-name }}-${DOCKER_TAG}" >> "${GITHUB_OUTPUT}" - - name: Build docker image - id: build-docker-image - timeout-minutes: 145 - uses: pytorch/test-infra/.github/actions/calculate-docker-image@main + # No step timeout: the action retries a cold BuildKit pool for up to two + # hours, and the job's timeout-minutes is what bounds that. + - name: Build and push to ECR + uses: pytorch/test-infra/.github/actions/docker-build-remote-buildkit@main with: - docker-image-name: ci-image:${{ matrix.docker-image-name }} - always-rebuild: true - push: true - force-push: true - - - name: Teardown Linux - uses: pytorch/test-infra/.github/actions/teardown-linux@main - if: always() + command: cd .ci/docker && REMOTE_BUILDKIT=1 ./build.sh ${{ matrix.docker-image-name }} -t ${{ steps.tag.outputs.docker-image }} From 8634528b03ed0f3cd5213731ba2baf4ac183d84f Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Aug 2026 12:30:09 -0700 Subject: [PATCH 2/7] Add the ciflow/docker trigger A fork PR gets no OIDC token, so on OSDC a pull_request-triggered build cannot assume role/arc to push. Applying the ciflow/docker label tags the PR head and fires the push trigger with the PR's own SHA, which does work. pull_request stays for now and comes out once that path is proven. Authored with Claude Code. --- .github/workflows/docker-builds.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index e941ae94052..6009f2dbdb3 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -1,5 +1,9 @@ name: docker-builds +# The ciflow/docker tag is the path that works on OSDC: a fork PR gets no OIDC +# token, so a pull_request-triggered build cannot assume role/arc to push. The +# label tags the PR head, which fires the push trigger below with the PR's own +# SHA. pull_request is kept for now and comes out once that path is proven. on: workflow_dispatch: pull_request: @@ -11,6 +15,8 @@ on: branches: - main - release/* + tags: + - ciflow/docker/* paths: - .ci/docker/** - .github/workflows/docker-builds.yml From 651f35ab8fb7867c27e17bbda42dc05e5e896359 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Aug 2026 18:20:06 -0700 Subject: [PATCH 3/7] Register ciflow/docker Applying the label only pushes a tag if the tag is listed here, so the trigger added in the previous commit is inert without it. Authored with Claude Code. --- .github/pytorch-probot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pytorch-probot.yml b/.github/pytorch-probot.yml index 87f4832c7da..8fce4158042 100644 --- a/.github/pytorch-probot.yml +++ b/.github/pytorch-probot.yml @@ -5,6 +5,7 @@ ciflow_push_tags: - ciflow/apple - ciflow/cuda - ciflow/cuda-perf +- ciflow/docker - ciflow/metal - ciflow/mlx - ciflow/rocm From 92c7c8415b5db100026c40ed1bd2644f38bbc30f Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Aug 2026 18:23:02 -0700 Subject: [PATCH 4/7] Auto-label ciflow/docker on the paths that need an image rebuild Removing the pull_request trigger otherwise made rebuilding the images a thing contributors have to remember. The labeler applies ciflow/docker on the same paths docker-builds triggers on, which pytorchbot turns into the tag push. Authored with Claude Code. --- .github/labeler.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/labeler.yml b/.github/labeler.yml index e0be91fdaca..aeaf81423dc 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -13,3 +13,12 @@ ciflow/trunk: - any-glob-to-any-file: - 'backends/arm/**' - 'examples/arm/**' + +# Same paths docker-builds triggers on, so a PR that touches the images still +# rebuilds them without the pull_request trigger it used to rely on. +ciflow/docker: +- changed-files: + - any-glob-to-any-file: + - '.ci/docker/**' + - '.github/workflows/docker-builds.yml' + - 'requirements-lintrunner.txt' From 5723f8760ea736eccb44b00f2c4d9a0da60db158 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Aug 2026 18:16:24 -0700 Subject: [PATCH 5/7] Use mt-l-arm64g4-16-62 for the arm64 image builds arc.yaml maps linux.arm64.2xlarge to l-arm64g2-6-32 and ci-infra renamed that def to l-arm64g2-6-25, but neither name schedules: the three arm64 legs of the first run of this workflow sat queued for nearly four hours and were cancelled without a runner ever picking them up. mt-l-arm64g4-16-62 is the only arm64 label with live production evidence, and it is what pytorch/pytorch's own OSDC docker-builds uses. Authored with Claude Code. --- .github/workflows/docker-builds.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index 6009f2dbdb3..9210af92685 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -45,9 +45,9 @@ jobs: matrix: # BuildKit runs out of cluster on the OSDC BuildKit pool; this runner is # only an orchestrator that streams the (tiny) .ci/docker context and - # waits, so the smallest label of the right architecture is enough. The - # architecture still has to match, because the helper picks the - # per-architecture buildkitd address off `uname -m`. + # waits, so it can be small. The architecture still has to match, because + # the helper picks the per-architecture buildkitd address off `uname -m`, + # and on arm64 this is the smallest label that actually schedules. runner: [mt-l-x86iavx512-8-64] docker-image-name: [ executorch-ubuntu-22.04-gcc11, @@ -68,11 +68,11 @@ jobs: ] include: - docker-image-name: executorch-ubuntu-22.04-gcc11-aarch64 - runner: mt-l-arm64g2-6-25 + runner: mt-l-arm64g4-16-62 - docker-image-name: executorch-ubuntu-22.04-gcc11-aarch64-android - runner: mt-l-arm64g2-6-25 + runner: mt-l-arm64g4-16-62 - docker-image-name: executorch-ubuntu-22.04-gcc11-aarch64-arm-sdk - runner: mt-l-arm64g2-6-25 + runner: mt-l-arm64g4-16-62 runs-on: ${{ matrix.runner }} container: @@ -106,7 +106,7 @@ jobs: DOCKER_TAG=$(git rev-parse HEAD:.ci/docker) echo "docker-image=${ECR_REGISTRY}/${ECR_REPOSITORY}:${{ matrix.docker-image-name }}-${DOCKER_TAG}" >> "${GITHUB_OUTPUT}" - # No step timeout: the action retries a cold BuildKit pool for up to two + # No step timeout: the action retries a cold BuildKit pool for at least two # hours, and the job's timeout-minutes is what bounds that. - name: Build and push to ECR uses: pytorch/test-infra/.github/actions/docker-build-remote-buildkit@main From 9ebfb7a9e289c50162aee52c0de75bbe08076b9a Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Aug 2026 18:52:51 -0700 Subject: [PATCH 6/7] Give the image build sccache credentials on OSDC The compiler stubs route through sccache, which resolves its S3 credentials from EC2 instance metadata. A BuildKit pod has none, so every compiler invocation died on a metadata timeout and took the PyTorch build's CMake compiler check with it. Dropping the S3 backend was the smaller change but not a viable one: the PyTorch build layer takes ~19 minutes against a 240 minute job timeout, which only holds because sccache is serving nearly all of it. So docker-builds hands the assumed-role credentials to the build as a BuildKit secret instead, which role/arc already has write access to the bucket for. A secret mount never lands in a layer. install_pytorch.sh falls back to a local cache when the secret is absent, so a build outside CI still works rather than failing on the first compile. Authored with Claude Code. --- .ci/docker/common/install_pytorch.sh | 13 +++++++++++++ .ci/docker/ubuntu/Dockerfile | 7 ++++++- .github/workflows/docker-builds.yml | 24 +++++++++++++++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/.ci/docker/common/install_pytorch.sh b/.ci/docker/common/install_pytorch.sh index a71698c6318..0ac5e79cf4a 100755 --- a/.ci/docker/common/install_pytorch.sh +++ b/.ci/docker/common/install_pytorch.sh @@ -10,6 +10,19 @@ set -ex # shellcheck source=/dev/null source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" +# The compiler stubs run through sccache, which fails hard rather than +# compiling uncached when it cannot reach its S3 bucket. Use the credentials +# docker-builds mounts, or fall back to a local cache so a build without them +# still works. +SCCACHE_CREDENTIALS=/run/secrets/aws-credentials +if [[ -s "${SCCACHE_CREDENTIALS}" ]]; then + export AWS_SHARED_CREDENTIALS_FILE="${SCCACHE_CREDENTIALS}" +else + echo "No sccache credentials; caching compiler output locally" >&2 + unset SCCACHE_BUCKET SCCACHE_S3_KEY_PREFIX + export SCCACHE_DIR=/tmp/sccache +fi + install_domains() { echo "Install torchvision and torchaudio" pip_install --no-build-isolation --user "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION}" diff --git a/.ci/docker/ubuntu/Dockerfile b/.ci/docker/ubuntu/Dockerfile index 5acbd8181d1..27faf7d688d 100644 --- a/.ci/docker/ubuntu/Dockerfile +++ b/.ci/docker/ubuntu/Dockerfile @@ -70,7 +70,12 @@ ARG SKIP_PYTORCH ARG PYTORCH_BUILD_MAX_JOBS COPY ./common/install_pytorch.sh install_pytorch.sh COPY ./common/utils.sh utils.sh -RUN if [ -z "${SKIP_PYTORCH}" ]; then bash ./install_pytorch.sh; fi && rm install_pytorch.sh utils.sh +# A host docker build let sccache reach its S3 bucket with the EC2 instance +# role; a BuildKit pod has no instance metadata, so docker-builds passes the +# credentials in as this secret. World-readable because the sccache server runs +# as ci-user; a secret mount never lands in a layer. +RUN --mount=type=secret,id=aws-credentials,mode=0444 \ + if [ -z "${SKIP_PYTORCH}" ]; then bash ./install_pytorch.sh; fi && rm install_pytorch.sh utils.sh ARG LINTRUNNER # Install lintrunner if needed diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index 9210af92685..ec74dad2f4c 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -95,6 +95,28 @@ jobs: with: registries: "308535385114" + # sccache cannot source S3 credentials from instance metadata inside a + # BuildKit pod, so hand the assumed-role ones to the build as a secret. An + # empty file is a valid outcome: install_pytorch.sh then caches locally. + - name: Stage sccache credentials + id: sccache-credentials + shell: bash + run: | + set -euo pipefail + creds="${RUNNER_TEMP}/aws-credentials" + install -m 600 /dev/null "${creds}" + if [[ -n "${AWS_ACCESS_KEY_ID:-}" ]]; then + { + echo "[default]" + echo "aws_access_key_id=${AWS_ACCESS_KEY_ID}" + echo "aws_secret_access_key=${AWS_SECRET_ACCESS_KEY}" + echo "aws_session_token=${AWS_SESSION_TOKEN}" + } > "${creds}" + else + echo "::warning::No AWS credentials; the image build will not use the sccache S3 cache" + fi + echo "path=${creds}" >> "${GITHUB_OUTPUT}" + - name: Compute the image tag id: tag shell: bash @@ -111,4 +133,4 @@ jobs: - name: Build and push to ECR uses: pytorch/test-infra/.github/actions/docker-build-remote-buildkit@main with: - command: cd .ci/docker && REMOTE_BUILDKIT=1 ./build.sh ${{ matrix.docker-image-name }} -t ${{ steps.tag.outputs.docker-image }} + command: cd .ci/docker && REMOTE_BUILDKIT=1 ./build.sh ${{ matrix.docker-image-name }} -t ${{ steps.tag.outputs.docker-image }} --secret id=aws-credentials,src=${{ steps.sccache-credentials.outputs.path }} From d4c09e305830b125481ebb012c342ebe7119682d Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Aug 2026 19:53:19 -0700 Subject: [PATCH 7/7] Address review: drop pull_request, restore the build retry Removing the pull_request trigger settles the duplicate-run problem: the labeler applies ciflow/docker on the same paths, so every PR touching them was building all 16 images twice, and the pull_request half was always red on a fork for want of an OIDC token. calculate-docker-image ran the build under three retries because it "frequently fails with network error downloading various stuffs". docker-build-remote-buildkit only retries failures from before BuildKit starts, so a hiccup downloading the QNN, MediaTek, Arm or Zephyr SDKs would now fail the job outright. Put an equivalent retry back around the build. Authored with Claude Code. --- .github/labeler.yml | 4 ++-- .github/workflows/docker-builds.yml | 32 +++++++++++++++++++---------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index aeaf81423dc..889c7fe70fc 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -14,8 +14,8 @@ ciflow/trunk: - 'backends/arm/**' - 'examples/arm/**' -# Same paths docker-builds triggers on, so a PR that touches the images still -# rebuilds them without the pull_request trigger it used to rely on. +# Same paths docker-builds triggers on. It has no pull_request trigger, so this +# label is what rebuilds the images for a PR that changes them. ciflow/docker: - changed-files: - any-glob-to-any-file: diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index ec74dad2f4c..ed732243d2d 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -1,16 +1,11 @@ name: docker-builds -# The ciflow/docker tag is the path that works on OSDC: a fork PR gets no OIDC -# token, so a pull_request-triggered build cannot assume role/arc to push. The -# label tags the PR head, which fires the push trigger below with the PR's own -# SHA. pull_request is kept for now and comes out once that path is proven. +# No pull_request trigger: a fork PR gets no OIDC token, so the build could not +# assume role/arc to push and every run would fail. A PR that needs fresh images +# gets the ciflow/docker label instead, which tags the PR head and fires the push +# trigger below with the PR's own SHA. Matches pytorch/pytorch. on: workflow_dispatch: - pull_request: - paths: - - .ci/docker/** - - .github/workflows/docker-builds.yml - - requirements-lintrunner.txt push: branches: - main @@ -25,7 +20,7 @@ on: - cron: 1 3 * * 3 concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} cancel-in-progress: true env: @@ -133,4 +128,19 @@ jobs: - name: Build and push to ECR uses: pytorch/test-infra/.github/actions/docker-build-remote-buildkit@main with: - command: cd .ci/docker && REMOTE_BUILDKIT=1 ./build.sh ${{ matrix.docker-image-name }} -t ${{ steps.tag.outputs.docker-image }} --secret id=aws-credentials,src=${{ steps.sccache-credentials.outputs.path }} + # calculate-docker-image wrapped the build in three retries because it + # "frequently fails with network error downloading various stuffs". + # The action only retries failures from before BuildKit starts, so keep + # a retry of our own around the SDK downloads. + command: | + cd .ci/docker + for attempt in 1 2 3; do + if REMOTE_BUILDKIT=1 ./build.sh ${{ matrix.docker-image-name }} \ + -t ${{ steps.tag.outputs.docker-image }} \ + --secret id=aws-credentials,src=${{ steps.sccache-credentials.outputs.path }}; then + exit 0 + fi + echo "::warning::docker build attempt ${attempt} of 3 failed" + if [ "${attempt}" -lt 3 ]; then sleep 90; fi + done + exit 1