From 0a8e483f38e65f11329e1edcc677e94b1aa0a984 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 24 Aug 2026 12:07:43 -0700 Subject: [PATCH] Move the remaining native Linux CI jobs off EC2 These are the Linux jobs that write their own steps instead of calling linux_job, so the v3 migration did not reach them. The two Cadence Xtensa workflows were native specifically because the OIDC token had to be minted on the host: the ACTIONS_ID_TOKEN_REQUEST_* variables did not cross into the docker exec, so the job assumed the Cadence role outside the container and passed the credentials in through docker run. OSDC has no host docker daemon, and a job there already runs entirely inside its container, so the workaround is no longer needed and the docker run wrapper collapses into plain steps. The reason these stay native rather than becoming linux_job_v3 is now only that they assume the Cadence artifacts role rather than role/arc. Three jobs did no ExecuTorch work at all and move to GitHub-hosted runners instead of OSDC: the stale-PR bot, which only calls the GitHub API, and the two jobs that repackage an artifact into S3, which now assume role/arc through OIDC rather than relying on the EC2 instance role they used to inherit. apple.yml's test-demo-ios keeps its EC2 label, because mobile_job.yml resolves its actions by relative path and that does not work on OSDC. This builds on the linux_job_v3 migration for _docker-image.yml, which resolves the CI image the Xtensa jobs now name in `container:`. Authored with Claude Code. --- .github/workflows/_test_backend.yml | 15 +++++- .github/workflows/_xtensa_build.yml | 77 +++++++++++++---------------- .github/workflows/_xtensa_test.yml | 70 ++++++++++++-------------- .github/workflows/apple.yml | 18 ++++++- .github/workflows/stale.yml | 3 +- 5 files changed, 99 insertions(+), 84 deletions(-) diff --git a/.github/workflows/_test_backend.yml b/.github/workflows/_test_backend.yml index 5355a286cb1..d5440b463a1 100644 --- a/.github/workflows/_test_backend.yml +++ b/.github/workflows/_test_backend.yml @@ -86,7 +86,13 @@ jobs: package-golden-artifacts: if: ${{ inputs.run-linux }} needs: test-backend-linux - runs-on: linux.2xlarge + # Repackaging artifacts needs no ExecuTorch toolchain, so this runs on a + # GitHub-hosted runner rather than moving to OSDC; the S3 upload that used + # to ride on the EC2 instance role assumes role/arc through OIDC instead. + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read steps: - name: Download model test artifacts uses: actions/download-artifact@v4 @@ -127,6 +133,13 @@ jobs: path: golden_artifacts_*.zip if-no-files-found: ignore + - name: Configure AWS credentials + if: ${{ hashFiles('golden_artifacts_*.zip') != '' }} + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::308535385114:role/arc + aws-region: us-east-1 + - name: Upload golden artifacts to S3 uses: seemethere/upload-artifact-s3@v5 if: ${{ hashFiles('golden_artifacts_*.zip') != '' }} diff --git a/.github/workflows/_xtensa_build.yml b/.github/workflows/_xtensa_build.yml index ac78323aa3e..4d1d577a41d 100644 --- a/.github/workflows/_xtensa_build.yml +++ b/.github/workflows/_xtensa_build.yml @@ -1,9 +1,8 @@ # Reusable: cross-compile cadence_executor_runner for one Cadence Xtensa core. # -# A native job (not linux_job_v2) because the GitHub OIDC token must be minted on -# the runner host: the ACTIONS_ID_TOKEN_REQUEST_* vars do not cross into -# linux_job_v2's docker exec. So the role is assumed on the host, then the build -# runs inside the CI image via docker run with the creds passed in. Binding the +# A native job rather than linux_job_v3, because the build has to assume the +# Cadence artifacts role rather than the role/arc that linux_job_v3 assumes for +# itself, and only a native job can run configure-aws-credentials. Binding the # environment also gives the OIDC token the environment claim. The licensed # toolchain + core configs are fetched at runtime from an auth-gated store; # role/region/store come from CI variables and are not committed. @@ -23,30 +22,37 @@ on: default: "" jobs: + # The runner pod pulls the container before any step runs, so the image has to + # be a fully qualified reference resolved by a job this one depends on. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + build: name: ${{ inputs.backend }} - runs-on: linux.2xlarge + needs: docker-image + runs-on: mt-l-x86iavx512-8-64 + container: + image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} environment: cadence permissions: id-token: write contents: read steps: + - name: Clean workspace + shell: bash + run: | + set -eux + rm -rf "${GITHUB_WORKSPACE}" + mkdir -p "${GITHUB_WORKSPACE}" + - name: Checkout executorch uses: actions/checkout@v4 with: submodules: recursive ref: ${{ inputs.ref }} - - name: Calculate docker image - id: calculate-docker-image - uses: pytorch/test-infra/.github/actions/calculate-docker-image@main - with: - docker-image-name: ci-image:executorch-ubuntu-22.04-clang12 - - - name: Pull docker image - run: docker pull "${{ steps.calculate-docker-image.outputs.docker-image }}" - - - name: Assume Cadence artifacts role (host OIDC) + - name: Assume Cadence artifacts role uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ vars.CADENCE_CI_AWS_ROLE }} @@ -54,37 +60,24 @@ jobs: - name: Cross-compile cadence_executor_runner env: - DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }} BACKEND: ${{ inputs.backend }} XTENSA_S3_BUCKET: ${{ vars.CADENCE_CI_S3_BUCKET }} shell: bash run: | - set -eux - # OIDC/role assumption already happened on the host above; pass the - # resulting AWS creds and the store/backend into the CI image, where - # the toolchain download + cross-compile run. - docker run --rm \ - -e BACKEND -e XTENSA_S3_BUCKET \ - -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ - -e AWS_DEFAULT_REGION -e AWS_REGION \ - -v "${GITHUB_WORKSPACE}:/work/executorch" -w /work/executorch \ - "${DOCKER_IMAGE}" \ - bash -c ' - set -exo pipefail - eval "$(/opt/conda/bin/conda shell.bash hook)" - conda activate "$(conda env list --json | jq -r ".envs | .[-1]")" - ./install_requirements.sh > /dev/null - pip install --quiet awscli - # hifi4/fusion_g3 optimized kernels need the foss-xtensa nnlib - # sources, which are not vendored in executorch; the cadence - # installer clones them. vision has no nnlib dependency. - if [ "${BACKEND}" != "vision" ]; then - backends/cadence/install_requirements.sh - fi - source .ci/scripts/setup-xtensa-tools.sh "${BACKEND}" - .ci/scripts/build-cadence-xtensa.sh --no-run - chmod -R a+rX cmake-out - ' + set -exo pipefail + eval "$(/opt/conda/bin/conda shell.bash hook)" + conda activate "$(conda env list --json | jq -r ".envs | .[-1]")" + ./install_requirements.sh > /dev/null + pip install --quiet awscli + # hifi4/fusion_g3 optimized kernels need the foss-xtensa nnlib + # sources, which are not vendored in executorch; the cadence + # installer clones them. vision has no nnlib dependency. + if [ "${BACKEND}" != "vision" ]; then + backends/cadence/install_requirements.sh + fi + source .ci/scripts/setup-xtensa-tools.sh "${BACKEND}" + .ci/scripts/build-cadence-xtensa.sh --no-run + chmod -R a+rX cmake-out - name: Upload runner uses: actions/upload-artifact@v4 diff --git a/.github/workflows/_xtensa_test.yml b/.github/workflows/_xtensa_test.yml index ae65b19f24e..4f05c3e63a2 100644 --- a/.github/workflows/_xtensa_test.yml +++ b/.github/workflows/_xtensa_test.yml @@ -1,6 +1,6 @@ # Reusable: build + run the Cadence Xtensa op-level gtest tests for one core on # the Instruction Set Simulator (xt-run). Mirrors _xtensa_build.yml's native -# OIDC + docker-run skeleton (running xt-run needs the same licensed toolchain), +# OIDC skeleton (running xt-run needs the same licensed toolchain), # then builds the gtest op-test ELF and runs it on the simulator. The runner # cross-compile and these op tests are separate build configs (the tests need # exceptions/RTTI that the runner build disables), so this is a self-contained @@ -21,30 +21,37 @@ on: default: "" jobs: + # The runner pod pulls the container before any step runs, so the image has to + # be a fully qualified reference resolved by a job this one depends on. + docker-image: + name: Resolve CI docker image + uses: ./.github/workflows/_docker-image.yml + test: name: ${{ inputs.backend }} - runs-on: linux.2xlarge + needs: docker-image + runs-on: mt-l-x86iavx512-8-64 + container: + image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/ci-image:executorch-ubuntu-22.04-clang12-${{ needs.docker-image.outputs.ci-docker-hash }} environment: cadence permissions: id-token: write contents: read steps: + - name: Clean workspace + shell: bash + run: | + set -eux + rm -rf "${GITHUB_WORKSPACE}" + mkdir -p "${GITHUB_WORKSPACE}" + - name: Checkout executorch uses: actions/checkout@v4 with: submodules: recursive ref: ${{ inputs.ref }} - - name: Calculate docker image - id: calculate-docker-image - uses: pytorch/test-infra/.github/actions/calculate-docker-image@main - with: - docker-image-name: ci-image:executorch-ubuntu-22.04-clang12 - - - name: Pull docker image - run: docker pull "${{ steps.calculate-docker-image.outputs.docker-image }}" - - - name: Assume Cadence artifacts role (host OIDC) + - name: Assume Cadence artifacts role uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ vars.CADENCE_CI_AWS_ROLE }} @@ -52,33 +59,20 @@ jobs: - name: Build and run op tests on xt-run env: - DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }} BACKEND: ${{ inputs.backend }} XTENSA_S3_BUCKET: ${{ vars.CADENCE_CI_S3_BUCKET }} shell: bash run: | - set -eux - # OIDC/role assumption already happened on the host above; pass the - # resulting AWS creds and the store/backend into the CI image, where - # the toolchain download + op-test build + xt-run happen. - docker run --rm \ - -e BACKEND -e XTENSA_S3_BUCKET \ - -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ - -e AWS_DEFAULT_REGION -e AWS_REGION \ - -v "${GITHUB_WORKSPACE}:/work/executorch" -w /work/executorch \ - "${DOCKER_IMAGE}" \ - bash -c ' - set -exo pipefail - eval "$(/opt/conda/bin/conda shell.bash hook)" - conda activate "$(conda env list --json | jq -r ".envs | .[-1]")" - ./install_requirements.sh > /dev/null - pip install --quiet awscli - # hifi4/fusion_g3 optimized kernels need the foss-xtensa nnlib - # sources, which are not vendored in executorch; the cadence - # installer clones them. vision has no nnlib dependency. - if [ "${BACKEND}" != "vision" ]; then - backends/cadence/install_requirements.sh - fi - source .ci/scripts/setup-xtensa-tools.sh "${BACKEND}" - .ci/scripts/test-cadence-xtensa.sh - ' + set -exo pipefail + eval "$(/opt/conda/bin/conda shell.bash hook)" + conda activate "$(conda env list --json | jq -r ".envs | .[-1]")" + ./install_requirements.sh > /dev/null + pip install --quiet awscli + # hifi4/fusion_g3 optimized kernels need the foss-xtensa nnlib + # sources, which are not vendored in executorch; the cadence + # installer clones them. vision has no nnlib dependency. + if [ "${BACKEND}" != "vision" ]; then + backends/cadence/install_requirements.sh + fi + source .ci/scripts/setup-xtensa-tools.sh "${BACKEND}" + .ci/scripts/test-cadence-xtensa.sh diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index a93b5a9559c..d7320d7beb5 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -122,7 +122,13 @@ jobs: # Upload the test demo app to S3 upload-demo-ios: needs: build-demo-ios - runs-on: linux.2xlarge + # Moving an artifact from GitHub to S3 needs no ExecuTorch toolchain, so + # this runs on a GitHub-hosted runner rather than moving to OSDC; the upload + # that used to ride on the EC2 instance role assumes role/arc through OIDC. + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read steps: - name: Download the artifacts from GitHub uses: actions/download-artifact@v4 @@ -138,6 +144,12 @@ jobs: set -eux ls -lah ./ + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::308535385114:role/arc + aws-region: us-east-1 + - name: Upload the artifacts to S3 uses: seemethere/upload-artifact-s3@v5 with: @@ -159,7 +171,9 @@ jobs: uses: pytorch/test-infra/.github/workflows/mobile_job.yml@main with: device-type: ios - # For iOS testing, the runner just needs to call AWS Device Farm, so there is no need to run this on macOS + # For iOS testing, the runner just needs to call AWS Device Farm, so there is no need to run this on macOS. + # Still an EC2 label: mobile_job.yml resolves its actions by relative path + # (./test-infra/.github/actions/...), which does not work on OSDC. runner: linux.2xlarge test-infra-ref: main # This is the ARN of ExecuTorch project on AWS diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index ae7cbe6857b..8ead5efb4a5 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -14,7 +14,8 @@ on: jobs: stale: if: ${{ github.repository == 'pytorch/executorch' }} - runs-on: linux.large + # Pure GitHub API work, so it needs neither an EC2 nor an OSDC runner. + runs-on: ubuntu-latest permissions: contents: read pull-requests: write