From b25387fbe197e0af1e2104905b8fb6215478fe62 Mon Sep 17 00:00:00 2001
From: myurasov-nv <168484206+myurasov-nv@users.noreply.github.com>
Date: Wed, 26 Aug 2026 07:50:40 -0700
Subject: [PATCH] Fixes CI base image pulls when NGC refuses the inherited
 credential (#7332)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Currently `Build Base Docker Image` and `Build cuRobo Docker Image` fail
with `denied: Access Denied` while resolving the Isaac Sim base image
from public ("nvidia") NGC org.

**Cause.** `ecr-build-push-pull` copies the runner's persistent
`nvcr.io` credentials for a private NGC org into the job. NGC declines
that identity a pull token for the public `nvidia/*` catalog, so an
image that needs no credentials becomes unpullable.

**Fix.** The setup step now confirms those credentials can read the
configured base image, and retries anonymously when they cannot. The
deps-hash step fails loudly on an unreadable digest, which until now
corrupted the cache key in silence. `docker-build` receives the same
fallback, and its login moves to `--password-stdin`.

**Why this branch.** Only `release/3.0.0-beta2` pins a public base
image, so the bug lives here alone. `develop` keeps this logic in
`.github/actions/_lib/`, a path this branch does not have, so a fix
there could never run here.

| Before | After |
| ------ | ----- |
| Base image pull uses only the runner's inherited private-org
credentials, which NGC refuses for the public catalog, so the job fails
with `denied: Access Denied` | Credentials are tried first; if the
registry refuses them the pull retries anonymously, and the public base
image resolves |

## Type of change

- Bug fix (non-breaking change which fixes an issue)

## Release backport

- [ ] <!-- backport-active-release --> Backport this pull request to the
active release branch after it merges into `develop` (not applicable:
this targets a release branch directly, and the equivalent code on
`develop` is already correct)

## Screenshots

Not applicable; the change affects CI credential handling only.

## Checklist

- [x] I have read and understood the [contribution
guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html)
- [ ] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format` (not run locally, no dev setup on this machine;
this PR's own `pre-commit` check passes)
- [ ] I have made corresponding changes to the documentation (no
documented interface changes)
- [ ] My changes generate no new warnings (not assessable for a
composite action)
- [ ] I have added tests that prove my fix is effective or that my
feature works (no harness executes composite actions; the CI run on this
PR is the check)
- [ ] I have added a changelog fragment under
`source/<pkg>/changelog.d/` for every touched package (do **not** edit
`CHANGELOG.rst` or bump `extension.toml` — CI handles that) (no package
under `source/` touched)
- [x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there
---
 .github/actions/docker-build/action.yml       | 21 ++++++-
 .../actions/ecr-build-push-pull/action.yml    | 60 +++++++++++++++----
 2 files changed, 67 insertions(+), 14 deletions(-)

diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml
index 7f88241cfb8c..a588de519fd6 100644
--- a/.github/actions/docker-build/action.yml
+++ b/.github/actions/docker-build/action.yml
@@ -30,17 +30,34 @@ runs:
   steps:
     - name: NGC Login
       shell: sh
+      env:
+        BASE_IMAGE_REF: ${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}
       run: |
         # Only attempt NGC login if API key is available
-        if [ -n "${{ env.NGC_API_KEY }}" ]; then
+        if [ -n "${NGC_API_KEY:-}" ]; then
           echo "Logging into NGC registry..."
-          docker login -u \$oauthtoken -p ${{ env.NGC_API_KEY }} nvcr.io
+          printf '%s' "${NGC_API_KEY}" | docker login -u '$oauthtoken' --password-stdin nvcr.io
           echo "✅ Successfully logged into NGC registry"
         else
           echo "⚠️ NGC_API_KEY not available - skipping NGC login"
           echo "This is normal for PRs from forks or when secrets are not configured"
         fi
 
+        # NGC declines the runner's inherited private-org credentials a pull token
+        # for the public nvidia/* catalog, so keep those credentials only while
+        # they can read the base image this build was asked for.
+        if ! docker buildx imagetools inspect "${BASE_IMAGE_REF}" >/dev/null 2>&1; then
+          ANON_CONFIG_DIR=$(mktemp -d)
+          echo '{"credsStore":"","auths":{}}' > "${ANON_CONFIG_DIR}/config.json"
+          if DOCKER_CONFIG="${ANON_CONFIG_DIR}" docker buildx imagetools inspect \
+            "${BASE_IMAGE_REF}" >/dev/null 2>&1; then
+            echo "⚠️ Inherited credentials cannot read ${BASE_IMAGE_REF}; using anonymous access"
+            echo "DOCKER_CONFIG=${ANON_CONFIG_DIR}" >> "$GITHUB_ENV"
+          else
+            rm -rf "${ANON_CONFIG_DIR}"
+          fi
+        fi
+
     - name: Build Docker Image
       shell: sh
       run: |
diff --git a/.github/actions/ecr-build-push-pull/action.yml b/.github/actions/ecr-build-push-pull/action.yml
index b661d4b9fd62..fdf338abe3b5 100644
--- a/.github/actions/ecr-build-push-pull/action.yml
+++ b/.github/actions/ecr-build-push-pull/action.yml
@@ -51,6 +51,8 @@ runs:
 
     - name: Setup docker config and login to nvcr.io
       shell: bash
+      env:
+        BASE_IMAGE_REF: ${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}
       run: |
         DOCKER_CONFIG_DIR=$(mktemp -d)
         if [ -f "${HOME}/.docker/config.json" ]; then
@@ -58,16 +60,34 @@ runs:
         else
           echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json"
         fi
-        echo "DOCKER_CONFIG=${DOCKER_CONFIG_DIR}" >> "$GITHUB_ENV"
         export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}"
 
-        if [ -n "${{ env.NGC_API_KEY }}" ]; then
+        if [ -n "${NGC_API_KEY:-}" ]; then
           echo "🔵 Logging into nvcr.io..."
-          docker login -u \$oauthtoken -p ${{ env.NGC_API_KEY }} nvcr.io
+          printf '%s' "${NGC_API_KEY}" | docker login -u '$oauthtoken' --password-stdin nvcr.io
         else
           echo "🟠 NGC_API_KEY not set - skipping nvcr.io login (normal for fork PRs)"
         fi
 
+        # NGC refuses the runner's inherited private-org credentials a pull token
+        # for the public nvidia/* catalog rather than downgrading to anonymous,
+        # so keep them only if they can actually read the base image.
+        if ! docker buildx imagetools inspect "${BASE_IMAGE_REF}" >/dev/null 2>&1; then
+          ANON_CONFIG_DIR=$(mktemp -d)
+          echo '{"credsStore":"","auths":{}}' > "${ANON_CONFIG_DIR}/config.json"
+          if DOCKER_CONFIG="${ANON_CONFIG_DIR}" docker buildx imagetools inspect \
+            "${BASE_IMAGE_REF}" >/dev/null 2>&1; then
+            echo "🟠 Inherited credentials cannot read ${BASE_IMAGE_REF}; using anonymous access"
+            rm -rf "${DOCKER_CONFIG_DIR}"
+            DOCKER_CONFIG_DIR="${ANON_CONFIG_DIR}"
+            export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}"
+          else
+            rm -rf "${ANON_CONFIG_DIR}"
+          fi
+        fi
+
+        echo "DOCKER_CONFIG=${DOCKER_CONFIG_DIR}" >> "$GITHUB_ENV"
+
     ##### 2: Resolve ECR URL #####
 
     # Tries: explicit input >> ECR_CACHE_URL env var >> SSM parameter on EC2.
@@ -208,16 +228,32 @@ runs:
         DEPS_MANIFEST_PATTERN='(setup\.py|pyproject\.toml|setup\.cfg|extension\.toml|requirements[^/]*\.txt|uv\.lock)$'
 
         # Resolve the actual base image digest so a new push of a mutable tag
-        # (e.g. latest-develop) invalidates the deps cache automatically.
-        BASE_IMAGE_DIGEST=$(docker buildx imagetools inspect \
-          "${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}" \
-          --format '{{json .Manifest.Digest}}' 2>/dev/null | tr -d '"' || true)
-        if [ -n "${BASE_IMAGE_DIGEST}" ]; then
-          BASE_IMAGE_UNIQ_ID="${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}:${BASE_IMAGE_DIGEST}"
-        else
-          echo "🟠 Could not resolve base image digest, falling back to tag string"
-          BASE_IMAGE_UNIQ_ID="${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}"
+        # (e.g. latest-develop) invalidates the deps cache. Failing is deliberate;
+        # stderr is kept off stdout so diagnostics can never reach the digest.
+        BASE_IMAGE_DIGEST=""
+        INSPECT_ERR="$(mktemp)"
+        for attempt in 1 2 3; do
+          INSPECT_OUT=$(docker buildx imagetools inspect \
+            "${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}" \
+            --format '{{json .Manifest.Digest}}' 2>"${INSPECT_ERR}" || true)
+          CANDIDATE=$(printf '%s' "${INSPECT_OUT}" | tr -d '"')
+          case "${CANDIDATE}" in
+            sha256:*)
+              BASE_IMAGE_DIGEST="${CANDIDATE}"
+              break
+              ;;
+          esac
+          echo "🟠 Base image manifest read attempt ${attempt}/3 failed: $(tr '\n' ' ' < "${INSPECT_ERR}")"
+          if [ "${attempt}" -lt 3 ]; then
+            sleep $((attempt * 5))
+          fi
+        done
+        rm -f "${INSPECT_ERR}"
+        if [ -z "${BASE_IMAGE_DIGEST}" ]; then
+          echo "::error::Cannot read the manifest for ${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }} after 3 attempts (see the attempt logs above)."
+          exit 1
         fi
+        BASE_IMAGE_UNIQ_ID="${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}:${BASE_IMAGE_DIGEST}"
 
         echo "🔵 Base image ID: ${BASE_IMAGE_UNIQ_ID}"