fix: add retry with backoff and jitter for image pulls

Author: maxday

aws-lambda-java-runtime-interface-client/Makefile

@@ -26,9 +26,10 @@ test:
 
 .PHONY: setup-codebuild-agent
 setup-codebuild-agent:
-	docker build -t codebuild-agent \
+	test/integration/codebuild-local/docker-retry.sh docker build --load -t codebuild-agent \
 	 --build-arg ARCHITECTURE=$(ARCHITECTURE_ALIAS) \
-	  - < test/integration/codebuild-local/Dockerfile.agent
+	 -f test/integration/codebuild-local/Dockerfile.agent \
+	 test/integration/codebuild-local
 
 .PHONY: test-smoke
 test-smoke: setup-codebuild-agent

aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/docker-retry.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+set -uo pipefail
+
+MAX_ATTEMPTS="${RETRY_MAX_ATTEMPTS:-5}"
+BASE_DELAY="${RETRY_BASE_DELAY:-5}"
+MAX_DELAY="${RETRY_MAX_DELAY:-60}"
+
+if (( $# == 0 )); then
+    >&2 echo "usage: docker-retry.sh <command> [args...]"
+    exit 2
+fi
+
+attempt=1
+while true; do
+    "$@" && exit 0
+    status=$?
+
+    if (( attempt >= MAX_ATTEMPTS )); then
+        >&2 echo "docker-retry: '$*' failed after ${attempt} attempt(s) (exit ${status}); giving up."
+        exit "$status"
+    fi
+
+    # Exponential backoff: BASE_DELAY * 2^(attempt-1), capped at MAX_DELAY.
+    backoff=$(( BASE_DELAY * (2 ** (attempt - 1)) ))
+    (( backoff > MAX_DELAY )) && backoff=$MAX_DELAY
+    # Full jitter: wait a random duration in [0, backoff] so concurrent jobs
+    # spread out instead of retrying at the same moment.
+    delay=$(( RANDOM % (backoff + 1) ))
+
+    >&2 echo "docker-retry: '$*' failed (exit ${status}); attempt ${attempt}/${MAX_ATTEMPTS}, retrying in ${delay}s."
+    sleep "$delay"
+    attempt=$(( attempt + 1 ))
+done

aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/test_one.sh

@@ -60,14 +60,18 @@ main() {
 
     ARTIFACTS_DIR="$CODEBUILD_TEMP_DIR/artifacts"
     mkdir -p "$ARTIFACTS_DIR"
+
+    LOCAL_AGENT_IMAGE="$(get_local_agent_image)"
+    "$(dirname "$0")"/docker-retry.sh docker pull "$LOCAL_AGENT_IMAGE" || true
+
     # Run CodeBuild local agent.
     "$(dirname "$0")"/codebuild_build.sh \
         -i "$CODEBUILD_IMAGE_TAG" \
         -a "$ARTIFACTS_DIR" \
         -e "$ENVFILE" \
         -b "$BUILDSPEC_YML" \
         -s "$(dirname $PWD)" \
-        -l "$(get_local_agent_image)"
+        -l "$LOCAL_AGENT_IMAGE"
 }
 
 main "$@"

aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.alpine.yml

@@ -72,7 +72,7 @@ phases:
           "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp"
       - echo "Building image ${IMAGE_TAG}"
       - >
-        docker build . \
+        aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/docker-retry.sh docker build . \
           -f "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp" \
           -t "${IMAGE_TAG}" \
           --platform="${PLATFORM}" \

aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazoncorretto.yml

@@ -68,7 +68,7 @@ phases:
           "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp"
       - echo "Building image ${IMAGE_TAG}"
       - >
-        docker build . \
+        aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/docker-retry.sh docker build . \
           -f "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp" \
           -t "${IMAGE_TAG}" \
           --platform="${PLATFORM}" \

aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazonlinux.1.yml

@@ -55,7 +55,7 @@ phases:
           "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp"
       - echo "Building image ${IMAGE_TAG}"
       - >
-        docker build . \
+        aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/docker-retry.sh docker build . \
           -f "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp" \
           -t "${IMAGE_TAG}" \
           --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \

aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazonlinux.2.yml

@@ -67,7 +67,7 @@ phases:
           "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp"
       - echo "Building image ${IMAGE_TAG}"
       - >
-        docker build . \
+        aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/docker-retry.sh docker build . \
           -f "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp" \
           -t "${IMAGE_TAG}" \
           --platform="${PLATFORM}" \

aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.debian.yml

@@ -71,7 +71,7 @@ phases:
           "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp"
       - echo "Building image ${IMAGE_TAG}"
       - >
-        docker build . \
+        aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/docker-retry.sh docker build . \
           -f "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp" \
           -t "${IMAGE_TAG}" \
           --platform="${PLATFORM}" \

aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.ubuntu.yml

@@ -71,7 +71,7 @@ phases:
           "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp"
       - echo "Building image ${IMAGE_TAG}"
       - >
-        docker build . \
+        aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/docker-retry.sh docker build . \
           -f "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp" \
           -t "${IMAGE_TAG}" \
           --platform="${PLATFORM}" \

aws-lambda-java-runtime-interface-client/test/integration/codebuild/scripts/configure_multi_arch_env.sh

@@ -15,7 +15,7 @@ else
 fi
 
 echo "Installing ${TARGET_EMULATOR} emulator"
-docker pull public.ecr.aws/eks-distro-build-tooling/binfmt-misc:qemu-v6.1.0
+"$(dirname "$0")"/../../codebuild-local/docker-retry.sh docker pull public.ecr.aws/eks-distro-build-tooling/binfmt-misc:qemu-v6.1.0
 docker run --rm --privileged public.ecr.aws/eks-distro-build-tooling/binfmt-misc:qemu-v6.1.0 --install ${TARGET_EMULATOR}
 echo "Setting docker build command to default to buildx"
 echo "Docker buildx version: $(docker buildx version)"