diff --git a/ci/tasks/light-aws/cleanup-ami.sh b/ci/tasks/light-aws/cleanup-ami.sh index 9d21e2f041..99b24ed427 100755 --- a/ci/tasks/light-aws/cleanup-ami.sh +++ b/ci/tasks/light-aws/cleanup-ami.sh @@ -30,6 +30,28 @@ if [ -n "${ami_role_arn:-}" ]; then export AWS_PROFILE=resource_account fi +# Every stemcell pipeline runs the published sweep against the same account, so two +# pipelines can select the same AMI before either has deleted it. This handles +# expected potential errors that could occur on a collision. +delete_tolerating_missing() { + local output + if output="$("$@" 2>&1)"; then + [ -n "${output}" ] && echo "${output}" + return 0 + fi + + case "${output}" in + *InvalidAMIID.Unavailable*|*InvalidAMIID.NotFound*|*InvalidSnapshot.NotFound*) + echo " already deleted by another run, skipping" + return 0 + ;; + *) + echo "${output}" >&2 + return 1 + ;; + esac +} + past_due=$(date --date="${ami_older_than_days} days ago" +"%Y-%m-%d") # shellcheck disable=SC2016 past_due_query='sort_by(Images,&CreationDate)[?CreationDate<`'"${past_due}"'`].{ImageId: ImageId, date:CreationDate, SnapshotId: BlockDeviceMappings[0].Ebs.SnapshotId,Version: Tags[?Key==`name`]|[0].Value}' @@ -59,16 +81,6 @@ for region in ${ami_destinations}; do ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}")) fi - if [ -n "${snapshot_id:-}" ]; then - results=$(aws ec2 describe-images \ - --owners self \ - --output json \ - --region "${region}" \ - --filters "Name=block-device-mapping.snapshot-id,Values=${snapshot_id}" \ - --query "${past_due_query}" | jq 'reverse | del(.[range(env.ami_keep_latest|tonumber)])') - ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}")) - fi - # 'ami_list' is a json array of objects, each object is an ami and its snapshot for row in $(echo "${ami_list}" | jq -r '.[] | @base64'); do _jq() { @@ -83,14 +95,12 @@ for region in ${ami_destinations}; do Snapshot id: $(_jq '.SnapshotId') " - aws ec2 deregister-image \ + delete_tolerating_missing aws ec2 deregister-image \ --image-id "$(_jq '.ImageId')" \ --region "${region}" - if [ "${snapshot_id:-}" != "$(_jq '.SnapshotId')" ]; then - aws ec2 delete-snapshot \ - --snapshot-id "$(_jq '.SnapshotId')" \ - --region "${region}" - fi + delete_tolerating_missing aws ec2 delete-snapshot \ + --snapshot-id "$(_jq '.SnapshotId')" \ + --region "${region}" done done diff --git a/ci/tasks/light-aws/cleanup-ami.yml b/ci/tasks/light-aws/cleanup-ami.yml index e30236167c..ff0cbde1c4 100644 --- a/ci/tasks/light-aws/cleanup-ami.yml +++ b/ci/tasks/light-aws/cleanup-ami.yml @@ -14,5 +14,4 @@ params: ami_older_than_days: "60" # Number of days AMI to keep excluding those currently being running ami_keep_latest: "5" # Number of previous AMI to keep excluding those currently being running os_name: "" # e.g ubuntu-jammy - snapshot_id: "" # Snapshot id to delete remove_public_images: "false"