diff --git a/.github/workflows/cloud-smoke.yml b/.github/workflows/cloud-smoke.yml index a90d8dc..25dca1d 100644 --- a/.github/workflows/cloud-smoke.yml +++ b/.github/workflows/cloud-smoke.yml @@ -31,7 +31,9 @@ jobs: run: make terraform-lint-check - name: Validate fallback cleanup behavior - run: bash ci/cloud-smoke-cleanup-contract.sh + run: | + bash ci/cloud-smoke-cleanup-contract.sh + make hosted-cleanup-retry-contract config-management-lint: name: Ansible and Salt lint diff --git a/Makefile b/Makefile index 8759482..5aea108 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lint lint-check shell-lint config-management-smoke +.PHONY: lint lint-check shell-lint hosted-cleanup-retry-contract config-management-smoke .PHONY: terraform-fmt terraform-fmt-check terraform-validate terraform-lint-check terraform-docs .PHONY: config-management-cloud-smoke config-management-cloud-smoke-ansible-drupal config-management-cloud-smoke-salt-drupal .PHONY: destroy-config-management-cloud-smoke destroy-config-management-cloud-smoke-ansible-drupal destroy-config-management-cloud-smoke-salt-drupal @@ -10,9 +10,9 @@ DOCS_IMAGE ?= cloud-compose-docs DOCS_PORT ?= 8888 DOCS_DOCKER_USER ?= $(shell id -u):$(shell id -g) -lint: terraform-fmt shell-lint terraform-validate +lint: terraform-fmt shell-lint hosted-cleanup-retry-contract terraform-validate -lint-check: terraform-fmt-check shell-lint terraform-validate +lint-check: terraform-fmt-check shell-lint hosted-cleanup-retry-contract terraform-validate terraform-fmt: terraform fmt -recursive @@ -31,6 +31,9 @@ shell-lint: -path "./docs/site" -prune -o \ -type f -name "*.sh" -print0 | xargs -0 shellcheck +hosted-cleanup-retry-contract: + bash ci/hosted-cleanup-retry-contract.sh + config-management-smoke: ci/config-management-smoke.sh diff --git a/ci/cloud-smoke.sh b/ci/cloud-smoke.sh index c5f6a8a..efeee9b 100755 --- a/ci/cloud-smoke.sh +++ b/ci/cloud-smoke.sh @@ -165,12 +165,18 @@ api_request() { ;; esac - if response="$(curl -sS -X "$method" -H "Authorization: Bearer ${token}" -w $'\n%{http_code}' "${base_url}${path}")"; then + if response="$(curl -sS \ + --connect-timeout 10 \ + --max-time 45 \ + -X "$method" \ + -H "Authorization: Bearer ${token}" \ + -w $'\n%{http_code}' \ + "${base_url}${path}")"; then http_code="${response##*$'\n'}" body="${response%$'\n'"$http_code"}" else echo "${provider} API request failed for ${method} ${path}; check ${token_name} and network access." >&2 - return 1 + return 75 fi case "$http_code" in @@ -185,6 +191,37 @@ api_request() { echo "${provider} API rejected ${token_name} with HTTP 403 for ${method} ${path}; verify the token has the permissions required by smoke cleanup and Terraform." >&2 return 22 ;; + 404 | 410) + if [[ "$method" == "DELETE" ]]; then + return 0 + fi + echo "${provider} API returned HTTP ${http_code} for ${method} ${path}." >&2 + if [[ -n "$body" ]]; then + printf '%s\n' "$body" >&2 + fi + return 22 + ;; + 408 | 425 | 429 | 5??) + echo "${provider} API returned retryable HTTP ${http_code} for ${method} ${path}." >&2 + if [[ -n "$body" ]]; then + printf '%s\n' "$body" >&2 + fi + return 75 + ;; + 409 | 423) + if [[ "$method" == "DELETE" ]]; then + echo "${provider} API returned retryable HTTP ${http_code} for ${method} ${path}." >&2 + if [[ -n "$body" ]]; then + printf '%s\n' "$body" >&2 + fi + return 75 + fi + echo "${provider} API returned HTTP ${http_code} for ${method} ${path}." >&2 + if [[ -n "$body" ]]; then + printf '%s\n' "$body" >&2 + fi + return 22 + ;; *) echo "${provider} API returned HTTP ${http_code} for ${method} ${path}." >&2 if [[ -n "$body" ]]; then @@ -200,7 +237,25 @@ api_delete() { } api_get() { - api_request "$1" GET "$2" + local provider="$1" path="$2" attempt=1 delay=2 status + + while true; do + if api_request "$provider" GET "$path"; then + return 0 + else + status=$? + fi + if [[ "$status" -ne 75 || "$attempt" -ge 6 ]]; then + return "$status" + fi + echo "Retrying ${provider} API GET ${path} in ${delay}s (attempt $((attempt + 1)) of 6)" >&2 + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + if [[ "$delay" -gt 30 ]]; then + delay=30 + fi + done } gcp_region() { @@ -212,7 +267,7 @@ gcp_zone() { } delete_ids() { - local provider="$1" path_prefix="$2" id attempt + local provider="$1" path_prefix="$2" id attempt status local failed=0 deleted while IFS= read -r id; do @@ -225,11 +280,18 @@ delete_ids() { if api_delete "$provider" "${path_prefix}/${id}"; then deleted=true break + else + status=$? + fi + if [[ "$status" -ne 75 ]]; then + break + fi + if [[ "$attempt" -lt 12 ]]; then + sleep 10 fi - sleep 10 done if [[ "$deleted" != "true" ]]; then - echo "Failed to delete ${provider} ${path_prefix}/${id} after 12 attempts" >&2 + echo "Failed to delete ${provider} ${path_prefix}/${id} after ${attempt} attempt(s)" >&2 failed=1 fi done @@ -453,9 +515,9 @@ target_name_prefix() { printf 'cc-%s-%s\n' "$(provider_slug "$provider")" "$(template_slug "$template")" } -provider_tag_cleanup() { - local target="$1" run_id="${2:-}" run_tag run_fragment provider name_prefix - local cleanup_status=0 +provider_resource_ids() { + local target="$1" run_id="${2:-}" kind="$3" + local run_tag run_fragment provider name_prefix run_tag="$(smoke_run_tag "$run_id")" run_fragment="" @@ -465,30 +527,107 @@ provider_tag_cleanup() { provider="$(target_provider "$target")" name_prefix="$(target_name_prefix "$target")" - case "$provider" in - digitalocean) + case "${provider}:${kind}" in + digitalocean:firewalls) api_get digitalocean "/firewalls?per_page=200" | - jq -r --arg name_prefix "${name_prefix}-" --arg run_fragment "$run_fragment" '.firewalls[]? | select(.name | startswith($name_prefix)) | select($run_fragment == "" or (.name | contains($run_fragment))) | .id' | - delete_ids digitalocean "/firewalls" || cleanup_status=1 + jq -r --arg name_prefix "${name_prefix}-" --arg run_fragment "$run_fragment" 'if (.firewalls | type) != "array" then error("DigitalOcean firewalls response is not an array") else .firewalls[] | select(.name | startswith($name_prefix)) | select($run_fragment == "" or (.name | contains($run_fragment))) | .id end' + ;; + digitalocean:droplets) api_get digitalocean "/droplets?tag_name=cloud-compose-smoke&per_page=200" | - jq -r --arg target "$target" --arg run_tag "$run_tag" '.droplets[]? | select((.tags // []) | index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id' | - delete_ids digitalocean "/droplets" || cleanup_status=1 - sleep 10 + jq -r --arg target "$target" --arg run_tag "$run_tag" 'if (.droplets | type) != "array" then error("DigitalOcean droplets response is not an array") else .droplets[] | select((.tags // []) | index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id end' + ;; + digitalocean:volumes) api_get digitalocean "/volumes?tag_name=cloud-compose-smoke&per_page=200" | - jq -r --arg target "$target" --arg run_tag "$run_tag" '.volumes[]? | select((.tags // []) | index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id' | - delete_ids digitalocean "/volumes" || cleanup_status=1 + jq -r --arg target "$target" --arg run_tag "$run_tag" 'if (.volumes | type) != "array" then error("DigitalOcean volumes response is not an array") else .volumes[] | select((.tags // []) | index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id end' ;; - linode) + linode:firewalls) api_get linode "/networking/firewalls?page_size=500" | - jq -r --arg target "$target" --arg run_tag "$run_tag" '.data[]? | select((.tags // []) | index("cloud-compose-smoke") and index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id' | - delete_ids linode "/networking/firewalls" || cleanup_status=1 + jq -r --arg target "$target" --arg run_tag "$run_tag" 'if (.data | type) != "array" then error("Linode firewalls response data is not an array") else .data[] | select((.tags // []) | index("cloud-compose-smoke") and index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id end' + ;; + linode:instances) api_get linode "/linode/instances?page_size=500" | - jq -r --arg target "$target" --arg run_tag "$run_tag" '.data[]? | select((.tags // []) | index("cloud-compose-smoke") and index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id' | - delete_ids linode "/linode/instances" || cleanup_status=1 - sleep 10 + jq -r --arg target "$target" --arg run_tag "$run_tag" 'if (.data | type) != "array" then error("Linode instances response data is not an array") else .data[] | select((.tags // []) | index("cloud-compose-smoke") and index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id end' + ;; + linode:volumes) api_get linode "/volumes?page_size=500" | - jq -r --arg target "$target" --arg run_tag "$run_tag" '.data[]? | select((.tags // []) | index("cloud-compose-smoke") and index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id' | - delete_ids linode "/volumes" || cleanup_status=1 + jq -r --arg target "$target" --arg run_tag "$run_tag" 'if (.data | type) != "array" then error("Linode volumes response data is not an array") else .data[] | select((.tags // []) | index("cloud-compose-smoke") and index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id end' + ;; + *) + echo "Unknown provider resource kind: ${provider}:${kind}" >&2 + return 2 + ;; + esac +} + +provider_cleanup_residuals() { + local target="$1" run_id="${2:-}" provider kind ids id index + local -a kinds path_prefixes + + provider="$(target_provider "$target")" + case "$provider" in + digitalocean) + kinds=(firewalls droplets volumes) + path_prefixes=(/firewalls /droplets /volumes) + ;; + linode) + kinds=(firewalls instances volumes) + path_prefixes=(/networking/firewalls /linode/instances /volumes) + ;; + *) + return 0 + ;; + esac + + for index in "${!kinds[@]}"; do + kind="${kinds[$index]}" + if ! ids="$(provider_resource_ids "$target" "$run_id" "$kind")"; then + return 1 + fi + while IFS= read -r id; do + [[ -n "$id" ]] || continue + printf '%s%s\n' "${path_prefixes[$index]}/" "$id" + done <<<"$ids" + done +} + +verify_no_provider_resources() { + local target="$1" run_id="${2:-}" attempt residuals + + for attempt in {1..6}; do + if ! residuals="$(provider_cleanup_residuals "$target" "$run_id")"; then + echo "Could not verify provider cleanup for ${target}" >&2 + return 1 + fi + if [[ -z "$residuals" ]]; then + echo "Verified that no matching ${target} smoke resources remain" + return 0 + fi + echo "Matching ${target} smoke resources remain after cleanup verification attempt ${attempt}:" >&2 + printf '%s\n' "$residuals" >&2 + if [[ "$attempt" -lt 6 ]]; then + sleep 10 + fi + done + + echo "Provider cleanup left matching ${target} smoke resources" >&2 + return 1 +} + +provider_tag_cleanup() { + local target="$1" run_id="${2:-}" provider kind index name_prefix + local cleanup_status=0 + local -a kinds path_prefixes + + provider="$(target_provider "$target")" + name_prefix="$(target_name_prefix "$target")" + case "$provider" in + digitalocean) + kinds=(firewalls droplets volumes) + path_prefixes=(/firewalls /droplets /volumes) + ;; + linode) + kinds=(firewalls instances volumes) + path_prefixes=(/networking/firewalls /linode/instances /volumes) ;; gcp) local project name_filter region cloud_run_services instance_rows firewall_names disk_rows @@ -674,6 +813,20 @@ provider_tag_cleanup() { ;; esac + if [[ "$provider" == "digitalocean" || "$provider" == "linode" ]]; then + for index in "${!kinds[@]}"; do + kind="${kinds[$index]}" + provider_resource_ids "$target" "$run_id" "$kind" | + delete_ids "$provider" "${path_prefixes[$index]}" || cleanup_status=1 + if [[ "$kind" == "droplets" || "$kind" == "instances" ]]; then + sleep 10 + fi + done + if [[ "$cleanup_status" -eq 0 ]] && ! verify_no_provider_resources "$target" "$run_id"; then + cleanup_status=1 + fi + fi + return "$cleanup_status" } @@ -967,7 +1120,7 @@ run_target() ( set -euo pipefail local target="$1" - local root workdir key_path home_dir output_json public_key + local root workdir key_path home_dir output_json public_key run_id local -a auto_args var_args root="$(target_root "$target")" @@ -981,6 +1134,7 @@ run_target() ( chmod 0700 "$home_dir/.ssh" ensure_key "$key_path" + run_id="$(smoke_run_id)" mapfile -d '' -t var_args < <(target_var_args "$root" "$key_path" "$target") auto_args=() @@ -991,7 +1145,7 @@ run_target() ( cleanup_started=false # shellcheck disable=SC2317 cleanup() { - local status=$? + local status="$1" local destroy_status destroy_timeout cleanup_status trap - EXIT INT TERM HUP @@ -1020,14 +1174,18 @@ run_target() ( fi if [[ "$destroy_status" -ne 0 ]]; then echo "Terraform destroy failed for ${target}; attempting provider tag cleanup" - provider_tag_cleanup "$target" || cleanup_status=$? + provider_tag_cleanup "$target" "$run_id" || cleanup_status=$? fi if [[ "$status" -eq 0 && "$destroy_status" -ne 0 && "$cleanup_status" -ne 0 ]]; then + echo "Terraform destroy and provider tag cleanup both failed for ${target}" >&2 exit "$destroy_status" fi exit "$status" } - trap cleanup EXIT INT TERM HUP + trap 'cleanup "$?"' EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM echo "Initializing ${target}" TF_DATA_DIR="$workdir/.terraform" terraform -chdir="$root" init -input=false diff --git a/ci/config-management-cloud-smoke.sh b/ci/config-management-cloud-smoke.sh index 5cafa97..6e6ca1f 100755 --- a/ci/config-management-cloud-smoke.sh +++ b/ci/config-management-cloud-smoke.sh @@ -130,12 +130,18 @@ api_request() { local method="$1" path="$2" local body http_code response - if response="$(curl -sS -X "$method" -H "Authorization: Bearer ${LINODE_TOKEN}" -w $'\n%{http_code}' "https://api.linode.com/v4${path}")"; then + if response="$(curl -sS \ + --connect-timeout 10 \ + --max-time 45 \ + -X "$method" \ + -H "Authorization: Bearer ${LINODE_TOKEN}" \ + -w $'\n%{http_code}' \ + "https://api.linode.com/v4${path}")"; then http_code="${response##*$'\n'}" body="${response%$'\n'$http_code}" else echo "linode API request failed for ${method} ${path}; check LINODE_TOKEN and network access." >&2 - return 1 + return 75 fi case "$http_code" in @@ -150,6 +156,37 @@ api_request() { echo "linode API rejected LINODE_TOKEN with HTTP 403 for ${method} ${path}; verify the token has the permissions required by smoke cleanup and Terraform." >&2 return 22 ;; + 404 | 410) + if [[ "$method" == "DELETE" ]]; then + return 0 + fi + echo "linode API returned HTTP ${http_code} for ${method} ${path}." >&2 + if [[ -n "$body" ]]; then + printf '%s\n' "$body" >&2 + fi + return 22 + ;; + 408 | 425 | 429 | 5??) + echo "linode API returned retryable HTTP ${http_code} for ${method} ${path}." >&2 + if [[ -n "$body" ]]; then + printf '%s\n' "$body" >&2 + fi + return 75 + ;; + 409 | 423) + if [[ "$method" == "DELETE" ]]; then + echo "linode API returned retryable HTTP ${http_code} for ${method} ${path}." >&2 + if [[ -n "$body" ]]; then + printf '%s\n' "$body" >&2 + fi + return 75 + fi + echo "linode API returned HTTP ${http_code} for ${method} ${path}." >&2 + if [[ -n "$body" ]]; then + printf '%s\n' "$body" >&2 + fi + return 22 + ;; *) echo "linode API returned HTTP ${http_code} for ${method} ${path}." >&2 if [[ -n "$body" ]]; then @@ -161,7 +198,25 @@ api_request() { } api_get() { - api_request GET "$1" + local path="$1" attempt=1 delay=2 status + + while true; do + if api_request GET "$path"; then + return 0 + else + status=$? + fi + if [[ "$status" -ne 75 || "$attempt" -ge 6 ]]; then + return "$status" + fi + echo "Retrying linode API GET ${path} in ${delay}s (attempt $((attempt + 1)) of 6)" >&2 + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + if [[ "$delay" -gt 30 ]]; then + delay=30 + fi + done } api_delete() { @@ -169,36 +224,121 @@ api_delete() { } delete_ids() { - local path_prefix="$1" id attempt + local path_prefix="$1" id attempt status + local failed=0 deleted while IFS= read -r id; do [[ -n "$id" ]] || continue + deleted=false for attempt in {1..12}; do echo "Deleting Linode resource ${path_prefix}/${id} (attempt ${attempt})" if api_delete "${path_prefix}/${id}"; then + deleted=true break + else + status=$? + fi + if [[ "$status" -ne 75 ]]; then + break + fi + if [[ "$attempt" -lt 12 ]]; then + sleep 10 fi - sleep 10 done + if [[ "$deleted" != "true" ]]; then + echo "Failed to delete Linode resource ${path_prefix}/${id} after ${attempt} attempt(s)" >&2 + failed=1 + fi done + + return "$failed" } -provider_tag_cleanup() { - local target="$1" run_id="${2:-}" run_tag tag +provider_resource_ids() { + local target="$1" run_id="${2:-}" kind="$3" run_tag tag run_tag="$(smoke_run_tag "$run_id")" tag="$(target_tag "$target")" - api_get "/networking/firewalls?page_size=500" | - jq -r --arg tag "$tag" --arg run_tag "$run_tag" '.data[]? | select((.tags // []) | index("cloud-compose-smoke") and index("config-management-smoke") and index($tag)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id' | - delete_ids "/networking/firewalls" - api_get "/linode/instances?page_size=500" | - jq -r --arg tag "$tag" --arg run_tag "$run_tag" '.data[]? | select((.tags // []) | index("cloud-compose-smoke") and index("config-management-smoke") and index($tag)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id' | - delete_ids "/linode/instances" - sleep 10 - api_get "/volumes?page_size=500" | - jq -r --arg tag "$tag" --arg run_tag "$run_tag" '.data[]? | select((.tags // []) | index("cloud-compose-smoke") and index("config-management-smoke") and index($tag)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id' | - delete_ids "/volumes" + case "$kind" in + firewalls) + api_get "/networking/firewalls?page_size=500" | + jq -r --arg tag "$tag" --arg run_tag "$run_tag" 'if (.data | type) != "array" then error("Linode firewalls response data is not an array") else .data[] | select((.tags // []) | index("cloud-compose-smoke") and index("config-management-smoke") and index($tag)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id end' + ;; + instances) + api_get "/linode/instances?page_size=500" | + jq -r --arg tag "$tag" --arg run_tag "$run_tag" 'if (.data | type) != "array" then error("Linode instances response data is not an array") else .data[] | select((.tags // []) | index("cloud-compose-smoke") and index("config-management-smoke") and index($tag)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id end' + ;; + volumes) + api_get "/volumes?page_size=500" | + jq -r --arg tag "$tag" --arg run_tag "$run_tag" 'if (.data | type) != "array" then error("Linode volumes response data is not an array") else .data[] | select((.tags // []) | index("cloud-compose-smoke") and index("config-management-smoke") and index($tag)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id end' + ;; + *) + echo "Unknown Linode resource kind: ${kind}" >&2 + return 2 + ;; + esac +} + +provider_cleanup_residuals() { + local target="$1" run_id="${2:-}" kind ids id index + local -a kinds=(firewalls instances volumes) + local -a path_prefixes=(/networking/firewalls /linode/instances /volumes) + + for index in "${!kinds[@]}"; do + kind="${kinds[$index]}" + if ! ids="$(provider_resource_ids "$target" "$run_id" "$kind")"; then + return 1 + fi + while IFS= read -r id; do + [[ -n "$id" ]] || continue + printf '%s%s\n' "${path_prefixes[$index]}/" "$id" + done <<<"$ids" + done +} + +verify_no_provider_resources() { + local target="$1" run_id="${2:-}" attempt residuals + + for attempt in {1..6}; do + if ! residuals="$(provider_cleanup_residuals "$target" "$run_id")"; then + echo "Could not verify provider cleanup for config-management ${target}" >&2 + return 1 + fi + if [[ -z "$residuals" ]]; then + echo "Verified that no matching config-management ${target} smoke resources remain" + return 0 + fi + echo "Matching config-management ${target} smoke resources remain after cleanup verification attempt ${attempt}:" >&2 + printf '%s\n' "$residuals" >&2 + if [[ "$attempt" -lt 6 ]]; then + sleep 10 + fi + done + + echo "Provider cleanup left matching config-management ${target} smoke resources" >&2 + return 1 +} + +provider_tag_cleanup() { + local target="$1" run_id="${2:-}" kind index + local cleanup_status=0 + local -a kinds=(firewalls instances volumes) + local -a path_prefixes=(/networking/firewalls /linode/instances /volumes) + + for index in "${!kinds[@]}"; do + kind="${kinds[$index]}" + provider_resource_ids "$target" "$run_id" "$kind" | + delete_ids "${path_prefixes[$index]}" || cleanup_status=1 + if [[ "$kind" == "instances" ]]; then + sleep 10 + fi + done + if [[ "$cleanup_status" -eq 0 ]] && ! verify_no_provider_resources "$target" "$run_id"; then + cleanup_status=1 + fi + + return "$cleanup_status" } scan_host_key() { @@ -366,7 +506,7 @@ require_sweep_commands() { run_target() ( set -euo pipefail - local target="$1" root workdir key_path home_dir output_json host + local target="$1" root workdir key_path home_dir output_json host run_id local -a auto_args var_args valid_target "$target" || { @@ -384,6 +524,7 @@ run_target() ( chmod 0700 "$home_dir/.ssh" ensure_key "$key_path" + run_id="$(smoke_run_id)" mapfile -d '' -t var_args < <(target_var_args "$key_path" "$target") auto_args=() @@ -392,9 +533,10 @@ run_target() ( fi cleanup_started=false + # shellcheck disable=SC2317 cleanup() { - local status=$? - local destroy_status=0 destroy_timeout + local status="$1" + local cleanup_status=0 destroy_status=0 destroy_timeout trap - EXIT INT TERM HUP if [[ "$cleanup_started" == "true" ]]; then @@ -420,11 +562,18 @@ run_target() ( destroy_status=$? fi if [[ "$destroy_status" -ne 0 ]]; then - provider_tag_cleanup "$target" || true + provider_tag_cleanup "$target" "$run_id" || cleanup_status=$? + fi + if [[ "$status" -eq 0 && "$destroy_status" -ne 0 && "$cleanup_status" -ne 0 ]]; then + echo "Terraform destroy and provider tag cleanup both failed for ${target}" >&2 + exit "$destroy_status" fi exit "$status" } - trap cleanup EXIT INT TERM HUP + trap 'cleanup "$?"' EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM echo "Initializing ${target} raw Linode smoke" TF_DATA_DIR="$workdir/.terraform" terraform -chdir="$root" init -input=false @@ -455,7 +604,7 @@ run_target() ( destroy_target() ( set -euo pipefail - local target="$1" root workdir key_path destroy_status=0 destroy_timeout + local target="$1" root workdir key_path cleanup_status=0 destroy_status=0 destroy_timeout local -a auto_args var_args valid_target "$target" || exit 2 @@ -491,8 +640,16 @@ destroy_target() ( set -e fi - provider_tag_cleanup "$target" "$(smoke_run_id)" - return "$destroy_status" + provider_tag_cleanup "$target" "$(smoke_run_id)" || cleanup_status=$? + + if [[ "$destroy_status" -ne 0 && "$cleanup_status" -eq 0 ]]; then + echo "Provider tag cleanup completed for ${target} after Terraform destroy failed" + return 0 + fi + if [[ "$destroy_status" -ne 0 ]]; then + return "$destroy_status" + fi + return "$cleanup_status" ) main() { diff --git a/ci/hosted-cleanup-retry-contract.sh b/ci/hosted-cleanup-retry-contract.sh new file mode 100755 index 0000000..aaa5da2 --- /dev/null +++ b/ci/hosted-cleanup-retry-contract.sh @@ -0,0 +1,698 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +tmp="$(mktemp -d "${TMPDIR:-/tmp}/cloud-compose-hosted-cleanup.XXXXXX")" +trap 'rm -rf "$tmp"' EXIT + +fail() { + echo "hosted cleanup retry contract: $*" >&2 + exit 1 +} + +mkdir -p "$tmp/bin" "$tmp/state" + +cat >"$tmp/bin/sleep" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf 'sleep %s\n' "$*" >>"$FAKE_API_LOG" +EOF + +cat >"$tmp/bin/curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +printf 'curl' >>"$FAKE_API_LOG" +printf ' %q' "$@" >>"$FAKE_API_LOG" +printf '\n' >>"$FAKE_API_LOG" + +method="" +url="" +connect_timeout="" +max_time="" +while [[ "$#" -gt 0 ]]; do + case "$1" in + -sS) + shift + ;; + --connect-timeout) + connect_timeout="${2:-}" + shift 2 + ;; + --max-time) + max_time="${2:-}" + shift 2 + ;; + -X) + method="${2:-}" + shift 2 + ;; + -H | -w) + shift 2 + ;; + https://*) + url="$1" + shift + ;; + *) + echo "unexpected fake curl argument: $1" >&2 + exit 64 + ;; + esac +done + +[[ "$connect_timeout" == "10" ]] || { + echo "cleanup request omitted the 10-second connect timeout" >&2 + exit 64 +} +[[ "$max_time" == "45" ]] || { + echo "cleanup request omitted the 45-second total timeout" >&2 + exit 64 +} + +state_count() { + local key="$1" + + if [[ -f "$FAKE_API_STATE/$key.count" ]]; then + cat "$FAKE_API_STATE/$key.count" + else + printf '0\n' + fi +} + +increment_count() { + local key="$1" count + + count="$(state_count "$key")" + count=$((count + 1)) + printf '%s\n' "$count" >"$FAKE_API_STATE/$key.count" + printf '%s\n' "$count" +} + +respond() { + local body="$1" code="$2" + + printf '%s\n%s\n' "$body" "$code" +} + +mark_deleted() { + printf '%s\n' "$url" >>"$FAKE_API_STATE/deleted.log" +} + +resource_visible() { + local delete_url="$1" primary_count + + case "$FAKE_API_MODE" in + residual-persistent) + return 0 + ;; + residual-delayed) + primary_count="$(state_count primary)" + if [[ "$primary_count" -le 3 ]]; then + return 0 + fi + return 1 + ;; + esac + + if [[ -f "$FAKE_API_STATE/deleted.log" ]] && grep -Fxq "$delete_url" "$FAKE_API_STATE/deleted.log"; then + return 1 + fi + return 0 +} + +normal_get_body() { + local single=false + + if [[ "$FAKE_API_MODE" == delete-* ]]; then + single=true + fi + + case "$FAKE_API_DRIVER:$url" in + app-linode:*api.linode.com*/networking/firewalls\?*) + if resource_visible 'https://api.linode.com/v4/networking/firewalls/101'; then + printf '%s' '{"data":[{"id":101,"tags":["cloud-compose-smoke","linode-wp","gha-run-123456789"]},{"id":901,"tags":["cloud-compose-smoke","linode-wp","gha-run-foreign"]},{"id":902,"tags":["cloud-compose-smoke","linode-isle","gha-run-123456789"]}]}' + else + printf '%s' '{"data":[{"id":901,"tags":["cloud-compose-smoke","linode-wp","gha-run-foreign"]},{"id":902,"tags":["cloud-compose-smoke","linode-isle","gha-run-123456789"]}]}' + fi + ;; + app-linode:*api.linode.com*/linode/instances\?*) + if [[ "$single" == "true" ]]; then + printf '%s' '{"data":[]}' + elif resource_visible 'https://api.linode.com/v4/linode/instances/102'; then + printf '%s' '{"data":[{"id":102,"tags":["cloud-compose-smoke","linode-wp","gha-run-123456789"]},{"id":903,"tags":["cloud-compose-smoke","linode-wp","gha-run-foreign"]}]}' + else + printf '%s' '{"data":[{"id":903,"tags":["cloud-compose-smoke","linode-wp","gha-run-foreign"]}]}' + fi + ;; + app-linode:*api.linode.com*/volumes\?*) + if [[ "$single" == "true" ]]; then + printf '%s' '{"data":[]}' + elif resource_visible 'https://api.linode.com/v4/volumes/103'; then + printf '%s' '{"data":[{"id":103,"tags":["cloud-compose-smoke","linode-wp","gha-run-123456789"]},{"id":904,"tags":["cloud-compose-smoke","linode-isle","gha-run-123456789"]}]}' + else + printf '%s' '{"data":[{"id":904,"tags":["cloud-compose-smoke","linode-isle","gha-run-123456789"]}]}' + fi + ;; + config-management:*api.linode.com*/networking/firewalls\?*) + if resource_visible 'https://api.linode.com/v4/networking/firewalls/201'; then + printf '%s' '{"data":[{"id":201,"tags":["cloud-compose-smoke","config-management-smoke","config-management-ansible-drupal","gha-run-123456789"]},{"id":911,"tags":["cloud-compose-smoke","config-management-smoke","config-management-ansible-drupal","gha-run-foreign"]},{"id":912,"tags":["cloud-compose-smoke","config-management-smoke","config-management-salt-drupal","gha-run-123456789"]}]}' + else + printf '%s' '{"data":[{"id":911,"tags":["cloud-compose-smoke","config-management-smoke","config-management-ansible-drupal","gha-run-foreign"]},{"id":912,"tags":["cloud-compose-smoke","config-management-smoke","config-management-salt-drupal","gha-run-123456789"]}]}' + fi + ;; + config-management:*api.linode.com*/linode/instances\?*) + if [[ "$single" == "true" ]]; then + printf '%s' '{"data":[]}' + elif resource_visible 'https://api.linode.com/v4/linode/instances/202'; then + printf '%s' '{"data":[{"id":202,"tags":["cloud-compose-smoke","config-management-smoke","config-management-ansible-drupal","gha-run-123456789"]},{"id":913,"tags":["cloud-compose-smoke","config-management-smoke","config-management-ansible-drupal","gha-run-foreign"]}]}' + else + printf '%s' '{"data":[{"id":913,"tags":["cloud-compose-smoke","config-management-smoke","config-management-ansible-drupal","gha-run-foreign"]}]}' + fi + ;; + config-management:*api.linode.com*/volumes\?*) + if [[ "$single" == "true" ]]; then + printf '%s' '{"data":[]}' + elif resource_visible 'https://api.linode.com/v4/volumes/203'; then + printf '%s' '{"data":[{"id":203,"tags":["cloud-compose-smoke","config-management-smoke","config-management-ansible-drupal","gha-run-123456789"]},{"id":914,"tags":["cloud-compose-smoke","config-management-smoke","config-management-salt-drupal","gha-run-123456789"]}]}' + else + printf '%s' '{"data":[{"id":914,"tags":["cloud-compose-smoke","config-management-smoke","config-management-salt-drupal","gha-run-123456789"]}]}' + fi + ;; + app-digitalocean:*api.digitalocean.com*/firewalls\?*) + if resource_visible 'https://api.digitalocean.com/v2/firewalls/do-firewall'; then + printf '%s' '{"firewalls":[{"id":"do-firewall","name":"cc-do-isle-123456789-abcdef-cloud-compose"},{"id":"foreign-firewall-run","name":"cc-do-isle-987654321-abcdef-cloud-compose"},{"id":"foreign-firewall-target","name":"cc-do-wp-123456789-abcdef-cloud-compose"}]}' + else + printf '%s' '{"firewalls":[{"id":"foreign-firewall-run","name":"cc-do-isle-987654321-abcdef-cloud-compose"},{"id":"foreign-firewall-target","name":"cc-do-wp-123456789-abcdef-cloud-compose"}]}' + fi + ;; + app-digitalocean:*api.digitalocean.com*/droplets\?*) + if [[ "$single" == "true" ]]; then + printf '%s' '{"droplets":[]}' + elif resource_visible 'https://api.digitalocean.com/v2/droplets/301'; then + printf '%s' '{"droplets":[{"id":301,"tags":["cloud-compose-smoke","digitalocean-isle","gha-run-123456789"]},{"id":931,"tags":["cloud-compose-smoke","digitalocean-isle","gha-run-foreign"]}]}' + else + printf '%s' '{"droplets":[{"id":931,"tags":["cloud-compose-smoke","digitalocean-isle","gha-run-foreign"]}]}' + fi + ;; + app-digitalocean:*api.digitalocean.com*/volumes\?*) + if [[ "$single" == "true" ]]; then + printf '%s' '{"volumes":[]}' + elif resource_visible 'https://api.digitalocean.com/v2/volumes/do-volume'; then + printf '%s' '{"volumes":[{"id":"do-volume","tags":["cloud-compose-smoke","digitalocean-isle","gha-run-123456789"]},{"id":"foreign-volume","tags":["cloud-compose-smoke","digitalocean-wp","gha-run-123456789"]}]}' + else + printf '%s' '{"volumes":[{"id":"foreign-volume","tags":["cloud-compose-smoke","digitalocean-wp","gha-run-123456789"]}]}' + fi + ;; + *) + echo "unexpected fake API URL for $FAKE_API_DRIVER: $url" >&2 + exit 64 + ;; + esac +} + +if [[ "$method" == "DELETE" ]]; then + printf '%s\n' "$url" >>"$FAKE_API_STATE/deletes.log" + count="$(increment_count delete)" + case "$FAKE_API_MODE:$count" in + delete-transient:1) + exit 7 + ;; + delete-transient:2) + respond '{"error":"request timeout"}' 408 + ;; + delete-transient:3) + respond '{"error":"too early"}' 425 + ;; + delete-transient:4) + respond '{"error":"rate limited"}' 429 + ;; + delete-transient:5) + respond '{"error":"server error"}' 500 + ;; + delete-transient:6) + mark_deleted + respond '' 204 + ;; + delete-persistent:*) + respond '{"error":"unavailable"}' 503 + ;; + delete-auth:*) + respond '{"error":"unauthorized"}' 401 + ;; + delete-forbidden:*) + respond '{"error":"forbidden"}' 403 + ;; + delete-permanent:*) + respond '{"error":"bad request"}' 400 + ;; + delete-conflict:1) + respond '{"error":"conflict"}' 409 + ;; + delete-conflict:2) + respond '{"error":"locked"}' 423 + ;; + delete-conflict:3) + mark_deleted + respond '' 204 + ;; + delete-absent-404:*) + mark_deleted + respond '{"error":"not found"}' 404 + ;; + delete-absent-410:*) + mark_deleted + respond '{"error":"gone"}' 410 + ;; + *) + mark_deleted + respond '' 204 + ;; + esac + exit 0 +fi + +[[ "$method" == "GET" ]] || { + echo "unexpected fake curl method: $method" >&2 + exit 64 +} + +case "$url" in + *api.linode.com*/networking/firewalls\?* | *api.digitalocean.com*/firewalls\?*) primary=true ;; + *) primary=false ;; +esac + +if [[ "$primary" == "true" ]]; then + count="$(increment_count primary)" + case "$FAKE_API_MODE:$count" in + get-transient:1) + exit 7 + ;; + get-transient:2) + respond '{"error":"request timeout"}' 408 + exit 0 + ;; + get-transient:3) + respond '{"error":"too early"}' 425 + exit 0 + ;; + get-transient:4) + respond '{"error":"rate limited"}' 429 + exit 0 + ;; + get-transient:5) + respond '{"error":"server error"}' 500 + exit 0 + ;; + get-persistent:*) + respond '{"error":"unavailable"}' 503 + exit 0 + ;; + get-unauthorized:*) + respond '{"error":"unauthorized"}' 401 + exit 0 + ;; + get-forbidden:*) + respond '{"error":"forbidden"}' 403 + exit 0 + ;; + get-client-error:*) + respond '{"error":"bad request"}' 400 + exit 0 + ;; + get-not-found:*) + respond '{"error":"not found"}' 404 + exit 0 + ;; + get-gone:*) + respond '{"error":"gone"}' 410 + exit 0 + ;; + get-conflict:*) + respond '{"error":"conflict"}' 409 + exit 0 + ;; + get-locked:*) + respond '{"error":"locked"}' 423 + exit 0 + ;; + esac +fi + +if [[ "$FAKE_API_MODE" == "malformed" ]]; then + respond '{}' 200 +else + respond "$(normal_get_body)" 200 +fi +EOF + +cat >"$tmp/bin/terraform" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +printf 'terraform' >>"$FAKE_TERRAFORM_LOG" +printf ' %q' "$@" >>"$FAKE_TERRAFORM_LOG" +printf '\n' >>"$FAKE_TERRAFORM_LOG" + +command_name="" +for argument in "$@"; do + case "$argument" in + init | validate | apply | output | destroy) + command_name="$argument" + break + ;; + esac +done + +case "$command_name" in + init | validate) + exit 0 + ;; + apply) + if [[ -n "${FAKE_BODY_SIGNAL:-}" ]]; then + kill -s "$FAKE_BODY_SIGNAL" "$PPID" + exit 0 + fi + exit "${FAKE_BODY_STATUS:-0}" + ;; + output) + case "$FAKE_API_DRIVER" in + app-linode) + printf '%s\n' '{"host":"127.0.0.1","ssh_port":22,"ssh_user":"tester","project_dir":"/home/cloud-compose/app","context_name":"smoke","plugin":"wordpress","environment":"test","site":"smoke","project_name":"smoke","compose_project_name":"smoke","provider":"linode"}' + ;; + config-management) + printf '%s\n' '{"host":"127.0.0.1","method":"ansible","cloud_compose_name":"smoke","app":"drupal","environment":"test","project_dir":"/home/cloud-compose/app"}' + ;; + *) + echo "unexpected terraform output driver: $FAKE_API_DRIVER" >&2 + exit 64 + ;; + esac + ;; + destroy) + if [[ "${FAKE_DESTROY_MODE:-success}" == "failure" ]]; then + exit 42 + fi + ;; + *) + echo "unexpected fake terraform invocation: $*" >&2 + exit 64 + ;; +esac +EOF + +cat >"$tmp/bin/ssh-keygen" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +path="" +while [[ "$#" -gt 0 ]]; do + if [[ "$1" == "-f" ]]; then + path="${2:-}" + break + fi + shift +done +[[ -n "$path" ]] || exit 64 +printf '%s\n' 'fake-private-key' >"$path" +printf '%s\n' 'ssh-ed25519 fake-public-key cloud-compose-smoke' >"${path}.pub" +EOF + +cat >"$tmp/bin/ssh-keyscan" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' '127.0.0.1 ssh-ed25519 fake-host-key' +EOF + +cat >"$tmp/bin/ssh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +case "$*" in + *cloud-compose-bootstrap-complete*) printf 'complete\n' ;; + *cloud-init\ status*) printf 'cloud-init not installed\n' ;; +esac +EOF + +cat >"$tmp/bin/sitectl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +exit 0 +EOF + +cat >"$tmp/bin/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +cat >/dev/null +EOF + +cat >"$tmp/bin/tar" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf 'fake archive' +EOF + +chmod +x "$tmp/bin/"* + +driver_command() { + local driver="$1" operation="$2" + + case "$operation:$driver" in + sweep:app-linode) printf '%s\0%s\0%s\0' bash "$repo_root/ci/cloud-smoke.sh" sweep-linode-wp ;; + sweep:app-digitalocean) printf '%s\0%s\0%s\0' bash "$repo_root/ci/cloud-smoke.sh" sweep-digitalocean-isle ;; + sweep:config-management) printf '%s\0%s\0%s\0' bash "$repo_root/ci/config-management-cloud-smoke.sh" sweep-ansible-drupal ;; + flow:app-linode) printf '%s\0%s\0%s\0' bash "$repo_root/ci/cloud-smoke.sh" linode-wp ;; + flow:config-management) printf '%s\0%s\0%s\0' bash "$repo_root/ci/config-management-cloud-smoke.sh" ansible-drupal ;; + destroy:app-linode) printf '%s\0%s\0%s\0' bash "$repo_root/ci/cloud-smoke.sh" destroy-linode-wp ;; + destroy:config-management) printf '%s\0%s\0%s\0' bash "$repo_root/ci/config-management-cloud-smoke.sh" destroy-ansible-drupal ;; + *) fail "unknown driver operation ${operation}:${driver}" ;; + esac +} + +run_case() { + local operation="$1" driver="$2" mode="$3" + local destroy_mode="${4:-success}" body_signal="${5:-}" body_status="${6:-0}" + local output state status + local -a command + + state="$tmp/state/${operation}-${driver}-${mode}-${destroy_mode}-${body_signal:-none}-${body_status}" + mkdir -p "$state" + case "$operation:$driver" in + destroy:app-linode) mkdir -p "$state/work/linode-wp/.terraform" ;; + destroy:config-management) mkdir -p "$state/work/config-management-linode-ansible-drupal/.terraform" ;; + esac + output="$state/output.log" + mapfile -d '' -t command < <(driver_command "$driver" "$operation") + + if PATH="$tmp/bin:$PATH" \ + CLOUD_COMPOSE_SMOKE_AUTO_APPROVE=true \ + CLOUD_COMPOSE_SMOKE_BOOT_TIMEOUT=10 \ + CLOUD_COMPOSE_SMOKE_DESTROY_TIMEOUT=10 \ + CLOUD_COMPOSE_SMOKE_RUN_ID=123456789 \ + CLOUD_COMPOSE_SMOKE_WORKDIR="$state/work" \ + DIGITALOCEAN_TOKEN=test-token \ + FAKE_API_DRIVER="$driver" \ + FAKE_API_LOG="$state/api.log" \ + FAKE_API_MODE="$mode" \ + FAKE_API_STATE="$state" \ + FAKE_BODY_SIGNAL="$body_signal" \ + FAKE_BODY_STATUS="$body_status" \ + FAKE_DESTROY_MODE="$destroy_mode" \ + FAKE_TERRAFORM_LOG="$state/terraform.log" \ + GITHUB_ACTIONS=true \ + LINODE_TOKEN=test-token \ + "${command[@]}" >"$output" 2>&1; then + status=0 + else + status=$? + fi + printf '%s\n' "$status" >"$state/status" + printf '%s\n' "$state" +} + +assert_status() { + local state="$1" want="$2" got + + got="$(<"$state/status")" + if [[ "$want" == "success" && "$got" -ne 0 ]]; then + cat "$state/output.log" >&2 + fail "$(basename "$state") failed with status $got" + fi + if [[ "$want" == "failure" && "$got" -eq 0 ]]; then + cat "$state/output.log" >&2 + fail "$(basename "$state") unexpectedly succeeded" + fi + if [[ "$want" =~ ^[0-9]+$ && "$got" -ne "$want" ]]; then + cat "$state/output.log" >&2 + fail "$(basename "$state") returned $got, want $want" + fi +} + +count_file() { + local state="$1" key="$2" + + if [[ -f "$state/$key.count" ]]; then + cat "$state/$key.count" + else + printf '0\n' + fi +} + +owned_delete_urls() { + case "$1" in + app-linode) + printf '%s\n' \ + 'https://api.linode.com/v4/networking/firewalls/101' \ + 'https://api.linode.com/v4/linode/instances/102' \ + 'https://api.linode.com/v4/volumes/103' + ;; + config-management) + printf '%s\n' \ + 'https://api.linode.com/v4/networking/firewalls/201' \ + 'https://api.linode.com/v4/linode/instances/202' \ + 'https://api.linode.com/v4/volumes/203' + ;; + app-digitalocean) + printf '%s\n' \ + 'https://api.digitalocean.com/v2/firewalls/do-firewall' \ + 'https://api.digitalocean.com/v2/droplets/301' \ + 'https://api.digitalocean.com/v2/volumes/do-volume' + ;; + esac +} + +assert_owned_only() { + local state="$1" driver="$2" + local actual expected + + expected="$(owned_delete_urls "$driver")" + if [[ -f "$state/deletes.log" ]]; then + actual="$(<"$state/deletes.log")" + else + actual="" + fi + [[ "$actual" == "$expected" ]] || { + printf 'expected deletes:\n%s\nactual deletes:\n%s\n' "$expected" "$actual" >&2 + fail "$driver did not delete exactly its run-owned resources" + } + if grep -Eq 'foreign|/9[0-9][0-9]$' "$state/deletes.log"; then + fail "$driver selected a foreign resource" + fi +} + +assert_all_linode_queries() { + local state="$1" + + for path in networking/firewalls linode/instances volumes; do + grep -Fq "$path" "$state/api.log" || fail "$(basename "$state") skipped the $path cleanup pipeline" + done +} + +sleep_count() { + local state="$1" seconds="$2" count + + count="$(grep -c "^sleep ${seconds}$" "$state/api.log" 2>/dev/null || true)" + printf '%s\n' "$count" +} + +for driver in app-linode config-management; do + state="$(run_case sweep "$driver" get-transient)" + assert_status "$state" success + [[ "$(count_file "$state" primary)" == "7" ]] || fail "$driver did not recover on GET attempt six and verify cleanup" + assert_owned_only "$state" "$driver" + mapfile -t delays < <(awk '$1 == "sleep" {print $2}' "$state/api.log" | sed -n '1,5p') + [[ "${delays[*]}" == "2 4 8 16 30" ]] || fail "$driver used unexpected GET backoff: ${delays[*]}" + + state="$(run_case sweep "$driver" get-persistent)" + assert_status "$state" failure + [[ "$(count_file "$state" primary)" == "6" ]] || fail "$driver did not stop after six GET attempts" + + for mode in get-unauthorized get-forbidden get-client-error get-not-found get-gone get-conflict get-locked; do + state="$(run_case sweep "$driver" "$mode")" + assert_status "$state" failure + [[ "$(count_file "$state" primary)" == "1" ]] || fail "$driver retried fail-fast GET mode $mode" + done + + state="$(run_case sweep "$driver" malformed)" + assert_status "$state" failure + assert_all_linode_queries "$state" + [[ "$(count_file "$state" delete)" == "0" ]] || fail "$driver deleted resources from malformed API data" +done + +state="$(run_case sweep app-digitalocean get-transient)" +assert_status "$state" success +assert_owned_only "$state" app-digitalocean +state="$(run_case sweep app-digitalocean malformed)" +assert_status "$state" failure + +for driver in app-linode config-management app-digitalocean; do + state="$(run_case sweep "$driver" residual-delayed)" + assert_status "$state" success + assert_owned_only "$state" "$driver" + [[ "$(count_file "$state" primary)" == "4" ]] || fail "$driver did not wait for delayed residual visibility to clear" + [[ "$(sleep_count "$state" 10)" == "3" ]] || fail "$driver used unexpected delayed-residual backoff" + + state="$(run_case sweep "$driver" residual-persistent)" + assert_status "$state" failure + assert_owned_only "$state" "$driver" + [[ "$(count_file "$state" primary)" == "7" ]] || fail "$driver did not bound persistent residual verification at six attempts" + [[ "$(sleep_count "$state" 10)" == "6" ]] || fail "$driver used unexpected persistent-residual backoff" +done + +for driver in app-linode config-management; do + while read -r mode want_status want_attempts retry_sleeps; do + state="$(run_case sweep "$driver" "$mode")" + assert_status "$state" "$want_status" + [[ "$(count_file "$state" delete)" == "$want_attempts" ]] || \ + fail "$driver $mode used $(count_file "$state" delete) DELETE attempts, want $want_attempts" + # Each Linode sweep has one dependency-order pause between instances and volumes. + [[ "$(sleep_count "$state" 10)" == "$((retry_sleeps + 1))" ]] || \ + fail "$driver $mode used $(sleep_count "$state" 10) ten-second sleeps, want $((retry_sleeps + 1))" + assert_all_linode_queries "$state" + done <<'EOF' +delete-transient success 6 5 +delete-persistent failure 12 11 +delete-auth failure 1 0 +delete-forbidden failure 1 0 +delete-permanent failure 1 0 +delete-conflict success 3 2 +delete-absent-404 success 1 0 +delete-absent-410 success 1 0 +EOF +done + +for driver in app-linode config-management; do + state="$(run_case flow "$driver" owned failure)" + assert_status "$state" success + assert_owned_only "$state" "$driver" + grep -Fq ' destroy ' "$state/terraform.log" || fail "$driver EXIT cleanup did not run Terraform destroy" + + state="$(run_case flow "$driver" delete-persistent failure)" + assert_status "$state" 42 + [[ "$(count_file "$state" delete)" == "12" ]] || fail "$driver EXIT fallback did not exhaust DELETE retries" + + state="$(run_case flow "$driver" delete-persistent failure '' 37)" + assert_status "$state" 37 + + while read -r signal want_status; do + state="$(run_case flow "$driver" owned success "$signal")" + assert_status "$state" "$want_status" + grep -Fq ' destroy ' "$state/terraform.log" || fail "$driver $signal path skipped EXIT destroy" + done <<'EOF' +HUP 129 +INT 130 +TERM 143 +EOF +done + +for driver in app-linode config-management; do + state="$(run_case destroy "$driver" owned failure)" + assert_status "$state" success + assert_owned_only "$state" "$driver" + + state="$(run_case destroy "$driver" residual-persistent failure)" + assert_status "$state" 42 +done + +echo "Hosted cleanup retry contracts passed"