diff --git a/.github/workflows/cloud-smoke-cleanup.yml b/.github/workflows/cloud-smoke-cleanup.yml index c06ce57..06117bd 100644 --- a/.github/workflows/cloud-smoke-cleanup.yml +++ b/.github/workflows/cloud-smoke-cleanup.yml @@ -9,43 +9,92 @@ on: permissions: contents: read - id-token: write jobs: cleanup: name: Destroy ${{ matrix.name }} - if: github.event.workflow_run.head_repository.full_name == github.repository && contains(fromJSON('["failure","cancelled","timed_out"]'), github.event.workflow_run.conclusion) + if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.head_repository.full_name == github.repository && contains(fromJSON('["failure","cancelled","timed_out"]'), github.event.workflow_run.conclusion) runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: include: - name: DigitalOcean ISLE + kind: app provider: digitalocean template: isle + environment: cloud-smoke-cleanup-digitalocean + concurrency_group: cloud-compose-smoke-digitalocean-isle - name: Linode WordPress + kind: app provider: linode template: wp - - name: GCP WordPress - provider: gcp - template: wp + environment: cloud-smoke-cleanup-linode + concurrency_group: cloud-compose-smoke-linode-wp + - name: Linode Ansible Drupal config-management + kind: config-management + provider: linode + method: ansible + environment: cloud-smoke-cleanup-linode + concurrency_group: cloud-compose-config-management-ansible-drupal + - name: Linode Salt Drupal config-management + kind: config-management + provider: linode + method: salt + environment: cloud-smoke-cleanup-linode + concurrency_group: cloud-compose-config-management-salt-drupal + environment: ${{ matrix.environment }} + concurrency: + group: ${{ matrix.concurrency_group }} + cancel-in-progress: false + env: + CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.event.workflow_run.id }} + DIGITALOCEAN_TOKEN: ${{ matrix.provider == 'digitalocean' && secrets.DIGITALOCEAN_TOKEN || '' }} + LINODE_TOKEN: ${{ matrix.provider == 'linode' && secrets.LINODE_TOKEN || '' }} + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + ref: ${{ github.sha }} + + - name: Sweep provider smoke resources + if: matrix.kind == 'app' + run: ci/cloud-smoke.sh sweep-${{ matrix.provider }}-${{ matrix.template }} + + - name: Sweep config-management smoke resources + if: matrix.kind == 'config-management' + run: ci/config-management-cloud-smoke.sh sweep-${{ matrix.method }}-drupal + + gcp-cleanup: + name: Destroy GCP WordPress + if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.head_repository.full_name == github.repository && contains(fromJSON('["failure","cancelled","timed_out"]'), github.event.workflow_run.conclusion) + runs-on: ubuntu-24.04 + environment: cloud-smoke-cleanup-gcp + permissions: + contents: read + id-token: write concurrency: - group: cloud-compose-smoke-${{ matrix.provider }}-${{ matrix.template }} + group: cloud-compose-smoke-gcp-wp cancel-in-progress: false env: CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.event.workflow_run.id }} - DIGITALOCEAN_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }} GCLOUD_OIDC_POOL: ${{ vars.GCLOUD_OIDC_POOL || secrets.GCLOUD_OIDC_POOL }} GCLOUD_PROJECT: ${{ vars.GCLOUD_PROJECT || secrets.GCLOUD_PROJECT }} GCLOUD_REGION: ${{ vars.GCLOUD_REGION || secrets.GCLOUD_REGION }} GSA: ${{ vars.GSA || secrets.GSA }} - LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + ref: ${{ github.sha }} + + - name: Check GCP cleanup configuration + run: | + test -n "$GCLOUD_OIDC_POOL" || { echo "GCLOUD_OIDC_POOL environment variable or secret is required for GCP cleanup"; exit 1; } + test -n "$GSA" || { echo "GSA environment variable or secret is required for GCP cleanup"; exit 1; } + test -n "$GCLOUD_PROJECT" || { echo "GCLOUD_PROJECT environment variable or secret is required for GCP cleanup"; exit 1; } - name: Authenticate to Google Cloud - if: matrix.provider == 'gcp' && env.GCLOUD_OIDC_POOL != '' && env.GSA != '' && env.GCLOUD_PROJECT != '' uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 with: workload_identity_provider: ${{ env.GCLOUD_OIDC_POOL }} @@ -53,13 +102,7 @@ jobs: project_id: ${{ env.GCLOUD_PROJECT }} - name: Install gcloud - if: matrix.provider == 'gcp' && env.GCLOUD_OIDC_POOL != '' && env.GSA != '' && env.GCLOUD_PROJECT != '' uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3 - - name: Sweep smoke resources - if: matrix.provider != 'gcp' || (env.GCLOUD_OIDC_POOL != '' && env.GSA != '' && env.GCLOUD_PROJECT != '') - run: ci/cloud-smoke.sh sweep-${{ matrix.provider }}-${{ matrix.template }} - - - name: Skip GCP cleanup - if: matrix.provider == 'gcp' && (env.GCLOUD_OIDC_POOL == '' || env.GSA == '' || env.GCLOUD_PROJECT == '') - run: echo "Skipping GCP cleanup because GCP smoke secrets are not configured." + - name: Sweep GCP smoke resources + run: ci/cloud-smoke.sh sweep-gcp-wp diff --git a/.github/workflows/cloud-smoke.yml b/.github/workflows/cloud-smoke.yml index ce8c3b1..a90d8dc 100644 --- a/.github/workflows/cloud-smoke.yml +++ b/.github/workflows/cloud-smoke.yml @@ -30,6 +30,9 @@ jobs: - name: Run Terraform lint run: make terraform-lint-check + - name: Validate fallback cleanup behavior + run: bash ci/cloud-smoke-cleanup-contract.sh + config-management-lint: name: Ansible and Salt lint runs-on: ubuntu-24.04 @@ -159,80 +162,3 @@ jobs: - name: Skip GCP destroy if: always() && matrix.provider == 'gcp' && (env.GCLOUD_OIDC_POOL == '' || env.GSA == '' || env.GCLOUD_PROJECT == '') run: echo "Skipping GCP destroy because GCP smoke secrets are not configured and no GCP resources were applied." - - config-management-cleanup: - name: Destroy Linode ${{ matrix.method }} Drupal config-management - needs: config-management-cloud-smoke - if: always() && (failure() || cancelled()) && github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - method: - - ansible - - salt - concurrency: - group: cloud-compose-config-management-${{ matrix.method }}-drupal - cancel-in-progress: false - env: - CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.run_id }} - LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }} - steps: - - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - - - name: Sweep Linode config-management smoke resources - run: ci/config-management-cloud-smoke.sh sweep-${{ matrix.method }}-drupal - - cleanup: - name: Destroy ${{ matrix.name }} - needs: smoke - if: always() && (failure() || cancelled()) - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - include: - - name: DigitalOcean ISLE - provider: digitalocean - template: isle - - name: Linode WordPress - provider: linode - template: wp - - name: GCP WordPress - provider: gcp - template: wp - concurrency: - group: cloud-compose-smoke-${{ matrix.provider }}-${{ matrix.template }} - cancel-in-progress: false - env: - CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.run_id }} - DIGITALOCEAN_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }} - GCLOUD_OIDC_POOL: ${{ vars.GCLOUD_OIDC_POOL || secrets.GCLOUD_OIDC_POOL }} - GCLOUD_PROJECT: ${{ vars.GCLOUD_PROJECT || secrets.GCLOUD_PROJECT }} - GCLOUD_REGION: ${{ vars.GCLOUD_REGION || secrets.GCLOUD_REGION }} - GSA: ${{ vars.GSA || secrets.GSA }} - LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }} - steps: - - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - - - name: Authenticate to Google Cloud - if: matrix.provider == 'gcp' && env.GCLOUD_OIDC_POOL != '' && env.GSA != '' && env.GCLOUD_PROJECT != '' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - workload_identity_provider: ${{ env.GCLOUD_OIDC_POOL }} - service_account: ${{ env.GSA }} - project_id: ${{ env.GCLOUD_PROJECT }} - - - name: Install gcloud - if: matrix.provider == 'gcp' && env.GCLOUD_OIDC_POOL != '' && env.GSA != '' && env.GCLOUD_PROJECT != '' - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3 - - - name: Sweep smoke resources - if: matrix.provider != 'gcp' || (env.GCLOUD_OIDC_POOL != '' && env.GSA != '' && env.GCLOUD_PROJECT != '') - run: ci/cloud-smoke.sh sweep-${{ matrix.provider }}-${{ matrix.template }} - - - name: Skip GCP cleanup - if: matrix.provider == 'gcp' && (env.GCLOUD_OIDC_POOL == '' || env.GSA == '' || env.GCLOUD_PROJECT == '') - run: echo "Skipping GCP cleanup because GCP smoke secrets are not configured." diff --git a/ci/cloud-smoke-cleanup-contract.sh b/ci/cloud-smoke-cleanup-contract.sh new file mode 100644 index 0000000..fabfbb5 --- /dev/null +++ b/ci/cloud-smoke-cleanup-contract.sh @@ -0,0 +1,336 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +tmp="$(mktemp -d "${TMPDIR:-/tmp}/cloud-compose-cleanup.XXXXXX")" +trap 'rm -rf "$tmp"' EXIT + +fail() { + echo "cloud smoke cleanup contract: $*" >&2 + exit 1 +} + +mkdir -p "$tmp/bin" "$tmp/state" + +cat >"$tmp/bin/sleep" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf 'sleep %s\n' "$*" >>"$FAKE_GCLOUD_LOG" +EOF + +cat >"$tmp/bin/gcloud" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +printf 'gcloud' >>"$FAKE_GCLOUD_LOG" +printf ' %q' "$@" >>"$FAKE_GCLOUD_LOG" +printf '\n' >>"$FAKE_GCLOUD_LOG" + +state_count() { + local key="$1" + + if [[ -f "$FAKE_GCLOUD_STATE/$key" ]]; then + cat "$FAKE_GCLOUD_STATE/$key" + else + printf '0\n' + fi +} + +mutation_done() { + [[ -f "$FAKE_GCLOUD_STATE/$1.done" ]] +} + +emit_project_binding() { + local state_key="$1" role="$2" member="$3" separator_name="$4" + local -n separator_ref="$separator_name" + + if mutation_done "$state_key"; then + return 0 + fi + printf '%s{"role":"%s","members":["%s"]}' "$separator_ref" "$role" "$member" + separator_ref=, +} + +command_name="" +case "${1:-} ${2:-} ${3:-} ${4:-}" in + "run services list "*) + if [[ "${FAKE_GCLOUD_MODE:-success}" == "residual-cloud-run" ]] || ! mutation_done run-services-delete; then + printf 'cc-g-wp-12345678-abcd\n' + fi + exit 0 + ;; + "run services get-iam-policy "*) + if mutation_done run-invoker-remove; then + printf '{"bindings":[]}\n' + else + printf '{"bindings":[{"role":"roles/run.invoker","members":["allUsers"]}]}\n' + fi + exit 0 + ;; + "compute instances list "*) + if ! mutation_done instances-delete; then + if [[ " $* " == *" --format=json "* ]]; then + printf '[{"zone":"https://www.googleapis.com/compute/v1/projects/test-project/zones/us-east5-b","name":"cc-g-wp-12345678-abcd"}]\n' + else + printf 'cc-g-wp-12345678-abcd\n' + fi + fi + exit 0 + ;; + "compute firewall-rules list "*) + if ! mutation_done firewalls-delete; then + printf 'allow-ssh-ipv4-cc-g-wp-12345678-abcd\n' + fi + exit 0 + ;; + "compute disks list "*) + if ! mutation_done disks-delete; then + if [[ " $* " == *" --format=json "* ]]; then + printf '[{"zone":"https://www.googleapis.com/compute/v1/projects/test-project/zones/us-east5-b","name":"cc-g-wp-12345678-abcd-data-disk"}]\n' + else + printf 'cc-g-wp-12345678-abcd-data-disk\n' + fi + fi + exit 0 + ;; + "iam service-accounts list "*) + if ! mutation_done service-accounts-delete; then + printf '%s\n' \ + 'vm-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com' \ + 'internal-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com' \ + 'ppb-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com' \ + 'cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com' + fi + exit 0 + ;; + "compute networks subnets list") + if ! mutation_done subnets-delete; then + if [[ " $* " == *" --format=json "* ]]; then + printf '[{"region":"https://www.googleapis.com/compute/v1/projects/test-project/regions/us-east5","name":"cc-g-wp-12345678-abcd"}]\n' + else + printf 'cc-g-wp-12345678-abcd\n' + fi + fi + exit 0 + ;; + "compute networks list "*) + if ! mutation_done networks-delete; then + printf 'cc-g-wp-12345678-abcd\n' + fi + exit 0 + ;; + "projects get-iam-policy test-project "*) + separator="" + printf '{"bindings":[' + emit_project_binding iam-log-remove roles/logging.logWriter \ + serviceAccount:vm-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com separator + emit_project_binding iam-monitoring-remove roles/monitoring.metricWriter \ + serviceAccount:internal-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com separator + emit_project_binding iam-suspend-remove projects/test-project/roles/suspendVM \ + serviceAccount:internal-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com separator + emit_project_binding iam-start-remove projects/test-project/roles/startVM \ + serviceAccount:ppb-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com separator + printf ']}\n' + exit 0 + ;; + "run services remove-iam-policy-binding "*) + command_name="run-invoker-remove" + ;; + "run services delete "*) + command_name="run-services-delete" + ;; + "projects remove-iam-policy-binding "*) + case " $* " in + *" --role roles/logging.logWriter "*) command_name="iam-log-remove" ;; + *" --role roles/monitoring.metricWriter "*) command_name="iam-monitoring-remove" ;; + *" --role projects/test-project/roles/suspendVM "*) command_name="iam-suspend-remove" ;; + *" --role projects/test-project/roles/startVM "*) command_name="iam-start-remove" ;; + *) echo "unexpected project IAM removal: $*" >&2; exit 64 ;; + esac + ;; + "compute instances delete "*) + command_name="instances-delete" + ;; + "compute firewall-rules delete "*) + command_name="firewalls-delete" + ;; + "compute disks delete "*) + command_name="disks-delete" + ;; + "iam service-accounts delete "*) + command_name="service-accounts-delete" + ;; + "compute networks subnets delete") + command_name="subnets-delete" + ;; + "compute networks delete "*) + command_name="networks-delete" + ;; + *) + echo "unexpected gcloud invocation: $*" >&2 + exit 64 + ;; +esac + +count_file="$FAKE_GCLOUD_STATE/$command_name" +count="$(state_count "$command_name")" +count=$((count + 1)) +printf '%s\n' "$count" >"$count_file" + +case "${FAKE_GCLOUD_MODE:-success}:$command_name:$count" in + retry-transient:run-services-delete:1 | retry-transient:iam-log-remove:1 | retry-transient:networks-delete:1) + exit 1 + ;; +esac +if [[ "${FAKE_GCLOUD_MODE:-success}" == "aggregate-failure" && + ( "$command_name" == "run-services-delete" || "$command_name" == "instances-delete" || "$command_name" == "subnets-delete" ) ]]; then + exit 1 +fi + +touch "$FAKE_GCLOUD_STATE/$command_name.done" +EOF + +chmod +x "$tmp/bin/gcloud" "$tmp/bin/sleep" + +run_cleanup() { + local mode="$1" log="$2" state="$3" + + mkdir -p "$state" + PATH="$tmp/bin:$PATH" \ + FAKE_GCLOUD_LOG="$log" \ + FAKE_GCLOUD_MODE="$mode" \ + FAKE_GCLOUD_STATE="$state" \ + GCLOUD_PROJECT=test-project \ + CLOUD_COMPOSE_SMOKE_RUN_ID=123456789 \ + bash "$repo_root/ci/cloud-smoke.sh" sweep-gcp-wp +} + +success_log="$tmp/success.log" +run_cleanup retry-transient "$success_log" "$tmp/state/success" + +[[ "$(<"$tmp/state/success/run-services-delete")" == "2" ]] || \ + fail "Cloud Run cleanup did not retry a transient delete failure" +[[ "$(<"$tmp/state/success/iam-log-remove")" == "2" ]] || \ + fail "GCP IAM cleanup did not retry a transient removal failure" +[[ "$(<"$tmp/state/success/networks-delete")" == "2" ]] || \ + fail "GCP network cleanup did not retry a transient delete failure" + +for command_name in \ + "run services remove-iam-policy-binding" \ + "run services delete" \ + "projects remove-iam-policy-binding" \ + "compute instances delete" \ + "compute firewall-rules delete" \ + "compute disks delete" \ + "iam service-accounts delete" \ + "compute networks subnets delete" \ + "compute networks delete"; do + grep -Fq "gcloud ${command_name}" "$success_log" || \ + fail "GCP fallback omitted ${command_name}" +done + +grep -F 'gcloud compute instances list' "$success_log" | grep -Fq 'cc-g-wp-12345678-' || \ + fail "GCP cleanup did not constrain resources to the originating workflow run" +grep -F 'gcloud iam service-accounts list' "$success_log" | grep -Fq 'ppb-' || \ + fail "GCP cleanup does not select the power-button service account" +for role in \ + roles/logging.logWriter \ + roles/monitoring.metricWriter \ + projects/test-project/roles/suspendVM \ + projects/test-project/roles/startVM; do + grep -F 'gcloud projects remove-iam-policy-binding' "$success_log" | grep -Fq "$role" || \ + fail "GCP cleanup omitted project IAM role ${role}" +done +for account in \ + vm-cc-g-wp-12345678-abcd \ + internal-cc-g-wp-12345678-abcd \ + ppb-cc-g-wp-12345678-abcd \ + cc-g-wp-12345678-abcd; do + grep -F 'gcloud iam service-accounts delete' "$success_log" | grep -Fq "$account@test-project.iam.gserviceaccount.com" || \ + fail "GCP cleanup omitted service account ${account}" +done +grep -F 'gcloud run services remove-iam-policy-binding' "$success_log" | grep -Fq -- '--condition=None' || \ + fail "Cloud Run invoker cleanup did not target the unconditional binding" +if grep -F 'gcloud projects remove-iam-policy-binding' "$success_log" | grep -Fv -- '--condition=None' >/dev/null; then + fail "Project IAM cleanup did not target unconditional bindings explicitly" +fi + +run_invoker_line="$(grep -nF 'gcloud run services remove-iam-policy-binding' "$success_log" | head -n1 | cut -d: -f1)" +run_delete_line="$(grep -nF 'gcloud run services delete' "$success_log" | tail -n1 | cut -d: -f1)" +instance_line="$(grep -nF 'gcloud compute instances delete' "$success_log" | head -n1 | cut -d: -f1)" +[[ "$run_invoker_line" -lt "$run_delete_line" && "$run_delete_line" -lt "$instance_line" ]] || \ + fail "GCP fallback did not remove Cloud Run ingress before deleting the service and VM" + +last_project_iam_line="$(grep -nF 'gcloud projects remove-iam-policy-binding' "$success_log" | tail -n1 | cut -d: -f1)" +first_service_account_line="$(grep -nF 'gcloud iam service-accounts delete' "$success_log" | head -n1 | cut -d: -f1)" +[[ "$last_project_iam_line" -lt "$first_service_account_line" ]] || \ + fail "GCP fallback deleted service accounts before removing their project IAM bindings" + +subnet_line="$(grep -nF 'gcloud compute networks subnets delete' "$success_log" | tail -n1 | cut -d: -f1)" +network_line="$(grep -nF 'gcloud compute networks delete' "$success_log" | head -n1 | cut -d: -f1)" +[[ "$subnet_line" -lt "$network_line" ]] || \ + fail "GCP fallback did not delete subnetworks before their parent networks" +last_verification_line="$(grep -nF 'gcloud projects get-iam-policy' "$success_log" | tail -n1 | cut -d: -f1)" +[[ "$network_line" -lt "$last_verification_line" ]] || \ + fail "GCP fallback did not verify residual resources after all deletions" + +[[ "$(<"$tmp/state/success/service-accounts-delete")" == "4" ]] || \ + fail "GCP cleanup did not delete VM, internal, power-button, and app service accounts" + +failure_log="$tmp/failure.log" +if run_cleanup aggregate-failure "$failure_log" "$tmp/state/failure" >/dev/null 2>&1; then + fail "GCP fallback reported success after permanent resource deletion failures" +fi +[[ "$(<"$tmp/state/failure/run-services-delete")" == "12" ]] || \ + fail "Cloud Run deletion did not exhaust its retry budget" +[[ "$(<"$tmp/state/failure/instances-delete")" == "12" ]] || \ + fail "GCP instance deletion did not exhaust its retry budget" +[[ "$(<"$tmp/state/failure/subnets-delete")" == "12" ]] || \ + fail "GCP subnetwork deletion did not exhaust its retry budget" +[[ -f "$tmp/state/failure/networks-delete" ]] || \ + fail "GCP cleanup stopped instead of aggregating failures across resource kinds" +[[ -f "$tmp/state/failure/service-accounts-delete" ]] || \ + fail "GCP cleanup skipped unrelated resources after an earlier failure" + +residual_log="$tmp/residual.log" +if run_cleanup residual-cloud-run "$residual_log" "$tmp/state/residual" >/dev/null 2>&1; then + fail "GCP fallback reported success while a matching Cloud Run service remained" +fi +[[ "$(grep -cF 'gcloud run services list' "$residual_log")" -ge 13 ]] || \ + fail "GCP cleanup did not exhaust residual-resource verification retries" + +pr_workflow="$repo_root/.github/workflows/cloud-smoke.yml" +cleanup_workflow="$repo_root/.github/workflows/cloud-smoke-cleanup.yml" +docs="$repo_root/docs/runtime-contracts.md" + +grep -Fq "if: always()" "$pr_workflow" || \ + fail "PR smoke jobs no longer destroy resources in the already-approved job" +if grep -Eq '^ (config-management-cleanup|cleanup|gcp-cleanup):' "$pr_workflow"; then + fail "PR-controlled workflow still contains a second secret-bearing cleanup job" +fi +grep -Fq "workflow_run:" "$cleanup_workflow" || \ + fail "trusted default-branch fallback cleanup is missing" +grep -Fq "github.event.workflow_run.event == 'pull_request'" "$cleanup_workflow" || \ + fail "fallback cleanup is not restricted to pull-request workflow runs" +grep -Fq "github.event.workflow_run.head_repository.full_name == github.repository" "$cleanup_workflow" || \ + fail "fallback cleanup is not restricted to same-repository workflow runs" +grep -Fq "ref: \${{ github.sha }}" "$cleanup_workflow" || \ + fail "fallback cleanup does not check out the trusted default-branch revision" +if grep -Fq "github.event.workflow_run.head_sha" "$cleanup_workflow"; then + fail "fallback cleanup checks out pull-request-controlled code" +fi +for environment in \ + cloud-smoke-cleanup-digitalocean \ + cloud-smoke-cleanup-linode \ + cloud-smoke-cleanup-gcp; do + if grep -Fq "$environment" "$pr_workflow"; then + fail "pull-request-controlled workflow can request cleanup environment ${environment}" + fi + grep -Fq "$environment" "$cleanup_workflow" || \ + fail "fallback cleanup is missing dedicated environment ${environment}" + grep -Fq "$environment" "$docs" || \ + fail "runtime documentation omits cleanup environment ${environment}" +done + +echo "Cloud smoke cleanup contracts passed" diff --git a/ci/cloud-smoke.sh b/ci/cloud-smoke.sh index 2db6f2b..c5f6a8a 100755 --- a/ci/cloud-smoke.sh +++ b/ci/cloud-smoke.sh @@ -167,7 +167,7 @@ api_request() { if response="$(curl -sS -X "$method" -H "Authorization: Bearer ${token}" -w $'\n%{http_code}' "${base_url}${path}")"; then http_code="${response##*$'\n'}" - body="${response%$'\n'$http_code}" + body="${response%$'\n'"$http_code"}" else echo "${provider} API request failed for ${method} ${path}; check ${token_name} and network access." >&2 return 1 @@ -213,19 +213,198 @@ gcp_zone() { delete_ids() { local provider="$1" path_prefix="$2" id attempt + local failed=0 deleted while IFS= read -r id; do if [[ -z "$id" ]]; then continue fi + deleted=false for attempt in {1..12}; do echo "Deleting ${provider} ${path_prefix}/${id} (attempt ${attempt})" if api_delete "$provider" "${path_prefix}/${id}"; then + deleted=true break fi sleep 10 done + if [[ "$deleted" != "true" ]]; then + echo "Failed to delete ${provider} ${path_prefix}/${id} after 12 attempts" >&2 + failed=1 + fi + done + + return "$failed" +} + +gcp_command_with_retry() { + local operation="$1" description="$2" attempt + shift 2 + + for attempt in {1..12}; do + echo "${operation} gcp ${description} (attempt ${attempt})" + if gcloud "$@"; then + return 0 + fi + if ((attempt < 12)); then + sleep 10 + fi done + + echo "Failed while ${operation,,} gcp ${description} after 12 attempts" >&2 + return 1 +} + +gcp_delete_with_retry() { + local description="$1" + shift + + gcp_command_with_retry "Deleting" "$description" "$@" +} + +gcp_remove_with_retry() { + local description="$1" + shift + + gcp_command_with_retry "Removing" "$description" "$@" +} + +gcp_project_iam_rows() { + local project="$1" name_filter="$2" account_pattern + + account_pattern="(vm-|internal-|ppb-)?${name_filter#^}.*@${project}\\.iam\\.gserviceaccount\\.com" + gcloud projects get-iam-policy "$project" --format=json | + jq -r \ + --arg account_pattern "$account_pattern" \ + --arg start_role "projects/${project}/roles/startVM" \ + --arg suspend_role "projects/${project}/roles/suspendVM" ' + [$start_role, $suspend_role, "roles/logging.logWriter", "roles/monitoring.metricWriter"] as $managed_roles | + .bindings[]? | + select(.condition == null) | + select(.role as $role | $managed_roles | index($role)) | + .role as $role | + .members[]? | + select(test("^(deleted:)?serviceAccount:" + $account_pattern + "(\\?uid=[^[:space:]]+)?$")) | + [$role, .] | @tsv + ' +} + +gcp_smoke_residuals() { + local project="$1" name_filter="$2" region="$3" + local output name email role member + + if ! output="$(gcloud run services list \ + --project "$project" \ + --region "$region" \ + --filter="metadata.name~'${name_filter}'" \ + --format='value(metadata.name)')"; then + echo "Could not verify Cloud Run cleanup" >&2 + return 1 + fi + while IFS= read -r name; do + [[ -n "$name" ]] && printf 'cloud-run\t%s\n' "$name" + done <<<"$output" + + if ! output="$(gcloud compute instances list \ + --project "$project" \ + --filter="name~'${name_filter}'" \ + --format='value(name)')"; then + echo "Could not verify GCP instance cleanup" >&2 + return 1 + fi + while IFS= read -r name; do + [[ -n "$name" ]] && printf 'instance\t%s\n' "$name" + done <<<"$output" + + if ! output="$(gcloud compute firewall-rules list \ + --project "$project" \ + --filter="name~'^(allow-ssh-ipv4-|allow-ssh-ipv6-|allow-rollout-ipv4-)${name_filter#^}'" \ + --format='value(name)')"; then + echo "Could not verify GCP firewall cleanup" >&2 + return 1 + fi + while IFS= read -r name; do + [[ -n "$name" ]] && printf 'firewall\t%s\n' "$name" + done <<<"$output" + + if ! output="$(gcloud compute disks list \ + --project "$project" \ + --filter="name~'${name_filter}'" \ + --format='value(name)')"; then + echo "Could not verify GCP disk cleanup" >&2 + return 1 + fi + while IFS= read -r name; do + [[ -n "$name" ]] && printf 'disk\t%s\n' "$name" + done <<<"$output" + + if ! output="$(gcloud iam service-accounts list \ + --project "$project" \ + --filter="email~'^(vm-|internal-|ppb-)?${name_filter#^}.*@${project}\\.iam\\.gserviceaccount\\.com$'" \ + --format='value(email)')"; then + echo "Could not verify GCP service-account cleanup" >&2 + return 1 + fi + while IFS= read -r email; do + [[ -n "$email" ]] && printf 'service-account\t%s\n' "$email" + done <<<"$output" + + if ! output="$(gcloud compute networks subnets list \ + --project "$project" \ + --filter="name~'${name_filter}'" \ + --format='value(name)')"; then + echo "Could not verify GCP subnetwork cleanup" >&2 + return 1 + fi + while IFS= read -r name; do + [[ -n "$name" ]] && printf 'subnetwork\t%s\n' "$name" + done <<<"$output" + + if ! output="$(gcloud compute networks list \ + --project "$project" \ + --filter="name~'${name_filter}'" \ + --format='value(name)')"; then + echo "Could not verify GCP network cleanup" >&2 + return 1 + fi + while IFS= read -r name; do + [[ -n "$name" ]] && printf 'network\t%s\n' "$name" + done <<<"$output" + + if ! output="$(gcp_project_iam_rows "$project" "$name_filter")"; then + echo "Could not verify GCP project-IAM cleanup" >&2 + return 1 + fi + while IFS=$'\t' read -r role member; do + [[ -n "$role" && -n "$member" ]] && printf 'project-iam\t%s\t%s\n' "$role" "$member" + done <<<"$output" + + return 0 +} + +gcp_verify_no_smoke_resources() { + local project="$1" name_filter="$2" region="$3" + local attempt residuals + + for attempt in {1..12}; do + residuals="" + if residuals="$(gcp_smoke_residuals "$project" "$name_filter" "$region")"; then + if [[ -z "$residuals" ]]; then + echo "Verified that no matching gcp smoke resources remain" + return 0 + fi + echo "Matching gcp smoke resources remain after cleanup attempt ${attempt}:" >&2 + printf '%s\n' "$residuals" >&2 + else + echo "Could not complete gcp residual-resource verification (attempt ${attempt})" >&2 + fi + if ((attempt < 12)); then + sleep 10 + fi + done + + echo "GCP smoke cleanup left matching resources or could not verify their removal" >&2 + return 1 } smoke_run_tag() { @@ -276,6 +455,7 @@ target_name_prefix() { provider_tag_cleanup() { local target="$1" run_id="${2:-}" run_tag run_fragment provider name_prefix + local cleanup_status=0 run_tag="$(smoke_run_tag "$run_id")" run_fragment="" @@ -289,96 +469,212 @@ provider_tag_cleanup() { digitalocean) api_get digitalocean "/firewalls?per_page=200" | jq -r --arg name_prefix "${name_prefix}-" --arg run_fragment "$run_fragment" '.firewalls[]? | select(.name | startswith($name_prefix)) | select($run_fragment == "" or (.name | contains($run_fragment))) | .id' | - delete_ids digitalocean "/firewalls" + delete_ids digitalocean "/firewalls" || cleanup_status=1 api_get digitalocean "/droplets?tag_name=cloud-compose-smoke&per_page=200" | jq -r --arg target "$target" --arg run_tag "$run_tag" '.droplets[]? | select((.tags // []) | index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id' | - delete_ids digitalocean "/droplets" + delete_ids digitalocean "/droplets" || cleanup_status=1 sleep 10 api_get digitalocean "/volumes?tag_name=cloud-compose-smoke&per_page=200" | jq -r --arg target "$target" --arg run_tag "$run_tag" '.volumes[]? | select((.tags // []) | index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id' | - delete_ids digitalocean "/volumes" + delete_ids digitalocean "/volumes" || cleanup_status=1 ;; linode) api_get linode "/networking/firewalls?page_size=500" | jq -r --arg target "$target" --arg run_tag "$run_tag" '.data[]? | select((.tags // []) | index("cloud-compose-smoke") and index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id' | - delete_ids linode "/networking/firewalls" + delete_ids linode "/networking/firewalls" || cleanup_status=1 api_get linode "/linode/instances?page_size=500" | jq -r --arg target "$target" --arg run_tag "$run_tag" '.data[]? | select((.tags // []) | index("cloud-compose-smoke") and index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id' | - delete_ids linode "/linode/instances" + delete_ids linode "/linode/instances" || cleanup_status=1 sleep 10 api_get linode "/volumes?page_size=500" | jq -r --arg target "$target" --arg run_tag "$run_tag" '.data[]? | select((.tags // []) | index("cloud-compose-smoke") and index($target)) | select($run_tag == "" or ((.tags // []) | index($run_tag))) | .id' | - delete_ids linode "/volumes" + delete_ids linode "/volumes" || cleanup_status=1 ;; gcp) - local project name_filter + local project name_filter region cloud_run_services instance_rows firewall_names disk_rows + local project_iam_rows service_accounts subnetwork_rows network_names + local zone_url region_url name email role member cloud_run_policy project="$GCLOUD_PROJECT" + region="$(gcp_region)" name_filter="^${name_prefix}-" if [[ -n "$run_id" ]]; then name_filter="^${name_prefix}-$(printf '%s' "$run_id" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | cut -c1-8)-" fi - gcloud compute instances list \ + if ! cloud_run_services="$(gcloud run services list \ + --project "$project" \ + --region "$region" \ + --filter="metadata.name~'${name_filter}'" \ + --format='value(metadata.name)')"; then + cleanup_status=1 + cloud_run_services="" + fi + while IFS= read -r name; do + [[ -n "$name" ]] || continue + cloud_run_policy="" + if cloud_run_policy="$(gcloud run services get-iam-policy "$name" \ + --project "$project" \ + --region "$region" \ + --format=json)"; then + if ! jq -e 'type == "object" and ((has("bindings") | not) or (.bindings | type == "array"))' \ + <<<"$cloud_run_policy" >/dev/null; then + echo "Cloud Run IAM policy for ${name} was not valid JSON policy data" >&2 + cleanup_status=1 + elif jq -e ' + any(.bindings[]?; + .role == "roles/run.invoker" and + .condition == null and + any(.members[]?; . == "allUsers") + ) + ' <<<"$cloud_run_policy" >/dev/null; then + if ! gcp_remove_with_retry "Cloud Run invoker from ${name}" \ + run services remove-iam-policy-binding "$name" \ + --project "$project" \ + --region "$region" \ + --member allUsers \ + --role roles/run.invoker \ + --condition=None \ + --quiet; then + cleanup_status=1 + fi + fi + else + echo "Could not inspect Cloud Run IAM policy for ${name}" >&2 + cleanup_status=1 + fi + if ! gcp_delete_with_retry "Cloud Run service ${name}" run services delete "$name" \ + --project "$project" \ + --region "$region" \ + --quiet; then + cleanup_status=1 + fi + done <<<"$cloud_run_services" + + if ! instance_rows="$(gcloud compute instances list \ --project "$project" \ --filter="name~'${name_filter}'" \ --format=json | - jq -r '.[]? | [.zone, .name] | @tsv' | - while IFS=$'\t' read -r zone_url name; do - [[ -n "$name" ]] || continue - echo "Deleting gcp instance ${name}" - gcloud compute instances delete "$name" \ - --project "$project" \ - --zone "${zone_url##*/}" \ - --quiet || true - done - - gcloud compute firewall-rules list \ + jq -r '.[]? | [.zone, .name] | @tsv')"; then + cleanup_status=1 + instance_rows="" + fi + while IFS=$'\t' read -r zone_url name; do + [[ -n "$name" ]] || continue + if ! gcp_delete_with_retry "instance ${name}" compute instances delete "$name" \ + --project "$project" \ + --zone "${zone_url##*/}" \ + --quiet; then + cleanup_status=1 + fi + done <<<"$instance_rows" + + if ! firewall_names="$(gcloud compute firewall-rules list \ --project "$project" \ - --filter="name~'^(allow-ssh-ipv4-|allow-ssh-ipv6-)${name_filter#^}'" \ - --format='value(name)' | - while IFS= read -r name; do - [[ -n "$name" ]] || continue - echo "Deleting gcp firewall ${name}" - gcloud compute firewall-rules delete "$name" --project "$project" --quiet || true - done - - gcloud compute disks list \ + --filter="name~'^(allow-ssh-ipv4-|allow-ssh-ipv6-|allow-rollout-ipv4-)${name_filter#^}'" \ + --format='value(name)')"; then + cleanup_status=1 + firewall_names="" + fi + while IFS= read -r name; do + [[ -n "$name" ]] || continue + if ! gcp_delete_with_retry "firewall ${name}" compute firewall-rules delete "$name" \ + --project "$project" \ + --quiet; then + cleanup_status=1 + fi + done <<<"$firewall_names" + + if ! disk_rows="$(gcloud compute disks list \ --project "$project" \ --filter="name~'${name_filter}'" \ --format=json | - jq -r '.[]? | [.zone, .name] | @tsv' | - while IFS=$'\t' read -r zone_url name; do - [[ -n "$name" ]] || continue - echo "Deleting gcp disk ${name}" - gcloud compute disks delete "$name" \ - --project "$project" \ - --zone "${zone_url##*/}" \ - --quiet || true - done - - gcloud iam service-accounts list \ + jq -r '.[]? | [.zone, .name] | @tsv')"; then + cleanup_status=1 + disk_rows="" + fi + while IFS=$'\t' read -r zone_url name; do + [[ -n "$name" ]] || continue + if ! gcp_delete_with_retry "disk ${name}" compute disks delete "$name" \ + --project "$project" \ + --zone "${zone_url##*/}" \ + --quiet; then + cleanup_status=1 + fi + done <<<"$disk_rows" + + if ! project_iam_rows="$(gcp_project_iam_rows "$project" "$name_filter")"; then + cleanup_status=1 + project_iam_rows="" + fi + while IFS=$'\t' read -r role member; do + [[ -n "$role" && -n "$member" ]] || continue + if ! gcp_remove_with_retry "project IAM binding ${role} for ${member}" \ + projects remove-iam-policy-binding "$project" \ + --member "$member" \ + --role "$role" \ + --condition=None \ + --quiet; then + cleanup_status=1 + fi + done <<<"$project_iam_rows" + + if ! service_accounts="$(gcloud iam service-accounts list \ + --project "$project" \ + --filter="email~'^(vm-|internal-|ppb-)?${name_filter#^}.*@${project}\\.iam\\.gserviceaccount\\.com$'" \ + --format='value(email)')"; then + cleanup_status=1 + service_accounts="" + fi + while IFS= read -r email; do + [[ -n "$email" ]] || continue + if ! gcp_delete_with_retry "service account ${email}" iam service-accounts delete "$email" \ + --project "$project" \ + --quiet; then + cleanup_status=1 + fi + done <<<"$service_accounts" + + if ! subnetwork_rows="$(gcloud compute networks subnets list \ --project "$project" \ - --filter="email~'^(vm-|internal-)?${name_filter#^}.*@${project}\\.iam\\.gserviceaccount\\.com$'" \ - --format='value(email)' | - while IFS= read -r email; do - [[ -n "$email" ]] || continue - echo "Deleting gcp service account ${email}" - gcloud projects remove-iam-policy-binding "$project" \ - --member "serviceAccount:${email}" \ - --role roles/logging.logWriter \ - --quiet >/dev/null 2>&1 || true - gcloud projects remove-iam-policy-binding "$project" \ - --member "serviceAccount:${email}" \ - --role roles/monitoring.metricWriter \ - --quiet >/dev/null 2>&1 || true - gcloud projects remove-iam-policy-binding "$project" \ - --member "serviceAccount:${email}" \ - --role "projects/${project}/roles/suspendVM" \ - --quiet >/dev/null 2>&1 || true - gcloud iam service-accounts delete "$email" --project "$project" --quiet || true - done + --filter="name~'${name_filter}'" \ + --format=json | + jq -r '.[]? | [.region, .name] | @tsv')"; then + cleanup_status=1 + subnetwork_rows="" + fi + while IFS=$'\t' read -r region_url name; do + [[ -n "$name" ]] || continue + if ! gcp_delete_with_retry "subnetwork ${name}" compute networks subnets delete "$name" \ + --project "$project" \ + --region "${region_url##*/}" \ + --quiet; then + cleanup_status=1 + fi + done <<<"$subnetwork_rows" + + if ! network_names="$(gcloud compute networks list \ + --project "$project" \ + --filter="name~'${name_filter}'" \ + --format='value(name)')"; then + cleanup_status=1 + network_names="" + fi + while IFS= read -r name; do + [[ -n "$name" ]] || continue + if ! gcp_delete_with_retry "network ${name}" compute networks delete "$name" \ + --project "$project" \ + --quiet; then + cleanup_status=1 + fi + done <<<"$network_names" + + if ! gcp_verify_no_smoke_resources "$project" "$name_filter" "$region"; then + cleanup_status=1 + fi ;; esac + + return "$cleanup_status" } maybe_sweep_orphans() { diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index 30099d0..cfe81dd 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -89,3 +89,18 @@ for every app context. Backups are written under DigitalOcean and Linode deployments should leave power management disabled because stopped VMs do not provide the same cost profile. + +## Hosted smoke cleanup credentials + +Pull-request smoke jobs destroy resources in the same already-approved job. +When a runner is cancelled or lost, the `workflow_run` fallback checks out only +trusted default-branch code and uses separate GitHub Environments: + +- `cloud-smoke-cleanup-digitalocean` +- `cloud-smoke-cleanup-linode` +- `cloud-smoke-cleanup-gcp` + +Configure these cleanup environments without reviewer gates, restrict them to +the default branch, and give each a distinct least-privilege cleanup credential. +Do not expose a cleanup environment to pull-request-controlled code. The +ordinary smoke environments and their create/test credentials remain separate.