diff --git a/.github/workflows/cloud-smoke-cleanup.yml b/.github/workflows/cloud-smoke-cleanup.yml index 06117bd..62971c4 100644 --- a/.github/workflows/cloud-smoke-cleanup.yml +++ b/.github/workflows/cloud-smoke-cleanup.yml @@ -94,6 +94,15 @@ jobs: test -n "$GSA" || { echo "GSA environment variable or secret is required for GCP cleanup"; exit 1; } test -n "$GCLOUD_PROJECT" || { echo "GCLOUD_PROJECT environment variable or secret is required for GCP cleanup"; exit 1; } + - name: Install Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: "1.26.x" + cache: false + + - name: Build trusted cloud CI runner + run: make cloud-compose-ci + - name: Authenticate to Google Cloud uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 with: diff --git a/.github/workflows/cloud-smoke.yml b/.github/workflows/cloud-smoke.yml index fee7a45..4e8a67a 100644 --- a/.github/workflows/cloud-smoke.yml +++ b/.github/workflows/cloud-smoke.yml @@ -43,10 +43,8 @@ jobs: - name: Run repository lint run: make lint-check - - name: Validate fallback cleanup behavior - run: | - bash ci/cloud-smoke-cleanup-contract.sh - make hosted-cleanup-retry-contract + - name: Validate hosted cleanup retry behavior + run: make hosted-cleanup-retry-contract config-management-lint: name: Ansible and Salt lint @@ -195,6 +193,15 @@ jobs: terraform_version: 1.14.6 terraform_wrapper: false + - name: Install Go for cloud CI runner + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: "1.26.x" + cache: false + + - name: Build cloud CI runner + run: make cloud-compose-ci + - name: Check GCP smoke configuration run: | test -n "$GCLOUD_OIDC_POOL" || { echo "GCLOUD_OIDC_POOL environment variable or secret is required for GCP smoke tests"; exit 1; } diff --git a/.gitignore b/.gitignore index 6ac0c2b..02f0f0b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ site/ docs/site/ docs/.cache/ .cloud-compose-smoke/ +.bin/ diff --git a/Makefile b/Makefile index 2e3749a..0bb855b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke +.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract sitectl-version-contract go-fmt-check go-vet go-contracts template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke cloud-compose-ci .PHONY: terraform-fmt terraform-fmt-check terraform-validate terraform-validate-contract terraform-lint-check terraform-docs terraform-docs-check .PHONY: config-management-cloud-smoke config-management-cloud-smoke-ansible-drupal config-management-cloud-smoke-salt-drupal .PHONY: destroy-config-management-cloud-smoke destroy-config-management-cloud-smoke-ansible-drupal destroy-config-management-cloud-smoke-salt-drupal @@ -11,6 +11,10 @@ DOCS_PORT ?= 8888 DOCS_DOCKER_USER ?= $(shell id -u):$(shell id -g) ACTIONLINT_VERSION ?= v1.7.12 TERRAFORM_DOCS_VERSION ?= v0.21.0 +CLOUD_COMPOSE_CI_BIN ?= $(CURDIR)/.bin/cloud-compose-ci +export CLOUD_COMPOSE_CI_BIN +GO_MODULE_FILES := $(wildcard go.mod go.sum) +GO_SOURCES := $(shell find cmd internal -type f -name '*.go') lint: terraform-fmt actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check @@ -75,11 +79,16 @@ systemd-contract: sitectl-version-contract: bash ci/sitectl-version-contract.sh -template-version-contract: - bash ci/template-version-contract.sh +go-fmt-check: + @files="$$(gofmt -l .)"; test -z "$$files" || { echo "Go files require formatting:"; printf '%s\n' "$$files"; exit 1; } -rollout-parity-contract: - bash ci/rollout-parity-contract.sh +go-vet: + go vet ./... + +go-contracts: go-fmt-check go-vet + go test -count=1 ./... + +template-version-contract rollout-parity-contract: go-contracts rootfs-package-contract: bash ci/rootfs-package-contract.sh @@ -93,8 +102,13 @@ cos-jq-portability-contract: source-trust-contract: bash ci/source-trust-contract.sh -cloud-smoke-cleanup-contract: - bash ci/cloud-smoke-cleanup-contract.sh +cloud-smoke-cleanup-contract: go-contracts + +cloud-compose-ci: $(CLOUD_COMPOSE_CI_BIN) + +$(CLOUD_COMPOSE_CI_BIN): $(GO_MODULE_FILES) $(GO_SOURCES) + @mkdir -p "$(dir $(CLOUD_COMPOSE_CI_BIN))" + go build -trimpath -o "$(CLOUD_COMPOSE_CI_BIN)" ./cmd/cloud-compose-ci gcp-upgrade-smoke-contract: bash ci/gcp-upgrade-smoke-contract.sh @@ -135,12 +149,13 @@ terraform-docs: terraform-docs-check: go run github.com/terraform-docs/terraform-docs@$(TERRAFORM_DOCS_VERSION) markdown table --sort-by required --output-file README.md --output-check . -smoke-test-clouds: +smoke-test-clouds: cloud-compose-ci ci/cloud-smoke.sh all smoke-test: @test -n "$(PROVIDER)" || { echo "PROVIDER is required"; exit 2; } @test -n "$(TEMPLATE)" || { echo "TEMPLATE is required"; exit 2; } + @if [ "$(PROVIDER)" = "gcp" ]; then $(MAKE) --no-print-directory cloud-compose-ci; fi ci/cloud-smoke.sh $(PROVIDER)-$(TEMPLATE) smoke-test-digitalocean-isle: @@ -155,6 +170,7 @@ smoke-test-gcp-wp: destroy-smoke: @test -n "$(PROVIDER)" || { echo "PROVIDER is required"; exit 2; } @test -n "$(TEMPLATE)" || { echo "TEMPLATE is required"; exit 2; } + @if [ "$(PROVIDER)" = "gcp" ]; then $(MAKE) --no-print-directory cloud-compose-ci; fi ci/cloud-smoke.sh destroy-$(PROVIDER)-$(TEMPLATE) destroy-smoke-digitalocean-isle: diff --git a/ci/cloud-smoke-cleanup-contract.sh b/ci/cloud-smoke-cleanup-contract.sh deleted file mode 100644 index e12683c..0000000 --- a/ci/cloud-smoke-cleanup-contract.sh +++ /dev/null @@ -1,340 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -tmp="$(mktemp -d "${TMPDIR:-/tmp}/cloud-compose-cleanup.XXXXXX")" -trap 'rm -rf "$tmp"' EXIT - -fail() { - echo "cloud smoke cleanup contract: $*" >&2 - exit 1 -} - -mkdir -p "$tmp/bin" "$tmp/state" - -cat >"$tmp/bin/sleep" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail -printf 'sleep %s\n' "$*" >>"$FAKE_GCLOUD_LOG" -EOF - -cat >"$tmp/bin/gcloud" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail - -printf 'gcloud' >>"$FAKE_GCLOUD_LOG" -printf ' %q' "$@" >>"$FAKE_GCLOUD_LOG" -printf '\n' >>"$FAKE_GCLOUD_LOG" - -state_count() { - local key="$1" - - if [[ -f "$FAKE_GCLOUD_STATE/$key" ]]; then - cat "$FAKE_GCLOUD_STATE/$key" - else - printf '0\n' - fi -} - -mutation_done() { - [[ -f "$FAKE_GCLOUD_STATE/$1.done" ]] -} - -emit_project_binding() { - local state_key="$1" role="$2" member="$3" separator_name="$4" - local -n separator_ref="$separator_name" - - if mutation_done "$state_key"; then - return 0 - fi - printf '%s{"role":"%s","members":["%s"]}' "$separator_ref" "$role" "$member" - separator_ref=, -} - -command_name="" -case "${1:-} ${2:-} ${3:-} ${4:-}" in - "run services list "*) - if [[ "${FAKE_GCLOUD_MODE:-success}" == "residual-cloud-run" ]] || ! mutation_done run-services-delete; then - printf 'cc-g-wp-12345678-abcd\n' - fi - exit 0 - ;; - "run services get-iam-policy "*) - if mutation_done run-invoker-remove; then - printf '{"bindings":[]}\n' - else - printf '{"bindings":[{"role":"roles/run.invoker","members":["allUsers"]}]}\n' - fi - exit 0 - ;; - "compute instances list "*) - if ! mutation_done instances-delete; then - if [[ " $* " == *" --format=json "* ]]; then - printf '[{"zone":"https://www.googleapis.com/compute/v1/projects/test-project/zones/us-east5-b","name":"cc-g-wp-12345678-abcd"}]\n' - else - printf 'cc-g-wp-12345678-abcd\n' - fi - fi - exit 0 - ;; - "compute firewall-rules list "*) - if ! mutation_done firewalls-delete; then - printf '%s\n' \ - 'allow-ssh-ipv4-cc-g-wp-12345678-abcd' \ - 'allow-cloud-run-cc-g-wp-12345678-abcd' - fi - exit 0 - ;; - "compute disks list "*) - if ! mutation_done disks-delete; then - if [[ " $* " == *" --format=json "* ]]; then - printf '[{"zone":"https://www.googleapis.com/compute/v1/projects/test-project/zones/us-east5-b","name":"cc-g-wp-12345678-abcd-data-disk"}]\n' - else - printf 'cc-g-wp-12345678-abcd-data-disk\n' - fi - fi - exit 0 - ;; - "iam service-accounts list "*) - if ! mutation_done service-accounts-delete; then - printf '%s\n' \ - 'vm-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com' \ - 'internal-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com' \ - 'ppb-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com' \ - 'cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com' - fi - exit 0 - ;; - "compute networks subnets list") - if ! mutation_done subnets-delete; then - if [[ " $* " == *" --format=json "* ]]; then - printf '[{"region":"https://www.googleapis.com/compute/v1/projects/test-project/regions/us-east5","name":"cc-g-wp-12345678-abcd"}]\n' - else - printf 'cc-g-wp-12345678-abcd\n' - fi - fi - exit 0 - ;; - "compute networks list "*) - if ! mutation_done networks-delete; then - printf 'cc-g-wp-12345678-abcd\n' - fi - exit 0 - ;; - "projects get-iam-policy test-project "*) - separator="" - printf '{"bindings":[' - emit_project_binding iam-log-remove roles/logging.logWriter \ - serviceAccount:vm-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com separator - emit_project_binding iam-monitoring-remove roles/monitoring.metricWriter \ - serviceAccount:internal-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com separator - emit_project_binding iam-suspend-remove projects/test-project/roles/suspendVM \ - serviceAccount:internal-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com separator - emit_project_binding iam-start-remove projects/test-project/roles/startVM \ - serviceAccount:ppb-cc-g-wp-12345678-abcd@test-project.iam.gserviceaccount.com separator - printf ']}\n' - exit 0 - ;; - "run services remove-iam-policy-binding "*) - command_name="run-invoker-remove" - ;; - "run services delete "*) - command_name="run-services-delete" - ;; - "projects remove-iam-policy-binding "*) - case " $* " in - *" --role roles/logging.logWriter "*) command_name="iam-log-remove" ;; - *" --role roles/monitoring.metricWriter "*) command_name="iam-monitoring-remove" ;; - *" --role projects/test-project/roles/suspendVM "*) command_name="iam-suspend-remove" ;; - *" --role projects/test-project/roles/startVM "*) command_name="iam-start-remove" ;; - *) echo "unexpected project IAM removal: $*" >&2; exit 64 ;; - esac - ;; - "compute instances delete "*) - command_name="instances-delete" - ;; - "compute firewall-rules delete "*) - command_name="firewalls-delete" - ;; - "compute disks delete "*) - command_name="disks-delete" - ;; - "iam service-accounts delete "*) - command_name="service-accounts-delete" - ;; - "compute networks subnets delete") - command_name="subnets-delete" - ;; - "compute networks delete "*) - command_name="networks-delete" - ;; - *) - echo "unexpected gcloud invocation: $*" >&2 - exit 64 - ;; -esac - -count_file="$FAKE_GCLOUD_STATE/$command_name" -count="$(state_count "$command_name")" -count=$((count + 1)) -printf '%s\n' "$count" >"$count_file" - -case "${FAKE_GCLOUD_MODE:-success}:$command_name:$count" in - retry-transient:run-services-delete:1 | retry-transient:iam-log-remove:1 | retry-transient:networks-delete:1) - exit 1 - ;; -esac -if [[ "${FAKE_GCLOUD_MODE:-success}" == "aggregate-failure" && - ( "$command_name" == "run-services-delete" || "$command_name" == "instances-delete" || "$command_name" == "subnets-delete" ) ]]; then - exit 1 -fi - -touch "$FAKE_GCLOUD_STATE/$command_name.done" -EOF - -chmod +x "$tmp/bin/gcloud" "$tmp/bin/sleep" - -run_cleanup() { - local mode="$1" log="$2" state="$3" - - mkdir -p "$state" - PATH="$tmp/bin:$PATH" \ - FAKE_GCLOUD_LOG="$log" \ - FAKE_GCLOUD_MODE="$mode" \ - FAKE_GCLOUD_STATE="$state" \ - GCLOUD_PROJECT=test-project \ - CLOUD_COMPOSE_SMOKE_RUN_ID=123456789 \ - bash "$repo_root/ci/cloud-smoke.sh" sweep-gcp-wp -} - -success_log="$tmp/success.log" -run_cleanup retry-transient "$success_log" "$tmp/state/success" - -[[ "$(<"$tmp/state/success/run-services-delete")" == "2" ]] || \ - fail "Cloud Run cleanup did not retry a transient delete failure" -[[ "$(<"$tmp/state/success/iam-log-remove")" == "2" ]] || \ - fail "GCP IAM cleanup did not retry a transient removal failure" -[[ "$(<"$tmp/state/success/networks-delete")" == "2" ]] || \ - fail "GCP network cleanup did not retry a transient delete failure" - -for command_name in \ - "run services remove-iam-policy-binding" \ - "run services delete" \ - "projects remove-iam-policy-binding" \ - "compute instances delete" \ - "compute firewall-rules delete" \ - "compute disks delete" \ - "iam service-accounts delete" \ - "compute networks subnets delete" \ - "compute networks delete"; do - grep -Fq "gcloud ${command_name}" "$success_log" || \ - fail "GCP fallback omitted ${command_name}" -done - -grep -F 'gcloud compute instances list' "$success_log" | grep -Fq 'cc-g-wp-12345678-' || \ - fail "GCP cleanup did not constrain resources to the originating workflow run" -grep -F 'gcloud iam service-accounts list' "$success_log" | grep -Fq 'ppb-' || \ - fail "GCP cleanup does not select the power-button service account" -grep -F 'gcloud compute firewall-rules list' "$success_log" | grep -Fq 'allow-cloud-run-' || \ - fail "GCP cleanup does not select the VM ingress firewall for Direct VPC egress" -for role in \ - roles/logging.logWriter \ - roles/monitoring.metricWriter \ - projects/test-project/roles/suspendVM \ - projects/test-project/roles/startVM; do - grep -F 'gcloud projects remove-iam-policy-binding' "$success_log" | grep -Fq "$role" || \ - fail "GCP cleanup omitted project IAM role ${role}" -done -for account in \ - vm-cc-g-wp-12345678-abcd \ - internal-cc-g-wp-12345678-abcd \ - ppb-cc-g-wp-12345678-abcd \ - cc-g-wp-12345678-abcd; do - grep -F 'gcloud iam service-accounts delete' "$success_log" | grep -Fq "$account@test-project.iam.gserviceaccount.com" || \ - fail "GCP cleanup omitted service account ${account}" -done -grep -F 'gcloud run services remove-iam-policy-binding' "$success_log" | grep -Fq -- '--condition=None' || \ - fail "Cloud Run invoker cleanup did not target the unconditional binding" -if grep -F 'gcloud projects remove-iam-policy-binding' "$success_log" | grep -Fv -- '--condition=None' >/dev/null; then - fail "Project IAM cleanup did not target unconditional bindings explicitly" -fi - -run_invoker_line="$(grep -nF 'gcloud run services remove-iam-policy-binding' "$success_log" | head -n1 | cut -d: -f1)" -run_delete_line="$(grep -nF 'gcloud run services delete' "$success_log" | tail -n1 | cut -d: -f1)" -instance_line="$(grep -nF 'gcloud compute instances delete' "$success_log" | head -n1 | cut -d: -f1)" -[[ "$run_invoker_line" -lt "$run_delete_line" && "$run_delete_line" -lt "$instance_line" ]] || \ - fail "GCP fallback did not remove Cloud Run ingress before deleting the service and VM" - -last_project_iam_line="$(grep -nF 'gcloud projects remove-iam-policy-binding' "$success_log" | tail -n1 | cut -d: -f1)" -first_service_account_line="$(grep -nF 'gcloud iam service-accounts delete' "$success_log" | head -n1 | cut -d: -f1)" -[[ "$last_project_iam_line" -lt "$first_service_account_line" ]] || \ - fail "GCP fallback deleted service accounts before removing their project IAM bindings" - -subnet_line="$(grep -nF 'gcloud compute networks subnets delete' "$success_log" | tail -n1 | cut -d: -f1)" -network_line="$(grep -nF 'gcloud compute networks delete' "$success_log" | head -n1 | cut -d: -f1)" -[[ "$subnet_line" -lt "$network_line" ]] || \ - fail "GCP fallback did not delete subnetworks before their parent networks" -last_verification_line="$(grep -nF 'gcloud projects get-iam-policy' "$success_log" | tail -n1 | cut -d: -f1)" -[[ "$network_line" -lt "$last_verification_line" ]] || \ - fail "GCP fallback did not verify residual resources after all deletions" - -[[ "$(<"$tmp/state/success/service-accounts-delete")" == "4" ]] || \ - fail "GCP cleanup did not delete VM, internal, power-button, and app service accounts" - -failure_log="$tmp/failure.log" -if run_cleanup aggregate-failure "$failure_log" "$tmp/state/failure" >/dev/null 2>&1; then - fail "GCP fallback reported success after permanent resource deletion failures" -fi -[[ "$(<"$tmp/state/failure/run-services-delete")" == "12" ]] || \ - fail "Cloud Run deletion did not exhaust its retry budget" -[[ "$(<"$tmp/state/failure/instances-delete")" == "12" ]] || \ - fail "GCP instance deletion did not exhaust its retry budget" -[[ "$(<"$tmp/state/failure/subnets-delete")" == "12" ]] || \ - fail "GCP subnetwork deletion did not exhaust its retry budget" -[[ -f "$tmp/state/failure/networks-delete" ]] || \ - fail "GCP cleanup stopped instead of aggregating failures across resource kinds" -[[ -f "$tmp/state/failure/service-accounts-delete" ]] || \ - fail "GCP cleanup skipped unrelated resources after an earlier failure" - -residual_log="$tmp/residual.log" -if run_cleanup residual-cloud-run "$residual_log" "$tmp/state/residual" >/dev/null 2>&1; then - fail "GCP fallback reported success while a matching Cloud Run service remained" -fi -[[ "$(grep -cF 'gcloud run services list' "$residual_log")" -ge 13 ]] || \ - fail "GCP cleanup did not exhaust residual-resource verification retries" - -pr_workflow="$repo_root/.github/workflows/cloud-smoke.yml" -cleanup_workflow="$repo_root/.github/workflows/cloud-smoke-cleanup.yml" -docs="$repo_root/docs/runtime-contracts.md" - -grep -Fq "if: always()" "$pr_workflow" || \ - fail "PR smoke jobs no longer destroy resources in the already-approved job" -if grep -Eq '^ (config-management-cleanup|cleanup|gcp-cleanup):' "$pr_workflow"; then - fail "PR-controlled workflow still contains a second secret-bearing cleanup job" -fi -grep -Fq "workflow_run:" "$cleanup_workflow" || \ - fail "trusted default-branch fallback cleanup is missing" -grep -Fq "github.event.workflow_run.event == 'pull_request'" "$cleanup_workflow" || \ - fail "fallback cleanup is not restricted to pull-request workflow runs" -grep -Fq "github.event.workflow_run.head_repository.full_name == github.repository" "$cleanup_workflow" || \ - fail "fallback cleanup is not restricted to same-repository workflow runs" -grep -Fq "ref: \${{ github.sha }}" "$cleanup_workflow" || \ - fail "fallback cleanup does not check out the trusted default-branch revision" -if grep -Fq "github.event.workflow_run.head_sha" "$cleanup_workflow"; then - fail "fallback cleanup checks out pull-request-controlled code" -fi -for environment in \ - cloud-smoke-cleanup-digitalocean \ - cloud-smoke-cleanup-linode \ - cloud-smoke-cleanup-gcp; do - if grep -Fq "$environment" "$pr_workflow"; then - fail "pull-request-controlled workflow can request cleanup environment ${environment}" - fi - grep -Fq "$environment" "$cleanup_workflow" || \ - fail "fallback cleanup is missing dedicated environment ${environment}" - grep -Fq "$environment" "$docs" || \ - fail "runtime documentation omits cleanup environment ${environment}" -done - -echo "Cloud smoke cleanup contracts passed" diff --git a/ci/cloud-smoke.sh b/ci/cloud-smoke.sh index bfd68cf..99c5c8c 100755 --- a/ci/cloud-smoke.sh +++ b/ci/cloud-smoke.sh @@ -32,7 +32,9 @@ Optional environment: Seconds allowed for Terraform destroy during cleanup. CLOUD_COMPOSE_SMOKE_SWEEP_ORPHANS=true Remove prior smoke resources for the same target before apply. - CLOUD_COMPOSE_SMOKE_RUN_ID Optional run id used to target provider cleanup. + CLOUD_COMPOSE_SMOKE_ALLOW_ALL_RUNS=true + Permit an explicit sweep command to remove every run for a target. + CLOUD_COMPOSE_SMOKE_RUN_ID Run id used to target provider cleanup; required for scoped GCP cleanup. CLOUD_COMPOSE_SMOKE_TARGETS Space-separated targets used by "all" and "sweep". DIGITALOCEAN_API_TOKEN Backward-compatible alias for DIGITALOCEAN_TOKEN. GCLOUD_REGION=us-east5 Google Cloud region for gcp targets. @@ -148,6 +150,25 @@ smoke_run_id() { printf '%s\n' "${CLOUD_COMPOSE_SMOKE_RUN_ID:-${GITHUB_RUN_ID:-}}" } +gcp_run_namespace() { + local target="$1" run_id="$2" runner + + if [[ "$(target_provider "$target")" != "gcp" ]]; then + return 0 + fi + if [[ -z "$run_id" ]]; then + echo "CLOUD_COMPOSE_SMOKE_RUN_ID is required for every GCP smoke apply" >&2 + return 1 + fi + + runner="${CLOUD_COMPOSE_CI_BIN:-$repo_root/.bin/cloud-compose-ci}" + if [[ ! -x "$runner" ]]; then + echo "Missing compiled CI runner: ${runner}; run 'make cloud-compose-ci' first" >&2 + return 1 + fi + "$runner" gcp namespace --run-id "$run_id" +} + api_request() { local provider="$1" method="$2" path="$3" local base_url body http_code response token token_name @@ -299,176 +320,6 @@ delete_ids() { return "$failed" } -gcp_command_with_retry() { - local operation="$1" description="$2" attempt - shift 2 - - for attempt in {1..12}; do - echo "${operation} gcp ${description} (attempt ${attempt})" - if gcloud "$@"; then - return 0 - fi - if ((attempt < 12)); then - sleep 10 - fi - done - - echo "Failed while ${operation,,} gcp ${description} after 12 attempts" >&2 - return 1 -} - -gcp_delete_with_retry() { - local description="$1" - shift - - gcp_command_with_retry "Deleting" "$description" "$@" -} - -gcp_remove_with_retry() { - local description="$1" - shift - - gcp_command_with_retry "Removing" "$description" "$@" -} - -gcp_project_iam_rows() { - local project="$1" name_filter="$2" account_pattern - - account_pattern="(vm-|internal-|ppb-)?${name_filter#^}.*@${project}\\.iam\\.gserviceaccount\\.com" - gcloud projects get-iam-policy "$project" --format=json | - jq -r \ - --arg account_pattern "$account_pattern" \ - --arg start_role "projects/${project}/roles/startVM" \ - --arg suspend_role "projects/${project}/roles/suspendVM" ' - [$start_role, $suspend_role, "roles/logging.logWriter", "roles/monitoring.metricWriter"] as $managed_roles | - .bindings[]? | - select(.condition == null) | - select(.role as $role | $managed_roles | index($role)) | - .role as $role | - .members[]? | - select(test("^(deleted:)?serviceAccount:" + $account_pattern + "(\\?uid=[^[:space:]]+)?$")) | - [$role, .] | @tsv - ' -} - -gcp_smoke_residuals() { - local project="$1" name_filter="$2" region="$3" - local output name email role member - - if ! output="$(gcloud run services list \ - --project "$project" \ - --region "$region" \ - --filter="metadata.name~'${name_filter}'" \ - --format='value(metadata.name)')"; then - echo "Could not verify Cloud Run cleanup" >&2 - return 1 - fi - while IFS= read -r name; do - [[ -n "$name" ]] && printf 'cloud-run\t%s\n' "$name" - done <<<"$output" - - if ! output="$(gcloud compute instances list \ - --project "$project" \ - --filter="name~'${name_filter}'" \ - --format='value(name)')"; then - echo "Could not verify GCP instance cleanup" >&2 - return 1 - fi - while IFS= read -r name; do - [[ -n "$name" ]] && printf 'instance\t%s\n' "$name" - done <<<"$output" - - if ! output="$(gcloud compute firewall-rules list \ - --project "$project" \ - --filter="name~'^(allow-ssh-ipv4-|allow-ssh-ipv6-|allow-rollout-ipv4-|allow-cloud-run-)${name_filter#^}'" \ - --format='value(name)')"; then - echo "Could not verify GCP firewall cleanup" >&2 - return 1 - fi - while IFS= read -r name; do - [[ -n "$name" ]] && printf 'firewall\t%s\n' "$name" - done <<<"$output" - - if ! output="$(gcloud compute disks list \ - --project "$project" \ - --filter="name~'${name_filter}'" \ - --format='value(name)')"; then - echo "Could not verify GCP disk cleanup" >&2 - return 1 - fi - while IFS= read -r name; do - [[ -n "$name" ]] && printf 'disk\t%s\n' "$name" - done <<<"$output" - - if ! output="$(gcloud iam service-accounts list \ - --project "$project" \ - --filter="email~'^(vm-|internal-|ppb-)?${name_filter#^}.*@${project}\\.iam\\.gserviceaccount\\.com$'" \ - --format='value(email)')"; then - echo "Could not verify GCP service-account cleanup" >&2 - return 1 - fi - while IFS= read -r email; do - [[ -n "$email" ]] && printf 'service-account\t%s\n' "$email" - done <<<"$output" - - if ! output="$(gcloud compute networks subnets list \ - --project "$project" \ - --filter="name~'${name_filter}'" \ - --format='value(name)')"; then - echo "Could not verify GCP subnetwork cleanup" >&2 - return 1 - fi - while IFS= read -r name; do - [[ -n "$name" ]] && printf 'subnetwork\t%s\n' "$name" - done <<<"$output" - - if ! output="$(gcloud compute networks list \ - --project "$project" \ - --filter="name~'${name_filter}'" \ - --format='value(name)')"; then - echo "Could not verify GCP network cleanup" >&2 - return 1 - fi - while IFS= read -r name; do - [[ -n "$name" ]] && printf 'network\t%s\n' "$name" - done <<<"$output" - - if ! output="$(gcp_project_iam_rows "$project" "$name_filter")"; then - echo "Could not verify GCP project-IAM cleanup" >&2 - return 1 - fi - while IFS=$'\t' read -r role member; do - [[ -n "$role" && -n "$member" ]] && printf 'project-iam\t%s\t%s\n' "$role" "$member" - done <<<"$output" - - return 0 -} - -gcp_verify_no_smoke_resources() { - local project="$1" name_filter="$2" region="$3" - local attempt residuals - - for attempt in {1..12}; do - residuals="" - if residuals="$(gcp_smoke_residuals "$project" "$name_filter" "$region")"; then - if [[ -z "$residuals" ]]; then - echo "Verified that no matching gcp smoke resources remain" - return 0 - fi - echo "Matching gcp smoke resources remain after cleanup attempt ${attempt}:" >&2 - printf '%s\n' "$residuals" >&2 - else - echo "Could not complete gcp residual-resource verification (attempt ${attempt})" >&2 - fi - if ((attempt < 12)); then - sleep 10 - fi - done - - echo "GCP smoke cleanup left matching resources or could not verify their removal" >&2 - return 1 -} - smoke_run_tag() { local run_id="$1" @@ -614,7 +465,7 @@ verify_no_provider_resources() { } provider_tag_cleanup() { - local target="$1" run_id="${2:-}" provider kind index name_prefix + local target="$1" run_id="${2:-}" allow_all_runs="${3:-false}" provider kind index name_prefix local cleanup_status=0 local -a kinds path_prefixes @@ -630,184 +481,29 @@ provider_tag_cleanup() { path_prefixes=(/networking/firewalls /linode/instances /volumes) ;; gcp) - local project name_filter region cloud_run_services instance_rows firewall_names disk_rows - local project_iam_rows service_accounts subnetwork_rows network_names - local zone_url region_url name email role member cloud_run_policy - project="$GCLOUD_PROJECT" - region="$(gcp_region)" - name_filter="^${name_prefix}-" - if [[ -n "$run_id" ]]; then - name_filter="^${name_prefix}-$(printf '%s' "$run_id" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | cut -c1-8)-" - fi - - if ! cloud_run_services="$(gcloud run services list \ - --project "$project" \ - --region "$region" \ - --filter="metadata.name~'${name_filter}'" \ - --format='value(metadata.name)')"; then - cleanup_status=1 - cloud_run_services="" - fi - while IFS= read -r name; do - [[ -n "$name" ]] || continue - cloud_run_policy="" - if cloud_run_policy="$(gcloud run services get-iam-policy "$name" \ - --project "$project" \ - --region "$region" \ - --format=json)"; then - if ! jq -e 'type == "object" and ((has("bindings") | not) or (.bindings | type == "array"))' \ - <<<"$cloud_run_policy" >/dev/null; then - echo "Cloud Run IAM policy for ${name} was not valid JSON policy data" >&2 - cleanup_status=1 - elif jq -e ' - any(.bindings[]?; - .role == "roles/run.invoker" and - .condition == null and - any(.members[]?; . == "allUsers") - ) - ' <<<"$cloud_run_policy" >/dev/null; then - if ! gcp_remove_with_retry "Cloud Run invoker from ${name}" \ - run services remove-iam-policy-binding "$name" \ - --project "$project" \ - --region "$region" \ - --member allUsers \ - --role roles/run.invoker \ - --condition=None \ - --quiet; then - cleanup_status=1 - fi - fi - else - echo "Could not inspect Cloud Run IAM policy for ${name}" >&2 - cleanup_status=1 - fi - if ! gcp_delete_with_retry "Cloud Run service ${name}" run services delete "$name" \ - --project "$project" \ - --region "$region" \ - --quiet; then - cleanup_status=1 - fi - done <<<"$cloud_run_services" - - if ! instance_rows="$(gcloud compute instances list \ - --project "$project" \ - --filter="name~'${name_filter}'" \ - --format=json | - jq -r '.[]? | [.zone, .name] | @tsv')"; then - cleanup_status=1 - instance_rows="" - fi - while IFS=$'\t' read -r zone_url name; do - [[ -n "$name" ]] || continue - if ! gcp_delete_with_retry "instance ${name}" compute instances delete "$name" \ - --project "$project" \ - --zone "${zone_url##*/}" \ - --quiet; then - cleanup_status=1 - fi - done <<<"$instance_rows" + local cleanup_binary + local -a cleanup_args - if ! firewall_names="$(gcloud compute firewall-rules list \ - --project "$project" \ - --filter="name~'^(allow-ssh-ipv4-|allow-ssh-ipv6-|allow-rollout-ipv4-|allow-cloud-run-)${name_filter#^}'" \ - --format='value(name)')"; then - cleanup_status=1 - firewall_names="" - fi - while IFS= read -r name; do - [[ -n "$name" ]] || continue - if ! gcp_delete_with_retry "firewall ${name}" compute firewall-rules delete "$name" \ - --project "$project" \ - --quiet; then - cleanup_status=1 - fi - done <<<"$firewall_names" - - if ! disk_rows="$(gcloud compute disks list \ - --project "$project" \ - --filter="name~'${name_filter}'" \ - --format=json | - jq -r '.[]? | [.zone, .name] | @tsv')"; then - cleanup_status=1 - disk_rows="" - fi - while IFS=$'\t' read -r zone_url name; do - [[ -n "$name" ]] || continue - if ! gcp_delete_with_retry "disk ${name}" compute disks delete "$name" \ - --project "$project" \ - --zone "${zone_url##*/}" \ - --quiet; then - cleanup_status=1 - fi - done <<<"$disk_rows" - - if ! project_iam_rows="$(gcp_project_iam_rows "$project" "$name_filter")"; then - cleanup_status=1 - project_iam_rows="" - fi - while IFS=$'\t' read -r role member; do - [[ -n "$role" && -n "$member" ]] || continue - if ! gcp_remove_with_retry "project IAM binding ${role} for ${member}" \ - projects remove-iam-policy-binding "$project" \ - --member "$member" \ - --role "$role" \ - --condition=None \ - --quiet; then - cleanup_status=1 - fi - done <<<"$project_iam_rows" - - if ! service_accounts="$(gcloud iam service-accounts list \ - --project "$project" \ - --filter="email~'^(vm-|internal-|ppb-)?${name_filter#^}.*@${project}\\.iam\\.gserviceaccount\\.com$'" \ - --format='value(email)')"; then - cleanup_status=1 - service_accounts="" - fi - while IFS= read -r email; do - [[ -n "$email" ]] || continue - if ! gcp_delete_with_retry "service account ${email}" iam service-accounts delete "$email" \ - --project "$project" \ - --quiet; then - cleanup_status=1 - fi - done <<<"$service_accounts" - - if ! subnetwork_rows="$(gcloud compute networks subnets list \ - --project "$project" \ - --filter="name~'${name_filter}'" \ - --format=json | - jq -r '.[]? | [.region, .name] | @tsv')"; then - cleanup_status=1 - subnetwork_rows="" + cleanup_binary="${CLOUD_COMPOSE_CI_BIN:-$repo_root/.bin/cloud-compose-ci}" + if [[ ! -x "$cleanup_binary" ]]; then + echo "Missing compiled CI runner: ${cleanup_binary}; run 'make cloud-compose-ci' first" >&2 + return 1 fi - while IFS=$'\t' read -r region_url name; do - [[ -n "$name" ]] || continue - if ! gcp_delete_with_retry "subnetwork ${name}" compute networks subnets delete "$name" \ - --project "$project" \ - --region "${region_url##*/}" \ - --quiet; then - cleanup_status=1 - fi - done <<<"$subnetwork_rows" - - if ! network_names="$(gcloud compute networks list \ - --project "$project" \ - --filter="name~'${name_filter}'" \ - --format='value(name)')"; then - cleanup_status=1 - network_names="" + cleanup_args=( + gcp sweep + --project "$GCLOUD_PROJECT" + --region "$(gcp_region)" + --target "$target" + ) + if [[ -n "$run_id" ]]; then + cleanup_args+=(--run-id "$run_id") + elif [[ "$allow_all_runs" == "true" ]]; then + cleanup_args+=(--all-runs) + else + echo "GCP cleanup requires CLOUD_COMPOSE_SMOKE_RUN_ID; set CLOUD_COMPOSE_SMOKE_ALLOW_ALL_RUNS=true only for an intentional target-wide orphan sweep" >&2 + return 1 fi - while IFS= read -r name; do - [[ -n "$name" ]] || continue - if ! gcp_delete_with_retry "network ${name}" compute networks delete "$name" \ - --project "$project" \ - --quiet; then - cleanup_status=1 - fi - done <<<"$network_names" - - if ! gcp_verify_no_smoke_resources "$project" "$name_filter" "$region"; then + if ! "$cleanup_binary" "${cleanup_args[@]}"; then cleanup_status=1 fi ;; @@ -837,7 +533,7 @@ maybe_sweep_orphans() { return 0 fi echo "Sweeping prior ${target} smoke-test resources" - provider_tag_cleanup "$target" + provider_tag_cleanup "$target" "" true } ensure_key() { @@ -1165,6 +861,9 @@ run_target() ( ensure_key "$key_path" run_id="$(smoke_run_id)" + # Validate GCP cleanup ownership before Terraform can create resources. The + # reader phase intentionally keeps writing the legacy resource namespace. + gcp_run_namespace "$target" "$run_id" >/dev/null mapfile -d '' -t var_args < <(target_var_args "$root" "$key_path" "$target") auto_args=() @@ -1312,44 +1011,44 @@ main() { sweep) for target in $(default_targets); do provider="$(target_provider "$target")" - require_cmd jq case "$provider" in digitalocean | linode) require_cmd curl + require_cmd jq ;; gcp) require_cmd gcloud ;; esac target_env "$target" - provider_tag_cleanup "$target" "${CLOUD_COMPOSE_SMOKE_RUN_ID:-}" + provider_tag_cleanup "$target" "${CLOUD_COMPOSE_SMOKE_RUN_ID:-}" "${CLOUD_COMPOSE_SMOKE_ALLOW_ALL_RUNS:-false}" done exit 0 ;; sweep-*) target="${1#sweep-}" provider="$(target_provider "$target")" - require_cmd jq case "$provider" in digitalocean | linode) require_cmd curl + require_cmd jq ;; gcp) require_cmd gcloud ;; esac target_env "$target" - provider_tag_cleanup "$target" "${CLOUD_COMPOSE_SMOKE_RUN_ID:-}" + provider_tag_cleanup "$target" "${CLOUD_COMPOSE_SMOKE_RUN_ID:-}" "${CLOUD_COMPOSE_SMOKE_ALLOW_ALL_RUNS:-false}" exit 0 ;; destroy-*) target="${1#destroy-}" provider="$(target_provider "$target")" - require_cmd jq require_cmd terraform case "$provider" in digitalocean | linode) require_cmd curl + require_cmd jq ;; gcp) require_cmd gcloud diff --git a/ci/rollout-parity-contract.sh b/ci/rollout-parity-contract.sh deleted file mode 100755 index 9218dae..0000000 --- a/ci/rollout-parity-contract.sh +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" - -python3 - "$repo_root" <<'PY' -import ast -import json -import re -import sys -from pathlib import Path - -root = Path(sys.argv[1]) - -expected_rollout = [ - 'TARGET_REF="${GIT_REF:-${GIT_BRANCH:-}}"', - 'if [ -n "$TARGET_REF" ]; then sitectl deploy --context "${SITECTL_CONTEXT_NAME}" --ref "$TARGET_REF"; else sitectl deploy --context "${SITECTL_CONTEXT_NAME}" --skip-git; fi', - 'sitectl healthcheck --context "${SITECTL_CONTEXT_NAME}" --persist', - 'if [ "${SITECTL_ENVIRONMENT}" != "production" ]; then sitectl verify --context "${SITECTL_CONTEXT_NAME}" ${SITECTL_VERIFY_ARGS:-}; fi', -] - - -def fail(message): - raise SystemExit(f"rollout parity contract: {message}") - - -def require_match(pattern, text, label, flags=0): - match = re.search(pattern, text, flags) - if match is None: - fail(f"could not find {label}") - return match - - -def parse_hcl_rollout(path, pattern, label): - text = path.read_text() - block = require_match(pattern, text, label, re.MULTILINE | re.DOTALL).group("items") - commands = [] - for raw_line in block.splitlines(): - line = raw_line.strip() - if not line: - continue - if line.endswith(","): - line = line[:-1] - try: - command = json.loads(line) - except json.JSONDecodeError as exc: - fail(f"could not parse {label} command {line!r}: {exc}") - commands.append(command.replace("$${", "${")) - return commands - - -def parse_ansible_rollout(path): - text = path.read_text() - block = require_match( - r"^cloud_compose_default_rollout:\n(?P(?:^ - .*\n)+)", - text, - "Ansible default rollout list", - re.MULTILINE, - ).group("items") - commands = [] - for raw_line in block.splitlines(): - match = re.fullmatch(r" - '(.*)'", raw_line) - if match is None: - fail(f"could not parse Ansible rollout command {raw_line!r}") - commands.append(match.group(1)) - return commands - - -def parse_salt_rollout(path): - text = path.read_text() - block = require_match( - r"^\{% set default_rollout = \[\n(?P.*?)^\] %\}", - text, - "Salt default rollout list", - re.MULTILINE | re.DOTALL, - ).group("items") - commands = [] - for raw_line in block.splitlines(): - line = raw_line.strip().removesuffix(",") - if not line: - continue - try: - command = ast.literal_eval(line) - except (SyntaxError, ValueError) as exc: - fail(f"could not parse Salt rollout command {line!r}: {exc}") - if not isinstance(command, str): - fail(f"Salt rollout entry is not a string: {line!r}") - commands.append(command) - return commands - - -rollout_sources = { - "GCP Terraform": ( - root / "modules/gcp/variables.tf", - parse_hcl_rollout( - root / "modules/gcp/variables.tf", - r'^variable "docker_compose_rollout" \{.*?^ default = \[\n(?P.*?)^ \]\n', - "GCP Terraform default rollout list", - ), - ), - "Linux VM Terraform": ( - root / "modules/linux-vm-runtime/variables.tf", - parse_hcl_rollout( - root / "modules/linux-vm-runtime/variables.tf", - r'^variable "docker_compose_rollout" \{.*?^ default = \[\n(?P.*?)^ \]\n', - "Linux VM Terraform default rollout list", - ), - ), - "Ansible": ( - root / "ansible/roles/cloud_compose/defaults/main.yml", - parse_ansible_rollout(root / "ansible/roles/cloud_compose/defaults/main.yml"), - ), - "Salt": ( - root / "salt/cloud-compose/init.sls", - parse_salt_rollout(root / "salt/cloud-compose/init.sls"), - ), - "rollout documentation": ( - root / "docs/rollout.md", - parse_hcl_rollout( - root / "docs/rollout.md", - r"^ rollout = \[\n(?P.*?)^ \]\n", - "documented rollout list", - ), - ), -} - -for label, (path, commands) in rollout_sources.items(): - if commands != expected_rollout: - fail( - f"{label} rollout list in {path.relative_to(root)} diverged:\n" - f"expected {json.dumps(expected_rollout, indent=2)}\n" - f"actual {json.dumps(commands, indent=2)}" - ) - -legacy_rollout_script = "scripts/" + "rollout.sh" -for label, (path, _) in rollout_sources.items(): - if legacy_rollout_script in path.read_text(): - fail(f"{label} still delegates lifecycle ownership to {legacy_rollout_script}") - -compose_versions = {} -version_patterns = { - "root Terraform": ( - "variables.tf", - r'compose_version\s*=\s*optional\(string,\s*"(?P[^"]+)"\)', - ), - "GCP provider": ( - "providers/gcp/variables.tf", - r'compose_version\s*=\s*optional\(string,\s*"(?P[^"]+)"\)', - ), - "DigitalOcean provider": ( - "providers/do/variables.tf", - r'compose_version\s*=\s*optional\(string,\s*"(?P[^"]+)"\)', - ), - "Linode provider": ( - "providers/linode/variables.tf", - r'compose_version\s*=\s*optional\(string,\s*"(?P[^"]+)"\)', - ), - "DigitalOcean module": ( - "modules/digitalocean/variables.tf", - r'compose_version\s*=\s*optional\(string,\s*"(?P[^"]+)"\)', - ), - "Linode module": ( - "modules/linode/variables.tf", - r'compose_version\s*=\s*optional\(string,\s*"(?P[^"]+)"\)', - ), - "GCP module": ( - "modules/gcp/variables.tf", - r'variable "docker_compose_version" \{.*?default\s*=\s*"(?P[^"]+)"', - ), - "Linux VM module": ( - "modules/linux-vm-runtime/variables.tf", - r'variable "docker_compose_version" \{.*?default\s*=\s*"(?P[^"]+)"', - ), - "Ansible": ( - "ansible/roles/cloud_compose/defaults/main.yml", - r'^cloud_compose_docker_compose_version:\s*(?P\S+)\s*$', - ), - "Salt": ( - "salt/cloud-compose/init.sls", - r"docker\.get\('compose_version',\s*cc\.get\('docker_compose_version',\s*'(?P[^']+)'\)\)", - ), - "host installer fallback": ( - "rootfs/home/cloud-compose/install-docker-plugins.sh", - r'DOCKER_COMPOSE_VERSION="\$\{DOCKER_COMPOSE_VERSION:-(?P[^}]+)\}"', - ), -} - -for label, (relative_path, pattern) in version_patterns.items(): - path = root / relative_path - version = require_match( - pattern, - path.read_text(), - f"{label} Docker Compose default", - re.MULTILINE | re.DOTALL, - ).group("version") - if re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+", version) is None: - fail(f"{label} has an invalid Docker Compose release tag: {version!r}") - compose_versions[label] = version - -if len(set(compose_versions.values())) != 1: - fail(f"Docker Compose defaults diverged: {json.dumps(compose_versions, indent=2)}") - -renovate = (root / "renovate.json5").read_text() -for marker in ( - "Update Docker Compose configuration-management defaults", - "ansible/roles/cloud_compose/defaults/main", - "salt/cloud-compose/init", -): - if marker not in renovate: - fail(f"Renovate configuration is missing configuration-management marker {marker!r}") - -print("Rollout and Docker Compose adapter parity contracts passed") -PY diff --git a/ci/template-version-contract.sh b/ci/template-version-contract.sh deleted file mode 100755 index c4a2dda..0000000 --- a/ci/template-version-contract.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -repo_root="$(cd "$(dirname "$0")/.." && pwd)" -registry="$repo_root/templates/apps.json" - -command -v jq >/dev/null 2>&1 || { - echo "Missing required command: jq" >&2 - exit 1 -} - -jq -e ' - .default.package_versions == { - "sitectl": "v0.39.0" - } and - .templates.archivesspace.package_versions == { - "sitectl": "v0.39.0", - "sitectl-archivesspace": "v0.6.0" - } and - .templates.drupal.package_versions == { - "sitectl": "v0.39.0", - "sitectl-drupal": "v0.11.0" - } and - .templates.isle.package_versions == { - "sitectl": "v0.39.0", - "sitectl-drupal": "v0.11.0", - "sitectl-isle": "v0.18.0" - } and - .templates.ojs.package_versions == { - "sitectl": "v0.39.0", - "sitectl-ojs": "v0.6.0" - } and - .templates["omeka-classic"].package_versions == { - "sitectl": "v0.39.0", - "sitectl-omeka-classic": "v0.6.0" - } and - .templates["omeka-s"].package_versions == { - "sitectl": "v0.39.0", - "sitectl-omeka-s": "v0.6.0" - } and - .templates.wp.package_versions == { - "sitectl": "v0.39.0", - "sitectl-wp": "v0.5.0" - } and - all([.default, .templates[]][]; (.packages | sort) == (.package_versions | keys | sort)) -' "$registry" >/dev/null - -for entrypoint in \ - "$repo_root/main.tf" \ - "$repo_root/providers/do/main.tf" \ - "$repo_root/providers/gcp/main.tf" \ - "$repo_root/providers/linode/main.tf"; do - grep -Fq 'if contains(keys(local.template.package_versions), package)' "$entrypoint" || { - echo "Template package versions are not filtered in ${entrypoint#"$repo_root"/}" >&2 - exit 1 - } - grep -Fq 'package_versions = merge(local.template_sitectl_package_versions, local.input_sitectl.package_versions)' "$entrypoint" || { - echo "Explicit package versions do not override template defaults in ${entrypoint#"$repo_root"/}" >&2 - exit 1 - } - grep -Fq 'local.input_sitectl.packages == null ? local.template.packages : local.input_sitectl.packages' "$entrypoint" || { - echo "Template package omission is not distinguished from an explicit package set in ${entrypoint#"$repo_root"/}" >&2 - exit 1 - } -done - -echo "Template sitectl package-version contract passed" diff --git a/cmd/cloud-compose-ci/main.go b/cmd/cloud-compose-ci/main.go new file mode 100644 index 0000000..4cb1dcd --- /dev/null +++ b/cmd/cloud-compose-ci/main.go @@ -0,0 +1,155 @@ +// Command cloud-compose-ci runs compiled cloud-compose CI lifecycle operations. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log/slog" + "os" + "os/signal" + "strings" + "syscall" + + "github.com/libops/cloud-compose/internal/gcpcleanup" + "github.com/libops/cloud-compose/internal/runnamespace" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + redactor := gcpcleanup.NewEnvironmentRedactor(os.Environ()) + commander := gcpcleanup.ExecCommander{Stderr: os.Stderr, Redactor: redactor} + os.Exit(run(ctx, os.Args[1:], os.Getenv, os.Stdout, os.Stderr, commander, redactor)) +} + +func run( + ctx context.Context, + args []string, + getenv func(string) string, + stdout io.Writer, + stderr io.Writer, + commander gcpcleanup.Commander, + redactor *gcpcleanup.Redactor, +) int { + if len(args) < 2 || args[0] != "gcp" { + printUsage(stderr) + return 2 + } + + switch args[1] { + case "namespace": + return runGCPNamespace(args[2:], stdout, stderr) + case "sweep": + return runGCPSweep(ctx, args[2:], getenv, stderr, commander, redactor) + default: + printUsage(stderr) + return 2 + } +} + +func runGCPNamespace(args []string, stdout, stderr io.Writer) int { + flags := flag.NewFlagSet("cloud-compose-ci gcp namespace", flag.ContinueOnError) + flags.SetOutput(stderr) + runID := flags.String("run-id", "", "canonical decimal GitHub Actions run ID") + if err := flags.Parse(args); err != nil { + return 2 + } + if flags.NArg() != 0 { + fmt.Fprintln(stderr, "cloud-compose-ci gcp namespace does not accept positional arguments") + return 2 + } + + namespace, err := runnamespace.Encode(*runID) + if err != nil { + fmt.Fprintf(stderr, "invalid --run-id: %s\n", err) + return 2 + } + if _, err := fmt.Fprintln(stdout, namespace); err != nil { + fmt.Fprintf(stderr, "write namespace: %s\n", err) + return 1 + } + return 0 +} + +func runGCPSweep( + ctx context.Context, + args []string, + getenv func(string) string, + stderr io.Writer, + commander gcpcleanup.Commander, + redactor *gcpcleanup.Redactor, +) int { + flags := flag.NewFlagSet("cloud-compose-ci gcp sweep", flag.ContinueOnError) + flags.SetOutput(stderr) + project := flags.String("project", getenv("GCLOUD_PROJECT"), "GCP project containing smoke resources") + region := flags.String("region", environmentDefault(getenv, "GCLOUD_REGION", "us-east5"), "GCP region containing Cloud Run resources") + target := flags.String("target", "gcp-wp", "cloud-compose smoke target") + runID := flags.String("run-id", getenv("CLOUD_COMPOSE_SMOKE_RUN_ID"), "originating workflow run ID") + allRuns := flags.Bool("all-runs", false, "sweep every run for the target; intended only for explicit orphan cleanup") + if err := flags.Parse(args); err != nil { + return 2 + } + if flags.NArg() != 0 { + fmt.Fprintln(stderr, "cloud-compose-ci gcp sweep does not accept positional arguments") + return 2 + } + if strings.TrimSpace(*project) == "" { + fmt.Fprintln(stderr, "--project or GCLOUD_PROJECT is required") + return 2 + } + runIDExplicit := false + flags.Visit(func(parsed *flag.Flag) { + if parsed.Name == "run-id" { + runIDExplicit = true + } + }) + if *allRuns && runIDExplicit { + fmt.Fprintln(stderr, "--run-id and --all-runs are mutually exclusive") + return 2 + } + effectiveRunID := *runID + if *allRuns { + // An explicit broad sweep overrides a run ID inherited from the + // environment. Only an explicitly supplied --run-id is a conflict. + effectiveRunID = "" + } + if effectiveRunID == "" && !*allRuns { + fmt.Fprintln(stderr, "--run-id or CLOUD_COMPOSE_SMOKE_RUN_ID is required unless --all-runs is set") + return 2 + } + + logger := slog.New(slog.NewTextHandler(stderr, &slog.HandlerOptions{Level: slog.LevelInfo})) + runner := gcpcleanup.Runner{ + Command: commander, + Logger: logger, + Redactor: redactor, + } + err := runner.Sweep(ctx, gcpcleanup.Config{ + Project: *project, + Region: *region, + Target: *target, + RunID: effectiveRunID, + AllowAllRuns: *allRuns, + }) + if err != nil { + fmt.Fprintf(stderr, "GCP cleanup failed: %s\n", redactor.String(err.Error())) + return 1 + } + return 0 +} + +func environmentDefault(getenv func(string) string, name, fallback string) string { + if value := getenv(name); value != "" { + return value + } + return fallback +} + +func printUsage(writer io.Writer) { + fmt.Fprintln(writer, "Usage:") + fmt.Fprintln(writer, " cloud-compose-ci gcp namespace --run-id RUN_ID") + fmt.Fprintln(writer, " cloud-compose-ci gcp sweep [--project PROJECT] [--region REGION] [--target gcp-wp] (--run-id RUN_ID | --all-runs)") +} diff --git a/cmd/cloud-compose-ci/main_test.go b/cmd/cloud-compose-ci/main_test.go new file mode 100644 index 0000000..caabb00 --- /dev/null +++ b/cmd/cloud-compose-ci/main_test.go @@ -0,0 +1,264 @@ +package main + +import ( + "bytes" + "context" + "errors" + "slices" + "strings" + "testing" + + "github.com/libops/cloud-compose/internal/gcpcleanup" +) + +type emptyGCloud struct { + calls [][]string +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { + return 0, errors.New("closed output") +} + +func (f *emptyGCloud) Run(_ context.Context, args ...string) ([]byte, error) { + f.calls = append(f.calls, slices.Clone(args)) + if len(args) >= 2 && args[0] == "projects" && args[1] == "get-iam-policy" { + return []byte(`{}`), nil + } + return []byte(`[]`), nil +} + +func TestRunGCPSweepUsesEnvironmentOwnership(t *testing.T) { + t.Parallel() + command := &emptyGCloud{} + environment := map[string]string{ + "GCLOUD_PROJECT": "test-project", + "GCLOUD_REGION": "us-east5", + "CLOUD_COMPOSE_SMOKE_RUN_ID": "123456789", + } + var stdout bytes.Buffer + var stderr bytes.Buffer + status := run(context.Background(), []string{"gcp", "sweep"}, environmentGetter(environment), &stdout, &stderr, command, gcpcleanup.NewRedactor()) + if status != 0 { + t.Fatalf("run() status = %d, stderr = %s", status, stderr.String()) + } + + var filter string + for _, call := range command.calls { + if len(call) >= 3 && slices.Equal(call[:3], []string{"compute", "instances", "list"}) { + for index := range call { + if call[index] == "--filter" && index+1 < len(call) { + filter = call[index+1] + } + } + break + } + } + if filter != "name~'^cc-g-wp-12345678-'" { + t.Errorf("instance ownership filter = %q", filter) + } +} + +func TestRunGCPSweepRequiresRunID(t *testing.T) { + t.Parallel() + command := &emptyGCloud{} + var stdout bytes.Buffer + var stderr bytes.Buffer + status := run( + context.Background(), + []string{"gcp", "sweep", "--project", "test-project"}, + environmentGetter(nil), + &stdout, + &stderr, + command, + gcpcleanup.NewRedactor(), + ) + if status != 2 { + t.Errorf("run() status = %d; want 2", status) + } + if !strings.Contains(stderr.String(), "run-id") { + t.Errorf("stderr omits run-id requirement: %s", stderr.String()) + } + if len(command.calls) != 0 { + t.Fatal("run() called gcloud without an ownership scope") + } +} + +func TestRunGCPSweepAllowsExplicitOrphanSweep(t *testing.T) { + t.Parallel() + command := &emptyGCloud{} + var stdout bytes.Buffer + var stderr bytes.Buffer + status := run( + context.Background(), + []string{"gcp", "sweep", "--project", "test-project", "--all-runs"}, + environmentGetter(nil), + &stdout, + &stderr, + command, + gcpcleanup.NewRedactor(), + ) + if status != 0 { + t.Fatalf("run() status = %d, stderr = %s", status, stderr.String()) + } +} + +func TestRunGCPSweepAllRunsOverridesInheritedRunID(t *testing.T) { + t.Parallel() + command := &emptyGCloud{} + environment := map[string]string{ + "CLOUD_COMPOSE_SMOKE_RUN_ID": "123456789", + } + var stdout bytes.Buffer + var stderr bytes.Buffer + status := run( + context.Background(), + []string{"gcp", "sweep", "--project", "test-project", "--all-runs"}, + environmentGetter(environment), + &stdout, + &stderr, + command, + gcpcleanup.NewRedactor(), + ) + if status != 0 { + t.Fatalf("run() status = %d, stderr = %s", status, stderr.String()) + } + + instanceList := firstCommandCall(t, command.calls, "compute", "instances", "list") + if got := commandArgumentValue(t, instanceList, "--filter"); got != "name~'^cc-g-wp-'" { + t.Errorf("instance ownership filter = %q; want an all-run target filter", got) + } +} + +func TestRunGCPSweepRejectsExplicitRunIDWithAllRuns(t *testing.T) { + t.Parallel() + command := &emptyGCloud{} + var stdout bytes.Buffer + var stderr bytes.Buffer + status := run( + context.Background(), + []string{"gcp", "sweep", "--project", "test-project", "--run-id", "123456789", "--all-runs"}, + environmentGetter(nil), + &stdout, + &stderr, + command, + gcpcleanup.NewRedactor(), + ) + if status != 2 { + t.Errorf("run() status = %d; want 2", status) + } + if !strings.Contains(stderr.String(), "mutually exclusive") { + t.Errorf("stderr omits explicit ownership conflict: %s", stderr.String()) + } + if len(command.calls) != 0 { + t.Fatal("run() called gcloud with conflicting ownership flags") + } +} + +func TestRunGCPNamespaceWritesOnlyNamespaceToStdout(t *testing.T) { + t.Parallel() + command := &emptyGCloud{} + var stdout bytes.Buffer + var stderr bytes.Buffer + + status := run( + context.Background(), + []string{"gcp", "namespace", "--run-id", "123456789"}, + environmentGetter(nil), + &stdout, + &stderr, + command, + gcpcleanup.NewRedactor(), + ) + if status != 0 { + t.Fatalf("run() status = %d, stderr = %s", status, stderr.String()) + } + if got := stdout.String(); got != "00021i3v9\n" { + t.Errorf("stdout = %q; want namespace only", got) + } + if stderr.Len() != 0 { + t.Errorf("stderr = %q; want empty", stderr.String()) + } + if len(command.calls) != 0 { + t.Fatal("namespace encoding unexpectedly called gcloud") + } +} + +func TestRunGCPNamespaceRejectsInvalidRunIDs(t *testing.T) { + t.Parallel() + for _, runID := range []string{"", "0123456789", "contract-run", "17592186044416"} { + runID := runID + t.Run(runID, func(t *testing.T) { + t.Parallel() + var stdout bytes.Buffer + var stderr bytes.Buffer + status := run( + context.Background(), + []string{"gcp", "namespace", "--run-id", runID}, + environmentGetter(nil), + &stdout, + &stderr, + &emptyGCloud{}, + gcpcleanup.NewRedactor(), + ) + if status != 2 { + t.Errorf("run() status = %d; want 2", status) + } + if stdout.Len() != 0 { + t.Errorf("stdout = %q; want empty", stdout.String()) + } + if !strings.Contains(stderr.String(), "invalid --run-id") { + t.Errorf("stderr omits invalid run ID error: %s", stderr.String()) + } + }) + } +} + +func TestRunGCPNamespaceReportsOutputFailure(t *testing.T) { + t.Parallel() + var stderr bytes.Buffer + status := run( + context.Background(), + []string{"gcp", "namespace", "--run-id", "123456789"}, + environmentGetter(nil), + failingWriter{}, + &stderr, + &emptyGCloud{}, + gcpcleanup.NewRedactor(), + ) + if status != 1 { + t.Errorf("run() status = %d; want 1", status) + } + if !strings.Contains(stderr.String(), "write namespace") { + t.Errorf("stderr omits output failure: %s", stderr.String()) + } +} + +func firstCommandCall(t testing.TB, calls [][]string, prefix ...string) []string { + t.Helper() + for _, call := range calls { + if len(call) >= len(prefix) && slices.Equal(call[:len(prefix)], prefix) { + return call + } + } + t.Fatalf("command prefix %q was not called", prefix) + return nil +} + +func commandArgumentValue(t testing.TB, args []string, name string) string { + t.Helper() + for index := range args { + if args[index] == name && index+1 < len(args) { + return args[index+1] + } + } + t.Fatalf("argument %q missing from %q", name, args) + return "" +} + +func environmentGetter(values map[string]string) func(string) string { + return func(name string) string { + return values[name] + } +} diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index 5f61c72..edc7200 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -742,10 +742,34 @@ reviewers**, and set its deployment branch policy to the selected branch `main` only. Those environments are consumed solely by the `workflow_run` workflow loaded from the default branch. It checks out `github.sha` (the trusted default-branch revision for that event), rejects fork-originated runs, and uses -the originating workflow run ID to constrain the sweep. This fallback runs -automatically after a failed, cancelled, or timed-out smoke workflow, including -cases where cancellation prevented the in-job destroy from finishing. Never -allow pull-request branches in a cleanup environment's deployment policy. +the originating workflow run ID to constrain the sweep. Only the GCP app-smoke +cleanup job builds `.bin/cloud-compose-ci`; it builds the binary once from that +trusted checkout and uses it for GCP discovery, retries, ordered deletion, and +residual verification. The GCP pull-request job likewise builds one binary +before apply and reuses that exact workspace binary from its `always()` cleanup +step. DigitalOcean and Linode app cleanup, plus Ansible and Salt +config-management cleanup, continue to use their hardened shell drivers; the Go +runner is not yet a general hosted-provider cleanup implementation. No cleanup +job executes a pull-request binary or downloads one as an artifact. This +fallback runs automatically after a failed, cancelled, or timed-out smoke +workflow, including cases where cancellation prevented the in-job destroy from +finishing. Never allow pull-request branches in a cleanup environment's +deployment policy. + +GCP run namespaces require a staged compatibility change because the privileged +fallback always executes the default branch. The compiled runner recognizes the +legacy first-eight-character namespace and the reserved exact namespace that +encodes a canonical numeric run ID as nine fixed-width base36 characters. That +width preserves both separators and the random suffix for every supported GCP +template within the 21-character resource-name limit. In this reader phase the +fresh GCP driver calls the pure `cloud-compose-ci gcp namespace` command before +Terraform argument construction to require a non-empty canonical run ID no +larger than 44 bits, while deliberately continuing to write legacy names. Manual GCP +smoke applies must therefore set `CLOUD_COMPOSE_SMOKE_RUN_ID`; non-GCP smoke +naming and invocation remain unchanged. Merge the +dual-reader before changing the smoke Terraform writer. Retain the legacy reader +until all branches and resources created by the old writer have expired; never +introduce a new writer that the trusted fallback cannot discover. Use separate provider identities for smoke and fallback cleanup: @@ -787,5 +811,15 @@ VPC networks; it must not own the singleton foundation or the deliberately persistent upgrade network. Every mutation is retried, failures are accumulated so one stuck resource does not prevent unrelated resources from being examined, and a final query requires zero matching disposable resources and IAM members. +Discovery reads are retried independently. Subnetwork and parent-network +deletion share a bounded two-hour-ten-minute exponential-backoff window because +Direct VPC addresses can remain allocated for one to two hours after Cloud Run +disconnects; exhausting that window still fails the job and reports the leak. The workflow fails if a mutation exhausts its retry budget or residual resources -remain, making leaks visible to operators. +remain, making leaks visible to operators. The compiled runner requires a run ID +by default; an operator can sweep every run for one disposable target only by +supplying the explicit `--all-runs` flag. That explicit flag overrides a run ID +inherited from `CLOUD_COMPOSE_SMOKE_RUN_ID`; supplying both `--run-id` and +`--all-runs` explicitly is an error. The compatibility shell exposes the broad +operation only when `CLOUD_COMPOSE_SMOKE_ALLOW_ALL_RUNS=true`; a missing run ID +otherwise fails closed. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2f6fd01 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/libops/cloud-compose + +go 1.26.0 diff --git a/internal/contracttest/cloud_smoke_cleanup_test.go b/internal/contracttest/cloud_smoke_cleanup_test.go new file mode 100644 index 0000000..0ce25e2 --- /dev/null +++ b/internal/contracttest/cloud_smoke_cleanup_test.go @@ -0,0 +1,236 @@ +package contracttest + +import ( + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" +) + +func TestCloudSmokeCleanupWorkflowTrust(t *testing.T) { + t.Parallel() + root := repositoryRoot(t) + pullRequestWorkflow := readRepositoryFile(t, root, ".github/workflows/cloud-smoke.yml") + cleanupWorkflow := readRepositoryFile(t, root, ".github/workflows/cloud-smoke-cleanup.yml") + docs := readRepositoryFile(t, root, "docs/runtime-contracts.md") + + for label, marker := range map[string]string{ + "pull-request event restriction": "github.event.workflow_run.event == 'pull_request'", + "same-repository restriction": "github.event.workflow_run.head_repository.full_name == github.repository", + "trusted checkout": "ref: ${{ github.sha }}", + "originating workflow run ID": "CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.event.workflow_run.id }}", + "compiled runner build": "run: make cloud-compose-ci", + } { + requireContains(t, cleanupWorkflow, marker, label) + } + if strings.Contains(cleanupWorkflow, "github.event.workflow_run.head_sha") { + t.Fatal("fallback cleanup checks out pull-request-controlled code") + } + if strings.Contains(cleanupWorkflow, "actions/download-artifact") { + t.Fatal("privileged fallback cleanup consumes a pull-request artifact") + } + + for _, environment := range []string{ + "cloud-smoke-cleanup-digitalocean", + "cloud-smoke-cleanup-linode", + "cloud-smoke-cleanup-gcp", + } { + if strings.Contains(pullRequestWorkflow, environment) { + t.Errorf("pull-request-controlled workflow can request cleanup environment %s", environment) + } + requireContains(t, cleanupWorkflow, environment, "dedicated cleanup environment "+environment) + requireContains(t, docs, environment, "cleanup environment documentation for "+environment) + } + + requireContains(t, pullRequestWorkflow, "if: always()", "same-job cleanup") + cleanupJob := regexp.MustCompile(`(?m)^ (config-management-cleanup|cleanup|gcp-cleanup):`) + if cleanupJob.MatchString(pullRequestWorkflow) { + t.Fatal("pull-request workflow contains a second secret-bearing cleanup job") + } + + buildIndex := strings.Index(pullRequestWorkflow, "- name: Build cloud CI runner") + freshIndex := strings.Index(pullRequestWorkflow, "- name: Run fresh smoke test") + upgradeIndex := strings.Index(pullRequestWorkflow, "- name: Run 0.10.2 upgrade smoke test") + destroyIndex := strings.Index(pullRequestWorkflow, "- name: Destroy fresh smoke resources") + if buildIndex < 0 || freshIndex < 0 || upgradeIndex < 0 || destroyIndex < 0 || + buildIndex >= freshIndex || buildIndex >= upgradeIndex || buildIndex >= destroyIndex { + t.Fatal("GCP smoke job does not build one cleanup runner before every lifecycle path") + } +} + +func TestCloudSmokeGCPWrapperUsesCompiledRunner(t *testing.T) { + t.Parallel() + root := repositoryRoot(t) + driver := readRepositoryFile(t, root, "ci/cloud-smoke.sh") + makefile := readRepositoryFile(t, root, "Makefile") + + for label, marker := range map[string]string{ + "compiled runner path": "CLOUD_COMPOSE_CI_BIN:-$repo_root/.bin/cloud-compose-ci", + "canonical run validator": `"$runner" gcp namespace --run-id "$run_id"`, + "GCP sweep subcommand": "gcp sweep", + "owned-run flag": `cleanup_args+=(--run-id "$run_id")`, + "explicit orphan flag": "cleanup_args+=(--all-runs)", + "broad-sweep gate": `CLOUD_COMPOSE_SMOKE_ALLOW_ALL_RUNS:-false`, + "Make build target": `go build -trimpath -o "$(CLOUD_COMPOSE_CI_BIN)" ./cmd/cloud-compose-ci`, + } { + requireContains(t, driver+makefile, marker, label) + } + for _, obsolete := range []string{ + "gcp_command_with_retry", + "gcp_project_iam_rows", + "gcp_smoke_residuals", + "gcp_verify_no_smoke_resources", + } { + if strings.Contains(driver, obsolete) { + t.Errorf("shell driver still owns migrated GCP behavior %s", obsolete) + } + } + if strings.Contains(makefile, "go run ./cmd/cloud-compose-ci") { + t.Fatal("cloud lifecycle uses go run instead of one compiled runner") + } + if _, err := os.Stat(filepath.Join(root, "ci/cloud-smoke-cleanup-contract.sh")); !os.IsNotExist(err) { + t.Fatalf("legacy fake-gcloud Bash contract still exists: %v", err) + } +} + +func TestCloudSmokeGCPApplyRequiresCanonicalRunID(t *testing.T) { + t.Parallel() + root := repositoryRoot(t) + driverPath := filepath.Join(root, "ci/cloud-smoke.sh") + driver := readRepositoryFile(t, root, "ci/cloud-smoke.sh") + + validation := `gcp_run_namespace "$target" "$run_id" >/dev/null` + validationIndex := strings.Index(driver, validation) + argumentsIndex := strings.Index(driver, `mapfile -d '' -t var_args < <(target_var_args`) + applyIndex := strings.Index(driver, `terraform -chdir="$root" apply`) + if validationIndex < 0 || argumentsIndex < 0 || applyIndex < 0 || + validationIndex >= argumentsIndex || validationIndex >= applyIndex { + t.Fatal("GCP smoke does not validate its canonical run ID before Terraform argument construction and apply") + } + + runHelper := func(t *testing.T, target, runID string) (string, error) { + t.Helper() + command := exec.Command( + "bash", + "-c", + `source "$1"; gcp_run_namespace "$2" "$3"`, + "cloud-compose-contract", + driverPath, + target, + runID, + ) + output, err := command.CombinedOutput() + return string(output), err + } + + output, err := runHelper(t, "gcp-wp", "") + if err == nil { + t.Fatal("GCP run-ID guard unexpectedly accepted an empty run ID") + } + if !strings.Contains(output, "required for every GCP smoke apply") { + t.Fatalf("GCP run-ID guard did not explain the failure: %s", output) + } + + if output, err := runHelper(t, "linode-wp", ""); err != nil { + t.Fatalf("non-GCP run-ID guard changed existing behavior: %v, output = %s", err, output) + } +} + +func TestCloudSmokeGCPWrapperFailsClosedWithoutRunID(t *testing.T) { + t.Parallel() + root := repositoryRoot(t) + temporaryDirectory := t.TempDir() + fakeRunner := filepath.Join(temporaryDirectory, "cloud-compose-ci") + fakeGCloud := filepath.Join(temporaryDirectory, "gcloud") + for path, content := range map[string]string{ + fakeRunner: `#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$@" >"$FAKE_CLEANUP_LOG" +`, + fakeGCloud: `#!/usr/bin/env bash +exit 0 +`, + } { + if err := os.WriteFile(path, []byte(content), 0o700); err != nil { + t.Fatalf("write fake executable %s: %v", path, err) + } + } + + runWrapper := func(t *testing.T, runID string, allowAll bool) (string, string, error) { + t.Helper() + logPath := filepath.Join(t.TempDir(), "cleanup.log") + overrides := map[string]string{ + "CLOUD_COMPOSE_CI_BIN": fakeRunner, + "CLOUD_COMPOSE_SMOKE_RUN_ID": runID, + "FAKE_CLEANUP_LOG": logPath, + "GCLOUD_PROJECT": "test-project", + "GCLOUD_REGION": "us-east5", + "PATH": temporaryDirectory + string(os.PathListSeparator) + os.Getenv("PATH"), + } + if allowAll { + overrides["CLOUD_COMPOSE_SMOKE_ALLOW_ALL_RUNS"] = "true" + } else { + overrides["CLOUD_COMPOSE_SMOKE_ALLOW_ALL_RUNS"] = "false" + } + command := exec.Command("bash", filepath.Join(root, "ci/cloud-smoke.sh"), "sweep-gcp-wp") + command.Env = overriddenEnvironment(overrides) + output, err := command.CombinedOutput() + logged, readErr := os.ReadFile(logPath) + if readErr != nil && !os.IsNotExist(readErr) { + t.Fatalf("read fake cleanup log: %v", readErr) + } + return string(output), string(logged), err + } + + t.Run("owned run", func(t *testing.T) { + output, logged, err := runWrapper(t, "123456789", false) + if err != nil { + t.Fatalf("wrapper error = %v, output = %s", err, output) + } + if !strings.Contains(logged, "--run-id\n123456789\n") || strings.Contains(logged, "--all-runs") { + t.Fatalf("wrapper did not preserve run ownership:\n%s", logged) + } + }) + + t.Run("missing scope", func(t *testing.T) { + output, logged, err := runWrapper(t, "", false) + if err == nil { + t.Fatal("wrapper unexpectedly allowed an unscoped cleanup") + } + if logged != "" { + t.Fatalf("wrapper invoked cleanup before establishing ownership:\n%s", logged) + } + if !strings.Contains(output, "requires CLOUD_COMPOSE_SMOKE_RUN_ID") { + t.Fatalf("wrapper did not explain the ownership failure:\n%s", output) + } + }) + + t.Run("explicit all runs", func(t *testing.T) { + output, logged, err := runWrapper(t, "", true) + if err != nil { + t.Fatalf("wrapper error = %v, output = %s", err, output) + } + if !strings.Contains(logged, "--all-runs\n") || strings.Contains(logged, "--run-id") { + t.Fatalf("wrapper did not pass the explicit broad-cleanup flag:\n%s", logged) + } + }) +} + +func overriddenEnvironment(overrides map[string]string) []string { + environment := make([]string, 0, len(os.Environ())+len(overrides)) + for _, entry := range os.Environ() { + name, _, found := strings.Cut(entry, "=") + if found { + if _, overridden := overrides[name]; overridden { + continue + } + } + environment = append(environment, entry) + } + for name, value := range overrides { + environment = append(environment, name+"="+value) + } + return environment +} diff --git a/internal/contracttest/helpers_test.go b/internal/contracttest/helpers_test.go new file mode 100644 index 0000000..c010df3 --- /dev/null +++ b/internal/contracttest/helpers_test.go @@ -0,0 +1,79 @@ +package contracttest + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +func repositoryRoot(t testing.TB) string { + t.Helper() + + directory, err := os.Getwd() + if err != nil { + t.Fatalf("resolve repository root: get working directory: %v", err) + } + + for { + if _, err := os.Stat(filepath.Join(directory, "go.mod")); err == nil { + return directory + } else if !os.IsNotExist(err) { + t.Fatalf("resolve repository root: inspect %s: %v", directory, err) + } + + parent := filepath.Dir(directory) + if parent == directory { + t.Fatal("resolve repository root: go.mod not found") + } + directory = parent + } +} + +func readRepositoryFile(t testing.TB, root, relativePath string) string { + t.Helper() + + content, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(relativePath))) + if err != nil { + t.Fatalf("read %s: %v", relativePath, err) + } + + return string(content) +} + +func requireContains(t testing.TB, text, expected, label string) { + t.Helper() + + if !strings.Contains(text, expected) { + t.Fatalf("could not find %s", label) + } +} + +func requireFirstSubmatch(t testing.TB, expression, text, label string) string { + t.Helper() + + pattern, err := regexp.Compile(expression) + if err != nil { + t.Fatalf("compile pattern for %s: %v", label, err) + } + + match := pattern.FindStringSubmatch(text) + if len(match) < 2 { + t.Fatalf("could not find %s", label) + } + + return match[1] +} + +func prettyJSON(t testing.TB, value any) string { + t.Helper() + + encoded, err := json.MarshalIndent(value, "", " ") + if err != nil { + t.Fatalf("format contract value: %v", err) + } + + return string(encoded) +} diff --git a/internal/contracttest/rollout_parity_test.go b/internal/contracttest/rollout_parity_test.go new file mode 100644 index 0000000..a774967 --- /dev/null +++ b/internal/contracttest/rollout_parity_test.go @@ -0,0 +1,284 @@ +package contracttest + +import ( + "encoding/json" + "fmt" + "regexp" + "slices" + "strconv" + "strings" + "testing" +) + +var expectedRollout = []string{ + `TARGET_REF="${GIT_REF:-${GIT_BRANCH:-}}"`, + `if [ -n "$TARGET_REF" ]; then sitectl deploy --context "${SITECTL_CONTEXT_NAME}" --ref "$TARGET_REF"; else sitectl deploy --context "${SITECTL_CONTEXT_NAME}" --skip-git; fi`, + `sitectl healthcheck --context "${SITECTL_CONTEXT_NAME}" --persist`, + `if [ "${SITECTL_ENVIRONMENT}" != "production" ]; then sitectl verify --context "${SITECTL_CONTEXT_NAME}" ${SITECTL_VERIFY_ARGS:-}; fi`, +} + +type rolloutSource struct { + label string + relativePath string + commands []string +} + +func TestRolloutParityContract(t *testing.T) { + root := repositoryRoot(t) + sources := []rolloutSource{ + { + label: "GCP Terraform", + relativePath: "modules/gcp/variables.tf", + commands: parseHCLRollout( + t, + root, + "modules/gcp/variables.tf", + `(?ms)^variable "docker_compose_rollout" \{.*?^ default = \[\n(.*?)^ \]\n`, + "GCP Terraform default rollout list", + ), + }, + { + label: "Linux VM Terraform", + relativePath: "modules/linux-vm-runtime/variables.tf", + commands: parseHCLRollout( + t, + root, + "modules/linux-vm-runtime/variables.tf", + `(?ms)^variable "docker_compose_rollout" \{.*?^ default = \[\n(.*?)^ \]\n`, + "Linux VM Terraform default rollout list", + ), + }, + { + label: "Ansible", + relativePath: "ansible/roles/cloud_compose/defaults/main.yml", + commands: parseAnsibleRollout(t, root), + }, + { + label: "Salt", + relativePath: "salt/cloud-compose/init.sls", + commands: parseSaltRollout(t, root), + }, + { + label: "rollout documentation", + relativePath: "docs/rollout.md", + commands: parseHCLRollout( + t, + root, + "docs/rollout.md", + `(?ms)^ rollout = \[\n(.*?)^ \]\n`, + "documented rollout list", + ), + }, + } + + for _, source := range sources { + if !slices.Equal(source.commands, expectedRollout) { + t.Errorf("%s rollout list in %s diverged:\nexpected %s\nactual %s", source.label, source.relativePath, prettyJSON(t, expectedRollout), prettyJSON(t, source.commands)) + } + + content := readRepositoryFile(t, root, source.relativePath) + if strings.Contains(content, "scripts/rollout.sh") { + t.Errorf("%s still delegates lifecycle ownership to scripts/rollout.sh", source.label) + } + } + + composeVersions := checkComposeVersions(t, root) + var expectedVersion string + for _, source := range composeVersions { + if expectedVersion == "" { + expectedVersion = source.version + continue + } + if source.version != expectedVersion { + t.Errorf("Docker Compose defaults diverged: %s is %s; expected %s", source.label, source.version, expectedVersion) + } + } + + renovate := readRepositoryFile(t, root, "renovate.json5") + for _, marker := range []string{ + "Update Docker Compose configuration-management defaults", + "ansible/roles/cloud_compose/defaults/main", + "salt/cloud-compose/init", + } { + requireContains(t, renovate, marker, fmt.Sprintf("Renovate configuration-management marker %q", marker)) + } +} + +func parseHCLRollout(t testing.TB, root, relativePath, expression, label string) []string { + t.Helper() + + content := readRepositoryFile(t, root, relativePath) + block := requireFirstSubmatch(t, expression, content, label) + commands := make([]string, 0, len(expectedRollout)) + for _, rawLine := range strings.Split(block, "\n") { + line := strings.TrimSpace(rawLine) + if line == "" { + continue + } + line = strings.TrimSuffix(line, ",") + + var command string + if err := json.Unmarshal([]byte(line), &command); err != nil { + t.Fatalf("could not parse %s command %q: %v", label, line, err) + } + commands = append(commands, strings.ReplaceAll(command, "$${", "${")) + } + + return commands +} + +func parseAnsibleRollout(t testing.TB, root string) []string { + t.Helper() + + const relativePath = "ansible/roles/cloud_compose/defaults/main.yml" + content := readRepositoryFile(t, root, relativePath) + block := requireFirstSubmatch(t, `(?m)^cloud_compose_default_rollout:\n((^ - .*\n)+)`, content, "Ansible default rollout list") + linePattern := regexp.MustCompile(`^ - '(.*)'$`) + commands := make([]string, 0, len(expectedRollout)) + for _, rawLine := range strings.Split(strings.TrimSuffix(block, "\n"), "\n") { + match := linePattern.FindStringSubmatch(rawLine) + if len(match) != 2 { + t.Fatalf("could not parse Ansible rollout command %q", rawLine) + } + commands = append(commands, match[1]) + } + + return commands +} + +func parseSaltRollout(t testing.TB, root string) []string { + t.Helper() + + const relativePath = "salt/cloud-compose/init.sls" + content := readRepositoryFile(t, root, relativePath) + block := requireFirstSubmatch(t, `(?ms)^\{% set default_rollout = \[\n(.*?)^\] %\}`, content, "Salt default rollout list") + commands := make([]string, 0, len(expectedRollout)) + for _, rawLine := range strings.Split(block, "\n") { + line := strings.TrimSuffix(strings.TrimSpace(rawLine), ",") + if line == "" { + continue + } + + command, err := unquoteSaltString(line) + if err != nil { + t.Fatalf("could not parse Salt rollout command %q: %v", line, err) + } + commands = append(commands, command) + } + + return commands +} + +func unquoteSaltString(literal string) (string, error) { + if len(literal) < 2 || literal[0] != literal[len(literal)-1] { + return "", fmt.Errorf("not a quoted string") + } + if literal[0] == '"' { + return strconv.Unquote(literal) + } + if literal[0] != '\'' { + return "", fmt.Errorf("unsupported string delimiter %q", literal[0]) + } + + inner := literal[1 : len(literal)-1] + var converted strings.Builder + converted.Grow(len(literal) + 2) + converted.WriteByte('"') + for index := 0; index < len(inner); index++ { + character := inner[index] + if character == '\\' && index+1 < len(inner) { + next := inner[index+1] + if next == '\'' { + converted.WriteByte('\'') + index++ + continue + } + converted.WriteByte(character) + converted.WriteByte(next) + index++ + continue + } + if character == '"' { + converted.WriteByte('\\') + } + converted.WriteByte(character) + } + converted.WriteByte('"') + + return strconv.Unquote(converted.String()) +} + +func TestUnquoteSaltString(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + literal string + want string + wantErr bool + }{ + {name: "single quoted", literal: `'say "hello"'`, want: `say "hello"`}, + {name: "escaped single quote", literal: `'it\'s ready'`, want: `it's ready`}, + {name: "escaped backslash", literal: `'C:\\tmp'`, want: `C:\tmp`}, + {name: "double quoted", literal: `"line\nnext"`, want: "line\nnext"}, + {name: "unquoted", literal: "command", wantErr: true}, + {name: "mismatched delimiters", literal: `'command"`, wantErr: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + got, err := unquoteSaltString(test.literal) + if test.wantErr { + if err == nil { + t.Fatalf("unquoteSaltString(%q) unexpectedly succeeded with %q", test.literal, got) + } + return + } + if err != nil { + t.Fatalf("unquoteSaltString(%q): %v", test.literal, err) + } + if got != test.want { + t.Errorf("unquoteSaltString(%q) = %q; want %q", test.literal, got, test.want) + } + }) + } +} + +type composeVersionSource struct { + label string + relativePath string + expression string + version string +} + +func checkComposeVersions(t testing.TB, root string) []composeVersionSource { + t.Helper() + + sources := []composeVersionSource{ + {label: "root Terraform", relativePath: "variables.tf", expression: `compose_version\s*=\s*optional\(string,\s*"([^"]+)"\)`}, + {label: "GCP provider", relativePath: "providers/gcp/variables.tf", expression: `compose_version\s*=\s*optional\(string,\s*"([^"]+)"\)`}, + {label: "DigitalOcean provider", relativePath: "providers/do/variables.tf", expression: `compose_version\s*=\s*optional\(string,\s*"([^"]+)"\)`}, + {label: "Linode provider", relativePath: "providers/linode/variables.tf", expression: `compose_version\s*=\s*optional\(string,\s*"([^"]+)"\)`}, + {label: "DigitalOcean module", relativePath: "modules/digitalocean/variables.tf", expression: `compose_version\s*=\s*optional\(string,\s*"([^"]+)"\)`}, + {label: "Linode module", relativePath: "modules/linode/variables.tf", expression: `compose_version\s*=\s*optional\(string,\s*"([^"]+)"\)`}, + {label: "GCP module", relativePath: "modules/gcp/variables.tf", expression: `(?ms)variable "docker_compose_version" \{.*?default\s*=\s*"([^"]+)"`}, + {label: "Linux VM module", relativePath: "modules/linux-vm-runtime/variables.tf", expression: `(?ms)variable "docker_compose_version" \{.*?default\s*=\s*"([^"]+)"`}, + {label: "Ansible", relativePath: "ansible/roles/cloud_compose/defaults/main.yml", expression: `(?m)^cloud_compose_docker_compose_version:\s*(\S+)\s*$`}, + {label: "Salt", relativePath: "salt/cloud-compose/init.sls", expression: `docker\.get\('compose_version',\s*cc\.get\('docker_compose_version',\s*'([^']+)'\)\)`}, + {label: "host installer fallback", relativePath: "rootfs/home/cloud-compose/install-docker-plugins.sh", expression: `DOCKER_COMPOSE_VERSION="\$\{DOCKER_COMPOSE_VERSION:-([^}]+)\}"`}, + } + + releaseTag := regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+$`) + for index := range sources { + source := &sources[index] + content := readRepositoryFile(t, root, source.relativePath) + source.version = requireFirstSubmatch(t, source.expression, content, source.label+" Docker Compose default") + if !releaseTag.MatchString(source.version) { + t.Errorf("%s has an invalid Docker Compose release tag: %q", source.label, source.version) + } + } + + return sources +} diff --git a/internal/contracttest/template_version_test.go b/internal/contracttest/template_version_test.go new file mode 100644 index 0000000..44f4df1 --- /dev/null +++ b/internal/contracttest/template_version_test.go @@ -0,0 +1,120 @@ +package contracttest + +import ( + "encoding/json" + "maps" + "slices" + "testing" +) + +type templateRegistry struct { + Default templateDefinition `json:"default"` + Templates map[string]templateDefinition `json:"templates"` +} + +type templateDefinition struct { + Packages []string `json:"packages"` + PackageVersions map[string]string `json:"package_versions"` +} + +func TestTemplateVersionContract(t *testing.T) { + root := repositoryRoot(t) + registryContent := readRepositoryFile(t, root, "templates/apps.json") + + var registry templateRegistry + if err := json.Unmarshal([]byte(registryContent), ®istry); err != nil { + t.Fatalf("parse templates/apps.json: %v", err) + } + + expectedVersions := map[string]map[string]string{ + "default": { + "sitectl": "v0.39.0", + }, + "archivesspace": { + "sitectl": "v0.39.0", + "sitectl-archivesspace": "v0.6.0", + }, + "drupal": { + "sitectl": "v0.39.0", + "sitectl-drupal": "v0.11.0", + }, + "isle": { + "sitectl": "v0.39.0", + "sitectl-drupal": "v0.11.0", + "sitectl-isle": "v0.18.0", + }, + "ojs": { + "sitectl": "v0.39.0", + "sitectl-ojs": "v0.6.0", + }, + "omeka-classic": { + "sitectl": "v0.39.0", + "sitectl-omeka-classic": "v0.6.0", + }, + "omeka-s": { + "sitectl": "v0.39.0", + "sitectl-omeka-s": "v0.6.0", + }, + "wp": { + "sitectl": "v0.39.0", + "sitectl-wp": "v0.5.0", + }, + } + + if !maps.Equal(registry.Default.PackageVersions, expectedVersions["default"]) { + t.Errorf("template %q package versions diverged:\nexpected %s\nactual %s", "default", prettyJSON(t, expectedVersions["default"]), prettyJSON(t, registry.Default.PackageVersions)) + } + + for _, name := range slices.Sorted(maps.Keys(expectedVersions)) { + if name == "default" { + continue + } + expected := expectedVersions[name] + definition, ok := registry.Templates[name] + if !ok { + t.Errorf("template %q is missing", name) + continue + } + if !maps.Equal(definition.PackageVersions, expected) { + t.Errorf("template %q package versions diverged:\nexpected %s\nactual %s", name, prettyJSON(t, expected), prettyJSON(t, definition.PackageVersions)) + } + } + + checkPackageVersionKeys := func(name string, definition templateDefinition) { + t.Helper() + + if definition.Packages == nil { + t.Errorf("template %q packages must be an array", name) + return + } + if definition.PackageVersions == nil { + t.Errorf("template %q package_versions must be an object", name) + return + } + + packages := slices.Clone(definition.Packages) + slices.Sort(packages) + versionedPackages := slices.Sorted(maps.Keys(definition.PackageVersions)) + if !slices.Equal(packages, versionedPackages) { + t.Errorf("template %q packages do not match package_versions keys:\npackages %s\npackage_versions %s", name, prettyJSON(t, packages), prettyJSON(t, versionedPackages)) + } + } + + checkPackageVersionKeys("default", registry.Default) + for _, name := range slices.Sorted(maps.Keys(registry.Templates)) { + checkPackageVersionKeys(name, registry.Templates[name]) + } + + entrypoints := []string{ + "main.tf", + "providers/do/main.tf", + "providers/gcp/main.tf", + "providers/linode/main.tf", + } + for _, relativePath := range entrypoints { + content := readRepositoryFile(t, root, relativePath) + requireContains(t, content, "if contains(keys(local.template.package_versions), package)", relativePath+" template package-version filter") + requireContains(t, content, "package_versions = merge(local.template_sitectl_package_versions, local.input_sitectl.package_versions)", relativePath+" explicit package-version override") + requireContains(t, content, "local.input_sitectl.packages == null ? local.template.packages : local.input_sitectl.packages", relativePath+" omitted-package handling") + } +} diff --git a/internal/gcpcleanup/cleanup.go b/internal/gcpcleanup/cleanup.go new file mode 100644 index 0000000..431f388 --- /dev/null +++ b/internal/gcpcleanup/cleanup.go @@ -0,0 +1,837 @@ +package gcpcleanup + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "regexp" + "strings" + "time" + + "github.com/libops/cloud-compose/internal/runnamespace" +) + +const ( + defaultAttempts = 12 + defaultDelay = 10 * time.Second + defaultQueryAttempts = 3 + defaultQueryDelay = 2 * time.Second + defaultNetworkRetryWindow = 2*time.Hour + 10*time.Minute +) + +var templateSlugs = map[string]string{ + "archivesspace": "as", + "drupal": "dr", + "isle": "isle", + "ojs": "ojs", + "omeka-classic": "oc", + "omeka-s": "os", + "wp": "wp", +} + +// Config identifies the GCP smoke resources that one sweep owns. +type Config struct { + Project string + Region string + Target string + RunID string + Attempts int + RetryDelay time.Duration + QueryAttempts int + QueryRetryDelay time.Duration + NetworkRetryWindow time.Duration + AllowAllRuns bool +} + +// Runner performs ordered GCP smoke-resource cleanup. +type Runner struct { + Command Commander + Logger *slog.Logger + Redactor *Redactor + Sleep func(context.Context, time.Duration) error + queryAttempts int + queryRetryDelay time.Duration + networkRetryBudget *time.Duration +} + +type namedResource struct { + Name string `json:"name"` +} + +type zonalResource struct { + Name string `json:"name"` + Zone string `json:"zone"` +} + +type regionalResource struct { + Name string `json:"name"` + Region string `json:"region"` +} + +type serviceAccount struct { + Email string `json:"email"` +} + +type cloudRunService struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` +} + +type iamBinding struct { + Role string `json:"role"` + Members []string `json:"members"` + Condition json.RawMessage `json:"condition,omitempty"` +} + +type projectIAMBinding struct { + Role string + Member string +} + +type residual struct { + Kind string + Name string + Role string + Member string +} + +type failureList struct { + errors []error +} + +// NameFilter returns the legacy anchored resource-name filter shared with the current smoke Terraform fixture. +func NameFilter(target, runID string) (string, error) { + prefix, err := targetNamePrefix(target) + if err != nil { + return "", err + } + filter := "^" + prefix + "-" + if runID != "" { + filter += normalizeRunID(runID, 8) + "-" + } + return filter, nil +} + +func ownershipFilters(target, runID string) ([]string, error) { + legacy, err := NameFilter(target, runID) + if err != nil { + return nil, err + } + if runID == "" { + return []string{legacy}, nil + } + prefix, err := targetNamePrefix(target) + if err != nil { + return nil, err + } + namespace, err := runnamespace.Encode(runID) + if err != nil { + return nil, fmt.Errorf("encode run ID namespace: %w", err) + } + return []string{legacy, "^" + prefix + "-" + namespace + "-"}, nil +} + +// Sweep removes all disposable GCP resources selected by the configuration. +func (r Runner) Sweep(ctx context.Context, config Config) error { + config, err := normalizeConfig(config) + if err != nil { + return err + } + if r.Command == nil { + return errors.New("gcp cleanup command is required") + } + if config.RunID == "" && !config.AllowAllRuns { + return errors.New("run id is required unless all-run cleanup is explicitly enabled") + } + if config.RunID != "" && config.AllowAllRuns { + return errors.New("run id and all-run cleanup are mutually exclusive") + } + + redactor := r.Redactor + if redactor == nil { + redactor = NewRedactor() + } + logger := r.Logger + if logger == nil { + logger = slog.New(slog.NewTextHandler(io.Discard, nil)) + } + sleep := r.Sleep + if sleep == nil { + sleep = sleepContext + } + r.Redactor = redactor + r.Logger = logger + r.Sleep = sleep + r.queryAttempts = config.QueryAttempts + r.queryRetryDelay = config.QueryRetryDelay + networkRetryBudget := config.NetworkRetryWindow + r.networkRetryBudget = &networkRetryBudget + + nameFilters, err := ownershipFilters(config.Target, config.RunID) + if err != nil { + return err + } + failures := &failureList{} + + for _, nameFilter := range nameFilters { + ownedNames := regexp.MustCompile(nameFilter) + r.cleanupCloudRun(ctx, config, nameFilter, ownedNames, failures) + r.cleanupInstances(ctx, config, nameFilter, ownedNames, failures) + r.cleanupFirewalls(ctx, config, nameFilter, failures) + r.cleanupDisks(ctx, config, nameFilter, ownedNames, failures) + r.cleanupProjectIAM(ctx, config, nameFilter, failures) + r.cleanupServiceAccounts(ctx, config, nameFilter, failures) + r.cleanupSubnetworks(ctx, config, nameFilter, ownedNames, failures) + r.cleanupNetworks(ctx, config, nameFilter, ownedNames, failures) + } + + for _, nameFilter := range nameFilters { + if err := r.verifyNoResources(ctx, config, nameFilter); err != nil { + failures.add("verify residual resources for "+nameFilter, err, redactor) + } + } + if failures.empty() { + logger.Info("GCP smoke cleanup completed", "target", config.Target, "run_id", normalizeRunID(config.RunID, 8)) + return nil + } + return failures +} + +func (r Runner) cleanupCloudRun(ctx context.Context, config Config, nameFilter string, owned *regexp.Regexp, failures *failureList) { + services, err := r.listCloudRun(ctx, config, nameFilter) + if err != nil { + failures.add("list Cloud Run services", err, r.Redactor) + return + } + for _, service := range services { + name := service.Metadata.Name + if !owned.MatchString(name) { + failures.addMessage("Cloud Run list returned out-of-scope service " + name) + continue + } + policy, err := r.cloudRunPolicy(ctx, config, name) + if err != nil { + failures.add("inspect Cloud Run IAM policy for "+name, err, r.Redactor) + } else if hasUnconditionalMember(policy, "roles/run.invoker", "allUsers") { + args := []string{ + "run", "services", "remove-iam-policy-binding", name, + "--project", config.Project, + "--region", config.Region, + "--member", "allUsers", + "--role", "roles/run.invoker", + "--condition=None", + "--quiet", + } + if err := r.retry(ctx, config, "remove Cloud Run invoker from "+name, args); err != nil { + failures.add("remove Cloud Run invoker from "+name, err, r.Redactor) + } + } + + args := []string{ + "run", "services", "delete", name, + "--project", config.Project, + "--region", config.Region, + "--quiet", + } + if err := r.retry(ctx, config, "delete Cloud Run service "+name, args); err != nil { + failures.add("delete Cloud Run service "+name, err, r.Redactor) + } + } +} + +func (r Runner) cleanupInstances(ctx context.Context, config Config, nameFilter string, owned *regexp.Regexp, failures *failureList) { + resources, err := r.listZonal(ctx, "compute", "instances", config.Project, nameFilter) + if err != nil { + failures.add("list compute instances", err, r.Redactor) + return + } + for _, resource := range resources { + if !owned.MatchString(resource.Name) { + failures.addMessage("instance list returned out-of-scope resource " + resource.Name) + continue + } + zone, err := resourceLocation(resource.Zone) + if err != nil { + failures.add("resolve zone for instance "+resource.Name, err, r.Redactor) + continue + } + args := []string{"compute", "instances", "delete", resource.Name, "--project", config.Project, "--zone", zone, "--quiet"} + if err := r.retry(ctx, config, "delete instance "+resource.Name, args); err != nil { + failures.add("delete instance "+resource.Name, err, r.Redactor) + } + } +} + +func (r Runner) cleanupFirewalls(ctx context.Context, config Config, nameFilter string, failures *failureList) { + resources, err := r.listFirewalls(ctx, config, nameFilter) + if err != nil { + failures.add("list firewall rules", err, r.Redactor) + return + } + owned := firewallPattern(nameFilter) + for _, resource := range resources { + if !owned.MatchString(resource.Name) { + failures.addMessage("firewall list returned out-of-scope resource " + resource.Name) + continue + } + args := []string{"compute", "firewall-rules", "delete", resource.Name, "--project", config.Project, "--quiet"} + if err := r.retry(ctx, config, "delete firewall "+resource.Name, args); err != nil { + failures.add("delete firewall "+resource.Name, err, r.Redactor) + } + } +} + +func (r Runner) cleanupDisks(ctx context.Context, config Config, nameFilter string, owned *regexp.Regexp, failures *failureList) { + resources, err := r.listZonal(ctx, "compute", "disks", config.Project, nameFilter) + if err != nil { + failures.add("list compute disks", err, r.Redactor) + return + } + for _, resource := range resources { + if !owned.MatchString(resource.Name) { + failures.addMessage("disk list returned out-of-scope resource " + resource.Name) + continue + } + zone, err := resourceLocation(resource.Zone) + if err != nil { + failures.add("resolve zone for disk "+resource.Name, err, r.Redactor) + continue + } + args := []string{"compute", "disks", "delete", resource.Name, "--project", config.Project, "--zone", zone, "--quiet"} + if err := r.retry(ctx, config, "delete disk "+resource.Name, args); err != nil { + failures.add("delete disk "+resource.Name, err, r.Redactor) + } + } +} + +func (r Runner) cleanupProjectIAM(ctx context.Context, config Config, nameFilter string, failures *failureList) { + bindings, err := r.listProjectIAM(ctx, config.Project, nameFilter) + if err != nil { + failures.add("list project IAM bindings", err, r.Redactor) + return + } + for _, binding := range bindings { + args := []string{ + "projects", "remove-iam-policy-binding", config.Project, + "--member", binding.Member, + "--role", binding.Role, + "--condition=None", + "--quiet", + } + description := fmt.Sprintf("remove project IAM binding %s for %s", binding.Role, binding.Member) + if err := r.retry(ctx, config, description, args); err != nil { + failures.add(description, err, r.Redactor) + } + } +} + +func (r Runner) cleanupServiceAccounts(ctx context.Context, config Config, nameFilter string, failures *failureList) { + accounts, err := r.listServiceAccounts(ctx, config, nameFilter) + if err != nil { + failures.add("list service accounts", err, r.Redactor) + return + } + owned := serviceAccountPattern(config.Project, nameFilter) + for _, account := range accounts { + if !owned.MatchString("serviceAccount:" + account.Email) { + failures.addMessage("service-account list returned out-of-scope account " + account.Email) + continue + } + args := []string{"iam", "service-accounts", "delete", account.Email, "--project", config.Project, "--quiet"} + if err := r.retry(ctx, config, "delete service account "+account.Email, args); err != nil { + failures.add("delete service account "+account.Email, err, r.Redactor) + } + } +} + +func (r Runner) cleanupSubnetworks(ctx context.Context, config Config, nameFilter string, owned *regexp.Regexp, failures *failureList) { + resources, err := r.listSubnetworks(ctx, config.Project, nameFilter) + if err != nil { + failures.add("list subnetworks", err, r.Redactor) + return + } + for _, resource := range resources { + if !owned.MatchString(resource.Name) { + failures.addMessage("subnetwork list returned out-of-scope resource " + resource.Name) + continue + } + region, err := resourceLocation(resource.Region) + if err != nil { + failures.add("resolve region for subnetwork "+resource.Name, err, r.Redactor) + continue + } + args := []string{"compute", "networks", "subnets", "delete", resource.Name, "--project", config.Project, "--region", region, "--quiet"} + if err := r.retryNetwork(ctx, config, "delete subnetwork "+resource.Name, args); err != nil { + failures.add("delete subnetwork "+resource.Name, err, r.Redactor) + } + } +} + +func (r Runner) cleanupNetworks(ctx context.Context, config Config, nameFilter string, owned *regexp.Regexp, failures *failureList) { + resources, err := r.listNamed(ctx, []string{"compute", "networks", "list"}, config.Project, "name", nameFilter) + if err != nil { + failures.add("list networks", err, r.Redactor) + return + } + for _, resource := range resources { + if !owned.MatchString(resource.Name) { + failures.addMessage("network list returned out-of-scope resource " + resource.Name) + continue + } + args := []string{"compute", "networks", "delete", resource.Name, "--project", config.Project, "--quiet"} + if err := r.retryNetwork(ctx, config, "delete network "+resource.Name, args); err != nil { + failures.add("delete network "+resource.Name, err, r.Redactor) + } + } +} + +func (r Runner) verifyNoResources(ctx context.Context, config Config, nameFilter string) error { + var lastError error + for attempt := 1; attempt <= config.Attempts; attempt++ { + resources, err := r.residuals(ctx, config, nameFilter) + if err == nil && len(resources) == 0 { + r.Logger.Info("Verified that no matching GCP smoke resources remain") + return nil + } + if err != nil { + lastError = err + r.Logger.Warn("Could not verify GCP residual resources", "attempt", attempt, "error", r.Redactor.String(err.Error())) + } else { + lastError = fmt.Errorf("%d matching resources remain", len(resources)) + for _, resource := range resources { + r.Logger.Warn("Matching GCP smoke resource remains", "attempt", attempt, "kind", resource.Kind, "name", resource.Name, "role", resource.Role, "member", resource.Member) + } + } + if attempt < config.Attempts { + if err := r.Sleep(ctx, config.RetryDelay); err != nil { + return fmt.Errorf("wait before residual verification retry: %w", err) + } + } + } + return fmt.Errorf("GCP smoke cleanup left matching resources or could not verify their removal: %w", lastError) +} + +func (r Runner) residuals(ctx context.Context, config Config, nameFilter string) ([]residual, error) { + result := make([]residual, 0) + + services, err := r.listCloudRun(ctx, config, nameFilter) + if err != nil { + return nil, fmt.Errorf("verify Cloud Run services: %w", err) + } + for _, service := range services { + result = append(result, residual{Kind: "cloud-run", Name: service.Metadata.Name}) + } + + instances, err := r.listZonal(ctx, "compute", "instances", config.Project, nameFilter) + if err != nil { + return nil, fmt.Errorf("verify instances: %w", err) + } + for _, instance := range instances { + result = append(result, residual{Kind: "instance", Name: instance.Name}) + } + + firewalls, err := r.listFirewalls(ctx, config, nameFilter) + if err != nil { + return nil, fmt.Errorf("verify firewalls: %w", err) + } + for _, firewall := range firewalls { + result = append(result, residual{Kind: "firewall", Name: firewall.Name}) + } + + disks, err := r.listZonal(ctx, "compute", "disks", config.Project, nameFilter) + if err != nil { + return nil, fmt.Errorf("verify disks: %w", err) + } + for _, disk := range disks { + result = append(result, residual{Kind: "disk", Name: disk.Name}) + } + + accounts, err := r.listServiceAccounts(ctx, config, nameFilter) + if err != nil { + return nil, fmt.Errorf("verify service accounts: %w", err) + } + for _, account := range accounts { + result = append(result, residual{Kind: "service-account", Name: account.Email}) + } + + subnetworks, err := r.listSubnetworks(ctx, config.Project, nameFilter) + if err != nil { + return nil, fmt.Errorf("verify subnetworks: %w", err) + } + for _, subnetwork := range subnetworks { + result = append(result, residual{Kind: "subnetwork", Name: subnetwork.Name}) + } + + networks, err := r.listNamed(ctx, []string{"compute", "networks", "list"}, config.Project, "name", nameFilter) + if err != nil { + return nil, fmt.Errorf("verify networks: %w", err) + } + for _, network := range networks { + result = append(result, residual{Kind: "network", Name: network.Name}) + } + + bindings, err := r.listProjectIAM(ctx, config.Project, nameFilter) + if err != nil { + return nil, fmt.Errorf("verify project IAM: %w", err) + } + for _, binding := range bindings { + result = append(result, residual{Kind: "project-iam", Role: binding.Role, Member: binding.Member}) + } + + return result, nil +} + +func (r Runner) listCloudRun(ctx context.Context, config Config, nameFilter string) ([]cloudRunService, error) { + var services []cloudRunService + err := r.runJSON(ctx, &services, + "run", "services", "list", + "--project", config.Project, + "--region", config.Region, + "--filter", "metadata.name~'"+nameFilter+"'", + "--format=json", + ) + return services, err +} + +func (r Runner) cloudRunPolicy(ctx context.Context, config Config, name string) ([]iamBinding, error) { + output, err := r.query(ctx, "inspect Cloud Run IAM policy", + "run", "services", "get-iam-policy", name, + "--project", config.Project, + "--region", config.Region, + "--format=json", + ) + if err != nil { + return nil, err + } + return parseIAMPolicy(output) +} + +func (r Runner) listZonal(ctx context.Context, group, resource, project, nameFilter string) ([]zonalResource, error) { + var resources []zonalResource + err := r.runJSON(ctx, &resources, + group, resource, "list", + "--project", project, + "--filter", "name~'"+nameFilter+"'", + "--format=json", + ) + return resources, err +} + +func (r Runner) listFirewalls(ctx context.Context, config Config, nameFilter string) ([]namedResource, error) { + filter := "^(allow-ssh-ipv4-|allow-ssh-ipv6-|allow-rollout-ipv4-|allow-cloud-run-)" + strings.TrimPrefix(nameFilter, "^") + return r.listNamed(ctx, []string{"compute", "firewall-rules", "list"}, config.Project, "name", filter) +} + +func (r Runner) listServiceAccounts(ctx context.Context, config Config, nameFilter string) ([]serviceAccount, error) { + filter := "^(vm-|internal-|ppb-)?" + strings.TrimPrefix(nameFilter, "^") + ".*@" + regexp.QuoteMeta(config.Project) + `\.iam\.gserviceaccount\.com$` + var accounts []serviceAccount + err := r.runJSON(ctx, &accounts, + "iam", "service-accounts", "list", + "--project", config.Project, + "--filter", "email~'"+filter+"'", + "--format=json", + ) + return accounts, err +} + +func (r Runner) listSubnetworks(ctx context.Context, project, nameFilter string) ([]regionalResource, error) { + var resources []regionalResource + err := r.runJSON(ctx, &resources, + "compute", "networks", "subnets", "list", + "--project", project, + "--filter", "name~'"+nameFilter+"'", + "--format=json", + ) + return resources, err +} + +func (r Runner) listNamed(ctx context.Context, prefix []string, project, field, filter string) ([]namedResource, error) { + args := append([]string{}, prefix...) + args = append(args, "--project", project, "--filter", field+"~'"+filter+"'", "--format=json") + var resources []namedResource + err := r.runJSON(ctx, &resources, args...) + return resources, err +} + +func (r Runner) listProjectIAM(ctx context.Context, project, nameFilter string) ([]projectIAMBinding, error) { + output, err := r.query(ctx, "list project IAM bindings", "projects", "get-iam-policy", project, "--format=json") + if err != nil { + return nil, err + } + bindings, err := parseIAMPolicy(output) + if err != nil { + return nil, err + } + + managedRoles := map[string]bool{ + "projects/" + project + "/roles/startVM": true, + "projects/" + project + "/roles/suspendVM": true, + "roles/logging.logWriter": true, + "roles/monitoring.metricWriter": true, + } + ownedAccount := serviceAccountPattern(project, nameFilter) + result := make([]projectIAMBinding, 0) + for _, binding := range bindings { + if !managedRoles[binding.Role] || !unconditional(binding.Condition) { + continue + } + for _, member := range binding.Members { + if ownedAccount.MatchString(member) { + result = append(result, projectIAMBinding{Role: binding.Role, Member: member}) + } + } + } + return result, nil +} + +func (r Runner) runJSON(ctx context.Context, destination any, args ...string) error { + description := strings.Join(args[:min(3, len(args))], " ") + output, err := r.query(ctx, description, args...) + if err != nil { + return err + } + if err := json.Unmarshal(output, destination); err != nil { + return fmt.Errorf("decode gcloud JSON: %w", err) + } + return nil +} + +func (r Runner) query(ctx context.Context, description string, args ...string) ([]byte, error) { + var lastError error + for attempt := 1; attempt <= r.queryAttempts; attempt++ { + output, err := r.Command.Run(ctx, args...) + if err == nil { + return output, nil + } + lastError = err + r.Logger.Warn("GCP cleanup query failed", "operation", description, "attempt", attempt, "error", r.Redactor.String(err.Error())) + if attempt < r.queryAttempts { + if err := r.Sleep(ctx, r.queryRetryDelay); err != nil { + return nil, fmt.Errorf("wait before query retry: %w", err) + } + } + } + return nil, fmt.Errorf("exhausted %d query attempts: %w", r.queryAttempts, lastError) +} + +func (r Runner) retry(ctx context.Context, config Config, description string, args []string) error { + var lastError error + for attempt := 1; attempt <= config.Attempts; attempt++ { + r.Logger.Info("Running GCP cleanup mutation", "operation", description, "attempt", attempt) + if _, err := r.Command.Run(ctx, args...); err == nil { + return nil + } else { + lastError = err + r.Logger.Warn("GCP cleanup mutation failed", "operation", description, "attempt", attempt, "error", r.Redactor.String(err.Error())) + } + if attempt < config.Attempts { + if err := r.Sleep(ctx, config.RetryDelay); err != nil { + return fmt.Errorf("wait before retry: %w", err) + } + } + } + return fmt.Errorf("exhausted %d attempts: %w", config.Attempts, lastError) +} + +func (r Runner) retryNetwork(ctx context.Context, config Config, description string, args []string) error { + var lastError error + delay := config.RetryDelay + attempt := 0 + for { + attempt++ + r.Logger.Info("Running GCP network cleanup mutation", "operation", description, "attempt", attempt) + if _, err := r.Command.Run(ctx, args...); err == nil { + return nil + } else { + lastError = err + r.Logger.Warn("GCP network cleanup mutation failed", "operation", description, "attempt", attempt, "error", r.Redactor.String(err.Error())) + } + + if r.networkRetryBudget == nil || *r.networkRetryBudget <= 0 { + break + } + wait := min(delay, *r.networkRetryBudget) + if err := r.Sleep(ctx, wait); err != nil { + return fmt.Errorf("wait before network cleanup retry: %w", err) + } + *r.networkRetryBudget -= wait + if delay < time.Minute { + delay = min(delay*2, time.Minute) + } + } + return fmt.Errorf("exhausted shared network retry window after %d attempts: %w", attempt, lastError) +} + +func normalizeConfig(config Config) (Config, error) { + config.Project = strings.TrimSpace(config.Project) + config.Region = strings.TrimSpace(config.Region) + if config.Project == "" { + return Config{}, errors.New("GCP project is required") + } + if config.Region == "" { + return Config{}, errors.New("GCP region is required") + } + if _, err := targetNamePrefix(config.Target); err != nil { + return Config{}, err + } + if config.Attempts <= 0 { + config.Attempts = defaultAttempts + } + if config.RetryDelay < 0 { + return Config{}, errors.New("retry delay cannot be negative") + } + if config.RetryDelay == 0 { + config.RetryDelay = defaultDelay + } + if config.QueryAttempts <= 0 { + config.QueryAttempts = defaultQueryAttempts + } + if config.QueryRetryDelay < 0 { + return Config{}, errors.New("query retry delay cannot be negative") + } + if config.QueryRetryDelay == 0 { + config.QueryRetryDelay = defaultQueryDelay + } + if config.NetworkRetryWindow < 0 { + return Config{}, errors.New("network retry window cannot be negative") + } + if config.NetworkRetryWindow == 0 { + config.NetworkRetryWindow = defaultNetworkRetryWindow + } + return config, nil +} + +func targetNamePrefix(target string) (string, error) { + provider, template, found := strings.Cut(target, "-") + if !found || provider != "gcp" { + return "", fmt.Errorf("unsupported GCP smoke target %q", target) + } + slug, ok := templateSlugs[template] + if !ok { + return "", fmt.Errorf("unsupported GCP smoke template %q", template) + } + return "cc-g-" + slug, nil +} + +func normalizeRunID(runID string, limit int) string { + var normalized strings.Builder + for _, character := range strings.ToLower(runID) { + if (character >= 'a' && character <= 'z') || (character >= '0' && character <= '9') || character == '-' { + normalized.WriteRune(character) + } else { + normalized.WriteByte('-') + } + if normalized.Len() >= limit { + break + } + } + return normalized.String() +} + +func parseIAMPolicy(output []byte) ([]iamBinding, error) { + trimmed := bytes.TrimSpace(output) + if len(trimmed) == 0 || trimmed[0] != '{' { + return nil, errors.New("IAM policy was not a JSON object") + } + var envelope map[string]json.RawMessage + if err := json.Unmarshal(trimmed, &envelope); err != nil { + return nil, fmt.Errorf("decode IAM policy: %w", err) + } + rawBindings, found := envelope["bindings"] + if !found { + return nil, nil + } + rawBindings = bytes.TrimSpace(rawBindings) + if len(rawBindings) == 0 || rawBindings[0] != '[' { + return nil, errors.New("IAM policy bindings were not an array") + } + var bindings []iamBinding + if err := json.Unmarshal(rawBindings, &bindings); err != nil { + return nil, fmt.Errorf("decode IAM policy bindings: %w", err) + } + return bindings, nil +} + +func hasUnconditionalMember(bindings []iamBinding, role, member string) bool { + for _, binding := range bindings { + if binding.Role != role || !unconditional(binding.Condition) { + continue + } + for _, candidate := range binding.Members { + if candidate == member { + return true + } + } + } + return false +} + +func unconditional(condition json.RawMessage) bool { + condition = bytes.TrimSpace(condition) + return len(condition) == 0 || bytes.Equal(condition, []byte("null")) +} + +func firewallPattern(nameFilter string) *regexp.Regexp { + return regexp.MustCompile("^(allow-ssh-ipv4-|allow-ssh-ipv6-|allow-rollout-ipv4-|allow-cloud-run-)" + strings.TrimPrefix(nameFilter, "^")) +} + +func serviceAccountPattern(project, nameFilter string) *regexp.Regexp { + return regexp.MustCompile(`^(deleted:)?serviceAccount:(vm-|internal-|ppb-)?` + strings.TrimPrefix(nameFilter, "^") + `.*@` + regexp.QuoteMeta(project) + `\.iam\.gserviceaccount\.com(\?uid=[^\s]+)?$`) +} + +func resourceLocation(value string) (string, error) { + value = strings.TrimSuffix(strings.TrimSpace(value), "/") + if value == "" { + return "", errors.New("resource location is empty") + } + if index := strings.LastIndexByte(value, '/'); index >= 0 { + value = value[index+1:] + } + if value == "" { + return "", errors.New("resource location is empty") + } + return value, nil +} + +func sleepContext(ctx context.Context, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (f *failureList) add(operation string, err error, redactor *Redactor) { + if err == nil { + return + } + f.errors = append(f.errors, fmt.Errorf("%s: %s", operation, redactor.String(err.Error()))) +} + +func (f *failureList) addMessage(message string) { + f.errors = append(f.errors, errors.New(message)) +} + +func (f *failureList) empty() bool { + return len(f.errors) == 0 +} + +func (f *failureList) Error() string { + messages := make([]string, 0, len(f.errors)) + for _, err := range f.errors { + messages = append(messages, err.Error()) + } + return "GCP smoke cleanup failed: " + strings.Join(messages, "; ") +} + +func (f *failureList) Unwrap() []error { + return f.errors +} diff --git a/internal/gcpcleanup/cleanup_test.go b/internal/gcpcleanup/cleanup_test.go new file mode 100644 index 0000000..a620b7f --- /dev/null +++ b/internal/gcpcleanup/cleanup_test.go @@ -0,0 +1,762 @@ +package gcpcleanup + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "regexp" + "slices" + "strings" + "testing" + "time" + + "github.com/libops/cloud-compose/internal/runnamespace" +) + +const ( + testProject = "test-project" + testName = "cc-g-wp-12345678-abcd" + testExactNamespace = "00021i3v9" + testExactName = "cc-g-wp-00021i3v9-abc" +) + +type fakeGCloud struct { + t *testing.T + mode string + calls [][]string + counts map[string]int + done map[string]bool + exact bool +} + +func newFakeGCloud(t *testing.T, mode string) *fakeGCloud { + t.Helper() + return &fakeGCloud{ + t: t, + mode: mode, + counts: make(map[string]int), + done: make(map[string]bool), + } +} + +func (f *fakeGCloud) Run(_ context.Context, args ...string) ([]byte, error) { + f.calls = append(f.calls, slices.Clone(args)) + if filter := argumentValueRaw(args, "--filter"); filter != "" { + exactFilter := strings.Contains(filter, testExactNamespace) + if exactFilter != f.exact { + return []byte(`[]`), nil + } + } + ownedName := f.ownedName() + + switch { + case hasPrefix(args, "run", "services", "list"): + f.counts["run-list"]++ + if f.mode == "retry-discovery" && f.counts["run-list"] == 1 { + return nil, errors.New("transient list failure") + } + if f.mode == "secret-list-failure" { + return nil, errors.New("authorization failed for top-secret-token") + } + if f.mode == "residual-cloud-run" || !f.done["run-delete"] { + service := cloudRunService{} + service.Metadata.Name = ownedName + return marshalJSON(f.t, []cloudRunService{service}), nil + } + return marshalJSON(f.t, []cloudRunService{}), nil + case hasPrefix(args, "run", "services", "get-iam-policy"): + bindings := []iamBinding{{ + Role: "roles/run.invoker", + Members: []string{"allUsers"}, + Condition: json.RawMessage(`{"title":"conditional"}`), + }} + if !f.done["run-invoker-remove"] { + bindings = append(bindings, iamBinding{Role: "roles/run.invoker", Members: []string{"allUsers"}}) + } + return marshalJSON(f.t, map[string]any{"bindings": bindings}), nil + case hasPrefix(args, "compute", "instances", "list"): + if f.mode == "out-of-scope-instance" { + return marshalJSON(f.t, []zonalResource{{Name: "production-instance", Zone: "us-east5-b"}}), nil + } + if !f.done["instances-delete"] { + return marshalJSON(f.t, []zonalResource{{Name: ownedName, Zone: "https://www.googleapis.com/compute/v1/projects/test-project/zones/us-east5-b"}}), nil + } + return marshalJSON(f.t, []zonalResource{}), nil + case hasPrefix(args, "compute", "firewall-rules", "list"): + if !f.done["firewalls-delete"] { + return marshalJSON(f.t, []namedResource{ + {Name: "allow-ssh-ipv4-" + ownedName}, + {Name: "allow-cloud-run-" + ownedName}, + }), nil + } + return marshalJSON(f.t, []namedResource{}), nil + case hasPrefix(args, "compute", "disks", "list"): + if !f.done["disks-delete"] { + return marshalJSON(f.t, []zonalResource{{Name: ownedName + "-data-disk", Zone: "us-east5-b"}}), nil + } + return marshalJSON(f.t, []zonalResource{}), nil + case hasPrefix(args, "iam", "service-accounts", "list"): + if !f.done["service-accounts-delete"] { + return marshalJSON(f.t, []serviceAccount{ + {Email: "vm-" + ownedName + "@test-project.iam.gserviceaccount.com"}, + {Email: "internal-" + ownedName + "@test-project.iam.gserviceaccount.com"}, + {Email: "ppb-" + ownedName + "@test-project.iam.gserviceaccount.com"}, + {Email: ownedName + "@test-project.iam.gserviceaccount.com"}, + }), nil + } + return marshalJSON(f.t, []serviceAccount{}), nil + case hasPrefix(args, "compute", "networks", "subnets", "list"): + if !f.done["subnets-delete"] { + return marshalJSON(f.t, []regionalResource{{Name: ownedName, Region: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-east5"}}), nil + } + return marshalJSON(f.t, []regionalResource{}), nil + case hasPrefix(args, "compute", "networks", "list"): + if !f.done["networks-delete"] { + return marshalJSON(f.t, []namedResource{{Name: ownedName}}), nil + } + return marshalJSON(f.t, []namedResource{}), nil + case hasPrefix(args, "projects", "get-iam-policy"): + return f.projectPolicy(), nil + } + + key, ok := mutationKey(args) + if !ok { + f.t.Fatalf("unexpected gcloud invocation: %q", args) + } + f.counts[key]++ + count := f.counts[key] + if f.mode == "retry-transient" && count == 1 && (key == "run-delete" || key == "iam-log-remove" || key == "networks-delete") { + return nil, errors.New("transient provider failure") + } + if f.mode == "aggregate-failure" && (key == "run-delete" || key == "instances-delete" || key == "subnets-delete") { + return nil, errors.New("permanent provider failure") + } + if f.mode == "network-retention" && key == "subnets-delete" && count < 5 { + return nil, errors.New("subnetwork is still reserved by serverless") + } + f.done[key] = true + return nil, nil +} + +func (f *fakeGCloud) ownedName() string { + if f.exact { + return testExactName + } + return testName +} + +func (f *fakeGCloud) projectPolicy() []byte { + ownedName := f.ownedName() + bindings := make([]iamBinding, 0, 7) + for _, entry := range []struct { + key string + role string + member string + }{ + {"iam-log-remove", "roles/logging.logWriter", "serviceAccount:vm-" + ownedName + "@test-project.iam.gserviceaccount.com"}, + {"iam-monitoring-remove", "roles/monitoring.metricWriter", "serviceAccount:internal-" + ownedName + "@test-project.iam.gserviceaccount.com"}, + {"iam-suspend-remove", "projects/test-project/roles/suspendVM", "serviceAccount:internal-" + ownedName + "@test-project.iam.gserviceaccount.com"}, + {"iam-start-remove", "projects/test-project/roles/startVM", "serviceAccount:ppb-" + ownedName + "@test-project.iam.gserviceaccount.com"}, + } { + if !f.done[entry.key] { + bindings = append(bindings, iamBinding{Role: entry.role, Members: []string{entry.member}}) + } + } + bindings = append(bindings, + iamBinding{ + Role: "roles/logging.logWriter", + Members: []string{"serviceAccount:vm-" + ownedName + "@test-project.iam.gserviceaccount.com"}, + Condition: json.RawMessage(`{"title":"keep-conditional"}`), + }, + iamBinding{Role: "roles/owner", Members: []string{"serviceAccount:" + ownedName + "@test-project.iam.gserviceaccount.com"}}, + iamBinding{Role: "roles/logging.logWriter", Members: []string{"serviceAccount:production@test-project.iam.gserviceaccount.com"}}, + ) + return marshalJSON(f.t, map[string]any{"bindings": bindings}) +} + +func TestSweepRetriesAndCoversOwnedResources(t *testing.T) { + t.Parallel() + fake := newFakeGCloud(t, "retry-transient") + runner, sleeps := testRunner(fake, nil) + + if err := runner.Sweep(context.Background(), testConfig()); err != nil { + t.Fatalf("Sweep() error = %v", err) + } + + for key, expected := range map[string]int{ + "run-invoker-remove": 1, + "run-delete": 2, + "instances-delete": 1, + "firewalls-delete": 2, + "disks-delete": 1, + "iam-log-remove": 2, + "iam-monitoring-remove": 1, + "iam-suspend-remove": 1, + "iam-start-remove": 1, + "service-accounts-delete": 4, + "subnets-delete": 1, + "networks-delete": 2, + } { + if actual := fake.counts[key]; actual != expected { + t.Errorf("mutation %s count = %d; want %d", key, actual, expected) + } + } + if *sleeps != 3 { + t.Errorf("retry sleep count = %d; want 3", *sleeps) + } + + instanceList := firstCall(t, fake.calls, "compute", "instances", "list") + if got := argumentValue(t, instanceList, "--filter"); got != "name~'^cc-g-wp-12345678-'" { + t.Errorf("instance filter = %q", got) + } + if !hasFilterContaining(fake.calls, testExactNamespace) { + t.Fatal("cleanup did not query the reserved exact run namespace") + } + serviceAccountList := firstCall(t, fake.calls, "iam", "service-accounts", "list") + if got := argumentValue(t, serviceAccountList, "--filter"); !strings.Contains(got, "ppb-") { + t.Errorf("service-account filter omits power-button identity: %q", got) + } + firewallList := firstCall(t, fake.calls, "compute", "firewall-rules", "list") + if got := argumentValue(t, firewallList, "--filter"); !strings.Contains(got, "allow-cloud-run-") { + t.Errorf("firewall filter omits Direct VPC ingress rule: %q", got) + } + + invokerIndex := firstCallIndex(t, fake.calls, "run", "services", "remove-iam-policy-binding") + runDeleteIndex := firstCallIndex(t, fake.calls, "run", "services", "delete") + instanceIndex := firstCallIndex(t, fake.calls, "compute", "instances", "delete") + if !(invokerIndex < runDeleteIndex && runDeleteIndex < instanceIndex) { + t.Errorf("Cloud Run ingress/service/instance cleanup order = %d, %d, %d", invokerIndex, runDeleteIndex, instanceIndex) + } + iamIndex := lastCallIndex(t, fake.calls, "projects", "remove-iam-policy-binding") + accountIndex := firstCallIndex(t, fake.calls, "iam", "service-accounts", "delete") + if iamIndex >= accountIndex { + t.Errorf("project IAM removal index %d is not before service-account deletion %d", iamIndex, accountIndex) + } + subnetIndex := lastCallIndex(t, fake.calls, "compute", "networks", "subnets", "delete") + networkIndex := firstCallIndex(t, fake.calls, "compute", "networks", "delete") + if subnetIndex >= networkIndex { + t.Errorf("subnetwork deletion index %d is not before network deletion %d", subnetIndex, networkIndex) + } + + for _, call := range matchingCalls(fake.calls, "projects", "remove-iam-policy-binding") { + if !slices.Contains(call, "--condition=None") { + t.Errorf("project IAM removal omits --condition=None: %q", call) + } + } + invokerCall := firstCall(t, fake.calls, "run", "services", "remove-iam-policy-binding") + if !slices.Contains(invokerCall, "--condition=None") { + t.Errorf("Cloud Run IAM removal omits --condition=None: %q", invokerCall) + } +} + +func TestSweepMutatesOrdersAndVerifiesExactNamespaceResources(t *testing.T) { + t.Parallel() + fake := newFakeGCloud(t, "success") + fake.exact = true + runner, _ := testRunner(fake, nil) + + if err := runner.Sweep(context.Background(), testConfig()); err != nil { + t.Fatalf("Sweep() error = %v", err) + } + + for key, expected := range map[string]int{ + "run-invoker-remove": 1, + "run-delete": 1, + "instances-delete": 1, + "firewalls-delete": 2, + "disks-delete": 1, + "iam-log-remove": 1, + "iam-monitoring-remove": 1, + "iam-suspend-remove": 1, + "iam-start-remove": 1, + "service-accounts-delete": 4, + "subnets-delete": 1, + "networks-delete": 1, + } { + if actual := fake.counts[key]; actual != expected { + t.Errorf("exact-namespace mutation %s count = %d; want %d", key, actual, expected) + } + } + if actual := fake.counts["run-list"]; actual != 2 { + t.Errorf("exact-namespace Cloud Run list count = %d; want initial discovery plus residual verification", actual) + } + + for _, prefix := range [][]string{ + {"run", "services", "delete"}, + {"compute", "instances", "delete"}, + {"compute", "disks", "delete"}, + {"compute", "networks", "subnets", "delete"}, + {"compute", "networks", "delete"}, + } { + call := firstCall(t, fake.calls, prefix...) + if !strings.Contains(strings.Join(call, "\x00"), testExactName) { + t.Errorf("exact-namespace mutation %q does not own %q: %q", prefix, testExactName, call) + } + } + + invokerIndex := firstCallIndex(t, fake.calls, "run", "services", "remove-iam-policy-binding") + runDeleteIndex := firstCallIndex(t, fake.calls, "run", "services", "delete") + instanceIndex := firstCallIndex(t, fake.calls, "compute", "instances", "delete") + if !(invokerIndex < runDeleteIndex && runDeleteIndex < instanceIndex) { + t.Errorf("exact Cloud Run ingress/service/instance cleanup order = %d, %d, %d", invokerIndex, runDeleteIndex, instanceIndex) + } + iamIndex := lastCallIndex(t, fake.calls, "projects", "remove-iam-policy-binding") + accountIndex := firstCallIndex(t, fake.calls, "iam", "service-accounts", "delete") + if iamIndex >= accountIndex { + t.Errorf("exact project IAM removal index %d is not before service-account deletion %d", iamIndex, accountIndex) + } + subnetIndex := lastCallIndex(t, fake.calls, "compute", "networks", "subnets", "delete") + networkIndex := firstCallIndex(t, fake.calls, "compute", "networks", "delete") + if subnetIndex >= networkIndex { + t.Errorf("exact subnetwork deletion index %d is not before network deletion %d", subnetIndex, networkIndex) + } +} + +func TestSweepReportsExactNamespaceResiduals(t *testing.T) { + t.Parallel() + fake := newFakeGCloud(t, "residual-cloud-run") + fake.exact = true + runner, _ := testRunner(fake, nil) + + err := runner.Sweep(context.Background(), testConfig()) + if err == nil || !strings.Contains(err.Error(), "verify residual resources") { + t.Fatalf("Sweep() error = %v; want exact-namespace residual failure", err) + } + if actual := fake.counts["run-list"]; actual != defaultAttempts+1 { + t.Errorf("exact-namespace Cloud Run list count = %d; want %d", actual, defaultAttempts+1) + } +} + +func TestSweepAggregatesPermanentFailures(t *testing.T) { + t.Parallel() + fake := newFakeGCloud(t, "aggregate-failure") + runner, _ := testRunner(fake, nil) + + err := runner.Sweep(context.Background(), testConfig()) + if err == nil { + t.Fatal("Sweep() unexpectedly succeeded") + } + for _, key := range []string{"run-delete", "instances-delete"} { + if actual := fake.counts[key]; actual != defaultAttempts { + t.Errorf("mutation %s count = %d; want %d", key, actual, defaultAttempts) + } + } + if actual := fake.counts["subnets-delete"]; actual < 2 { + t.Errorf("subnetwork mutation count = %d; want a bounded network-retention retry", actual) + } + for _, key := range []string{"networks-delete", "service-accounts-delete"} { + if fake.counts[key] == 0 { + t.Errorf("cleanup stopped before unrelated mutation %s", key) + } + } + message := err.Error() + for _, expected := range []string{"Cloud Run service", "instance", "subnetwork", "verify residual resources"} { + if !strings.Contains(message, expected) { + t.Errorf("aggregate error omits %q: %s", expected, message) + } + } +} + +func TestSweepRetriesTransientDiscovery(t *testing.T) { + t.Parallel() + fake := newFakeGCloud(t, "retry-discovery") + runner, sleeps := testRunner(fake, nil) + + if err := runner.Sweep(context.Background(), testConfig()); err != nil { + t.Fatalf("Sweep() error = %v", err) + } + if actual := fake.counts["run-list"]; actual != 3 { + t.Errorf("Cloud Run list count = %d; want initial retry plus final verification", actual) + } + if *sleeps != 1 { + t.Errorf("query retry sleep count = %d; want 1", *sleeps) + } +} + +func TestSweepUsesSharedDirectVPCReleaseWindow(t *testing.T) { + t.Parallel() + fake := newFakeGCloud(t, "network-retention") + runner, sleeps := testRunner(fake, nil) + + if err := runner.Sweep(context.Background(), testConfig()); err != nil { + t.Fatalf("Sweep() error = %v", err) + } + if actual := fake.counts["subnets-delete"]; actual != 5 { + t.Errorf("subnetwork mutation count = %d; want 5", actual) + } + if *sleeps != 4 { + t.Errorf("network retention sleep count = %d; want 4", *sleeps) + } +} + +func TestSweepExhaustsResidualVerification(t *testing.T) { + t.Parallel() + fake := newFakeGCloud(t, "residual-cloud-run") + runner, _ := testRunner(fake, nil) + + if err := runner.Sweep(context.Background(), testConfig()); err == nil { + t.Fatal("Sweep() unexpectedly succeeded with a residual Cloud Run service") + } + if actual := fake.counts["run-list"]; actual != defaultAttempts+1 { + t.Errorf("Cloud Run list count = %d; want %d", actual, defaultAttempts+1) + } +} + +func TestSweepRejectsOutOfScopeListResults(t *testing.T) { + t.Parallel() + fake := newFakeGCloud(t, "out-of-scope-instance") + runner, _ := testRunner(fake, nil) + + err := runner.Sweep(context.Background(), testConfig()) + if err == nil || !strings.Contains(err.Error(), "out-of-scope") { + t.Fatalf("Sweep() error = %v; want out-of-scope failure", err) + } + if fake.counts["instances-delete"] != 0 { + t.Fatal("cleanup deleted an out-of-scope instance") + } +} + +func TestSweepRedactsCommandFailures(t *testing.T) { + t.Parallel() + fake := newFakeGCloud(t, "secret-list-failure") + var logs bytes.Buffer + redactor := NewRedactor("top-secret-token") + runner, _ := testRunner(fake, &logs) + runner.Redactor = redactor + + err := runner.Sweep(context.Background(), testConfig()) + if err == nil { + t.Fatal("Sweep() unexpectedly succeeded") + } + combined := logs.String() + err.Error() + if strings.Contains(combined, "top-secret-token") { + t.Fatalf("cleanup output leaked a configured secret: %s", combined) + } + if !strings.Contains(combined, "[REDACTED]") { + t.Fatalf("cleanup output did not mark redacted data: %s", combined) + } +} + +func TestNameFilterMatchesTerraformOwnership(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + target string + runID string + want string + }{ + {name: "workflow run", target: "gcp-wp", runID: "123456789", want: "^cc-g-wp-12345678-"}, + {name: "normalized", target: "gcp-omeka-s", runID: "AB_cd!234567", want: "^cc-g-os-ab-cd-23-"}, + {name: "all runs", target: "gcp-wp", want: "^cc-g-wp-"}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := NameFilter(test.target, test.runID) + if err != nil { + t.Fatalf("NameFilter() error = %v", err) + } + if got != test.want { + t.Errorf("NameFilter() = %q; want %q", got, test.want) + } + }) + } +} + +func TestOwnershipFiltersReserveExactRunNamespace(t *testing.T) { + t.Parallel() + filters, err := ownershipFilters("gcp-wp", "123456789") + if err != nil { + t.Fatalf("ownershipFilters() error = %v", err) + } + want := []string{ + "^cc-g-wp-12345678-", + "^cc-g-wp-" + testExactNamespace + "-", + } + if !slices.Equal(filters, want) { + t.Errorf("ownershipFilters() = %q; want %q", filters, want) + } + + left, _ := ownershipFilters("gcp-wp", "123456780") + right, _ := ownershipFilters("gcp-wp", "123456789") + if left[0] != right[0] { + t.Fatal("test inputs no longer demonstrate the legacy eight-character collision") + } + if left[1] == right[1] { + t.Fatal("exact run namespaces collide") + } +} + +func TestExactOwnershipFilterFitsEverySupportedGCPTarget(t *testing.T) { + t.Parallel() + const ( + runID = "123456789" + otherRunID = "123456780" + resourceSuffix = "a" + gcpNameLimit = 21 + ) + targets := []struct { + target string + prefix string + }{ + {target: "gcp-archivesspace", prefix: "cc-g-as"}, + {target: "gcp-ojs", prefix: "cc-g-ojs"}, + {target: "gcp-isle", prefix: "cc-g-isle"}, + {target: "gcp-drupal", prefix: "cc-g-dr"}, + {target: "gcp-wp", prefix: "cc-g-wp"}, + {target: "gcp-omeka-s", prefix: "cc-g-os"}, + {target: "gcp-omeka-classic", prefix: "cc-g-oc"}, + } + + otherNamespace, err := runnamespace.Encode(otherRunID) + if err != nil { + t.Fatalf("runnamespace.Encode(%q) error = %v", otherRunID, err) + } + for _, test := range targets { + t.Run(test.target, func(t *testing.T) { + prefix, err := targetNamePrefix(test.target) + if err != nil { + t.Fatalf("targetNamePrefix() error = %v", err) + } + if prefix != test.prefix { + t.Fatalf("targetNamePrefix() = %q; want %q", prefix, test.prefix) + } + + filters, err := ownershipFilters(test.target, runID) + if err != nil { + t.Fatalf("ownershipFilters() error = %v", err) + } + if len(filters) != 2 { + t.Fatalf("ownershipFilters() returned %d filters; want legacy and exact", len(filters)) + } + + resourceName := strings.Join([]string{prefix, testExactNamespace, resourceSuffix}, "-") + if len(resourceName) > gcpNameLimit { + t.Fatalf("exact resource name %q is %d characters; limit is %d", resourceName, len(resourceName), gcpNameLimit) + } + if !strings.HasSuffix(resourceName, "-"+resourceSuffix) { + t.Fatalf("exact resource name %q lost its separator or random suffix", resourceName) + } + if !regexp.MustCompile(filters[1]).MatchString(resourceName) { + t.Fatalf("exact filter %q does not match resource %q", filters[1], resourceName) + } + + otherResource := strings.Join([]string{prefix, otherNamespace, resourceSuffix}, "-") + if regexp.MustCompile(filters[1]).MatchString(otherResource) { + t.Fatalf("exact filter %q matched another run's resource %q", filters[1], otherResource) + } + }) + } +} + +func TestSweepRequiresExplicitAllRunOwnership(t *testing.T) { + t.Parallel() + fake := newFakeGCloud(t, "success") + runner, _ := testRunner(fake, nil) + config := testConfig() + config.RunID = "" + + err := runner.Sweep(context.Background(), config) + if err == nil || !strings.Contains(err.Error(), "run id is required") { + t.Fatalf("Sweep() error = %v; want missing run-id error", err) + } + if len(fake.calls) != 0 { + t.Fatal("cleanup called gcloud before establishing resource ownership") + } +} + +func TestSweepRejectsConflictingOwnershipScopes(t *testing.T) { + t.Parallel() + fake := newFakeGCloud(t, "success") + runner, _ := testRunner(fake, nil) + config := testConfig() + config.AllowAllRuns = true + + err := runner.Sweep(context.Background(), config) + if err == nil || !strings.Contains(err.Error(), "mutually exclusive") { + t.Fatalf("Sweep() error = %v; want conflicting ownership error", err) + } + if len(fake.calls) != 0 { + t.Fatal("cleanup called gcloud before rejecting conflicting ownership scopes") + } +} + +func TestSweepRejectsInvalidRunIDBeforeCommand(t *testing.T) { + t.Parallel() + for _, runID := range []string{ + "12345678oops", + "012345678", + "17592186044416", + } { + t.Run(runID, func(t *testing.T) { + fake := newFakeGCloud(t, "success") + runner, _ := testRunner(fake, nil) + config := testConfig() + config.RunID = runID + + err := runner.Sweep(context.Background(), config) + if err == nil || !strings.Contains(err.Error(), "canonical decimal value no larger than 44 bits") { + t.Fatalf("Sweep() error = %v; want invalid run-id error", err) + } + if len(fake.calls) != 0 { + t.Fatalf("cleanup called gcloud before rejecting invalid run ID %q: %q", runID, fake.calls) + } + }) + } +} + +func testRunner(command Commander, logs *bytes.Buffer) (Runner, *int) { + sleeps := 0 + var logger *slog.Logger + if logs == nil { + logger = slog.New(slog.NewTextHandler(ioDiscard{}, nil)) + } else { + logger = slog.New(slog.NewTextHandler(logs, nil)) + } + return Runner{ + Command: command, + Logger: logger, + Sleep: func(context.Context, time.Duration) error { + sleeps++ + return nil + }, + }, &sleeps +} + +type ioDiscard struct{} + +func (ioDiscard) Write(data []byte) (int, error) { + return len(data), nil +} + +func testConfig() Config { + return Config{ + Project: testProject, + Region: "us-east5", + Target: "gcp-wp", + RunID: "123456789", + Attempts: defaultAttempts, + RetryDelay: time.Nanosecond, + QueryAttempts: defaultQueryAttempts, + QueryRetryDelay: time.Nanosecond, + NetworkRetryWindow: 15 * time.Nanosecond, + } +} + +func marshalJSON(t *testing.T, value any) []byte { + t.Helper() + result, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal fake gcloud response: %v", err) + } + return result +} + +func mutationKey(args []string) (string, bool) { + switch { + case hasPrefix(args, "run", "services", "remove-iam-policy-binding"): + return "run-invoker-remove", true + case hasPrefix(args, "run", "services", "delete"): + return "run-delete", true + case hasPrefix(args, "compute", "instances", "delete"): + return "instances-delete", true + case hasPrefix(args, "compute", "firewall-rules", "delete"): + return "firewalls-delete", true + case hasPrefix(args, "compute", "disks", "delete"): + return "disks-delete", true + case hasPrefix(args, "iam", "service-accounts", "delete"): + return "service-accounts-delete", true + case hasPrefix(args, "compute", "networks", "subnets", "delete"): + return "subnets-delete", true + case hasPrefix(args, "compute", "networks", "delete"): + return "networks-delete", true + case hasPrefix(args, "projects", "remove-iam-policy-binding"): + switch argumentValueRaw(args, "--role") { + case "roles/logging.logWriter": + return "iam-log-remove", true + case "roles/monitoring.metricWriter": + return "iam-monitoring-remove", true + case "projects/test-project/roles/suspendVM": + return "iam-suspend-remove", true + case "projects/test-project/roles/startVM": + return "iam-start-remove", true + } + } + return "", false +} + +func hasPrefix(args []string, prefix ...string) bool { + return len(args) >= len(prefix) && slices.Equal(args[:len(prefix)], prefix) +} + +func firstCall(t *testing.T, calls [][]string, prefix ...string) []string { + t.Helper() + index := firstCallIndex(t, calls, prefix...) + return calls[index] +} + +func firstCallIndex(t *testing.T, calls [][]string, prefix ...string) int { + t.Helper() + for index, call := range calls { + if hasPrefix(call, prefix...) { + return index + } + } + t.Fatalf("call %q not found", prefix) + return -1 +} + +func lastCallIndex(t *testing.T, calls [][]string, prefix ...string) int { + t.Helper() + for index := len(calls) - 1; index >= 0; index-- { + if hasPrefix(calls[index], prefix...) { + return index + } + } + t.Fatalf("call %q not found", prefix) + return -1 +} + +func matchingCalls(calls [][]string, prefix ...string) [][]string { + result := make([][]string, 0) + for _, call := range calls { + if hasPrefix(call, prefix...) { + result = append(result, call) + } + } + return result +} + +func argumentValue(t *testing.T, args []string, name string) string { + t.Helper() + value := argumentValueRaw(args, name) + if value == "" { + t.Fatalf("argument %s not found in %q", name, args) + } + return value +} + +func argumentValueRaw(args []string, name string) string { + for index := 0; index+1 < len(args); index++ { + if args[index] == name { + return args[index+1] + } + } + return "" +} + +func hasFilterContaining(calls [][]string, fragment string) bool { + for _, call := range calls { + if strings.Contains(argumentValueRaw(call, "--filter"), fragment) { + return true + } + } + return false +} + +func ExampleNameFilter() { + filter, _ := NameFilter("gcp-wp", "123456789") + fmt.Println(filter) + // Output: ^cc-g-wp-12345678- +} diff --git a/internal/gcpcleanup/command.go b/internal/gcpcleanup/command.go new file mode 100644 index 0000000..f5da8a8 --- /dev/null +++ b/internal/gcpcleanup/command.go @@ -0,0 +1,111 @@ +// Package gcpcleanup removes GCP resources owned by a cloud-compose smoke run. +package gcpcleanup + +import ( + "bytes" + "context" + "fmt" + "io" + "os/exec" + "slices" + "strings" +) + +// Commander executes one gcloud command and returns its standard output. +type Commander interface { + Run(context.Context, ...string) ([]byte, error) +} + +// ExecCommander invokes the installed gcloud CLI without a shell. +type ExecCommander struct { + Path string + Stderr io.Writer + Redactor *Redactor +} + +// Run executes gcloud with the supplied arguments. +func (c ExecCommander) Run(ctx context.Context, args ...string) ([]byte, error) { + path := c.Path + if path == "" { + path = "gcloud" + } + redactor := c.Redactor + if redactor == nil { + redactor = NewRedactor() + } + + // Arguments are passed directly to gcloud without a shell. Cleanup constructs + // every command from validated target prefixes, provider-returned owned names, + // and fixed flag names. + command := exec.CommandContext(ctx, path, args...) // #nosec G204 + var stdout bytes.Buffer + var stderr bytes.Buffer + command.Stdout = &stdout + command.Stderr = &stderr + + err := command.Run() + if stderr.Len() > 0 && c.Stderr != nil { + message := strings.TrimSpace(redactor.String(stderr.String())) + if message != "" { + _, _ = fmt.Fprintln(c.Stderr, message) + } + } + if err != nil { + return nil, fmt.Errorf("gcloud command failed: %w", err) + } + + return stdout.Bytes(), nil +} + +// Redactor replaces configured secret values before errors reach CI logs. +type Redactor struct { + secrets []string +} + +// NewRedactor constructs a redactor from non-empty secret values. +func NewRedactor(secrets ...string) *Redactor { + filtered := make([]string, 0, len(secrets)) + for _, secret := range secrets { + if secret != "" && !slices.Contains(filtered, secret) { + filtered = append(filtered, secret) + } + } + slices.SortFunc(filtered, func(left, right string) int { + return len(right) - len(left) + }) + return &Redactor{secrets: filtered} +} + +// NewEnvironmentRedactor discovers sensitive values in an environment snapshot. +func NewEnvironmentRedactor(environment []string) *Redactor { + secrets := make([]string, 0) + for _, entry := range environment { + name, value, found := strings.Cut(entry, "=") + if !found || value == "" || !sensitiveEnvironmentName(name) { + continue + } + secrets = append(secrets, value) + } + return NewRedactor(secrets...) +} + +// String returns text with all configured secret values replaced. +func (r *Redactor) String(value string) string { + if r == nil { + return value + } + for _, secret := range r.secrets { + value = strings.ReplaceAll(value, secret, "[REDACTED]") + } + return value +} + +func sensitiveEnvironmentName(name string) bool { + name = strings.ToUpper(name) + for _, marker := range []string{"TOKEN", "PASSWORD", "SECRET", "CREDENTIAL", "PRIVATE_KEY"} { + if strings.Contains(name, marker) { + return true + } + } + return false +} diff --git a/internal/gcpcleanup/command_test.go b/internal/gcpcleanup/command_test.go new file mode 100644 index 0000000..2370b56 --- /dev/null +++ b/internal/gcpcleanup/command_test.go @@ -0,0 +1,49 @@ +package gcpcleanup + +import ( + "bytes" + "context" + "fmt" + "os" + "strings" + "testing" +) + +func TestExecCommanderRedactsStderr(t *testing.T) { + if os.Getenv("GO_WANT_GCLOUD_HELPER") == "1" { + fmt.Fprintln(os.Stderr, "provider rejected top-secret-token") + os.Exit(1) + } + t.Setenv("GO_WANT_GCLOUD_HELPER", "1") + + var stderr bytes.Buffer + commander := ExecCommander{ + Path: os.Args[0], + Stderr: &stderr, + Redactor: NewRedactor("top-secret-token"), + } + _, err := commander.Run(context.Background(), "-test.run=TestExecCommanderRedactsStderr") + if err == nil { + t.Fatal("Run() unexpectedly succeeded") + } + if strings.Contains(stderr.String(), "top-secret-token") { + t.Fatalf("stderr leaked secret: %s", stderr.String()) + } + if !strings.Contains(stderr.String(), "[REDACTED]") { + t.Fatalf("stderr did not contain redaction marker: %s", stderr.String()) + } +} + +func TestNewEnvironmentRedactor(t *testing.T) { + t.Parallel() + redactor := NewEnvironmentRedactor([]string{ + "PATH=/usr/bin", + "CLOUDSDK_AUTH_ACCESS_TOKEN=access-value", + "GOOGLE_APPLICATION_CREDENTIALS=/private/credential.json", + }) + got := redactor.String("access-value /private/credential.json /usr/bin") + want := "[REDACTED] [REDACTED] /usr/bin" + if got != want { + t.Errorf("String() = %q; want %q", got, want) + } +} diff --git a/internal/runnamespace/namespace.go b/internal/runnamespace/namespace.go new file mode 100644 index 0000000..5fe61ce --- /dev/null +++ b/internal/runnamespace/namespace.go @@ -0,0 +1,28 @@ +// Package runnamespace encodes GitHub Actions run IDs into fixed-width GCP +// resource-name namespaces. +package runnamespace + +import ( + "errors" + "strconv" + "strings" +) + +const ( + // Width is the number of base36 characters reserved in GCP smoke names. + Width = 9 + maxBits = 44 +) + +// Encode converts a canonical decimal GitHub Actions run ID into a +// fixed-width, lowercase base36 namespace. The 44-bit limit guarantees that +// the result fits Width characters. +func Encode(runID string) (string, error) { + value, err := strconv.ParseUint(runID, 10, maxBits) + if err != nil || strconv.FormatUint(value, 10) != runID { + return "", errors.New("run ID must be a canonical decimal value no larger than 44 bits") + } + + encoded := strconv.FormatUint(value, 36) + return strings.Repeat("0", Width-len(encoded)) + encoded, nil +} diff --git a/internal/runnamespace/namespace_test.go b/internal/runnamespace/namespace_test.go new file mode 100644 index 0000000..433a613 --- /dev/null +++ b/internal/runnamespace/namespace_test.go @@ -0,0 +1,51 @@ +package runnamespace + +import "testing" + +func TestEncode(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + runID string + want string + }{ + {name: "workflow run", runID: "123456789", want: "00021i3v9"}, + {name: "zero", runID: "0", want: "000000000"}, + {name: "largest 44 bit value", runID: "17592186044415", want: "68hqlcqv3"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + got, err := Encode(test.runID) + if err != nil { + t.Fatalf("Encode(%q) error = %v", test.runID, err) + } + if got != test.want { + t.Errorf("Encode(%q) = %q; want %q", test.runID, got, test.want) + } + }) + } +} + +func TestEncodeRejectsNoncanonicalOrOversizedRunID(t *testing.T) { + t.Parallel() + + for _, runID := range []string{ + "", + "00", + "0123456789", + "+123", + "-123", + "123 ", + "123_456", + "contract-run", + "17592186044416", + } { + t.Run(runID, func(t *testing.T) { + t.Parallel() + if namespace, err := Encode(runID); err == nil { + t.Fatalf("Encode(%q) = %q; want an error", runID, namespace) + } + }) + } +}