[patch] Make Cloud Compose bootstrap retryable

Makefile

@@ -1,4 +1,4 @@
-.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract sitectl-version-contract go-fmt-check go-vet go-contracts template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke cloud-compose-ci
+.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract sitectl-version-contract go-fmt-check go-vet go-contracts template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke cloud-compose-ci
 .PHONY: terraform-fmt terraform-fmt-check terraform-validate terraform-validate-contract terraform-lint-check terraform-docs terraform-docs-check
 .PHONY: config-management-cloud-smoke config-management-cloud-smoke-ansible-drupal config-management-cloud-smoke-salt-drupal
 .PHONY: destroy-config-management-cloud-smoke destroy-config-management-cloud-smoke-ansible-drupal destroy-config-management-cloud-smoke-salt-drupal
@@ -16,9 +16,9 @@ export CLOUD_COMPOSE_CI_BIN
 GO_MODULE_FILES := $(wildcard go.mod go.sum)
 GO_SOURCES := $(shell find cmd internal -type f -name '*.go')
 
-lint: terraform-fmt actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check
+lint: terraform-fmt actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check
 
-lint-check: terraform-fmt-check actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check
+lint-check: terraform-fmt-check actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check
 
 actionlint:
 	go run github.com/rhysd/actionlint/cmd/actionlint@$(ACTIONLINT_VERSION)
@@ -79,6 +79,9 @@ config-management-input-contract:
 systemd-contract:
 	bash ci/systemd-contract.sh
 
+bootstrap-recovery-contract:
+	bash ci/bootstrap-recovery-contract.sh
+
 sitectl-version-contract:
 	bash ci/sitectl-version-contract.sh
 

ansible/roles/cloud_compose/defaults/main.yml

@@ -27,7 +27,7 @@ cloud_compose_reload_systemd: true
 cloud_compose_run_bootstrap: true
 cloud_compose_force_bootstrap: false
 cloud_compose_dedicated_host_acknowledged: false
-cloud_compose_bootstrap_timeout: 3600
+cloud_compose_bootstrap_timeout: 14400
 cloud_compose_bootstrap_poll_interval: 15
 
 cloud_compose_extra_env: {}

ansible/roles/cloud_compose/tasks/main.yml

@@ -580,7 +580,7 @@
 
 - name: Bootstrap cloud-compose runtime
   ansible.builtin.command:
-    cmd: bash "{{ cloud_compose_home }}/run.sh"
+    cmd: bash "{{ cloud_compose_home }}/start-cloud-compose-bootstrap.sh"
     creates: "{{ cloud_compose_home }}/.cloud-compose-bootstrap-complete"
   async: "{{ cloud_compose_bootstrap_timeout }}"
   poll: "{{ cloud_compose_bootstrap_poll_interval }}"

ci/app-filesystem-convergence-contract.sh

@@ -15,8 +15,8 @@ export COMPOSE_PROJECTS_FILE="$tmp/compose-projects.json"
 export COMPOSE_APPS_ENV_DIR="$tmp/apps"
 export COMPOSE_APPS_STATE_DIR="$tmp/state"
 export CLOUD_COMPOSE_DATA_ROOT="$tmp/data"
-project_dir="$CLOUD_COMPOSE_DATA_ROOT/repository/app"
-mkdir -p "$project_dir"
+fixture_project_dir="$CLOUD_COMPOSE_DATA_ROOT/repository/app"
+mkdir -p "$fixture_project_dir"
 
 write_manifest() {
   local path="$1"
@@ -32,30 +32,30 @@ write_manifest() {
 # shellcheck disable=SC1091
 source "$repo_root/rootfs/home/cloud-compose/compose-apps.sh"
 
-write_manifest "$project_dir"
-printf 'PRESERVED=value\n' >"$project_dir/.env"
-chmod 0400 "$project_dir/.env"
-original_inode="$(stat -c '%d:%i' "$project_dir/.env")"
+write_manifest "$fixture_project_dir"
+printf 'PRESERVED=value\n' >"$fixture_project_dir/.env"
+chmod 0400 "$fixture_project_dir/.env"
+original_inode="$(stat -c '%d:%i' "$fixture_project_dir/.env")"
 runtime_uid="$(id -u)"
 runtime_gid="$(id -g)"
 _converge_compose_app_filesystem_for_ids app "$runtime_uid" "$runtime_gid"
 
-[[ "$(<"$project_dir/.env")" == "PRESERVED=value" ]] ||
+[[ "$(<"$fixture_project_dir/.env")" == "PRESERVED=value" ]] ||
   fail "environment contents changed during metadata repair"
-[[ "$(stat -c '%d:%i' "$project_dir/.env")" == "$original_inode" ]] ||
+[[ "$(stat -c '%d:%i' "$fixture_project_dir/.env")" == "$original_inode" ]] ||
   fail "environment metadata repair replaced the file"
-[[ "$(stat -c '%u:%g:%a:%h' "$project_dir/.env")" == "${runtime_uid}:${runtime_gid}:640:1" ]] ||
+[[ "$(stat -c '%u:%g:%a:%h' "$fixture_project_dir/.env")" == "${runtime_uid}:${runtime_gid}:640:1" ]] ||
   fail "environment metadata did not converge"
-[[ "$(stat -c '%u:%g:%a' "$project_dir")" == "${runtime_uid}:${runtime_gid}:775" ]] ||
+[[ "$(stat -c '%u:%g:%a' "$fixture_project_dir")" == "${runtime_uid}:${runtime_gid}:775" ]] ||
   fail "exact manifest project directory metadata did not converge"
 
 # Convergence is idempotent and does not traverse application content.
-mkdir -p "$project_dir/preserved"
-printf 'unchanged\n' >"$project_dir/preserved/file"
-chmod 0600 "$project_dir/preserved/file"
-preserved_metadata="$(stat -c '%u:%g:%a:%i' "$project_dir/preserved/file")"
+mkdir -p "$fixture_project_dir/preserved"
+printf 'unchanged\n' >"$fixture_project_dir/preserved/file"
+chmod 0600 "$fixture_project_dir/preserved/file"
+preserved_metadata="$(stat -c '%u:%g:%a:%i' "$fixture_project_dir/preserved/file")"
 _converge_compose_app_filesystem_for_ids app "$runtime_uid" "$runtime_gid"
-[[ "$(stat -c '%u:%g:%a:%i' "$project_dir/preserved/file")" == "$preserved_metadata" ]] ||
+[[ "$(stat -c '%u:%g:%a:%i' "$fixture_project_dir/preserved/file")" == "$preserved_metadata" ]] ||
   fail "convergence changed a descendant outside the .env contract"
 
 # Hold convergence after its final read-only ownership transition while an
@@ -98,8 +98,8 @@ for _ in $(seq 1 500); do
   sleep 0.01
 done
 [[ -f "$CONVERGENCE_FROZEN" ]] || fail "convergence did not reach its frozen-directory state"
-if mv "$project_dir/.env" "$project_dir/.env.raced" 2>/dev/null; then
-  ln -s "$outside_target" "$project_dir/.env"
+if mv "$fixture_project_dir/.env" "$fixture_project_dir/.env.raced" 2>/dev/null; then
+  ln -s "$outside_target" "$fixture_project_dir/.env"
   : >"$tmp/adversary-swapped"
 fi
 : >"$CONVERGENCE_RELEASE"
@@ -114,27 +114,27 @@ wait "$convergence_pid"
 PATH="${PATH#"$tmp/bin:"}"
 export PATH
 
-mv "$project_dir/.env" "$project_dir/.env.regular"
-ln -s "$project_dir/.env.regular" "$project_dir/.env"
+mv "$fixture_project_dir/.env" "$fixture_project_dir/.env.regular"
+ln -s "$fixture_project_dir/.env.regular" "$fixture_project_dir/.env"
 if _converge_compose_app_filesystem_for_ids app "$runtime_uid" "$runtime_gid" >/dev/null 2>&1; then
   fail "symbolic-link environment file was accepted"
 fi
-chmod 0775 "$project_dir"
-rm "$project_dir/.env"
+chmod 0775 "$fixture_project_dir"
+rm "$fixture_project_dir/.env"
 
-mkdir "$project_dir/.env"
+mkdir "$fixture_project_dir/.env"
 if _converge_compose_app_filesystem_for_ids app "$runtime_uid" "$runtime_gid" >/dev/null 2>&1; then
   fail "non-regular environment path was accepted"
 fi
-chmod 0775 "$project_dir"
-rmdir "$project_dir/.env"
+chmod 0775 "$fixture_project_dir"
+rmdir "$fixture_project_dir/.env"
 
-ln "$project_dir/.env.regular" "$project_dir/.env"
+ln "$fixture_project_dir/.env.regular" "$fixture_project_dir/.env"
 if _converge_compose_app_filesystem_for_ids app "$runtime_uid" "$runtime_gid" >/dev/null 2>&1; then
   fail "hard-linked environment file was accepted"
 fi
-chmod 0775 "$project_dir"
-rm "$project_dir/.env"
+chmod 0775 "$fixture_project_dir"
+rm "$fixture_project_dir/.env"
 
 outside_project="$tmp/outside-project"
 mkdir "$outside_project"

ci/application-env-contract.sh

@@ -162,7 +162,10 @@ env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$host_env" \
     cd "$2"
     sync_compose_application_env
   ' application-env "$profile" "$tmp/alpha"
-! grep -Fq '# cloud-compose application: APPLICATION_LITERAL' "$tmp/alpha/.env"
+if grep -Fq '# cloud-compose application: APPLICATION_LITERAL' "$tmp/alpha/.env"; then
+  echo "Removed application environment entry remains in the Compose .env file" >&2
+  exit 1
+fi
 grep -Fq '# cloud-compose managed: COMPOSE_BIND_PORT' "$tmp/alpha/.env"
 
 fake_rollout="$tmp/fake-rollout"