From cd4cc1a41bdec3ba0c6cc68216a8063c8a3632cc Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Thu, 23 Jul 2026 16:12:28 -0400 Subject: [PATCH] [patch] Make Cloud Compose bootstrap retryable --- Makefile | 9 +- ansible/roles/cloud_compose/defaults/main.yml | 2 +- ansible/roles/cloud_compose/tasks/main.yml | 2 +- ci/app-filesystem-convergence-contract.sh | 54 +-- ci/application-env-contract.sh | 5 +- ci/bootstrap-recovery-contract.sh | 419 ++++++++++++++++++ ci/cloud-smoke.sh | 35 +- ci/config-management-cloud-smoke-inner.sh | 1 + ci/config-management-cloud-smoke.sh | 4 +- ci/config-management-input-contract.sh | 11 + ci/config-management-smoke-inner.sh | 2 + ci/filesystem-prep-contract.sh | 8 +- ci/gcp-upgrade-smoke-contract.sh | 35 +- ci/sitectl-version-contract.sh | 35 ++ ci/systemd-contract.sh | 29 +- ci/vault-runtime-contract.sh | 2 +- docs/non-gcp-providers.md | 9 +- docs/runtime-contracts.md | 15 + .../linux-vm-runtime/templates/cloud-init.yml | 2 +- .../system/cloud-compose-bootstrap.service | 30 ++ .../etc/systemd/system/cloud-compose.service | 6 +- rootfs/home/cloud-compose/app-init.sh | 3 + .../cloud-compose/assert-app-initialized.sh | 17 + .../home/cloud-compose/bootstrap-helpers.sh | 115 +++++ .../cloud-compose/libops-managed-runtime.sh | 8 +- rootfs/home/cloud-compose/run-bootstrap.sh | 10 + rootfs/home/cloud-compose/run.sh | 21 +- .../start-cloud-compose-bootstrap.sh | 30 ++ salt/cloud-compose/init.sls | 2 +- templates/cloud-init.yml | 2 +- 30 files changed, 859 insertions(+), 64 deletions(-) create mode 100644 ci/bootstrap-recovery-contract.sh create mode 100644 rootfs/etc/systemd/system/cloud-compose-bootstrap.service create mode 100644 rootfs/home/cloud-compose/assert-app-initialized.sh create mode 100644 rootfs/home/cloud-compose/bootstrap-helpers.sh create mode 100644 rootfs/home/cloud-compose/run-bootstrap.sh create mode 100644 rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh diff --git a/Makefile b/Makefile index 8611c4d..570386d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract sitectl-version-contract go-fmt-check go-vet go-contracts template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke cloud-compose-ci +.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract sitectl-version-contract go-fmt-check go-vet go-contracts template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke cloud-compose-ci .PHONY: terraform-fmt terraform-fmt-check terraform-validate terraform-validate-contract terraform-lint-check terraform-docs terraform-docs-check .PHONY: config-management-cloud-smoke config-management-cloud-smoke-ansible-drupal config-management-cloud-smoke-salt-drupal .PHONY: destroy-config-management-cloud-smoke destroy-config-management-cloud-smoke-ansible-drupal destroy-config-management-cloud-smoke-salt-drupal @@ -16,9 +16,9 @@ export CLOUD_COMPOSE_CI_BIN GO_MODULE_FILES := $(wildcard go.mod go.sum) GO_SOURCES := $(shell find cmd internal -type f -name '*.go') -lint: terraform-fmt actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check +lint: terraform-fmt actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check -lint-check: terraform-fmt-check actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check +lint-check: terraform-fmt-check actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check actionlint: go run github.com/rhysd/actionlint/cmd/actionlint@$(ACTIONLINT_VERSION) @@ -79,6 +79,9 @@ config-management-input-contract: systemd-contract: bash ci/systemd-contract.sh +bootstrap-recovery-contract: + bash ci/bootstrap-recovery-contract.sh + sitectl-version-contract: bash ci/sitectl-version-contract.sh diff --git a/ansible/roles/cloud_compose/defaults/main.yml b/ansible/roles/cloud_compose/defaults/main.yml index 4ddd92b..489df87 100644 --- a/ansible/roles/cloud_compose/defaults/main.yml +++ b/ansible/roles/cloud_compose/defaults/main.yml @@ -27,7 +27,7 @@ cloud_compose_reload_systemd: true cloud_compose_run_bootstrap: true cloud_compose_force_bootstrap: false cloud_compose_dedicated_host_acknowledged: false -cloud_compose_bootstrap_timeout: 3600 +cloud_compose_bootstrap_timeout: 14400 cloud_compose_bootstrap_poll_interval: 15 cloud_compose_extra_env: {} diff --git a/ansible/roles/cloud_compose/tasks/main.yml b/ansible/roles/cloud_compose/tasks/main.yml index 835786b..452a004 100644 --- a/ansible/roles/cloud_compose/tasks/main.yml +++ b/ansible/roles/cloud_compose/tasks/main.yml @@ -580,7 +580,7 @@ - name: Bootstrap cloud-compose runtime ansible.builtin.command: - cmd: bash "{{ cloud_compose_home }}/run.sh" + cmd: bash "{{ cloud_compose_home }}/start-cloud-compose-bootstrap.sh" creates: "{{ cloud_compose_home }}/.cloud-compose-bootstrap-complete" async: "{{ cloud_compose_bootstrap_timeout }}" poll: "{{ cloud_compose_bootstrap_poll_interval }}" diff --git a/ci/app-filesystem-convergence-contract.sh b/ci/app-filesystem-convergence-contract.sh index df09311..cc6a5f1 100755 --- a/ci/app-filesystem-convergence-contract.sh +++ b/ci/app-filesystem-convergence-contract.sh @@ -15,8 +15,8 @@ export COMPOSE_PROJECTS_FILE="$tmp/compose-projects.json" export COMPOSE_APPS_ENV_DIR="$tmp/apps" export COMPOSE_APPS_STATE_DIR="$tmp/state" export CLOUD_COMPOSE_DATA_ROOT="$tmp/data" -project_dir="$CLOUD_COMPOSE_DATA_ROOT/repository/app" -mkdir -p "$project_dir" +fixture_project_dir="$CLOUD_COMPOSE_DATA_ROOT/repository/app" +mkdir -p "$fixture_project_dir" write_manifest() { local path="$1" @@ -32,30 +32,30 @@ write_manifest() { # shellcheck disable=SC1091 source "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" -write_manifest "$project_dir" -printf 'PRESERVED=value\n' >"$project_dir/.env" -chmod 0400 "$project_dir/.env" -original_inode="$(stat -c '%d:%i' "$project_dir/.env")" +write_manifest "$fixture_project_dir" +printf 'PRESERVED=value\n' >"$fixture_project_dir/.env" +chmod 0400 "$fixture_project_dir/.env" +original_inode="$(stat -c '%d:%i' "$fixture_project_dir/.env")" runtime_uid="$(id -u)" runtime_gid="$(id -g)" _converge_compose_app_filesystem_for_ids app "$runtime_uid" "$runtime_gid" -[[ "$(<"$project_dir/.env")" == "PRESERVED=value" ]] || +[[ "$(<"$fixture_project_dir/.env")" == "PRESERVED=value" ]] || fail "environment contents changed during metadata repair" -[[ "$(stat -c '%d:%i' "$project_dir/.env")" == "$original_inode" ]] || +[[ "$(stat -c '%d:%i' "$fixture_project_dir/.env")" == "$original_inode" ]] || fail "environment metadata repair replaced the file" -[[ "$(stat -c '%u:%g:%a:%h' "$project_dir/.env")" == "${runtime_uid}:${runtime_gid}:640:1" ]] || +[[ "$(stat -c '%u:%g:%a:%h' "$fixture_project_dir/.env")" == "${runtime_uid}:${runtime_gid}:640:1" ]] || fail "environment metadata did not converge" -[[ "$(stat -c '%u:%g:%a' "$project_dir")" == "${runtime_uid}:${runtime_gid}:775" ]] || +[[ "$(stat -c '%u:%g:%a' "$fixture_project_dir")" == "${runtime_uid}:${runtime_gid}:775" ]] || fail "exact manifest project directory metadata did not converge" # Convergence is idempotent and does not traverse application content. -mkdir -p "$project_dir/preserved" -printf 'unchanged\n' >"$project_dir/preserved/file" -chmod 0600 "$project_dir/preserved/file" -preserved_metadata="$(stat -c '%u:%g:%a:%i' "$project_dir/preserved/file")" +mkdir -p "$fixture_project_dir/preserved" +printf 'unchanged\n' >"$fixture_project_dir/preserved/file" +chmod 0600 "$fixture_project_dir/preserved/file" +preserved_metadata="$(stat -c '%u:%g:%a:%i' "$fixture_project_dir/preserved/file")" _converge_compose_app_filesystem_for_ids app "$runtime_uid" "$runtime_gid" -[[ "$(stat -c '%u:%g:%a:%i' "$project_dir/preserved/file")" == "$preserved_metadata" ]] || +[[ "$(stat -c '%u:%g:%a:%i' "$fixture_project_dir/preserved/file")" == "$preserved_metadata" ]] || fail "convergence changed a descendant outside the .env contract" # Hold convergence after its final read-only ownership transition while an @@ -98,8 +98,8 @@ for _ in $(seq 1 500); do sleep 0.01 done [[ -f "$CONVERGENCE_FROZEN" ]] || fail "convergence did not reach its frozen-directory state" -if mv "$project_dir/.env" "$project_dir/.env.raced" 2>/dev/null; then - ln -s "$outside_target" "$project_dir/.env" +if mv "$fixture_project_dir/.env" "$fixture_project_dir/.env.raced" 2>/dev/null; then + ln -s "$outside_target" "$fixture_project_dir/.env" : >"$tmp/adversary-swapped" fi : >"$CONVERGENCE_RELEASE" @@ -114,27 +114,27 @@ wait "$convergence_pid" PATH="${PATH#"$tmp/bin:"}" export PATH -mv "$project_dir/.env" "$project_dir/.env.regular" -ln -s "$project_dir/.env.regular" "$project_dir/.env" +mv "$fixture_project_dir/.env" "$fixture_project_dir/.env.regular" +ln -s "$fixture_project_dir/.env.regular" "$fixture_project_dir/.env" if _converge_compose_app_filesystem_for_ids app "$runtime_uid" "$runtime_gid" >/dev/null 2>&1; then fail "symbolic-link environment file was accepted" fi -chmod 0775 "$project_dir" -rm "$project_dir/.env" +chmod 0775 "$fixture_project_dir" +rm "$fixture_project_dir/.env" -mkdir "$project_dir/.env" +mkdir "$fixture_project_dir/.env" if _converge_compose_app_filesystem_for_ids app "$runtime_uid" "$runtime_gid" >/dev/null 2>&1; then fail "non-regular environment path was accepted" fi -chmod 0775 "$project_dir" -rmdir "$project_dir/.env" +chmod 0775 "$fixture_project_dir" +rmdir "$fixture_project_dir/.env" -ln "$project_dir/.env.regular" "$project_dir/.env" +ln "$fixture_project_dir/.env.regular" "$fixture_project_dir/.env" if _converge_compose_app_filesystem_for_ids app "$runtime_uid" "$runtime_gid" >/dev/null 2>&1; then fail "hard-linked environment file was accepted" fi -chmod 0775 "$project_dir" -rm "$project_dir/.env" +chmod 0775 "$fixture_project_dir" +rm "$fixture_project_dir/.env" outside_project="$tmp/outside-project" mkdir "$outside_project" diff --git a/ci/application-env-contract.sh b/ci/application-env-contract.sh index 7fad2f5..303575f 100755 --- a/ci/application-env-contract.sh +++ b/ci/application-env-contract.sh @@ -162,7 +162,10 @@ env -i PATH=/usr/bin:/bin CLOUD_COMPOSE_ENV_FILE="$host_env" \ cd "$2" sync_compose_application_env ' application-env "$profile" "$tmp/alpha" -! grep -Fq '# cloud-compose application: APPLICATION_LITERAL' "$tmp/alpha/.env" +if grep -Fq '# cloud-compose application: APPLICATION_LITERAL' "$tmp/alpha/.env"; then + echo "Removed application environment entry remains in the Compose .env file" >&2 + exit 1 +fi grep -Fq '# cloud-compose managed: COMPOSE_BIND_PORT' "$tmp/alpha/.env" fake_rollout="$tmp/fake-rollout" diff --git a/ci/bootstrap-recovery-contract.sh b/ci/bootstrap-recovery-contract.sh new file mode 100644 index 0000000..39ae37b --- /dev/null +++ b/ci/bootstrap-recovery-contract.sh @@ -0,0 +1,419 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +helpers="$repo_root/rootfs/home/cloud-compose/bootstrap-helpers.sh" +assert_initialized="$repo_root/rootfs/home/cloud-compose/assert-app-initialized.sh" +start_bootstrap="$repo_root/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh" +run_script="$repo_root/rootfs/home/cloud-compose/run.sh" +app_init="$repo_root/rootfs/home/cloud-compose/app-init.sh" +run_bootstrap="$repo_root/rootfs/home/cloud-compose/run-bootstrap.sh" +bootstrap_unit="$repo_root/rootfs/etc/systemd/system/cloud-compose-bootstrap.service" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +fail() { + echo "bootstrap recovery contract: $*" >&2 + exit 1 +} + +assert_contains() { + local file="$1" value="$2" + grep -Fq -- "$value" "$file" || fail "$file does not contain: $value" +} + +mkdir -p "$tmp/bin" "$tmp/run" +systemctl_log="$tmp/systemctl.log" +active_calls="$tmp/active-calls" +: >"$systemctl_log" +printf '0\n' >"$active_calls" +cat >"$tmp/bin/systemctl" <<'EOF' +#!/usr/bin/env bash + +set -euo pipefail + +printf '%s\n' "$*" >>"${SYSTEMCTL_LOG:?}" +case "$1" in + show) + property="" + for arg in "$@"; do + case "$arg" in + --property=*) property="${arg#--property=}" ;; + esac + done + case "$property" in + LoadState) + printf 'loaded\n' + ;; + ActiveState) + calls="$(<"${ACTIVE_CALLS:?}")" + calls=$((calls + 1)) + printf '%s\n' "$calls" >"$ACTIVE_CALLS" + case "${SYSTEMD_SCENARIO:-recover}" in + active) printf 'active\n' ;; + forced) + if [[ -n "${UNIT_STARTED_FILE:-}" && + -f "$UNIT_STARTED_FILE" ]]; then + printf 'active\n' + elif [[ -n "${UNIT_STOPPED_FILE:-}" && + -f "$UNIT_STOPPED_FILE" ]]; then + printf 'inactive\n' + else + printf 'active\n' + fi + ;; + recover) + if ((calls == 1)); then + printf 'failed\n' + elif ((calls < 4)); then + printf 'activating\n' + else + printf 'active\n' + fi + ;; + failed) printf 'failed\n' ;; + *) exit 2 ;; + esac + ;; + *) exit 2 ;; + esac + ;; + stop) + if [[ -n "${UNIT_STOPPED_FILE:-}" ]]; then + : >"$UNIT_STOPPED_FILE" + rm -f -- "${UNIT_STARTED_FILE:-}" + fi + ;; + start) + if [[ -n "${UNIT_STARTED_FILE:-}" ]]; then + : >"$UNIT_STARTED_FILE" + fi + if [[ -n "${START_MARKER:-}" ]]; then + printf 'ready\n' >"$START_MARKER" + fi + ;; + enable | reset-failed | restart | daemon-reload) + ;; + status) + exit 3 + ;; + *) + exit 2 + ;; +esac +EOF +cat >"$tmp/bin/sleep" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +chmod +x "$tmp/bin/systemctl" "$tmp/bin/sleep" + +# shellcheck disable=SC1090 +source "$helpers" +PATH="$tmp/bin:/usr/bin:/bin" +export PATH SYSTEMCTL_LOG="$systemctl_log" ACTIVE_CALLS="$active_calls" + +SYSTEMD_SCENARIO=recover +export SYSTEMD_SCENARIO +cloud_compose_start_and_wait_for_oneshot cloud-compose.service 10 +grep -Fxq 'enable -- cloud-compose.service' "$systemctl_log" || + fail "failed application unit was not enabled" +grep -Fxq 'reset-failed -- cloud-compose.service' "$systemctl_log" || + fail "failed application unit was not reset" +grep -Fxq 'start --no-block -- cloud-compose.service' "$systemctl_log" || + fail "application unit was not started nonblocking" +[[ "$(<"$active_calls")" == "4" ]] || + fail "wait helper did not tolerate restart transitions until active" + +: >"$systemctl_log" +printf '0\n' >"$active_calls" +SYSTEMD_SCENARIO=active +cloud_compose_start_systemd_unit cloud-compose.service +if grep -Eq '^(reset-failed|start) ' "$systemctl_log"; then + fail "already active application unit was restarted" +fi + +if cloud_compose_start_systemd_unit ssh.service >/dev/null 2>&1; then + fail "start helper accepted an unrelated systemd unit" +fi +printf '0\n' >"$active_calls" +SYSTEMD_SCENARIO=failed +if CLOUD_COMPOSE_SYSTEMD_POLL_SECONDS=1 \ + cloud_compose_wait_for_oneshot cloud-compose.service 2 >/dev/null 2>&1; then + fail "wait helper accepted a unit that never became active" +fi + +durable_marker="$tmp/bootstrap-complete" +boot_marker="$tmp/run/app-init-complete" +cloud_compose_should_run_app_init "$durable_marker" "$boot_marker" || + fail "first boot would skip application initialization" +original_umask="$(umask)" +cloud_compose_publish_marker "$boot_marker" +[[ "$(umask)" == "$original_umask" ]] || + fail "publishing a marker changed the caller's umask" +if cloud_compose_should_run_app_init "$durable_marker" "$boot_marker"; then + fail "bootstrap retry would repeat successful application initialization" +fi +cloud_compose_publish_marker "$durable_marker" +cloud_compose_should_run_app_init "$durable_marker" "$boot_marker" || + fail "manual convergence with durable readiness would skip application initialization" +rm -f -- "$durable_marker" "$boot_marker" + +assert_env=( + "CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH=$helpers" + "CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER=$durable_marker" + "CLOUD_COMPOSE_APP_INIT_MARKER=$boot_marker" +) +if env "${assert_env[@]}" bash "$assert_initialized" >/dev/null 2>&1; then + fail "application preflight accepted missing initialization markers" +fi +cloud_compose_publish_marker "$boot_marker" +env "${assert_env[@]}" bash "$assert_initialized" +rm -f -- "$boot_marker" +cloud_compose_publish_marker "$durable_marker" +env "${assert_env[@]}" bash "$assert_initialized" +rm -f -- "$durable_marker" +ln -s /dev/null "$durable_marker" +if env "${assert_env[@]}" bash "$assert_initialized" >/dev/null 2>&1; then + fail "application preflight accepted a symlink marker" +fi + +assert_contains "$app_init" 'acquire_cloud_compose_lifecycle_lock "init"' +assert_contains "$app_init" 'trap release_cloud_compose_lifecycle_lock EXIT' +assert_contains "$run_script" 'if cloud_compose_should_run_app_init' +assert_contains "$run_script" 'cloud_compose_publish_marker "$current_boot_app_init_marker"' +assert_contains "$run_script" 'cloud_compose_start_and_wait_for_oneshot cloud-compose.service "$app_wait_seconds"' +assert_contains "$run_script" 'cloud_compose_publish_marker "$durable_bootstrap_marker"' +init_line="$(grep -nF 'run_as_cloud_compose bash /home/cloud-compose/app-init.sh' "$run_script" | cut -d: -f1)" +boot_marker_line="$(grep -nF 'cloud_compose_publish_marker "$current_boot_app_init_marker"' "$run_script" | cut -d: -f1)" +app_line="$(grep -nF 'cloud_compose_start_and_wait_for_oneshot cloud-compose.service' "$run_script" | cut -d: -f1)" +durable_marker_line="$(grep -nF 'cloud_compose_publish_marker "$durable_bootstrap_marker"' "$run_script" | cut -d: -f1)" +((init_line < boot_marker_line && boot_marker_line < app_line && app_line < durable_marker_line)) || + fail "bootstrap does not publish init, await the app, then publish durable readiness" + +assert_contains "$run_bootstrap" 'if ((EUID != 0)); then' +assert_contains "$run_bootstrap" 'exec bash /home/cloud-compose/run.sh' +if rg -n 'bootstrap\\.log|exec (>>|>)[^[:space:]]' "$run_bootstrap" >/dev/null; then + fail "bootstrap wrapper writes an independently unbounded log file" +fi +assert_contains "$bootstrap_unit" 'StandardOutput=journal' +assert_contains "$bootstrap_unit" 'StandardError=journal' +assert_contains "$bootstrap_unit" 'SyslogIdentifier=cloud-compose-bootstrap' +assert_contains "$bootstrap_unit" 'SyslogLevel=info' +assert_contains "$bootstrap_unit" 'SyslogLevelPrefix=no' +assert_contains "$bootstrap_unit" 'LogRateLimitIntervalSec=30s' +assert_contains "$bootstrap_unit" 'LogRateLimitBurst=1000' +assert_contains "$bootstrap_unit" 'UMask=0022' + +# The systemd bootstrap replaces cloud-init's direct execution and must retain +# its ordinary runtime-directory semantics. In particular, the unprivileged +# application user must be able to traverse the persistent target of the +# published sitectl symlink after a first-boot root install. +bootstrap_umask="$(sed -n 's/^UMask=//p' "$bootstrap_unit")" +[[ "$bootstrap_umask" == "0022" ]] || + fail "bootstrap unit does not declare exactly one compatible umask" +umask_fixture="$tmp/bootstrap-umask" +( + umask "$bootstrap_umask" + mkdir -p "$umask_fixture/libops-managed/bin" "$umask_fixture/published" + printf '#!/bin/sh\nexit 0\n' >"$umask_fixture/libops-managed/bin/sitectl" + chmod +x "$umask_fixture/libops-managed/bin/sitectl" + ln -s "$umask_fixture/libops-managed/bin/sitectl" "$umask_fixture/published/sitectl" +) +for traversable_dir in "$umask_fixture/libops-managed" "$umask_fixture/libops-managed/bin"; do + mode="$(stat -c '%a' "$traversable_dir")" + ((8#$mode & 0001)) || + fail "bootstrap umask makes the published sitectl target untraversable: $traversable_dir" +done +[[ -x "$umask_fixture/published/sitectl" ]] || + fail "published sitectl is not executable after first-boot installation" + +assert_contains "$start_bootstrap" 'CLOUD_COMPOSE_BOOTSTRAP_WAIT_SECONDS:-10800' +assert_contains "$start_bootstrap" 'if cloud_compose_marker_exists "$durable_marker"; then' +assert_contains "$start_bootstrap" 'systemctl daemon-reload' +assert_contains "$start_bootstrap" 'systemctl stop -- "$bootstrap_unit"' +assert_contains "$start_bootstrap" 'cloud_compose_start_and_wait_for_oneshot "$bootstrap_unit" "$wait_seconds"' +if rg -n '_SYSTEMD_UNIT=cloud-compose-bootstrap\\.service' \ + "$repo_root/rootfs/etc/fluent-bit" >/dev/null; then + fail "raw bootstrap output was added to Fluent Bit" +fi + +for cloud_init_template in \ + "$repo_root/templates/cloud-init.yml" \ + "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml"; do + assert_contains "$cloud_init_template" 'bash /home/cloud-compose/start-cloud-compose-bootstrap.sh' + if grep -Fq 'bash /home/cloud-compose/run.sh > /home/cloud-compose/run.log 2>&1' \ + "$cloud_init_template"; then + fail "cloud-init bypasses the retryable bootstrap unit" + fi +done + +: >"$systemctl_log" +printf '0\n' >"$active_calls" +SYSTEMD_SCENARIO=active +cloud_compose_publish_marker "$durable_marker" +CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH="$helpers" \ +CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER="$durable_marker" \ +SYSTEMD_SCENARIO="$SYSTEMD_SCENARIO" \ + bash "$start_bootstrap" +[[ ! -s "$systemctl_log" ]] || + fail "completed bootstrap was started again" + +: >"$systemctl_log" +printf '0\n' >"$active_calls" +unit_stopped="$tmp/unit-stopped" +unit_started="$tmp/unit-started" +rm -f -- "$durable_marker" "$unit_stopped" "$unit_started" +SYSTEMD_SCENARIO=forced \ +UNIT_STOPPED_FILE="$unit_stopped" \ +UNIT_STARTED_FILE="$unit_started" \ +START_MARKER="$durable_marker" \ +CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH="$helpers" \ +CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER="$durable_marker" \ + bash "$start_bootstrap" +grep -Fxq 'stop -- cloud-compose-bootstrap.service' "$systemctl_log" || + fail "removed readiness marker did not stop a stale active bootstrap unit" +grep -Fxq 'start --no-block -- cloud-compose-bootstrap.service' "$systemctl_log" || + fail "removed readiness marker did not start a fresh bootstrap unit" +[[ -f "$durable_marker" ]] || + fail "fresh bootstrap unit did not restore durable readiness" +rm -f -- "$durable_marker" + +# Execute the production run script from a relocated fixture so the contract can +# inject deterministic host commands without writing to /home or /run. The +# first application-service convergence fails after app-init succeeds. The +# second full run must reuse the current-boot init marker and publish durable +# readiness after the service recovers. +integration_root="$tmp/integration" +integration_home="$integration_root/home/cloud-compose" +integration_run="$integration_root/run" +integration_bin="$integration_root/bin" +integration_log="$integration_root/systemctl.log" +app_init_count="$integration_root/app-init-count" +first_output="$integration_root/first-attempt.log" +retry_output="$integration_root/retry-attempt.log" +mkdir -p "$integration_home" "$integration_run" "$integration_bin" +: >"$integration_log" +printf '0\n' >"$app_init_count" + +sed \ + -e "s#/home/cloud-compose#$integration_home#g" \ + -e "s#/run/cloud-compose-app-init-complete#$integration_run/app-init-complete#g" \ + "$run_script" >"$integration_home/run.sh" +cp "$helpers" "$integration_home/bootstrap-helpers.sh" +cat >"$integration_home/profile.sh" <<'EOF' +#!/usr/bin/env bash +: +EOF +cat >"$integration_home/noop.sh" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +for script in \ + host-conf.sh \ + host-init.sh \ + converge-app-filesystems.sh \ + prepare-app-sources.sh \ + vault-agent-init.sh; do + cp "$integration_home/noop.sh" "$integration_home/$script" +done +cat >"$integration_home/app-init.sh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +count="$(<"${APP_INIT_COUNT_FILE:?}")" +printf '%s\n' "$((count + 1))" >"$APP_INIT_COUNT_FILE" +EOF +cat >"$integration_bin/runuser" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +while (($# > 0)); do + if [[ "$1" == "--" ]]; then + shift + exec "$@" + fi + shift +done +exit 2 +EOF +cat >"$integration_bin/systemd-tmpfiles" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +cat >"$integration_bin/sleep" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +cat >"$integration_bin/systemctl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +printf '%s %s\n' "${RUN_ATTEMPT:?}" "$*" >>"${INTEGRATION_SYSTEMCTL_LOG:?}" +case "$1" in + show) + property="" + for arg in "$@"; do + case "$arg" in + --property=*) property="${arg#--property=}" ;; + esac + done + case "$property" in + LoadState) printf 'loaded\n' ;; + ActiveState) + if [[ "$RUN_ATTEMPT" == "fail" ]]; then + printf 'failed\n' + else + printf 'active\n' + fi + ;; + *) exit 2 ;; + esac + ;; + daemon-reload | disable | enable | reset-failed | restart | start) + ;; + status) + exit 3 + ;; + *) + exit 2 + ;; +esac +EOF +chmod +x "$integration_bin"/* + +integration_env=( + "APP_INIT_COUNT_FILE=$app_init_count" + "CLOUD_COMPOSE_APP_WAIT_SECONDS=1" + "CLOUD_COMPOSE_SYSTEMD_POLL_SECONDS=1" + "CLOUD_COMPOSE_PROVIDER=contract" + "CLOUD_COMPOSE_DOCKER_PRUNE_ENABLED=false" + "LIBOPS_INTERNAL_SERVICES_ENABLED=false" + "LIBOPS_MANAGED_RUNTIME_ENABLED=false" + "INTEGRATION_SYSTEMCTL_LOG=$integration_log" + "PATH=$integration_bin:/usr/bin:/bin" +) +if env "${integration_env[@]}" RUN_ATTEMPT=fail \ + bash "$integration_home/run.sh" >"$first_output" 2>&1; then + fail "first full bootstrap attempt accepted a failed application service" +fi +[[ "$(<"$app_init_count")" == "1" ]] || + fail "first full bootstrap attempt did not complete app-init exactly once" +[[ -f "$integration_run/app-init-complete" ]] || + fail "failed first attempt did not retain current-boot app-init readiness" +[[ ! -e "$integration_home/.cloud-compose-bootstrap-complete" ]] || + fail "failed first attempt published durable bootstrap readiness" + +env "${integration_env[@]}" RUN_ATTEMPT=recover \ + bash "$integration_home/run.sh" >"$retry_output" 2>&1 +[[ "$(<"$app_init_count")" == "1" ]] || + fail "bootstrap retry repeated successful app-init" +[[ -f "$integration_home/.cloud-compose-bootstrap-complete" ]] || + fail "bootstrap retry did not publish durable readiness" +grep -Fq 'Application initialization already completed during this boot' "$retry_output" || + fail "bootstrap retry did not report reuse of successful app-init" +grep -Fq 'fail reset-failed -- cloud-compose.service' "$integration_log" || + fail "first bootstrap attempt did not exercise failed service recovery" +grep -Fq 'recover enable -- cloud-compose.service' "$integration_log" || + fail "bootstrap retry did not converge the application service" + +echo "Bootstrap recovery contract passed" diff --git a/ci/cloud-smoke.sh b/ci/cloud-smoke.sh index c77650c..38e3e8f 100755 --- a/ci/cloud-smoke.sh +++ b/ci/cloud-smoke.sh @@ -385,7 +385,20 @@ if [ -f /home/cloud-compose/.cloud-compose-bootstrap-complete ]; then echo complete exit 0 fi -if systemctl is-active --quiet cloud-compose; then +bootstrap_load_state=\"\$(systemctl show --property=LoadState --value -- cloud-compose-bootstrap.service 2>/dev/null)\" +if [ \"\$bootstrap_load_state\" = loaded ]; then + bootstrap_active_state=\"\$(systemctl show --property=ActiveState --value -- cloud-compose-bootstrap.service 2>/dev/null)\" + bootstrap_sub_state=\"\$(systemctl show --property=SubState --value -- cloud-compose-bootstrap.service 2>/dev/null)\" + case \"\$bootstrap_active_state:\$bootstrap_sub_state\" in + active:* | activating:* | *:auto-restart) + echo active + exit 0 + ;; + esac +elif [ \"\$bootstrap_load_state\" = not-found ] && + systemctl is-active --quiet cloud-compose; then + # Releases before retryable bootstrap have no durable unit or marker; their + # active application service remains the compatibility completion signal. echo complete exit 0 fi @@ -443,8 +456,14 @@ wait_for_cloud_init() { ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" "bash -lc 'set +e echo \"--- active bootstrap processes ---\" ps -eo pid,ppid,stat,etime,args | grep -E \"cloud-init|runcmd|run.sh|host-conf|host-init|app-init|install-dependencies|apt-get|docker|sitectl|git clone\" | grep -v grep -echo \"--- /home/cloud-compose/run.log ---\" -sudo tail -n 160 /home/cloud-compose/run.log +echo \"--- cloud-compose bootstrap unit ---\" +sudo journalctl -u cloud-compose-bootstrap --no-pager -n 160 +echo \"--- legacy cloud-compose bootstrap log (when present) ---\" +if sudo test -f /home/cloud-compose/run.log && sudo test ! -L /home/cloud-compose/run.log; then + sudo tail -n 160 /home/cloud-compose/run.log +else + echo \"Legacy bootstrap log is not present\" +fi echo \"--- /var/log/cloud-init-output.log ---\" sudo tail -n 120 /var/log/cloud-init-output.log '" || true @@ -472,8 +491,14 @@ echo \"--- /var/log/cloud-init.log ---\" sudo tail -n 400 /var/log/cloud-init.log echo \"--- cloud-init runcmd ---\" sudo sed -n '1,240p' /var/lib/cloud/instance/scripts/runcmd -echo \"--- /home/cloud-compose/run.log ---\" -sudo tail -n 400 /home/cloud-compose/run.log +echo \"--- cloud-compose bootstrap unit ---\" +sudo journalctl -u cloud-compose-bootstrap --no-pager -n 400 +echo \"--- legacy cloud-compose bootstrap log (when present) ---\" +if sudo test -f /home/cloud-compose/run.log && sudo test ! -L /home/cloud-compose/run.log; then + sudo tail -n 400 /home/cloud-compose/run.log +else + echo \"Legacy bootstrap log is not present\" +fi echo \"--- cloud-compose unit ---\" sudo journalctl -u cloud-compose --no-pager -n 300 echo \"--- lifecycle lock permissions ---\" diff --git a/ci/config-management-cloud-smoke-inner.sh b/ci/config-management-cloud-smoke-inner.sh index 5ef5f00..f53111d 100755 --- a/ci/config-management-cloud-smoke-inner.sh +++ b/ci/config-management-cloud-smoke-inner.sh @@ -196,6 +196,7 @@ test -x /home/cloud-compose/up test -x /home/cloud-compose/down test -x /home/cloud-compose/rollout test -x /home/cloud-compose/run.sh +test -x /home/cloud-compose/start-cloud-compose-bootstrap.sh python3 -m json.tool /home/cloud-compose/compose-projects.json >/dev/null python3 -m json.tool /home/cloud-compose/application-env.json >/dev/null diff --git a/ci/config-management-cloud-smoke.sh b/ci/config-management-cloud-smoke.sh index 6d2ae86..13f596e 100755 --- a/ci/config-management-cloud-smoke.sh +++ b/ci/config-management-cloud-smoke.sh @@ -247,8 +247,8 @@ echo \"--- cloud-init status ---\" cloud-init status --long echo \"--- /var/log/cloud-init-output.log ---\" tail -n 300 /var/log/cloud-init-output.log -echo \"--- /home/cloud-compose/run.log ---\" -tail -n 300 /home/cloud-compose/run.log +echo \"--- cloud-compose bootstrap unit ---\" +sudo journalctl -u cloud-compose-bootstrap --no-pager -n 300 echo \"--- cloud-compose unit ---\" journalctl -u cloud-compose --no-pager -n 300 echo \"--- docker ps ---\" diff --git a/ci/config-management-input-contract.sh b/ci/config-management-input-contract.sh index 21a182c..b0dd25e 100755 --- a/ci/config-management-input-contract.sh +++ b/ci/config-management-input-contract.sh @@ -159,6 +159,7 @@ with tempfile.TemporaryDirectory(prefix="cloud-compose-input-contract.") as temp reject(label, payload, expected) ansible_tasks = (repo_root / "ansible/roles/cloud_compose/tasks/main.yml").read_text() +ansible_defaults = (repo_root / "ansible/roles/cloud_compose/defaults/main.yml").read_text() ansible_gate = ansible_tasks.find("Validate project directory and managed artifact host boundaries") ansible_first_mutation = ansible_tasks.find("Install Debian runtime dependencies") if ansible_gate < 0 or ansible_first_mutation < 0 or ansible_gate > ansible_first_mutation: @@ -167,6 +168,12 @@ if "files/validate-runtime-inputs.py" not in ansible_tasks: fail("Ansible does not execute the shared host-input validator") if "--data-root" in ansible_tasks: fail("Ansible makes the production project ownership boundary configurable") +if 'cmd: bash "{{ cloud_compose_home }}/start-cloud-compose-bootstrap.sh"' not in ansible_tasks: + fail("Ansible bypasses the retryable bootstrap service") +if 'cmd: bash "{{ cloud_compose_home }}/run.sh"' in ansible_tasks: + fail("Ansible still invokes the one-shot bootstrap script directly") +if "cloud_compose_bootstrap_timeout: 14400" not in ansible_defaults: + fail("Ansible bootstrap timeout does not cover the bounded retryable bootstrap wait") salt_state = (repo_root / "salt/cloud-compose/init.sls").read_text() cloud_smoke_driver = (repo_root / "ci/config-management-cloud-smoke.sh").read_text() @@ -175,6 +182,10 @@ salt_gate = salt_state.find("cloud-compose-host-inputs-valid:") salt_first_mutation = salt_state.find("cloud-compose-packages:") if salt_gate < 0 or salt_first_mutation < 0 or salt_gate > salt_first_mutation: fail("Salt host-input validation does not precede its first host mutation") +if "home ~ '/start-cloud-compose-bootstrap.sh'" not in salt_state: + fail("Salt bypasses the retryable bootstrap service") +if "home ~ '/run.sh'" in salt_state: + fail("Salt still invokes the one-shot bootstrap script directly") gate_block = salt_state[salt_gate:salt_first_mutation] for marker in ( "cmd.run:", diff --git a/ci/config-management-smoke-inner.sh b/ci/config-management-smoke-inner.sh index edcb3aa..611e5db 100755 --- a/ci/config-management-smoke-inner.sh +++ b/ci/config-management-smoke-inner.sh @@ -297,6 +297,7 @@ for path in [ "/home/cloud-compose/down", "/home/cloud-compose/rollout", "/home/cloud-compose/run.sh", + "/home/cloud-compose/start-cloud-compose-bootstrap.sh", ]: assert Path(path).exists(), path assert os.access(path, os.X_OK), path @@ -486,6 +487,7 @@ for path in [ "/home/cloud-compose/down", "/home/cloud-compose/rollout", "/home/cloud-compose/run.sh", + "/home/cloud-compose/start-cloud-compose-bootstrap.sh", ]: assert Path(path).exists(), path assert os.access(path, os.X_OK), path diff --git a/ci/filesystem-prep-contract.sh b/ci/filesystem-prep-contract.sh index 97ea748..d616186 100644 --- a/ci/filesystem-prep-contract.sh +++ b/ci/filesystem-prep-contract.sh @@ -386,14 +386,14 @@ for cloud_init_template in \ "$cloud_init_template" || fail "cloud-init does not verify every required mount before initialization" grep -Fq 'Required cloud-compose mount is unavailable:' "$cloud_init_template" || fail "cloud-init mount gate does not report the unavailable path" - grep -Fq ' bash /home/cloud-compose/run.sh > /home/cloud-compose/run.log 2>&1' \ - "$cloud_init_template" || fail "cloud-init application startup is outside the fail-closed mount block" + grep -Fq ' bash /home/cloud-compose/start-cloud-compose-bootstrap.sh' \ + "$cloud_init_template" || fail "cloud-init retryable bootstrap startup is outside the fail-closed mount block" marker_reset_line="$(grep -nF ' rm -f /home/cloud-compose/.cloud-compose-bootstrap-complete' \ "$cloud_init_template" | cut -d: -f1)" - run_line="$(grep -nF ' bash /home/cloud-compose/run.sh > /home/cloud-compose/run.log 2>&1' \ + run_line="$(grep -nF ' bash /home/cloud-compose/start-cloud-compose-bootstrap.sh' \ "$cloud_init_template" | cut -d: -f1)" [[ -n "$marker_reset_line" && -n "$run_line" && "$marker_reset_line" -lt "$run_line" ]] || - fail "cloud-init does not clear stale bootstrap readiness before application startup" + fail "cloud-init does not clear stale bootstrap readiness before retryable startup" done grep -Fq 'install -m 0600 /dev/null /run/cloud-compose-filesystems-ready' \ "$repo_root/templates/cloud-init.yml" || fail "GCP bootcmd does not publish filesystem readiness" diff --git a/ci/gcp-upgrade-smoke-contract.sh b/ci/gcp-upgrade-smoke-contract.sh index 4294e6d..6d02628 100755 --- a/ci/gcp-upgrade-smoke-contract.sh +++ b/ci/gcp-upgrade-smoke-contract.sh @@ -21,12 +21,13 @@ if grep -Fq 'google_compute_instance_iam_member.gce-start' <<<"$ppb_module"; the fi script="$repo_root/ci/gcp-upgrade-smoke.sh" +shared_smoke="$repo_root/ci/cloud-smoke.sh" fixture="$repo_root/tests/smoke/gcp-upgrade/main.tf" variables="$repo_root/tests/smoke/gcp-upgrade/variables.tf" workflow="$repo_root/.github/workflows/cloud-smoke.yml" docs="$repo_root/docs/runtime-contracts.md" -for required in "$script" "$fixture" "$variables" "$workflow" "$docs"; do +for required in "$script" "$shared_smoke" "$fixture" "$variables" "$workflow" "$docs"; do [[ -f "$required" ]] || fail "required file is missing: $required" done @@ -46,6 +47,16 @@ grep -Fq "provider_tag_cleanup gcp-wp \"\$run_id\"" "$script" || fail "upgrade runner does not finish cleanup with the verified provider sweep" grep -Fq 'target_env gcp-wp' "$script" || fail "upgrade cleanup does not load the concrete GCP WordPress target environment" +[[ "$(grep -Fc 'sudo test -f /home/cloud-compose/run.log && sudo test ! -L /home/cloud-compose/run.log' "$shared_smoke")" -eq 2 ]] || + fail "shared smoke diagnostics do not retain guarded legacy bootstrap logs for the pinned baseline" +grep -Fq 'bootstrap_load_state=\"\$(systemctl show --property=LoadState --value -- cloud-compose-bootstrap.service' "$shared_smoke" || + fail "shared smoke readiness does not distinguish retryable bootstrap from the legacy baseline" +grep -Fq 'active:* | activating:* | *:auto-restart)' "$shared_smoke" || + fail "shared smoke readiness can abandon the bootstrap unit during its restart delay" +grep -Fq 'elif [ \"\$bootstrap_load_state\" = not-found ] &&' "$shared_smoke" || + fail "shared smoke readiness does not restrict legacy fallback to an absent bootstrap unit" +grep -Fq 'systemctl is-active --quiet cloud-compose; then' "$shared_smoke" || + fail "shared smoke readiness lost its pre-bootstrap-unit compatibility signal" grep -Fq 'CLOUD_COMPOSE_SMOKE_RUN_ID must match GITHUB_RUN_ID in GitHub Actions' "$script" || fail "hosted cleanup ownership is not bound to the actual GitHub run id" grep -Fq 'CLOUD_COMPOSE_SMOKE_RUN_ID must be set explicitly outside GitHub Actions' "$script" || @@ -141,12 +152,22 @@ if grep -Fq 'runcmd = [' "$fixture"; then fi git -C "$repo_root" show f33117cdbbf4a9c7d59006a4db986baef118e6bb:templates/cloud-init.yml \ >"$tmp/baseline-cloud-init.yml" || fail "could not inspect the pinned baseline cloud-init" -for cloud_init_template in "$repo_root/templates/cloud-init.yml" "$tmp/baseline-cloud-init.yml"; do - run_script_line="$(grep -nF 'bash /home/cloud-compose/run.sh' "$cloud_init_template" | cut -d: -f1)" - template_initcmd_line="$(grep -nF 'for CMD in ADDITIONAL_INITCMD' "$cloud_init_template" | cut -d: -f1)" - [[ -n "$template_initcmd_line" && -n "$run_script_line" && "$template_initcmd_line" -lt "$run_script_line" ]] || - fail "baseline or current cloud-init does not execute fixture initcmd before run.sh" -done +baseline_run_line="$(grep -nF 'bash /home/cloud-compose/run.sh' "$tmp/baseline-cloud-init.yml" | cut -d: -f1 || true)" +baseline_initcmd_line="$(grep -nF 'for CMD in ADDITIONAL_INITCMD' "$tmp/baseline-cloud-init.yml" | cut -d: -f1 || true)" +[[ -n "$baseline_initcmd_line" && -n "$baseline_run_line" && "$baseline_initcmd_line" -lt "$baseline_run_line" ]] || + fail "baseline cloud-init does not execute fixture initcmd before run.sh" + +current_bootstrap_line="$( + grep -nF 'bash /home/cloud-compose/start-cloud-compose-bootstrap.sh' \ + "$repo_root/templates/cloud-init.yml" | + cut -d: -f1 || true +)" +current_initcmd_line="$( + grep -nF 'for CMD in ADDITIONAL_INITCMD' "$repo_root/templates/cloud-init.yml" | + cut -d: -f1 || true +)" +[[ -n "$current_initcmd_line" && -n "$current_bootstrap_line" && "$current_initcmd_line" -lt "$current_bootstrap_line" ]] || + fail "current cloud-init does not execute fixture initcmd before retryable bootstrap" grep -Fq 'for unit in internal-services.timer internal-services.service cloud-compose-internal-services.timer cloud-compose-internal-services.service' "$script" || fail "upgrade runner does not assert that both timer generations remain inactive after boot" grep -Fq 'run_direct_vpc_cold_start "$new_home" "$key_path" "$new_output" "$name" "$zone"' "$script" || diff --git a/ci/sitectl-version-contract.sh b/ci/sitectl-version-contract.sh index bd585f5..0ec2f60 100644 --- a/ci/sitectl-version-contract.sh +++ b/ci/sitectl-version-contract.sh @@ -9,6 +9,41 @@ trap 'rm -rf "$tmp"' EXIT touch "$tmp/profile.sh" +# A restrictive first-boot umask once left the root-owned managed binary path +# untraversable after application initialization dropped privileges. Exercise +# the production mkdirs implementation against that preexisting state and +# require every invocation to converge both public and private modes. +CLOUD_COMPOSE_PROFILE_PATH="$tmp/profile.sh" \ + bash --noprofile --norc -c ' + set -euo pipefail + source "$1" + STATE_DIR="$2/mode-state" + BIN_DIR="$STATE_DIR/bin" + TMP_DIR="$STATE_DIR/tmp" + PACKAGE_STATE_DIR="$STATE_DIR/packages" + ARTIFACT_STATE_DIR="$STATE_DIR/artifacts" + PUBLISHED_BIN_DIR="$2/published" + + umask 0077 + mkdir -p \ + "$BIN_DIR" \ + "$TMP_DIR" \ + "$PACKAGE_STATE_DIR" \ + "$ARTIFACT_STATE_DIR" \ + "$PUBLISHED_BIN_DIR" + chmod 0700 "$STATE_DIR" "$BIN_DIR" + chmod 0755 "$TMP_DIR" "$PACKAGE_STATE_DIR" "$ARTIFACT_STATE_DIR" + + mkdirs + mkdirs + + test "$(stat -c "%a" "$STATE_DIR")" = 755 + test "$(stat -c "%a" "$BIN_DIR")" = 755 + test "$(stat -c "%a" "$TMP_DIR")" = 700 + test "$(stat -c "%a" "$PACKAGE_STATE_DIR")" = 700 + test "$(stat -c "%a" "$ARTIFACT_STATE_DIR")" = 700 + ' cloud-compose-sitectl-modes "$runtime_script" "$tmp" + run_contract() { CLOUD_COMPOSE_PROFILE_PATH="$tmp/profile.sh" \ SITECTL_PACKAGES="$1" \ diff --git a/ci/systemd-contract.sh b/ci/systemd-contract.sh index 29e6a96..becdfaa 100755 --- a/ci/systemd-contract.sh +++ b/ci/systemd-contract.sh @@ -24,14 +24,31 @@ assert_contains() { fail "dedicated Docker prune units are missing" [[ -f "$unit_dir/cloud-compose-vault-agent.service" ]] || fail "namespaced Vault Agent unit is missing" [[ -f "$unit_dir/cloud-compose-overlay.service" ]] || fail "Docker overlay mount unit is missing" +[[ -f "$unit_dir/cloud-compose-bootstrap.service" ]] || fail "retryable bootstrap unit is missing" [[ -f "$unit_dir/cloud-compose-internal-services.service" && -f "$unit_dir/cloud-compose-internal-services.timer" ]] || \ fail "namespaced internal-services units are missing" assert_contains "$unit_dir/cloud-compose.service" 'Requires=docker.service cloud-compose-metadata-firewall.service' assert_contains "$unit_dir/cloud-compose.service" 'RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes' assert_contains "$unit_dir/cloud-compose.service" 'After=network-online.target docker.service cloud-compose-metadata-firewall.service' +assert_contains "$unit_dir/cloud-compose.service" 'ExecStartPre=/bin/bash /home/cloud-compose/assert-app-initialized.sh' +assert_contains "$unit_dir/cloud-compose.service" 'Restart=on-failure' +assert_contains "$unit_dir/cloud-compose.service" 'RestartSec=30s' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ConditionPathExists=!/home/cloud-compose/.cloud-compose-bootstrap-complete' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ExecStart=/bin/bash /home/cloud-compose/run-bootstrap.sh' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'Restart=on-failure' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'RestartSec=30s' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'TimeoutStartSec=2h' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'UMask=0022' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'StandardOutput=journal' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'StandardError=journal' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'SyslogIdentifier=cloud-compose-bootstrap' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'SyslogLevel=info' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'SyslogLevelPrefix=no' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'LogRateLimitIntervalSec=30s' +assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'LogRateLimitBurst=1000' assert_contains "$unit_dir/cloud-compose-internal-services.service" 'Requires=cloud-compose.service cloud-compose-metadata-firewall.service' -assert_contains "$unit_dir/cloud-compose.service" 'TimeoutStartSec=45min' +assert_contains "$unit_dir/cloud-compose.service" 'TimeoutStartSec=1h' assert_contains "$unit_dir/cloud-compose-mariadb-backup.service" 'TimeoutStartSec=12h' if grep -Fq 'Wants=cloud-compose.service' "$unit_dir/cloud-compose-mariadb-backup.service"; then fail "backup service starts an intentionally inactive application" @@ -58,9 +75,17 @@ assert_contains "$metadata_pre_unit" 'WantedBy=multi-user.target' assert_contains "$metadata_pre_unit" 'ExecStart=/bin/bash /home/cloud-compose/configure-metadata-firewall.sh pre-docker' assert_contains "$unit_dir/cloud-compose-overlay.service" 'Before=docker.service cloud-compose.service' assert_contains "$unit_dir/cloud-compose-overlay.service" 'ExecStart=/bin/bash /home/cloud-compose/mount-overlays.sh' +if grep -Eq '^(After|Before|BindsTo|PartOf|Requires|Requisite|Wants)=.*cloud-compose-bootstrap\\.service' \ + "$unit_dir/cloud-compose.service"; then + fail "application service has an ordering dependency on the bootstrap service" +fi +if grep -Eq '^(After|Before|BindsTo|PartOf|Requires|Requisite|Wants)=.*cloud-compose\\.service' \ + "$unit_dir/cloud-compose-bootstrap.service"; then + fail "bootstrap service has an ordering dependency on the application service" +fi run_script="$repo_root/rootfs/home/cloud-compose/run.sh" -assert_contains "$run_script" 'systemctl enable --now cloud-compose.service' +assert_contains "$run_script" 'cloud_compose_start_and_wait_for_oneshot cloud-compose.service "$app_wait_seconds"' assert_contains "$run_script" 'systemd-tmpfiles is required to prepare the cloud-compose lifecycle lock' assert_contains "$run_script" 'systemd-tmpfiles --create /etc/tmpfiles.d/cloud-compose.conf' assert_contains "$run_script" 'systemctl restart cloud-compose-overlay.service' diff --git a/ci/vault-runtime-contract.sh b/ci/vault-runtime-contract.sh index 78da7d2..521413c 100644 --- a/ci/vault-runtime-contract.sh +++ b/ci/vault-runtime-contract.sh @@ -138,7 +138,7 @@ vault_line="$(grep -nF 'bash /home/cloud-compose/vault-agent-init.sh' "$repo_roo source_line="$(grep -nF 'run_as_cloud_compose bash /home/cloud-compose/prepare-app-sources.sh' "$repo_root/rootfs/home/cloud-compose/run.sh" | cut -d: -f1)" rotation_line="$(grep -nF 'bash /home/cloud-compose/rotate-keys-daily.sh' "$repo_root/rootfs/home/cloud-compose/run.sh" | cut -d: -f1)" init_line="$(grep -nF 'run_as_cloud_compose bash /home/cloud-compose/app-init.sh' "$repo_root/rootfs/home/cloud-compose/run.sh" | cut -d: -f1)" -app_line="$(grep -nF 'systemctl enable --now cloud-compose.service' "$repo_root/rootfs/home/cloud-compose/run.sh" | cut -d: -f1)" +app_line="$(grep -nF 'cloud_compose_start_and_wait_for_oneshot cloud-compose.service' "$repo_root/rootfs/home/cloud-compose/run.sh" | cut -d: -f1)" [[ -n "$source_line" && -n "$rotation_line" && -n "$vault_line" && -n "$init_line" && -n "$app_line" ]] || \ fail "bootstrap is missing a source, rotation, Vault, init, or app-service phase" (( source_line < rotation_line && rotation_line < vault_line && vault_line < init_line && init_line < app_line )) || \ diff --git a/docs/non-gcp-providers.md b/docs/non-gcp-providers.md index 40622f9..15e9448 100644 --- a/docs/non-gcp-providers.md +++ b/docs/non-gcp-providers.md @@ -126,7 +126,14 @@ provider-specific entry wins when the same username appears in both maps. Use the Ansible role or Salt formula when Terraform should not create the VM, disk, firewall, or DNS. These adapters install the packaged cloud-compose rootfs, write `.env`, write `compose-projects.json`, reload systemd, and optionally run -the bootstrap. +the bootstrap. Like the Terraform providers, both adapters start +`cloud-compose-bootstrap.service` and wait for its durable readiness marker; +transient bootstrap and application-service failures retry after 30 seconds. +The Ansible role's four-hour async budget is intentionally longer than the +starter's bounded three-hour wait. Inspect failures in +`journalctl -u cloud-compose-bootstrap` and with +`systemctl status cloud-compose-bootstrap cloud-compose`; the systemd journal +is the canonical bootstrap log. These adapters are for a dedicated, empty application host. They install and restart Docker with `/mnt/disks/data/docker` as its data root, install diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md index 7b6f185..d711f22 100644 --- a/docs/runtime-contracts.md +++ b/docs/runtime-contracts.md @@ -142,6 +142,21 @@ Use a full commit for reproducible production rollouts. A commit pin fixes the repository contents but does not prove who authored them; protect the selected repository and review/sign commits according to your downstream governance. +First boot runs through `cloud-compose-bootstrap.service`. Both that bootstrap +and `cloud-compose.service` retry failures after 30 seconds, so a transient +registry, Vault, or Compose failure converges without an operator restarting +cloud-init. Application initialization is serialized by the normal lifecycle +lock and recorded for the current boot before the app starts; a bootstrap retry +reuses that successful initialization instead of repeating it. Durable +readiness is published only after the Compose `up` unit reaches its successful +oneshot state. The cloud-init caller waits for that result, so configured +post-initialization commands retain their ordering. Inspect the bounded system +journal with `sudo journalctl -u cloud-compose-bootstrap` and the unit states +with `systemctl status cloud-compose-bootstrap cloud-compose`. Bootstrap output +uses a fixed `info` priority and has no unit-specific Fluent Bit input, so raw +bootstrap output is not forwarded to Cloud Logging; systemd's own service +failures remain available to the existing warning-level collector. + ## Sitectl During init, the VM creates a `sitectl` context for every app. The default `up` diff --git a/modules/linux-vm-runtime/templates/cloud-init.yml b/modules/linux-vm-runtime/templates/cloud-init.yml index fdd24af..6557d7b 100644 --- a/modules/linux-vm-runtime/templates/cloud-init.yml +++ b/modules/linux-vm-runtime/templates/cloud-init.yml @@ -94,4 +94,4 @@ runcmd: chmod 0775 /mnt/disks/data /mnt/disks/volumes install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/data/libops rm -f /home/cloud-compose/.cloud-compose-bootstrap-complete - bash /home/cloud-compose/run.sh > /home/cloud-compose/run.log 2>&1 + bash /home/cloud-compose/start-cloud-compose-bootstrap.sh diff --git a/rootfs/etc/systemd/system/cloud-compose-bootstrap.service b/rootfs/etc/systemd/system/cloud-compose-bootstrap.service new file mode 100644 index 0000000..48db515 --- /dev/null +++ b/rootfs/etc/systemd/system/cloud-compose-bootstrap.service @@ -0,0 +1,30 @@ +[Unit] +Description=Converge Cloud Compose application bootstrap +Wants=network-online.target +After=network-online.target +RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes +ConditionPathExists=!/home/cloud-compose/.cloud-compose-bootstrap-complete +StartLimitIntervalSec=0 + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/bash /home/cloud-compose/run-bootstrap.sh +Restart=on-failure +RestartSec=30s +TimeoutStartSec=2h +# Match the cloud-init execution contract: runtime binaries and application +# scaffolding need traversable parent directories. Secret writers set their +# own restrictive modes. +UMask=0022 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=cloud-compose-bootstrap +# Keep raw command output out of the warning-only Fluent Bit journal input. +SyslogLevel=info +SyslogLevelPrefix=no +LogRateLimitIntervalSec=30s +LogRateLimitBurst=1000 + +[Install] +WantedBy=multi-user.target diff --git a/rootfs/etc/systemd/system/cloud-compose.service b/rootfs/etc/systemd/system/cloud-compose.service index bc7fd51..2cd1381 100644 --- a/rootfs/etc/systemd/system/cloud-compose.service +++ b/rootfs/etc/systemd/system/cloud-compose.service @@ -5,16 +5,20 @@ Requires=docker.service cloud-compose-metadata-firewall.service RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes Wants=network-online.target After=network-online.target docker.service cloud-compose-metadata-firewall.service cloud-compose-vault-agent.service +StartLimitIntervalSec=0 [Service] Type=oneshot RemainAfterExit=yes User=cloud-compose Group=cloud-compose +ExecStartPre=/bin/bash /home/cloud-compose/assert-app-initialized.sh ExecStartPre=/bin/bash /home/cloud-compose/assert-vault-ready.sh ExecStart=/bin/bash /home/cloud-compose/up ExecStop=/bin/bash /home/cloud-compose/down -TimeoutStartSec=45min +Restart=on-failure +RestartSec=30s +TimeoutStartSec=1h TimeoutStopSec=15min [Install] diff --git a/rootfs/home/cloud-compose/app-init.sh b/rootfs/home/cloud-compose/app-init.sh index 80d4d95..36bf261 100644 --- a/rootfs/home/cloud-compose/app-init.sh +++ b/rootfs/home/cloud-compose/app-init.sh @@ -10,6 +10,9 @@ cd /home/cloud-compose # shellcheck disable=SC1091 source /home/cloud-compose/compose-apps.sh +acquire_cloud_compose_lifecycle_lock "init" +trap release_cloud_compose_lifecycle_lock EXIT + apps=() compose_app_names_array apps for app in "${apps[@]}"; do diff --git a/rootfs/home/cloud-compose/assert-app-initialized.sh b/rootfs/home/cloud-compose/assert-app-initialized.sh new file mode 100644 index 0000000..6434942 --- /dev/null +++ b/rootfs/home/cloud-compose/assert-app-initialized.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# shellcheck disable=SC1090 +source "${CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH:-/home/cloud-compose/bootstrap-helpers.sh}" + +durable_marker="${CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/home/cloud-compose/.cloud-compose-bootstrap-complete}" +boot_marker="${CLOUD_COMPOSE_APP_INIT_MARKER:-/run/cloud-compose-app-init-complete}" + +if cloud_compose_marker_exists "$durable_marker" || + cloud_compose_marker_exists "$boot_marker"; then + exit 0 +fi + +echo "Cloud Compose application initialization has not completed for this boot" >&2 +exit 1 diff --git a/rootfs/home/cloud-compose/bootstrap-helpers.sh b/rootfs/home/cloud-compose/bootstrap-helpers.sh new file mode 100644 index 0000000..35a0256 --- /dev/null +++ b/rootfs/home/cloud-compose/bootstrap-helpers.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash + +cloud_compose_marker_exists() { + local marker="$1" + + [[ -f "$marker" && ! -L "$marker" ]] +} + +cloud_compose_should_run_app_init() { + local durable_marker="$1" + local boot_marker="$2" + + cloud_compose_marker_exists "$durable_marker" || + ! cloud_compose_marker_exists "$boot_marker" +} + +cloud_compose_publish_marker() ( + local marker="$1" + local marker_dir tmp_marker + + marker_dir="$(dirname -- "$marker")" + if [[ ! -d "$marker_dir" || -L "$marker_dir" ]]; then + echo "Unsafe Cloud Compose marker directory: $marker_dir" >&2 + return 1 + fi + + umask 022 + tmp_marker="$(mktemp "${marker}.tmp.XXXXXXXXXX")" || return 1 + if ! printf 'ready\n' >"$tmp_marker" || + ! chmod 0644 "$tmp_marker" || + ! mv -fT -- "$tmp_marker" "$marker"; then + rm -f -- "$tmp_marker" + return 1 + fi +) + +cloud_compose_validate_systemd_unit() { + case "$1" in + cloud-compose.service | cloud-compose-bootstrap.service) return 0 ;; + *) + echo "Unsupported Cloud Compose systemd unit: $1" >&2 + return 2 + ;; + esac +} + +cloud_compose_start_systemd_unit() { + local unit="$1" + local load_state active_state + + cloud_compose_validate_systemd_unit "$unit" || return + + load_state="$(systemctl show --property=LoadState --value -- "$unit")" || return 1 + if [[ "$load_state" != "loaded" ]]; then + echo "Cloud Compose systemd unit is not loaded: $unit ($load_state)" >&2 + return 1 + fi + + systemctl enable -- "$unit" + active_state="$(systemctl show --property=ActiveState --value -- "$unit")" || return 1 + if [[ "$active_state" == "active" || "$active_state" == "activating" ]]; then + return 0 + fi + + if [[ "$active_state" == "failed" ]]; then + systemctl reset-failed -- "$unit" + fi + systemctl start --no-block -- "$unit" +} + +cloud_compose_wait_for_oneshot() { + local unit="$1" + local timeout_seconds="$2" + local poll_seconds="${CLOUD_COMPOSE_SYSTEMD_POLL_SECONDS:-2}" + local elapsed=0 active_state load_state + + cloud_compose_validate_systemd_unit "$unit" || return + if [[ ! "$timeout_seconds" =~ ^[1-9][0-9]{0,4}$ ]] || + ((10#$timeout_seconds > 43200)); then + echo "Cloud Compose systemd wait must be from 1 through 43200 seconds" >&2 + return 2 + fi + if [[ ! "$poll_seconds" =~ ^[1-9][0-9]{0,2}$ ]] || + ((10#$poll_seconds > 300)); then + echo "CLOUD_COMPOSE_SYSTEMD_POLL_SECONDS must be from 1 through 300 seconds" >&2 + return 2 + fi + + while ((elapsed < 10#$timeout_seconds)); do + load_state="$(systemctl show --property=LoadState --value -- "$unit")" || return 1 + if [[ "$load_state" != "loaded" ]]; then + echo "Cloud Compose systemd unit stopped being loaded: $unit ($load_state)" >&2 + return 1 + fi + + active_state="$(systemctl show --property=ActiveState --value -- "$unit")" || return 1 + if [[ "$active_state" == "active" ]]; then + return 0 + fi + sleep "$poll_seconds" + elapsed=$((elapsed + 10#$poll_seconds)) + done + + echo "Timed out waiting ${timeout_seconds}s for $unit to become active" >&2 + systemctl status --no-pager --full -- "$unit" >&2 || true + return 1 +} + +cloud_compose_start_and_wait_for_oneshot() { + local unit="$1" + local timeout_seconds="$2" + + cloud_compose_start_systemd_unit "$unit" && + cloud_compose_wait_for_oneshot "$unit" "$timeout_seconds" +} diff --git a/rootfs/home/cloud-compose/libops-managed-runtime.sh b/rootfs/home/cloud-compose/libops-managed-runtime.sh index 17f9dff..94bc722 100644 --- a/rootfs/home/cloud-compose/libops-managed-runtime.sh +++ b/rootfs/home/cloud-compose/libops-managed-runtime.sh @@ -27,7 +27,13 @@ enabled() { } mkdirs() { - mkdir -p "$BIN_DIR" "$TMP_DIR" "$PACKAGE_STATE_DIR" "$ARTIFACT_STATE_DIR" "$PUBLISHED_BIN_DIR" + # The managed binary is published through /home/cloud-compose/bin and must + # remain traversable after a root-owned bootstrap drops to cloud-compose. + # Converge existing directories as well as new ones so a previously + # restrictive service umask cannot leave the published symlink unusable. + install -d -m 0755 "$STATE_DIR" "$BIN_DIR" + install -d -m 0700 "$TMP_DIR" "$PACKAGE_STATE_DIR" "$ARTIFACT_STATE_DIR" + mkdir -p "$PUBLISHED_BIN_DIR" } with_lock() { diff --git a/rootfs/home/cloud-compose/run-bootstrap.sh b/rootfs/home/cloud-compose/run-bootstrap.sh new file mode 100644 index 0000000..20e6ab4 --- /dev/null +++ b/rootfs/home/cloud-compose/run-bootstrap.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if ((EUID != 0)); then + echo "Cloud Compose bootstrap must run as root" >&2 + exit 1 +fi + +exec bash /home/cloud-compose/run.sh diff --git a/rootfs/home/cloud-compose/run.sh b/rootfs/home/cloud-compose/run.sh index ea2c52f..251e139 100644 --- a/rootfs/home/cloud-compose/run.sh +++ b/rootfs/home/cloud-compose/run.sh @@ -5,6 +5,9 @@ set -eou pipefail # shellcheck disable=SC1091 source /home/cloud-compose/profile.sh +# shellcheck disable=SC1091 +source /home/cloud-compose/bootstrap-helpers.sh + run_as_cloud_compose() ( # Configuration-management entrypoints commonly launch run.sh from /root. # Enter an accessible directory before dropping privileges so child scripts @@ -29,6 +32,10 @@ runtime_enabled() { esac } +durable_bootstrap_marker="/home/cloud-compose/.cloud-compose-bootstrap-complete" +current_boot_app_init_marker="/run/cloud-compose-app-init-complete" +app_wait_seconds="${CLOUD_COMPOSE_APP_WAIT_SECONDS:-4500}" + # The shared lifecycle lock must exist before the root-owned managed-runtime # installer and the unprivileged application service can contend for it. This # explicit creation covers first boot; systemd-tmpfiles recreates it thereafter. @@ -69,9 +76,16 @@ fi # An explicitly enabled Vault Agent is a startup dependency. Its systemd unit # also publishes a readiness marker before the application service may start. bash /home/cloud-compose/vault-agent-init.sh -run_as_cloud_compose bash /home/cloud-compose/app-init.sh +if cloud_compose_should_run_app_init \ + "$durable_bootstrap_marker" "$current_boot_app_init_marker"; then + rm -f -- "$current_boot_app_init_marker" + run_as_cloud_compose bash /home/cloud-compose/app-init.sh + cloud_compose_publish_marker "$current_boot_app_init_marker" +else + echo "Application initialization already completed during this boot; resuming service convergence" +fi -systemctl enable --now cloud-compose.service +cloud_compose_start_and_wait_for_oneshot cloud-compose.service "$app_wait_seconds" if runtime_enabled "${LIBOPS_INTERNAL_SERVICES_ENABLED:-false}"; then systemctl enable --now cloud-compose-internal-services.timer else @@ -88,5 +102,4 @@ else systemctl disable --now cloud-compose-docker-prune.timer cloud-compose-docker-prune.service >/dev/null 2>&1 || true fi systemctl enable --now cloud-compose-mariadb-backup.timer -touch /home/cloud-compose/.cloud-compose-bootstrap-complete -chown cloud-compose:cloud-compose /home/cloud-compose/.cloud-compose-bootstrap-complete +cloud_compose_publish_marker "$durable_bootstrap_marker" diff --git a/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh b/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh new file mode 100644 index 0000000..0e55092 --- /dev/null +++ b/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# shellcheck disable=SC1090 +source "${CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH:-/home/cloud-compose/bootstrap-helpers.sh}" + +durable_marker="${CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/home/cloud-compose/.cloud-compose-bootstrap-complete}" +wait_seconds="${CLOUD_COMPOSE_BOOTSTRAP_WAIT_SECONDS:-10800}" +bootstrap_unit="cloud-compose-bootstrap.service" + +if cloud_compose_marker_exists "$durable_marker"; then + exit 0 +fi + +systemctl daemon-reload +active_state="$(systemctl show --property=ActiveState --value -- "$bootstrap_unit")" +if [[ "$active_state" == "active" ]] && + ! cloud_compose_marker_exists "$durable_marker"; then + # RemainAfterExit keeps a completed oneshot active. Configuration + # management deliberately removes the durable marker when reviewed inputs + # change, so restart that stale active instance rather than treating it as + # an in-flight bootstrap. + systemctl stop -- "$bootstrap_unit" +fi +cloud_compose_start_and_wait_for_oneshot "$bootstrap_unit" "$wait_seconds" +if ! cloud_compose_marker_exists "$durable_marker"; then + echo "Cloud Compose bootstrap service became active without publishing readiness" >&2 + exit 1 +fi diff --git a/salt/cloud-compose/init.sls b/salt/cloud-compose/init.sls index e23a143..4fe5804 100644 --- a/salt/cloud-compose/init.sls +++ b/salt/cloud-compose/init.sls @@ -714,7 +714,7 @@ cloud-compose-clear-bootstrap-marker: {% if run_bootstrap is sameas true %} cloud-compose-bootstrap: cmd.run: - - name: bash {{ (home ~ '/run.sh') | json }} + - name: bash {{ (home ~ '/start-cloud-compose-bootstrap.sh') | json }} - creates: {{ (home ~ '/.cloud-compose-bootstrap-complete') | json }} - require: {% if install_packages %} diff --git a/templates/cloud-init.yml b/templates/cloud-init.yml index 705b11e..79ee4d8 100644 --- a/templates/cloud-init.yml +++ b/templates/cloud-init.yml @@ -92,7 +92,7 @@ ${ROOTFS_ARCHIVE_COMMAND} chmod 0775 /mnt/disks/data /mnt/disks/volumes install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/data/libops rm -f /home/cloud-compose/.cloud-compose-bootstrap-complete - bash /home/cloud-compose/run.sh > /home/cloud-compose/run.log 2>&1 + bash /home/cloud-compose/start-cloud-compose-bootstrap.sh %{ if length(ADDITIONAL_RUNCMD) > 0 ~} - | set -eu