diff --git a/migrations/db/20240502102550_easyengine_check_and_update_docker_one.php b/migrations/db/20240502102550_easyengine_check_and_update_docker_one.php index 9a25e20e4..d16b04f69 100644 --- a/migrations/db/20240502102550_easyengine_check_and_update_docker_one.php +++ b/migrations/db/20240502102550_easyengine_check_and_update_docker_one.php @@ -4,12 +4,12 @@ use EE; use EE\Migration\Base; -use Symfony\Component\Filesystem\Filesystem; class CheckAndUpdateDockerOne extends Base { public function __construct() { + parent::__construct(); if ( $this->is_first_execution ) { $this->skip_this_migration = true; } @@ -48,32 +48,14 @@ public function up() { } EE::debug( 'Checking docker-compose version' ); - $docker_compose_version = trim( EE::launch( 'docker-compose version --short' )->stdout ); - $docker_compose_path = EE::launch( 'command -v docker-compose' )->stdout; - $docker_compose_path = trim( $docker_compose_path ); - $docker_compose_backup_path = EE_BACKUP_DIR . '/docker-compose.backup'; - $docker_compose_new_path = EE_BACKUP_DIR . '/docker-compose'; - $fs = new Filesystem(); - if ( ! $fs->exists( EE_BACKUP_DIR ) ) { - $fs->mkdir( EE_BACKUP_DIR ); - } - $fs->copy( $docker_compose_path, $docker_compose_backup_path ); - - if ( version_compare( '1.29.2', $docker_compose_version, '!=' ) ) { - EE::exec( "curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m) -o $docker_compose_new_path && chmod +x $docker_compose_new_path" ); - EE::exec( "mv $docker_compose_new_path $docker_compose_path" ); - } + // Installs only when docker-compose is missing or too old; it never downgrades. + Containers::update_docker_compose(); // Check the version again post update. $docker_version = trim( EE::launch( 'docker version --format "{{.Server.Version}}"' )->stdout ); if ( version_compare( $docker_version, '20.10.10', '<' ) ) { EE::error( 'Docker version should be 20.10.10 or above. Please update Docker and try again.' ); } - - $docker_compose_version = trim( EE::launch( 'docker-compose version --short' )->stdout ); - if ( version_compare( '1.29.2', $docker_compose_version, '!=' ) ) { - EE::error( 'Docker-compose version should be 1.29.2. Please update Docker-compose and try again.' ); - } } /** diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 8c078e60d..c8898a93c 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -11,9 +11,21 @@ */ class Containers { + /** EasyEngine needs at least this docker-compose (see Runner::check_requirements()). */ + const DOCKER_COMPOSE_MIN_VERSION = '2.26.1'; + + /** Installed when docker-compose is missing or older than the minimum. */ + const DOCKER_COMPOSE_INSTALL_VERSION = '2.27.0'; + /** @var RevertableStepProcessor $rsp Keeps track of migration state. Reverts on error */ private static $rsp; + /** @var bool Whether the last update_docker_compose() call replaced the binary. */ + private static $replaced_docker_compose = false; + + /** @var string[] Images (without tag) the last get_current_docker_images_versions() found running. */ + private static $running_images = []; + /** * Main entry point for the script. Migrates all containers * @@ -22,12 +34,23 @@ class Containers { public static function start_container_migration() { EE\Utils\delem_log( 'Starting container migration' ); - self::$rsp = new RevertableStepProcessor(); + self::$rsp = new RevertableStepProcessor( true ); $img_versions = EE\Utils\get_image_versions(); $current_versions = self::get_current_docker_images_versions(); $updated_images = []; + // An interrupted upgrade recreated some global services without recording their tags. Count those by their + // recorded tags, so this run checks their readiness again and can roll them back to the pending snapshot. + if ( GlobalContainers::has_pending_compose_backup( EE_BACKUP_DIR . '/services/docker-compose.yml.backup' ) ) { + foreach ( array_keys( GlobalContainers::get_all_global_images_with_service_name() ) as $image ) { + $recorded = EE\Model\Option::get( $image ); + if ( $recorded ) { + $current_versions[ $image ] = $recorded; + } + } + } + $skip_download = [ 'easyengine/php5.6', 'easyengine/php7.0', @@ -40,34 +63,54 @@ public static function start_container_migration() { 'easyengine/php8.3', 'easyengine/php8.4', 'easyengine/php8.5', + 'easyengine/php', 'easyengine/newrelic-daemon', ]; foreach ( $img_versions as $img => $version ) { + if ( array_key_exists( $img, $current_versions ) && $current_versions[ $img ] !== $version ) { + $updated_images[] = $img; + } + } - if ( array_key_exists( $img, $current_versions ) ) { - if ( $current_versions[ $img ] !== $version ) { - $updated_images[] = $img; - if ( ! in_array( $img, $skip_download ) ) { - self::pull_or_error( $img, $version ); - } - } - } else { - if ( ! in_array( $img, $skip_download ) ) { - self::pull_or_error( $img, $version ); + if ( empty( $updated_images ) ) { + // Disabled sites an earlier upgrade skipped, or an interrupted one left behind, have no image in $updated_images. + if ( self::get_disabled_sites_to_update( $img_versions ) ) { + self::migrate_site_containers( [] ); + if ( ! self::$rsp->execute() ) { + throw new \Exception( 'Unable to migrate sites to newer version' ); } } + // Every tag is recorded (e.g. a run interrupted just after saving them): a pending snapshot is done with. + GlobalContainers::clear_pending_compose_backup( EE_BACKUP_DIR . '/services/docker-compose.yml.backup' ); + + return; } - if ( empty( $updated_images ) ) { + // Stopped global services keep their old image and tag (see migrate_global_containers()), so their images stay + // "updated" on every later run: when nothing running uses them and nothing else is left to do, stop here. + $by_state = GlobalContainers::get_updated_global_images_by_state( $updated_images, $img_versions ); + $idle = array_diff( $by_state['stopped'], self::$running_images ); + if ( ! array_diff( $updated_images, $idle ) && ! self::has_old_tags( EE_SERVICE_DIR . '/docker-compose.yml', $idle, $img_versions ) && ! self::get_disabled_sites_to_update( $img_versions ) ) { + EE::debug( 'Only stopped global services have new images, and their compose files have them already.' ); + // Nothing running needs a gate (that would count as updated), so an interrupted run's snapshot is done with. + GlobalContainers::clear_pending_compose_backup( EE_BACKUP_DIR . '/services/docker-compose.yml.backup' ); return; } - self::migrate_global_containers( $updated_images ); - self::migrate_site_containers( $updated_images ); + // An image no container runs and the DB doesn't track (e.g. mailhog, the amd64-only easyengine/php) has nothing + // to upgrade: pulling it only fails on hosts it isn't built for. Nor has one only stopped services use, which keep + // their old image. Compose pulls these when something first uses them. A stopped service on its new tag has it. + foreach ( array_diff( $updated_images, $idle, $by_state['current'], $skip_download ) as $img ) { + self::pull_or_error( $img, $img_versions[ $img ] ); + } + + // First: compose v1 can't recreate containers on Docker 25+, and on a rollback it is restored last. self::maybe_update_docker_compose(); - self::save_upgraded_image_versions( $current_versions, $img_versions, $updated_images ); + self::migrate_global_containers( $by_state ); + self::migrate_site_containers( $updated_images ); + self::save_upgraded_image_versions( $current_versions, $img_versions, array_values( array_diff( $updated_images, $by_state['stopped'] ) ) ); if ( ! self::$rsp->execute() ) { throw new \Exception( 'Unable to migrate sites to newer version' ); @@ -77,8 +120,8 @@ public static function start_container_migration() { } /** - * Maybe update docker-compose at the end of migration. - * Need to update to latest docker-compose version for new template changes. + * Maybe update docker-compose before any container is recreated, for the new templates and because v1 fails to + * recreate containers on current Docker. */ public static function maybe_update_docker_compose() { @@ -108,6 +151,15 @@ public static function save_upgraded_image_versions( $current_versions, $new_ver [ $current_versions, $updated_images ] ); + // The upgrade is committed: a later run takes a fresh snapshot of the global compose file. + self::$rsp->add_step( + 'clear-pending-global-compose-backup', + 'EE\Migration\GlobalContainers::clear_pending_compose_backup', + null, + [ EE_BACKUP_DIR . '/services/docker-compose.yml.backup' ], + null + ); + self::$rsp->add_step( 'prune-old-docker-images', 'EE\Migration\Containers::docker_cleanup', @@ -123,42 +175,152 @@ public static function save_upgraded_image_versions( $current_versions, $new_ver */ public static function docker_cleanup() { EE::exec( 'docker image prune -af --filter=label=org.label-schema.vendor="EasyEngine"' ); - EE::exec( 'docker network prune -f --filter=label=org.label-schema.vendor="EasyEngine"' ); + + // Not `docker network prune`: it also removes the networks of stopped containers, which then can't start, and the + // global networks that disabled sites join as external networks on `ee site enable`. + $keep = [ defined( 'GLOBAL_FRONTEND_NETWORK' ) ? GLOBAL_FRONTEND_NETWORK : 'ee-global-frontend-network', defined( 'GLOBAL_BACKEND_NETWORK' ) ? GLOBAL_BACKEND_NETWORK : 'ee-global-backend-network' ]; + $networks = array_filter( explode( "\n", trim( EE::launch( 'docker network ls --filter=label=org.label-schema.vendor="EasyEngine" --format "{{.Name}}"' )->stdout ) ) ); + foreach ( array_diff( $networks, $keep ) as $network ) { + $users = EE::launch( 'docker ps -aq --filter network=' . escapeshellarg( $network ) ); + if ( 0 === $users->return_code && '' === trim( $users->stdout ) ) { + EE::exec( 'docker network rm ' . escapeshellarg( $network ) ); + } + } } /** - * Update docker-compose to v2.27.0 if lower version is installed. + * Install docker-compose DOCKER_COMPOSE_INSTALL_VERSION when it is missing or older than DOCKER_COMPOSE_MIN_VERSION. + * A newer one is never replaced. A failed download or check only warns and keeps the installed binary. */ public static function update_docker_compose() { - $docker_compose_version = trim( EE::launch( 'docker-compose version --short' )->stdout ); + // Only this call's replacement may be reverted, not one made earlier in the run (e.g. by a DB migration, which has no undo). + self::$replaced_docker_compose = false; + $installed = self::get_docker_compose_version(); + if ( '' !== $installed && version_compare( $installed, self::DOCKER_COMPOSE_MIN_VERSION, '>=' ) ) { + EE::debug( "docker-compose $installed is recent enough, not replacing it." ); + + return; + } + + $version = self::DOCKER_COMPOSE_INSTALL_VERSION; + $url = self::get_docker_compose_download_url( $version, php_uname( 's' ), php_uname( 'm' ) ); + if ( '' === $url ) { + EE::warning( sprintf( 'EasyEngine requires docker-compose %s or newer, and there is no docker-compose %s build for %s %s. Please install it manually.', self::DOCKER_COMPOSE_MIN_VERSION, $version, php_uname( 's' ), php_uname( 'm' ) ) ); + + return; + } + $docker_compose_path = trim( EE::launch( 'command -v docker-compose' )->stdout ); + $docker_compose_path = '' === $docker_compose_path ? '/usr/local/bin/docker-compose' : $docker_compose_path; $docker_compose_backup_path = EE_BACKUP_DIR . '/docker-compose.backup'; $docker_compose_new_path = EE_BACKUP_DIR . '/docker-compose'; - if ( version_compare( '2.27.0', $docker_compose_version, '>' ) ) { + $fs = new Filesystem(); + $fs->mkdir( EE_BACKUP_DIR ); + $fs->remove( $docker_compose_new_path ); + + $failure = ''; + // --max-time and --retry-max-time: a stalled download must not hang the upgrade (curl retries timeouts too). + if ( ! EE::exec( sprintf( 'curl -fsSL --retry 3 --retry-max-time 120 --connect-timeout 10 --max-time 300 -o %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $url ) ) ) ) { + $failure = "the download of $url failed"; + } else { + $checksum = strtok( trim( EE::launch( sprintf( 'curl -fsSL --retry 3 --retry-max-time 60 --connect-timeout 10 --max-time 60 %s', escapeshellarg( "$url.sha256" ) ) )->stdout ), " \t" ); + $actual = is_file( $docker_compose_new_path ) ? hash_file( 'sha256', $docker_compose_new_path ) : ''; + if ( ! preg_match( '/^[0-9a-f]{64}$/', (string) $checksum ) || ! hash_equals( (string) $checksum, (string) $actual ) ) { + $failure = "the checksum of $url doesn't match"; + } else { + $fs->chmod( $docker_compose_new_path, 0755 ); + $new_version = self::normalize_version( EE::launch( escapeshellarg( $docker_compose_new_path ) . ' version --short' )->stdout ); + if ( ! version_compare( $new_version, self::DOCKER_COMPOSE_MIN_VERSION, '>=' ) ) { + $failure = 'the downloaded binary does not run'; + } + } + } - $fs = new Filesystem(); - if ( ! $fs->exists( EE_BACKUP_DIR ) ) { - $fs->mkdir( EE_BACKUP_DIR ); + if ( '' === $failure ) { + // A backup from an earlier run must never be restored over this install. + $fs->remove( $docker_compose_backup_path ); + if ( $fs->exists( $docker_compose_path ) ) { + $fs->copy( $docker_compose_path, $docker_compose_backup_path, true ); } - $fs->copy( $docker_compose_path, $docker_compose_backup_path ); + if ( EE::exec( sprintf( 'mv %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $docker_compose_path ) ) ) ) { + self::$replaced_docker_compose = true; + EE::debug( "Installed docker-compose $version at $docker_compose_path" ); - EE::exec( "curl -L https://github.com/docker/compose/releases/download/v2.27.0/docker-compose-$(uname -s)-$(uname -m) -o $docker_compose_new_path && chmod +x $docker_compose_new_path" ); - EE::exec( "mv $docker_compose_new_path $docker_compose_path" ); + return; + } + $failure = "it could not be moved to $docker_compose_path"; } + + $fs->remove( $docker_compose_new_path ); + EE::warning( sprintf( 'Could not install docker-compose %s (%s). EasyEngine requires docker-compose %s or newer, found %s.', $version, $failure, self::DOCKER_COMPOSE_MIN_VERSION, '' === $installed ? 'none' : $installed ) ); } /** - * Revert docker-compose to previous version. + * Revert docker-compose to the binary the last update_docker_compose() call replaced, if it did. + * An older backup from an earlier run is never restored. */ public static function revert_docker_compose() { - $docker_compose_path = EE::launch( 'command -v docker-compose' )->stdout; - $docker_compose_path = trim( $docker_compose_path ); + if ( ! self::$replaced_docker_compose ) { + return; + } + + $docker_compose_path = trim( EE::launch( 'command -v docker-compose' )->stdout ); $docker_compose_backup_path = EE_BACKUP_DIR . '/docker-compose.backup'; $fs = new Filesystem(); - $fs->copy( $docker_compose_backup_path, $docker_compose_path ); + if ( '' !== $docker_compose_path && $fs->exists( $docker_compose_backup_path ) ) { + $fs->copy( $docker_compose_backup_path, $docker_compose_path, true ); + } + self::$replaced_docker_compose = false; + } + + /** + * @return string Installed docker-compose version as x.y.z, or '' if there is none. + */ + public static function get_docker_compose_version() { + return self::normalize_version( EE::launch( 'docker-compose version --short' )->stdout ); + } + + /** + * Release asset URL of a docker-compose v2 build. + * + * @param string $version Version without `v`. + * @param string $os `uname -s`, e.g. Linux or Darwin. + * @param string $arch `uname -m`. + * + * @return string '' when there is no build for this platform. + */ + public static function get_docker_compose_download_url( $version, $os, $arch ) { + $os = strtolower( $os ); + $arches = [ + 'x86_64' => 'x86_64', + 'amd64' => 'x86_64', + 'aarch64' => 'aarch64', + 'arm64' => 'aarch64', + 'armv7l' => 'armv7', + 'armv6l' => 'armv6', + 'ppc64le' => 'ppc64le', + 's390x' => 's390x', + 'riscv64' => 'riscv64', + ]; + $arch = $arches[ strtolower( $arch ) ] ?? ''; + + if ( '' === $arch || ! ( 'linux' === $os || ( 'darwin' === $os && in_array( $arch, [ 'x86_64', 'aarch64' ], true ) ) ) ) { + return ''; + } + + return sprintf( 'https://github.com/docker/compose/releases/download/v%s/docker-compose-%s-%s', $version, $os, $arch ); + } + + /** + * @param string $output Output of `docker-compose version --short`, e.g. `2.27.0`, `v2.40.3` or `5.5.1-desktop.1`. + * + * @return string x.y.z, or ''. + */ + private static function normalize_version( $output ) { + return preg_match( '/(\d+\.\d+\.\d+)/', (string) $output, $matches ) ? $matches[1] : ''; } /** @@ -221,7 +383,7 @@ public static function get_current_docker_images_versions() { $dockerImages = EE::launch( 'docker ps --format "{{.Image}}" | grep "^easyengine/" | sort -u' )->stdout; $dockerImages = explode( "\n", trim( $dockerImages ) ); - $dockerImages = array_reduce( $dockerImages, function ( $result, $image ) { + $dockerImages = array_reduce( array_filter( $dockerImages ), function ( $result, $image ) { [ $imageName, $tag ] = explode( ':', $image, 2 ) + [ 1 => null ]; $result[ $imageName ] = $tag; @@ -229,7 +391,8 @@ public static function get_current_docker_images_versions() { return $result; }, [] ); - $mergedImages = $dockerImages + $dbImages; + self::$running_images = array_keys( $dockerImages ); + $mergedImages = $dockerImages + $dbImages; $mergedImages = array_filter( $mergedImages ); return $mergedImages; @@ -238,34 +401,44 @@ public static function get_current_docker_images_versions() { /** * Migrates global containers. These are container which are not created per site (i.e. ee-cron-scheduler). * - * @param $updated_images array of updated images. + * @param array $by_state Updated global images by state, from GlobalContainers::get_updated_global_images_by_state(). */ - private static function migrate_global_containers( $updated_images ) { + private static function migrate_global_containers( $by_state ) { + + $global_compose_file_path = EE_ROOT_DIR . '/services/docker-compose.yml'; - $updated_global_images = GlobalContainers::get_updated_global_images( $updated_images ); - if ( empty( $updated_global_images ) ) { + // A stopped service keeps its container (Docker restarts it on boot) and the old image the cleanup then keeps, so + // its old tag stays recorded; the regenerated compose file has the new one for `ee service enable`. A missing + // service is created from that file later. Without a file, one is generated with the new tags when needed. + $updated_global_images = $by_state['running']; + if ( empty( $updated_global_images ) && ( ! is_file( $global_compose_file_path ) || empty( $by_state['stopped'] ) && empty( $by_state['missing'] ) ) ) { return; } - $global_compose_file_path = EE_ROOT_DIR . '/services/docker-compose.yml'; $global_compose_file_backup_path = EE_BACKUP_DIR . '/services/docker-compose.yml.backup'; + $all_global_images = GlobalContainers::get_all_global_images_with_service_name(); + GlobalContainers::reset_recreated(); + + if ( in_array( 'easyengine/nginx-proxy', $updated_global_images, true ) ) { + // Before anything is recreated: a rollback after the DB moved to a new image may not get its datadir back. + self::$rsp->add_step( + 'check-running-proxy-config', + 'EE\Migration\GlobalContainers::check_running_proxy_config', + null, + [ $all_global_images['easyengine/nginx-proxy'] ], + null + ); + } self::$rsp->add_step( 'backup-global-docker-compose-file', - 'EE\Migration\SiteContainers::backup_restore', + 'EE\Migration\GlobalContainers::backup_compose_file', 'EE\Migration\GlobalContainers::revert_global_containers', [ $global_compose_file_path, $global_compose_file_backup_path ], [ $global_compose_file_backup_path, $global_compose_file_path, $updated_global_images ] ); - self::$rsp->add_step( - 'stop-global-containers', - 'EE\Migration\GlobalContainers::down_global_containers', - null, - [ $updated_global_images ], - null - ); - + // The services keep running until each is recreated in turn: no step stops them all at once. self::$rsp->add_step( 'generate-global-docker-compose-file', 'EE\Service\Utils\generate_global_docker_compose_yml', @@ -274,21 +447,54 @@ private static function migrate_global_containers( $updated_images ) { null ); - $all_global_images = GlobalContainers::get_all_global_images_with_service_name(); - foreach ( $updated_global_images as $image_name ) { + // No down steps: the backup step's down restores the old compose file and recreates these services from it. + foreach ( GlobalContainers::sort_for_upgrade( $updated_global_images ) as $image_name ) { $global_container_name = $all_global_images[ $image_name ]; - $global_service_name = ltrim( $global_container_name, 'services_' ); - $global_service_name = rtrim( $global_service_name, '_1' ); self::$rsp->add_step( "upgrade-$global_container_name-container", - "EE\Migration\GlobalContainers::global_service_up", - "EE\Migration\GlobalContainers::global_service_down", - [ $global_service_name ], - [ $global_service_name ] + 'EE\Migration\GlobalContainers::upgrade_global_service', + null, + [ GlobalContainers::get_service_name( $global_container_name ), $global_container_name ], + null ); } } + /** + * @param string $compose_file A docker-compose.yml. + * @param array $images Images to check. + * @param array $img_versions This release's tags. + * + * @return bool Whether the file sets one of $images to a tag other than this release's. + */ + private static function has_old_tags( $compose_file, $images, $img_versions ) { + foreach ( SiteContainers::get_compose_file_images( $compose_file ) as $image => $tag ) { + if ( in_array( $image, $images, true ) && isset( $img_versions[ $image ] ) && $img_versions[ $image ] !== $tag ) { + return true; + } + } + + return false; + } + + /** + * @param array $img_versions This release's tags. + * + * @return array Disabled sites whose docker-compose.yml sets an EasyEngine image to a tag other than this release's. + */ + private static function get_disabled_sites_to_update( $img_versions ) { + $sites = ( new \EE_DB() )->table( 'sites' )->all(); + + return array_values( + array_filter( + $sites, + function ( $site ) use ( $img_versions ) { + return ! $site['site_enabled'] && self::has_old_tags( $site['site_fs_path'] . '/docker-compose.yml', array_keys( $img_versions ), $img_versions ); + } + ) + ); + } + /** * Migrate site specific container. * @@ -298,15 +504,32 @@ private static function migrate_global_containers( $updated_images ) { */ public static function migrate_site_containers( $updated_images ) { - $db = new \EE_DB(); - $sites = ( $db->table( 'sites' )->all() ); + $db = new \EE_DB(); + $sites = ( $db->table( 'sites' )->all() ); + $img_versions = EE\Utils\get_image_versions(); foreach ( $sites as $site ) { $docker_yml = $site['site_fs_path'] . '/docker-compose.yml'; $docker_yml_backup = EE_BACKUP_DIR . '/' . $site['site_url'] . '/docker-compose.yml.backup'; - if ( ! SiteContainers::is_site_service_image_changed( $updated_images, $site ) ) { + // A disabled site has no containers, only its compose file, which nothing else would move to the new images. + // It is compared with this release's tags, not with this run's changed images: those can miss an image an + // interrupted run already moved for the enabled sites, or an earlier upgrade recorded without updating it. + if ( ! $site['site_enabled'] ) { + if ( self::has_old_tags( $docker_yml, array_keys( $img_versions ), $img_versions ) ) { + self::$rsp->add_step( + sprintf( 'update-disabled-%s-docker-compose', $site['site_url'] ), + 'EE\Migration\SiteContainers::update_disabled_site_compose_file', + 'EE\Migration\SiteContainers::restore_disabled_site_compose_file', + [ $site, $docker_yml_backup ], + [ $site ] + ); + } + continue; + } + + if ( ! $updated_images || ! SiteContainers::is_site_service_image_changed( $updated_images, $site ) ) { continue; } diff --git a/php/EE/Migration/CustomContainerMigrations.php b/php/EE/Migration/CustomContainerMigrations.php index 1fcb1f8bd..1f9b50981 100644 --- a/php/EE/Migration/CustomContainerMigrations.php +++ b/php/EE/Migration/CustomContainerMigrations.php @@ -34,7 +34,8 @@ public static function execute_migrations() { self::execute_migration_stack( $migrations ); } catch ( \Throwable $e ) { Utils\delem_log( 'ee migration ended abruptly' ); - exit( 1 ); + // Rethrown, not exit(1): the migration step processor reports it and rolls back. + throw $e; } EE::debug( 'Successfully migrated EasyEngine' ); @@ -160,7 +161,7 @@ private static function execute_migration_stack( $migrations ) { if ( ! file_exists( $migration_path ) ) { EE::error( "Unable to find migration file at $migration_path", false ); - throw new Exception(); + throw new \Exception( "Unable to find migration file at $migration_path" ); } require( $migration_path ); diff --git a/php/EE/Migration/Executor.php b/php/EE/Migration/Executor.php index 249e00df9..5749e3cf5 100644 --- a/php/EE/Migration/Executor.php +++ b/php/EE/Migration/Executor.php @@ -30,7 +30,8 @@ public static function execute_migrations() { self::execute_migration_stack( $migrations ); } catch ( \Throwable $e ) { Utils\delem_log( 'ee migration ended abruptly' ); - exit( 1 ); + // Rethrown, not exit(1): the migration step processor reports it and rolls back. + throw $e; } EE::debug( 'Successfully migrated EasyEngine' ); @@ -100,7 +101,7 @@ private static function execute_migration_stack( $migrations ) { if ( ! file_exists( $migration_path ) ) { EE::error( "Unable to find migration file at $migration_path", false ); - throw new Exception(); + throw new \Exception( "Unable to find migration file at $migration_path" ); } require( $migration_path ); diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index ca762bf54..c2f0e525d 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -4,12 +4,401 @@ use EE; use Symfony\Component\Filesystem\Filesystem; +use Mustangostang\Spyc; /** * Upgrade existing global containers to new docker-image */ class GlobalContainers { + /** + * Upgrade order. The proxy goes last so that a failing backend is rolled back before every site loses its proxy, + * and the database just before it, so of the globals only a failing proxy reverts its datadir onto the old image + * (a failure in a later site step still reverts every global). + */ + const UPGRADE_ORDER = [ + 'easyengine/cron', + 'easyengine/newrelic-daemon', + 'easyengine/redis', + 'easyengine/mariadb', + 'easyengine/nginx-proxy', + ]; + + /** Seconds a service may take to become ready (`global-service-ready-timeout` in config.yml overrides it). */ + const READY_TIMEOUT = 60; + + /** MariaDB may run crash recovery or a datadir upgrade first. */ + const DB_READY_TIMEOUT = 300; + + /** Seconds a service must stay up without restarting after its first successful probe. */ + const SETTLE_SECONDS = 2; + + /** Seconds one readiness probe may run inside the container before it counts as failed. */ + const PROBE_TIMEOUT = 10; + + /** Services with a readiness probe. The others (cron, newrelic-daemon) only need to have started. */ + const PROBED_SERVICES = [ 'global-nginx-proxy', 'global-db', 'global-redis' ]; + + /** @var callable|null Test seam: replaces sleep( $seconds ) while waiting for a service. */ + public static $sleeper; + + /** @var string[] Services this run has recreated, the only ones a rollback must recreate again. */ + private static $recreated = []; + + /** + * Sort updated global images into the order their services are upgraded in. + * + * @param array $images Updated global images. + * + * @return array + */ + public static function sort_for_upgrade( $images ) { + return array_values( array_merge( array_diff( $images, self::UPGRADE_ORDER ), array_intersect( self::UPGRADE_ORDER, $images ) ) ); + } + + /** + * Compose service name of a global container, e.g. `global-db` for `services_global-db_1`. + * + * @param string $container Container name. + * + * @return string + */ + public static function get_service_name( $container ) { + return preg_replace( '/^services_(.+)_1$/', '$1', $container ); + } + + /** + * Snapshot the global compose file before it is regenerated. A run that was interrupted after its snapshot left the + * services half upgraded, so its snapshot (marked pending until a run completes) is kept instead of the current file. + * + * @param string $source Global compose file. + * @param string $backup Snapshot path. + */ + public static function backup_compose_file( $source, $backup ) { + $fs = new Filesystem(); + if ( self::has_pending_compose_backup( $backup ) ) { + EE::debug( "Keeping $backup from an upgrade that did not complete" ); + + return; + } + $fs->copy( $source, $backup, true ); + self::pin_running_images( $backup ); + $fs->touch( $backup . '.pending' ); + } + + /** + * Set each global service in a compose snapshot to the image its container was created from. A stopped service an + * earlier upgrade left on its old image already has its new tag in the compose file, and a rollback must bring back + * the image that worked, not that untested one. + * + * @param string $compose_file Snapshot of the global compose file. + */ + private static function pin_running_images( $compose_file ) { + $config = (string) file_get_contents( $compose_file ); + $pinned = $config; + foreach ( self::get_all_global_images_with_service_name() as $image => $container ) { + $current = self::get_container_image( $container ); + if ( 0 === strpos( $current, "$image:" ) ) { + $pinned = preg_replace( '/^(\s*container_name:\s*' . preg_quote( $container, '/' ) . '\s*\n\s*image:\s*)' . preg_quote( $image, '/' ) . ':\S+[ \t]*$/m', '${1}' . $current, $pinned, 1 ); + } + } + if ( $pinned !== $config ) { + file_put_contents( $compose_file, $pinned ); + } + } + + /** + * @param string $backup Snapshot path. + * + * @return bool Whether an upgrade that did not complete left its snapshot. + */ + public static function has_pending_compose_backup( $backup ) { + return ( new Filesystem() )->exists( [ $backup, $backup . '.pending' ] ); + } + + /** + * Mark the compose snapshot as no longer needed by a later run: the upgrade completed or was rolled back. + * + * @param string $backup Snapshot path. + */ + public static function clear_pending_compose_backup( $backup ) { + ( new Filesystem() )->remove( $backup . '.pending' ); + } + + /** + * Start a new image migration with no service recorded as recreated. + */ + public static function reset_recreated() { + self::$recreated = []; + } + + /** + * Refuse to upgrade when the running proxy's config fails `nginx -t`: it lives on volumes, so the recreated proxy and + * the restored one would both fail to start. Runs before any global service is recreated. + * + * @param string $container Proxy container name. + * + * @throws \Exception + */ + public static function check_running_proxy_config( $container ) { + if ( 'running' !== self::get_container_state( $container )['status'] ) { + return; + } + $result = self::probe_exec( $container, 'nginx', '-t' ); + if ( self::timed_out( $result ) ) { + throw new \Exception( sprintf( '`nginx -t` in the running %s did not finish within %d s, so the upgrade cannot check its configuration. Check `docker exec %s nginx -t` and run the upgrade again.', $container, self::PROBE_TIMEOUT, $container ) ); + } + if ( $result->return_code ) { + throw new \Exception( "The configuration of the running $container fails `nginx -t`, so a recreated proxy would not start. Fix it (see `docker exec $container nginx -t`) and run the upgrade again." ); + } + } + + /** + * Recreate one global service with the image in the (already regenerated) global compose file, + * then wait until it is ready. Throws on failure, so the image migration rolls back. + * + * @param string $service Compose service, e.g. `global-db`. + * @param string $container Its container name. + * + * @throws \Exception + */ + public static function upgrade_global_service( $service, $container ) { + EE::debug( "Start upgrading $container" ); + + self::$recreated[] = $service; + if ( ! chdir( EE_SERVICE_DIR ) || ! self::compose_up( $service, [ $service ] ) ) { + throw new \Exception( "Unable to recreate $container with its new image. Please check logs for more details." ); + } + + self::wait_until_ready( $service, $container ); + EE::debug( "Complete upgrading $container" ); + } + + /** + * `up -d` global services in the current directory. docker-compose v1 can't recreate a container on Docker 25+ + * (KeyError: 'ContainerConfig', leaving it stopped), so with v1 the services to recreate are stopped and removed first. + * + * @param string $services Space-separated compose services. + * @param string[] $recreate Those whose container must be replaced; v1's `up -d` leaves the others alone. + * + * @return bool + */ + private static function compose_up( $services, $recreate ) { + $compose = \EE_DOCKER::docker_compose_with_custom(); + $up = "$compose up -d $services"; + $version = Containers::get_docker_compose_version(); + if ( $recreate && '' !== $version && version_compare( $version, '2.0.0', '<' ) ) { + $recreate = implode( ' ', $recreate ); + + return EE::exec( "$compose stop $recreate && $compose rm -f $recreate && $up" ); + } + + return EE::exec( $up ); + } + + /** + * Wait until a global container runs, isn't restarting, is healthy if it has a healthcheck, + * and passes its service's readiness probe, then stays so for a moment. Unprobed services only need to have started. + * + * @param string $service Compose service, e.g. `global-db`. + * @param string $container Its container name. + * @param int|null $timeout Seconds; defaults to the service's timeout. + * + * @throws \Exception When it isn't ready in time or its healthcheck reports unhealthy. + */ + public static function wait_until_ready( $service, $container, $timeout = null ) { + if ( null === $timeout ) { + $timeout = self::get_ready_timeout( $service ); + } + $start = microtime( true ); + $slept = 0; + $reason = ''; + + while ( true ) { + $state = self::get_container_state( $container ); + if ( 'unhealthy' === $state['health'] ) { + throw new \Exception( "$container is unhealthy according to its healthcheck." ); + } + + // No readiness signal, and cron restart-loops while it has no jobs (CRON-1): having started is enough. + if ( ! in_array( $service, self::PROBED_SERVICES, true ) && in_array( $state['status'], [ 'running', 'restarting' ], true ) && in_array( $state['health'], [ 'healthy', 'none' ], true ) ) { + if ( 'running' !== $state['status'] || 'false' !== $state['restarting'] ) { + EE::warning( "$container keeps restarting. Please check `docker logs $container`." ); + } + + return; + } + + $reason = self::not_ready_reason( $service, $container, $state ); + if ( '' === $reason ) { + self::pause( self::SETTLE_SECONDS ); + $slept += self::SETTLE_SECONDS; + $settled = self::get_container_state( $container ); + $reason = $settled['restart_count'] !== $state['restart_count'] ? 'it restarted' : self::not_ready_reason( $service, $container, $settled ); + if ( '' === $reason ) { + EE::debug( sprintf( '%s is ready after %.1f s', $container, microtime( true ) - $start ) ); + + return; + } + } + + // Time slept counts too, so a test sleeper that returns at once still ends the wait. + if ( max( microtime( true ) - $start, $slept ) >= $timeout ) { + throw new \Exception( sprintf( '%s is not ready after %d s (%s). Please check `docker logs %s`.', $container, $timeout, $reason, $container ) ); + } + self::pause( 1 ); + ++$slept; + } + } + + /** + * @param string $service Compose service. + * + * @return int Seconds. + */ + private static function get_ready_timeout( $service ) { + $configured = \EE\Utils\get_config_value( 'global-service-ready-timeout' ); + if ( is_numeric( $configured ) && $configured > 0 ) { + return (int) ceil( $configured ); + } + + return 'global-db' === $service ? self::DB_READY_TIMEOUT : self::READY_TIMEOUT; + } + + /** + * @param string $container Container name. + * + * @return array status, restarting, restart_count and health (`none` without a healthcheck); empty strings if it doesn't exist. + */ + private static function get_container_state( $container ) { + $format = '{{.State.Status}} {{.State.Restarting}} {{.RestartCount}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}'; + $result = EE::launch( sprintf( "docker inspect --format '%s' %s", $format, escapeshellarg( $container ) ) ); + $parts = 0 === $result->return_code ? preg_split( '/\s+/', trim( $result->stdout ) ) : []; + + return array_combine( [ 'status', 'restarting', 'restart_count', 'health' ], array_pad( array_slice( $parts, 0, 4 ), 4, '' ) ); + } + + /** + * @param string $container Container name. + * + * @return string The image it was created from, e.g. `easyengine/redis:v4.13.1`, or ''. + */ + private static function get_container_image( $container ) { + $result = EE::launch( sprintf( "docker inspect --format '{{.Config.Image}}' %s", escapeshellarg( $container ) ) ); + + return 0 === $result->return_code ? trim( $result->stdout ) : ''; + } + + /** + * @param string $service Compose service. + * @param string $container Container name. + * @param array $state From get_container_state(). + * + * @return string Why it isn't ready, or '' when it is. + */ + private static function not_ready_reason( $service, $container, $state ) { + if ( '' === $state['status'] ) { + return 'no such container'; + } + if ( 'running' !== $state['status'] || 'false' !== $state['restarting'] ) { + return "status {$state['status']}"; + } + if ( ! in_array( $state['health'], [ 'healthy', 'none' ], true ) ) { + return "health {$state['health']}"; + } + + return self::probe( $service, $container ); + } + + /** + * Service-specific readiness probe. + * + * @param string $service Compose service. + * @param string $container Container name. + * + * @return string Why it failed, or '' when it passed. + */ + private static function probe( $service, $container ) { + switch ( $service ) { + case 'global-nginx-proxy': + $result = self::probe_exec( $container, 'nginx', '-t' ); + if ( $result->return_code ) { + return self::probe_failure( 'nginx -t', $result ); + } + // Any HTTP status proves nginx answers through the published port; 443 is the same process. + $port = (string) \EE\Utils\get_config_value( 'proxy_80_port', '80' ); + $host = false === strpos( $port, ':' ) ? '127.0.0.1' : substr( $port, 0, strrpos( $port, ':' ) ); + $port = false === strpos( $port, ':' ) ? $port : substr( $port, strrpos( $port, ':' ) + 1 ); + // --noproxy: an http_proxy exported for pulls must not answer for the local proxy. + $status = trim( EE::launch( sprintf( "curl -s --noproxy '*' -o /dev/null -w '%%{http_code}' --max-time 5 %s", escapeshellarg( "http://$host:$port/" ) ) )->stdout ); + + return preg_match( '/^[1-5]\d\d$/', $status ) ? '' : "no HTTP response on port $port"; + + case 'global-db': + // Over TCP: the entrypoint's temporary init/upgrade server listens on the socket only. Access denied still means alive. + $ping = 'for c in mariadb-admin mysqladmin; do command -v $c >/dev/null 2>&1 && exec $c ping -h 127.0.0.1 --protocol=tcp --connect-timeout=2 --silent; done; exit 1'; + + $result = self::probe_exec( $container, 'sh', '-c', $ping ); + + return $result->return_code ? self::probe_failure( 'mysqladmin ping', $result ) : ''; + + case 'global-redis': + $result = self::probe_exec( $container, 'redis-cli', 'ping' ); + $pong = trim( $result->stdout ); + + // NOAUTH (a requirepass in redis.conf) still means redis answers. + return 'PONG' === $pong || 0 === strpos( $pong, 'NOAUTH' ) ? '' : self::probe_failure( 'redis-cli ping', $result ); + } + + return ''; + } + + /** + * Run a probe command in a container under `timeout`, so a service that accepts connections but never answers + * can't hang the upgrade. Where `timeout` doesn't work (busybox before 1.30 wants `-t`), the command runs unbounded. + * + * @param string $container Container name. + * @param string ...$args Command and its arguments. + * + * @return \EE\ProcessRun + */ + private static function probe_exec( $container, ...$args ) { + $bounded = sprintf( 'if timeout 1 true >/dev/null 2>&1; then exec timeout %d "$@"; fi; exec "$@"', self::PROBE_TIMEOUT ); + + return EE::launch( 'docker exec ' . escapeshellarg( $container ) . ' sh -c ' . escapeshellarg( $bounded ) . ' probe ' . implode( ' ', array_map( 'escapeshellarg', $args ) ) ); + } + + /** + * @param \EE\ProcessRun $result From probe_exec(). + * + * @return bool Whether `timeout` killed the probe: 124 from coreutils, 143 (SIGTERM) from busybox. + */ + private static function timed_out( $result ) { + return in_array( (int) $result->return_code, [ 124, 143 ], true ); + } + + /** + * @param string $probe Probe name. + * @param \EE\ProcessRun $result From probe_exec(). + * + * @return string Why the probe failed. + */ + private static function probe_failure( $probe, $result ) { + return self::timed_out( $result ) ? sprintf( '%s timed out after %d s', $probe, self::PROBE_TIMEOUT ) : "$probe failed"; + } + + /** + * @param int|float $seconds Seconds to sleep. + */ + private static function pause( $seconds ) { + if ( self::$sleeper ) { + call_user_func( self::$sleeper, $seconds ); + + return; + } + usleep( (int) ( $seconds * 1000000 ) ); + } + /** * Get global containers which has new image update. * @@ -18,16 +407,44 @@ class GlobalContainers { * @return array */ public static function get_updated_global_images( $updated_images ) { + return self::get_updated_global_images_by_state( $updated_images )['running']; + } - $global_images = self::get_all_global_images_with_service_name(); - $running_global_services = []; - foreach ( $global_images as $image => $container_name ) { - if ( 'running' === \EE_DOCKER::container_status( $container_name ) ) { - $running_global_services[] = $image; + /** + * Updated global images by the state of their service's container. + * + * @param array $updated_images Updated docker images. + * @param array|null $img_versions This release's tags; defaults to img-versions.json. + * + * @return array `running` (running or restart-looping), `stopped` (any other existing container, still on an old tag), + * `current` (stopped, but already on the new tag, e.g. recreated by an interrupted run) and `missing` images. + */ + public static function get_updated_global_images_by_state( $updated_images, $img_versions = null ) { + $img_versions = null === $img_versions ? EE\Utils\get_image_versions() : $img_versions; + $by_state = [ + 'running' => [], + 'stopped' => [], + 'current' => [], + 'missing' => [], + ]; + foreach ( self::get_all_global_images_with_service_name() as $image => $container_name ) { + if ( ! in_array( $image, $updated_images, true ) ) { + continue; + } + $status = self::get_container_state( $container_name )['status']; + // A restart-looping service is upgraded too (State.Running is false between its restarts). + if ( in_array( $status, [ 'running', 'restarting' ], true ) ) { + $by_state['running'][] = $image; + } elseif ( '' === $status ) { + $by_state['missing'][] = $image; + } elseif ( isset( $img_versions[ $image ] ) && self::get_container_image( $container_name ) === "$image:{$img_versions[ $image ]}" ) { + $by_state['current'][] = $image; + } else { + $by_state['stopped'][] = $image; } } - return array_intersect( $running_global_services, $updated_images ); + return $by_state; } /** @@ -40,27 +457,63 @@ public static function get_updated_global_images( $updated_images ) { */ public static function revert_global_containers( $source_path, $dest_path, $updated_images ) { - $services_to_regenerate = ''; - $all_global_images = self::get_all_global_images_with_service_name(); + $all_global_images = self::get_all_global_images_with_service_name(); + $services = []; foreach ( $updated_images as $image_name ) { - $global_container_name = $all_global_images[ $image_name ]; - $services_to_regenerate .= ltrim( rtrim( $global_container_name, '_1' ), 'services_' ) . ' '; - } - if ( empty( trim( $services_to_regenerate ) ) ) { - return; + $services[] = self::get_service_name( $all_global_images[ $image_name ] ); } + $recreated = self::$recreated; + self::$recreated = []; EE::debug( 'Start restoring global docker-compose.yml file from backup' ); $fs = new Filesystem(); + // Also when no running service changed: the regenerated file alone carries the new tags of the others. $fs->copy( $source_path, $dest_path, true ); + // Services an earlier, interrupted attempt moved to a new image aren't among this run's updates: put them back too. + $moved = array_diff( self::get_services_off_compose_file( $dest_path ), $services ); + $services = array_merge( $services, $moved ); + $recreate = array_values( array_merge( array_intersect( $services, $recreated ), $moved ) ); + chdir( EE_ROOT_DIR . '/services' ); - if ( ! EE::exec( \EE_DOCKER::docker_compose_with_custom() . ' up -d ' . $services_to_regenerate ) ) { + if ( $services && ! self::compose_up( implode( ' ', $services ), $recreate ) ) { throw new \Exception( 'Unable to downgrade global containers. Please check logs for more details.' ); } + $containers = array_combine( array_map( [ self::class, 'get_service_name' ], $all_global_images ), $all_global_images ); + foreach ( $services as $service ) { + try { + self::wait_until_ready( $service, $containers[ $service ] ); + } catch ( \Exception $e ) { + EE::warning( 'After restoring the previous image: ' . $e->getMessage() ); + } + } + self::clear_pending_compose_backup( $source_path ); EE::debug( 'Complete restoring global docker-compose.yml file from backup' ); } + /** + * @param string $compose_file Global compose file. + * + * @return string[] Global services that run an image other than the one the compose file sets for them. + */ + private static function get_services_off_compose_file( $compose_file ) { + // Spyc, not symfony/yaml: the phar doesn't ship symfony/yaml. + $config = is_file( $compose_file ) ? Spyc::YAMLLoad( $compose_file ) : []; + $services = []; + foreach ( self::get_all_global_images_with_service_name() as $container ) { + $service = self::get_service_name( $container ); + $image = is_array( $config ) && isset( $config['services'][ $service ]['image'] ) ? $config['services'][ $service ]['image'] : ''; + $running = EE::launch( sprintf( "docker inspect --format '{{.State.Status}} {{.Config.Image}}' %s", escapeshellarg( $container ) ) ); + $parts = 0 === $running->return_code ? preg_split( '/\s+/', trim( $running->stdout ) ) : []; + // A restart-looping container counts: it is the failed new image to put back. One never created doesn't. + if ( '' !== $image && 2 === count( $parts ) && in_array( $parts[0], [ 'running', 'restarting' ], true ) && $parts[1] !== $image ) { + $services[] = $service; + } + } + + return $services; + } + /** * Stop global container and remove them. * diff --git a/php/EE/Migration/SiteContainers.php b/php/EE/Migration/SiteContainers.php index e3a9d3bea..203dff56b 100644 --- a/php/EE/Migration/SiteContainers.php +++ b/php/EE/Migration/SiteContainers.php @@ -4,6 +4,7 @@ use EE; use Symfony\Component\Filesystem\Filesystem; +use Mustangostang\Spyc; /** * Migrate site specific containers to new images. @@ -93,6 +94,83 @@ public static function is_site_service_image_changed( $updated_images, $site_inf return false; } + /** @var string[] Disabled sites whose compose file this run backed up, the only ones a rollback restores. */ + private static $backed_up = []; + + /** + * EasyEngine images a docker-compose.yml sets, as image => tag. + * + * @param string $compose_file Path of the file. + * + * @return array + */ + public static function get_compose_file_images( $compose_file ) { + try { + // Spyc, not symfony/yaml: the phar doesn't ship symfony/yaml. + $config = is_file( $compose_file ) ? Spyc::YAMLLoad( $compose_file ) : []; + } catch ( \Exception $e ) { + $config = []; + } + $images = []; + if ( ! is_array( $config ) || empty( $config['services'] ) || ! is_array( $config['services'] ) ) { + return $images; + } + foreach ( $config['services'] as $service ) { + if ( is_array( $service ) && ! empty( $service['image'] ) && is_string( $service['image'] ) && 0 === strpos( $service['image'], 'easyengine/' ) ) { + list( $image, $tag ) = explode( ':', $service['image'], 2 ) + [ 1 => '' ]; + $images[ $image ] = $tag; + } + } + + return $images; + } + + /** + * Back up and regenerate a disabled site's docker-compose.yml with this release's images, without starting anything, + * so that `ee site enable` starts it on them. A failure only warns and keeps the old file: a disabled site must not + * block or roll back the upgrade of the others. + * + * @param array $site_info Site row. + * @param string $backup_path Where to back up its docker-compose.yml. + */ + public static function update_disabled_site_compose_file( $site_info, $backup_path ) { + $compose_file = $site_info['site_fs_path'] . '/docker-compose.yml'; + $fs = new Filesystem(); + unset( self::$backed_up[ $site_info['site_url'] ] ); + try { + $fs->copy( $compose_file, $backup_path, true ); + self::$backed_up[ $site_info['site_url'] ] = $backup_path; + self::generate_site_docker_compose_file( $site_info, self::get_site_object( $site_info['site_type'] ) ); + // A missing img-versions.json entry only raises a PHP warning and writes an image without a tag. + if ( preg_match( '/^\s*image:\s*\S+:\s*$/m', (string) file_get_contents( $compose_file ) ) ) { + throw new \Exception( 'an image has no tag' ); + } + } catch ( \Throwable $e ) { + try { + if ( isset( self::$backed_up[ $site_info['site_url'] ] ) ) { + $fs->copy( $backup_path, $compose_file, true ); + } + } catch ( \Exception $restore_error ) { + EE::warning( sprintf( 'Could not restore %s from %s: %s', $compose_file, $backup_path, $restore_error->getMessage() ) ); + } + unset( self::$backed_up[ $site_info['site_url'] ] ); + $reason = '' === $e->getMessage() ? 'see the error above' : $e->getMessage(); + EE::warning( sprintf( 'Could not update docker-compose.yml of the disabled site %1$s to the new images (%2$s). Run `ee site enable %1$s --refresh` to enable it on them.', $site_info['site_url'], $reason ) ); + } + } + + /** + * Rollback of update_disabled_site_compose_file(): restore the file it backed up in this run, if any. + * + * @param array $site_info Site row. + */ + public static function restore_disabled_site_compose_file( $site_info ) { + if ( isset( self::$backed_up[ $site_info['site_url'] ] ) ) { + ( new Filesystem() )->copy( self::$backed_up[ $site_info['site_url'] ], $site_info['site_fs_path'] . '/docker-compose.yml', true ); + unset( self::$backed_up[ $site_info['site_url'] ] ); + } + } + /** * Generate docker-compose.yml for specific site. * diff --git a/php/EE/RevertableStepProcessor.php b/php/EE/RevertableStepProcessor.php index fde925787..8e806b57c 100644 --- a/php/EE/RevertableStepProcessor.php +++ b/php/EE/RevertableStepProcessor.php @@ -18,6 +18,16 @@ class RevertableStepProcessor { /** @var int Keeps track of steps executed. All items in $steps till this index have been executed */ private $execution_index = 0; + /** @var bool Whether an `EE::error()`/`EE::halt()` inside a step throws, so the rollback still runs, instead of exiting. */ + private $capture_exit; + + /** + * @param bool $capture_exit Turn exits inside up and down steps into failures of that step. + */ + public function __construct( $capture_exit = false ) { + $this->capture_exit = (bool) $capture_exit; + } + /** * Adds a new step. * @@ -53,11 +63,11 @@ public function execute( $show_error = true ) { $context = $step['context']; try { EE::debug( "Executing $context... " ); - call_user_func_array( $step['up'], $step['up_params'] ?? [] ); + $this->call( $step['up'], $step['up_params'] ); $this->execution_index ++; EE::debug( "Executed $context." ); - } catch ( \Exception $e ) { - $exception_message = $e->getMessage(); + } catch ( \Throwable $e ) { + $exception_message = self::describe( $e ); $callable = EE\Utils\get_callable_name( $step['up'] ); if ( $show_error ) { EE::error( "Error while executing $context. $callable: $exception_message", false ); @@ -86,15 +96,50 @@ public function rollback() { try { EE::debug( "Reverting $context... " ); if ( null !== $step['down'] ) { - call_user_func_array( $step['down'], $step['down_params'] ?? [] ); + $this->call( $step['down'], $step['down_params'] ); } EE::debug( "Reverted $context" ); - } catch ( \Exception $e ) { - $exception_message = $e->getMessage(); + } catch ( \Throwable $e ) { + $exception_message = self::describe( $e ); EE::debug( "Encountered error while reverting $context: $exception_message. If possible, do it manually" ); } finally { $this->execution_index --; } } } + + /** + * Calls a step callable, capturing exits if this processor does. + * + * @param callable $callable Step callable. + * @param array|null $params Its parameters. + */ + private function call( $callable, $params ) { + if ( ! $this->capture_exit ) { + call_user_func_array( $callable, $params ?? [] ); + + return; + } + + $previous = EE::set_capture_exit( true ); + try { + call_user_func_array( $callable, $params ?? [] ); + } finally { + EE::set_capture_exit( $previous ); + } + } + + /** + * @param \Throwable $e Failure of a step. + * + * @return string + */ + private static function describe( \Throwable $e ) { + // EE::error() has already printed its message. + if ( $e instanceof ExitException ) { + return sprintf( 'exited with code %d', $e->getCode() ); + } + + return $e->getMessage(); + } } diff --git a/php/EE/Runner.php b/php/EE/Runner.php index 2b96b1c73..03f73a65d 100644 --- a/php/EE/Runner.php +++ b/php/EE/Runner.php @@ -167,7 +167,8 @@ private function migrate() { EE::error( 'EasyEngine update requires minimum 5GB disk space to run. Please free up some space and try again.' ); } - $rsp = new \EE\RevertableStepProcessor(); + // Capture EE::error() exits inside the steps so a failure still reverts what ran. + $rsp = new \EE\RevertableStepProcessor( true ); $rsp->add_step( 'ee-db-migrations', 'EE\Migration\Executor::execute_migrations' ); $rsp->add_step( 'ee-custom-container-migrations', 'EE\Migration\CustomContainerMigrations::execute_migrations', 'EE\Migration\CustomContainerMigrations::revert_executed_migrations' ); diff --git a/php/class-ee-db.php b/php/class-ee-db.php index 3d361198a..4c67656b1 100644 --- a/php/class-ee-db.php +++ b/php/class-ee-db.php @@ -32,17 +32,23 @@ public function __construct() { */ private static function init_db() { - if ( ! ( file_exists( DB ) ) ) { - self::create_required_tables(); - - return; - } + // PDO creates a missing file; an empty one (e.g. left by the sqlite3 CLI or an interrupted first run) has no schema either. + clearstatcache( true, DB ); + $is_new = ! file_exists( DB ) || 0 === filesize( DB ); try { self::$pdo = new PDO( 'sqlite:' . DB ); self::$pdo->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); + $tables = $is_new ? [] : self::$pdo->query( "SELECT name FROM sqlite_master WHERE type = 'table'" )->fetchAll( PDO::FETCH_COLUMN ); } catch ( PDOException $exception ) { - EE::error( 'Unable to initialize connection to EE sqlite database: ' . $exception->getMessage() ); + EE::error( sprintf( 'Unable to open the EasyEngine database %s: %s. If it is damaged, restore it from a backup, or move it away to start with an empty database.', DB, $exception->getMessage() ) ); + } + + if ( empty( $tables ) ) { + self::create_required_tables(); + } elseif ( array_diff( [ 'migrations', 'options' ], $tables ) ) { + // Tables but no base schema: not ours to initialise over. + EE::error( sprintf( 'The EasyEngine database %s lacks the %s table(s). Restore it from a backup, or move it away to start with an empty database.', DB, implode( ', ', array_diff( [ 'migrations', 'options' ], $tables ) ) ) ); } } @@ -50,30 +56,32 @@ private static function init_db() { * Sqlite database creation. */ private static function create_required_tables() { - try { - self::$pdo = new PDO( 'sqlite:' . DB ); - self::$pdo->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); - } catch ( PDOException $exception ) { - EE::error( $exception->getMessage() ); - } - $query = 'CREATE TABLE migrations ( + $queries = [ + 'CREATE TABLE IF NOT EXISTS migrations ( id INTEGER, migration VARCHAR, type VARCHAR, timestamp DATETIME, PRIMARY KEY (id) - );'; - - $query .= 'CREATE TABLE options ( + );', + 'CREATE TABLE IF NOT EXISTS options ( key VARCHAR NOT NULL, value VARCHAR NOT NULL, PRIMARY KEY (key) - );'; + );', + ]; try { - self::$pdo->exec( $query ); + self::$pdo->beginTransaction(); + foreach ( $queries as $query ) { + self::$pdo->exec( $query ); + } + self::$pdo->commit(); } catch ( PDOException $exception ) { + if ( self::$pdo->inTransaction() ) { + self::$pdo->rollBack(); + } EE::error( 'Encountered Error while creating table: ' . $exception->getMessage() ); } } diff --git a/php/class-ee.php b/php/class-ee.php index 72b455764..6156f9dde 100644 --- a/php/class-ee.php +++ b/php/class-ee.php @@ -790,6 +790,20 @@ public static function halt( $return_code ) { exit( $return_code ); } + /** + * Make `EE::error()` and `EE::halt()` throw `EE\ExitException` instead of exiting. + * + * @param bool $capture Whether to capture exits. + * + * @return bool The previous setting, to restore afterwards. + */ + public static function set_capture_exit( $capture ) { + $previous = self::$capture_exit; + self::$capture_exit = (bool) $capture; + + return $previous; + } + /** * Display a multi-line error message in a red box. Doesn't exit script. * @@ -1224,7 +1238,8 @@ public static function runcommand( $command, $options = array() ) { if ( -1 == $return_code ) { self::warning( 'Spawned process returned exit code -1, which could be caused by a custom compiled version of PHP that uses the --enable-sigchild option.' ); } elseif ( $return_code && $exit_error ) { - exit( $return_code ); + // halt(), not exit(): a migration step capturing exits must still roll back. + self::halt( $return_code ); } if ( true === $return || 'stdout' === $return ) { $retval = trim( $stdout ); @@ -1248,6 +1263,8 @@ public static function runcommand( $command, $options = array() ) { self::$logger = new EE\Loggers\Execution; self::$logger->ob_start(); } + $previous_capture_exit = self::$capture_exit; + $exit_exception = null; if ( ! $exit_error ) { self::$capture_exit = true; } @@ -1260,6 +1277,10 @@ public static function runcommand( $command, $options = array() ) { $return_code = 0; } catch ( ExitException $e ) { $return_code = $e->getCode(); + // With exit_error the caller would have exited; a caller that captures exits gets the exit instead. + if ( $exit_error && $return_code ) { + $exit_exception = $e; + } } if ( $return ) { $execution_logger = self::$logger; @@ -1281,8 +1302,10 @@ public static function runcommand( $command, $options = array() ) { ); } } - if ( ! $exit_error ) { - self::$capture_exit = false; + // Restore rather than clear: a caller (e.g. a migration step) may be capturing exits itself. + self::$capture_exit = $previous_capture_exit; + if ( $exit_exception ) { + throw $exit_exception; } } if ( ( true === $return || 'stdout' === $return )