From e25a8babd526b7a1ccc721979fe5fa2f12d66e3d Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 20:50:15 +0530 Subject: [PATCH 01/27] fix(migration): roll back when a migration step exits through EE::error() A helper that calls EE::error() inside a migration step exited the process, so none of the completed steps were reverted, and a PHP Error was fatal for the same reason. DB and container migration failures also called exit(1) directly, bypassing the step processor and its error message. RevertableStepProcessor can now capture exits: with it on, EE::error()/EE::halt() inside an up or down step throw EE\ExitException, which fails that step and runs the rollback. The migration pipeline and the image migration turn it on. Both processors now catch Throwable, the DB and container migration executors rethrow instead of exiting, and EE::runcommand() restores the previous capture setting instead of clearing it. --- php/EE/Migration/Containers.php | 2 +- .../Migration/CustomContainerMigrations.php | 3 +- php/EE/Migration/Executor.php | 3 +- php/EE/RevertableStepProcessor.php | 57 +++++++++++++++++-- php/EE/Runner.php | 3 +- php/class-ee.php | 20 ++++++- 6 files changed, 75 insertions(+), 13 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 8c078e60d..9965db1c1 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -22,7 +22,7 @@ class Containers { public static function start_container_migration() { EE\Utils\delem_log( 'Starting container migration' ); - self::$rsp = new RevertableStepProcessor(); + self::$rsp = new RevertableStepProcessor( true ); $img_versions = EE\Utils\get_image_versions(); $current_versions = self::get_current_docker_images_versions(); diff --git a/php/EE/Migration/CustomContainerMigrations.php b/php/EE/Migration/CustomContainerMigrations.php index 1fcb1f8bd..984ec0e9e 100644 --- a/php/EE/Migration/CustomContainerMigrations.php +++ b/php/EE/Migration/CustomContainerMigrations.php @@ -34,7 +34,8 @@ public static function execute_migrations() { self::execute_migration_stack( $migrations ); } catch ( \Throwable $e ) { Utils\delem_log( 'ee migration ended abruptly' ); - exit( 1 ); + // Rethrown, not exit(1): the migration step processor reports it and rolls back. + throw $e; } EE::debug( 'Successfully migrated EasyEngine' ); diff --git a/php/EE/Migration/Executor.php b/php/EE/Migration/Executor.php index 249e00df9..176158665 100644 --- a/php/EE/Migration/Executor.php +++ b/php/EE/Migration/Executor.php @@ -30,7 +30,8 @@ public static function execute_migrations() { self::execute_migration_stack( $migrations ); } catch ( \Throwable $e ) { Utils\delem_log( 'ee migration ended abruptly' ); - exit( 1 ); + // Rethrown, not exit(1): the migration step processor reports it and rolls back. + throw $e; } EE::debug( 'Successfully migrated EasyEngine' ); diff --git a/php/EE/RevertableStepProcessor.php b/php/EE/RevertableStepProcessor.php index fde925787..8e806b57c 100644 --- a/php/EE/RevertableStepProcessor.php +++ b/php/EE/RevertableStepProcessor.php @@ -18,6 +18,16 @@ class RevertableStepProcessor { /** @var int Keeps track of steps executed. All items in $steps till this index have been executed */ private $execution_index = 0; + /** @var bool Whether an `EE::error()`/`EE::halt()` inside a step throws, so the rollback still runs, instead of exiting. */ + private $capture_exit; + + /** + * @param bool $capture_exit Turn exits inside up and down steps into failures of that step. + */ + public function __construct( $capture_exit = false ) { + $this->capture_exit = (bool) $capture_exit; + } + /** * Adds a new step. * @@ -53,11 +63,11 @@ public function execute( $show_error = true ) { $context = $step['context']; try { EE::debug( "Executing $context... " ); - call_user_func_array( $step['up'], $step['up_params'] ?? [] ); + $this->call( $step['up'], $step['up_params'] ); $this->execution_index ++; EE::debug( "Executed $context." ); - } catch ( \Exception $e ) { - $exception_message = $e->getMessage(); + } catch ( \Throwable $e ) { + $exception_message = self::describe( $e ); $callable = EE\Utils\get_callable_name( $step['up'] ); if ( $show_error ) { EE::error( "Error while executing $context. $callable: $exception_message", false ); @@ -86,15 +96,50 @@ public function rollback() { try { EE::debug( "Reverting $context... " ); if ( null !== $step['down'] ) { - call_user_func_array( $step['down'], $step['down_params'] ?? [] ); + $this->call( $step['down'], $step['down_params'] ); } EE::debug( "Reverted $context" ); - } catch ( \Exception $e ) { - $exception_message = $e->getMessage(); + } catch ( \Throwable $e ) { + $exception_message = self::describe( $e ); EE::debug( "Encountered error while reverting $context: $exception_message. If possible, do it manually" ); } finally { $this->execution_index --; } } } + + /** + * Calls a step callable, capturing exits if this processor does. + * + * @param callable $callable Step callable. + * @param array|null $params Its parameters. + */ + private function call( $callable, $params ) { + if ( ! $this->capture_exit ) { + call_user_func_array( $callable, $params ?? [] ); + + return; + } + + $previous = EE::set_capture_exit( true ); + try { + call_user_func_array( $callable, $params ?? [] ); + } finally { + EE::set_capture_exit( $previous ); + } + } + + /** + * @param \Throwable $e Failure of a step. + * + * @return string + */ + private static function describe( \Throwable $e ) { + // EE::error() has already printed its message. + if ( $e instanceof ExitException ) { + return sprintf( 'exited with code %d', $e->getCode() ); + } + + return $e->getMessage(); + } } diff --git a/php/EE/Runner.php b/php/EE/Runner.php index 2b96b1c73..03f73a65d 100644 --- a/php/EE/Runner.php +++ b/php/EE/Runner.php @@ -167,7 +167,8 @@ private function migrate() { EE::error( 'EasyEngine update requires minimum 5GB disk space to run. Please free up some space and try again.' ); } - $rsp = new \EE\RevertableStepProcessor(); + // Capture EE::error() exits inside the steps so a failure still reverts what ran. + $rsp = new \EE\RevertableStepProcessor( true ); $rsp->add_step( 'ee-db-migrations', 'EE\Migration\Executor::execute_migrations' ); $rsp->add_step( 'ee-custom-container-migrations', 'EE\Migration\CustomContainerMigrations::execute_migrations', 'EE\Migration\CustomContainerMigrations::revert_executed_migrations' ); diff --git a/php/class-ee.php b/php/class-ee.php index 72b455764..390f2c263 100644 --- a/php/class-ee.php +++ b/php/class-ee.php @@ -790,6 +790,20 @@ public static function halt( $return_code ) { exit( $return_code ); } + /** + * Make `EE::error()` and `EE::halt()` throw `EE\ExitException` instead of exiting. + * + * @param bool $capture Whether to capture exits. + * + * @return bool The previous setting, to restore afterwards. + */ + public static function set_capture_exit( $capture ) { + $previous = self::$capture_exit; + self::$capture_exit = (bool) $capture; + + return $previous; + } + /** * Display a multi-line error message in a red box. Doesn't exit script. * @@ -1248,6 +1262,7 @@ public static function runcommand( $command, $options = array() ) { self::$logger = new EE\Loggers\Execution; self::$logger->ob_start(); } + $previous_capture_exit = self::$capture_exit; if ( ! $exit_error ) { self::$capture_exit = true; } @@ -1281,9 +1296,8 @@ public static function runcommand( $command, $options = array() ) { ); } } - if ( ! $exit_error ) { - self::$capture_exit = false; - } + // Restore rather than clear: a caller (e.g. a migration step) may be capturing exits itself. + self::$capture_exit = $previous_capture_exit; } if ( ( true === $return || 'stdout' === $return ) && 'json' === $parse ) { From 67464422b992670bc88c0e76e4a4a81b7fba335f Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 20:53:10 +0530 Subject: [PATCH 02/27] fix(migration): recreate global services one at a time behind a readiness gate The image migration stopped and removed every updated global service in one step, regenerated the compose file and only then brought them back, so all sites were down for the whole window, and an exit in between left the proxy and the DB removed. Nothing checked that a recreated service came back: a failed `up -d` or a crash-looping container counted as success, the new tags were saved and the old images pruned. Now the new compose file is written while the old containers keep running, and each updated service is recreated on its own with `up -d ` in the order cron, newrelic-daemon, redis, db, nginx-proxy. The proxy goes last so a failing backend is rolled back before every site loses its proxy; the DB goes just before it so its datadir is only reverted onto the old image when the proxy fails. Each recreate waits until the container runs without restarting, is healthy if the image has a healthcheck, passes its probe (nginx-proxy: `nginx -t` and an HTTP response on the published port 80; db: `mariadb-admin`/`mysqladmin ping` over TCP, which the entrypoint's temporary server doesn't answer; redis: `redis-cli ping`), and still does so 2 s later without a restart. cron and newrelic-daemon have no readiness signal and only need to have started: cron restart-loops while it has no jobs, which must not roll back every upgrade, so that only warns. The timeout is 60 s (300 s for the DB), overridable with `global-service-ready-timeout` in config.yml. A failure throws, so the existing revert restores the old compose file and recreates the services from it; that revert now checks their readiness too and warns. --- php/EE/Migration/Containers.php | 22 +-- php/EE/Migration/GlobalContainers.php | 231 ++++++++++++++++++++++++++ 2 files changed, 238 insertions(+), 15 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 9965db1c1..5c63523ae 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -258,14 +258,7 @@ private static function migrate_global_containers( $updated_images ) { [ $global_compose_file_backup_path, $global_compose_file_path, $updated_global_images ] ); - self::$rsp->add_step( - 'stop-global-containers', - 'EE\Migration\GlobalContainers::down_global_containers', - null, - [ $updated_global_images ], - null - ); - + // The services keep running until each is recreated in turn: no step stops them all at once. self::$rsp->add_step( 'generate-global-docker-compose-file', 'EE\Service\Utils\generate_global_docker_compose_yml', @@ -274,17 +267,16 @@ private static function migrate_global_containers( $updated_images ) { null ); + // No down steps: the backup step's down restores the old compose file and recreates these services from it. $all_global_images = GlobalContainers::get_all_global_images_with_service_name(); - foreach ( $updated_global_images as $image_name ) { + foreach ( GlobalContainers::sort_for_upgrade( $updated_global_images ) as $image_name ) { $global_container_name = $all_global_images[ $image_name ]; - $global_service_name = ltrim( $global_container_name, 'services_' ); - $global_service_name = rtrim( $global_service_name, '_1' ); self::$rsp->add_step( "upgrade-$global_container_name-container", - "EE\Migration\GlobalContainers::global_service_up", - "EE\Migration\GlobalContainers::global_service_down", - [ $global_service_name ], - [ $global_service_name ] + 'EE\Migration\GlobalContainers::upgrade_global_service', + null, + [ GlobalContainers::get_service_name( $global_container_name ), $global_container_name ], + null ); } } diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index ca762bf54..5c9ba6b97 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -10,6 +10,229 @@ */ class GlobalContainers { + /** + * Upgrade order. The proxy goes last so that a failing backend is rolled back before every site loses its proxy, + * and the database just before it, so its datadir is only reverted onto the old image if the proxy fails. + */ + const UPGRADE_ORDER = [ + 'easyengine/cron', + 'easyengine/newrelic-daemon', + 'easyengine/redis', + 'easyengine/mariadb', + 'easyengine/nginx-proxy', + ]; + + /** Seconds a service may take to become ready (`global-service-ready-timeout` in config.yml overrides it). */ + const READY_TIMEOUT = 60; + + /** MariaDB may run crash recovery or a datadir upgrade first. */ + const DB_READY_TIMEOUT = 300; + + /** Seconds a service must stay up without restarting after its first successful probe. */ + const SETTLE_SECONDS = 2; + + /** Services with a readiness probe. The others (cron, newrelic-daemon) only need to have started. */ + const PROBED_SERVICES = [ 'global-nginx-proxy', 'global-db', 'global-redis' ]; + + /** @var callable|null Test seam: replaces sleep( $seconds ) while waiting for a service. */ + public static $sleeper; + + /** + * Sort updated global images into the order their services are upgraded in. + * + * @param array $images Updated global images. + * + * @return array + */ + public static function sort_for_upgrade( $images ) { + return array_values( array_merge( array_diff( $images, self::UPGRADE_ORDER ), array_intersect( self::UPGRADE_ORDER, $images ) ) ); + } + + /** + * Compose service name of a global container, e.g. `global-db` for `services_global-db_1`. + * + * @param string $container Container name. + * + * @return string + */ + public static function get_service_name( $container ) { + return preg_replace( '/^services_(.+)_1$/', '$1', $container ); + } + + /** + * Recreate one global service with the image in the (already regenerated) global compose file, + * then wait until it is ready. Throws on failure, so the image migration rolls back. + * + * @param string $service Compose service, e.g. `global-db`. + * @param string $container Its container name. + * + * @throws \Exception + */ + public static function upgrade_global_service( $service, $container ) { + EE::debug( "Start upgrading $container" ); + + if ( ! chdir( EE_SERVICE_DIR ) || ! EE::exec( \EE_DOCKER::docker_compose_with_custom() . ' up -d ' . $service ) ) { + throw new \Exception( "Unable to recreate $container with its new image. Please check logs for more details." ); + } + + self::wait_until_ready( $service, $container ); + EE::debug( "Complete upgrading $container" ); + } + + /** + * Wait until a global container runs, isn't restarting, is healthy if it has a healthcheck, + * and passes its service's readiness probe, then stays so for a moment. Unprobed services only need to have started. + * + * @param string $service Compose service, e.g. `global-db`. + * @param string $container Its container name. + * @param int|null $timeout Seconds; defaults to the service's timeout. + * + * @throws \Exception When it isn't ready in time or its healthcheck reports unhealthy. + */ + public static function wait_until_ready( $service, $container, $timeout = null ) { + if ( null === $timeout ) { + $timeout = self::get_ready_timeout( $service ); + } + $start = microtime( true ); + $slept = 0; + $reason = ''; + + while ( true ) { + $state = self::get_container_state( $container ); + if ( 'unhealthy' === $state['health'] ) { + throw new \Exception( "$container is unhealthy according to its healthcheck." ); + } + + // No readiness signal, and cron restart-loops while it has no jobs (CRON-1): having started is enough. + if ( ! in_array( $service, self::PROBED_SERVICES, true ) && in_array( $state['status'], [ 'running', 'restarting' ], true ) && in_array( $state['health'], [ 'healthy', 'none' ], true ) ) { + if ( 'running' !== $state['status'] || 'false' !== $state['restarting'] ) { + EE::warning( "$container keeps restarting. Please check `docker logs $container`." ); + } + + return; + } + + $reason = self::not_ready_reason( $service, $container, $state ); + if ( '' === $reason ) { + self::pause( self::SETTLE_SECONDS ); + $slept += self::SETTLE_SECONDS; + $settled = self::get_container_state( $container ); + $reason = $settled['restart_count'] !== $state['restart_count'] ? 'it restarted' : self::not_ready_reason( $service, $container, $settled ); + if ( '' === $reason ) { + EE::debug( sprintf( '%s is ready after %.1f s', $container, microtime( true ) - $start ) ); + + return; + } + } + + // Time slept counts too, so a test sleeper that returns at once still ends the wait. + if ( max( microtime( true ) - $start, $slept ) >= $timeout ) { + throw new \Exception( sprintf( '%s is not ready after %d s (%s). Please check `docker logs %s`.', $container, $timeout, $reason, $container ) ); + } + self::pause( 1 ); + ++$slept; + } + } + + /** + * @param string $service Compose service. + * + * @return int Seconds. + */ + private static function get_ready_timeout( $service ) { + $configured = \EE\Utils\get_config_value( 'global-service-ready-timeout' ); + if ( is_numeric( $configured ) && $configured > 0 ) { + return (int) $configured; + } + + return 'global-db' === $service ? self::DB_READY_TIMEOUT : self::READY_TIMEOUT; + } + + /** + * @param string $container Container name. + * + * @return array status, restarting, restart_count and health (`none` without a healthcheck); empty strings if it doesn't exist. + */ + private static function get_container_state( $container ) { + $format = '{{.State.Status}} {{.State.Restarting}} {{.RestartCount}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}'; + $result = EE::launch( sprintf( "docker inspect --format '%s' %s", $format, escapeshellarg( $container ) ) ); + $parts = 0 === $result->return_code ? preg_split( '/\s+/', trim( $result->stdout ) ) : []; + + return array_combine( [ 'status', 'restarting', 'restart_count', 'health' ], array_pad( array_slice( $parts, 0, 4 ), 4, '' ) ); + } + + /** + * @param string $service Compose service. + * @param string $container Container name. + * @param array $state From get_container_state(). + * + * @return string Why it isn't ready, or '' when it is. + */ + private static function not_ready_reason( $service, $container, $state ) { + if ( '' === $state['status'] ) { + return 'no such container'; + } + if ( 'running' !== $state['status'] || 'false' !== $state['restarting'] ) { + return "status {$state['status']}"; + } + if ( ! in_array( $state['health'], [ 'healthy', 'none' ], true ) ) { + return "health {$state['health']}"; + } + + return self::probe( $service, $container ); + } + + /** + * Service-specific readiness probe. + * + * @param string $service Compose service. + * @param string $container Container name. + * + * @return string Why it failed, or '' when it passed. + */ + private static function probe( $service, $container ) { + $exec = 'docker exec ' . escapeshellarg( $container ) . ' '; + + switch ( $service ) { + case 'global-nginx-proxy': + if ( ! EE::exec( $exec . 'nginx -t' ) ) { + return 'nginx -t failed'; + } + // Any HTTP status proves nginx answers through the published port; 443 is the same process. + $port = (string) \EE\Utils\get_config_value( 'proxy_80_port', '80' ); + $host = false === strpos( $port, ':' ) ? '127.0.0.1' : substr( $port, 0, strrpos( $port, ':' ) ); + $port = false === strpos( $port, ':' ) ? $port : substr( $port, strrpos( $port, ':' ) + 1 ); + $status = trim( EE::launch( sprintf( "curl -s -o /dev/null -w '%%{http_code}' --max-time 5 %s", escapeshellarg( "http://$host:$port/" ) ) )->stdout ); + + return preg_match( '/^[1-5]\d\d$/', $status ) ? '' : "no HTTP response on port $port"; + + case 'global-db': + // Over TCP: the entrypoint's temporary init/upgrade server listens on the socket only. Access denied still means alive. + $ping = 'for c in mariadb-admin mysqladmin; do command -v $c >/dev/null 2>&1 && exec $c ping -h 127.0.0.1 --protocol=tcp --connect-timeout=2 --silent; done; exit 1'; + + return EE::exec( $exec . 'sh -c ' . escapeshellarg( $ping ) ) ? '' : 'mysqladmin ping failed'; + + case 'global-redis': + $pong = trim( EE::launch( $exec . 'redis-cli ping' )->stdout ); + + return 'PONG' === $pong ? '' : 'redis-cli ping failed'; + } + + return ''; + } + + /** + * @param int|float $seconds Seconds to sleep. + */ + private static function pause( $seconds ) { + if ( self::$sleeper ) { + call_user_func( self::$sleeper, $seconds ); + + return; + } + usleep( (int) ( $seconds * 1000000 ) ); + } + /** * Get global containers which has new image update. * @@ -58,6 +281,14 @@ public static function revert_global_containers( $source_path, $dest_path, $upda if ( ! EE::exec( \EE_DOCKER::docker_compose_with_custom() . ' up -d ' . $services_to_regenerate ) ) { throw new \Exception( 'Unable to downgrade global containers. Please check logs for more details.' ); } + foreach ( $updated_images as $image_name ) { + $container = $all_global_images[ $image_name ]; + try { + self::wait_until_ready( self::get_service_name( $container ), $container ); + } catch ( \Exception $e ) { + EE::warning( 'After restoring the previous image: ' . $e->getMessage() ); + } + } EE::debug( 'Complete restoring global docker-compose.yml file from backup' ); } From 5b935df28efa606c40e09faf24aec90318001e82 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 20:55:59 +0530 Subject: [PATCH 03/27] fix(migration): pull only images that something uses The image migration pulled every image in img-versions.json that no running container used and the DB didn't track, on every run, without ever recording it. That is easyengine/mailhog and the amd64-only easyengine/php (stable), which no site type uses: the pull costs time and Docker Hub quota everywhere, fails without registry access, and aborts the whole migration on arm64 hosts. Such images have nothing to upgrade, so they are no longer pulled; Compose pulls them when a container first needs them. easyengine/php also joins the images that are left to Compose when they do change. --- php/EE/Migration/Containers.php | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 5c63523ae..d52aae148 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -40,20 +40,16 @@ public static function start_container_migration() { 'easyengine/php8.3', 'easyengine/php8.4', 'easyengine/php8.5', + 'easyengine/php', 'easyengine/newrelic-daemon', ]; + // An image no container runs and the DB doesn't track (e.g. mailhog, the amd64-only easyengine/php) has nothing + // to upgrade: pulling it only fails on hosts it isn't built for. Compose pulls it when something first uses it. foreach ( $img_versions as $img => $version ) { - - if ( array_key_exists( $img, $current_versions ) ) { - if ( $current_versions[ $img ] !== $version ) { - $updated_images[] = $img; - if ( ! in_array( $img, $skip_download ) ) { - self::pull_or_error( $img, $version ); - } - } - } else { - if ( ! in_array( $img, $skip_download ) ) { + if ( array_key_exists( $img, $current_versions ) && $current_versions[ $img ] !== $version ) { + $updated_images[] = $img; + if ( ! in_array( $img, $skip_download, true ) ) { self::pull_or_error( $img, $version ); } } From b204545d7a1bc85d73cbc02021164796ad237682 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 20:58:18 +0530 Subject: [PATCH 04/27] fix(migration): install docker-compose only when missing or too old, and verify it CheckAndUpdateDockerOne never called the parent constructor, so its fresh-install skip was dead code: every first run replaced the host's docker-compose, whatever its version, with a downloaded 1.29.2 (a URL that 404s on aarch64), and the image migration then replaced that with 2.27.0. A host with compose v5 ended up on 2.27.0. The download had no --fail and no checksum and ignored its exit codes, so an HTTP error page could become the binary, and the revert never restored anything because it copied without overwriting. update_docker_compose() now installs 2.27.0 only when docker-compose is missing or older than 2.26.1, the minimum EasyEngine checks for, so it never downgrades. It resolves the release asset from the OS and architecture (x86_64, aarch64/arm64, armv6/7, ppc64le, s390x, riscv64), downloads with --fail, checks the published sha256 and that the new binary runs and reports at least 2.26.1, and only then moves it into place. A failure warns and keeps the installed binary. The revert restores the backup only if this run replaced the binary, never a stale one from an earlier run. The migration now calls the parent constructor, so it is skipped on fresh installs, and on upgrades it uses the same helper instead of forcing 1.29.2. --- ...easyengine_check_and_update_docker_one.php | 24 +--- php/EE/Migration/Containers.php | 130 ++++++++++++++++-- 2 files changed, 120 insertions(+), 34 deletions(-) diff --git a/migrations/db/20240502102550_easyengine_check_and_update_docker_one.php b/migrations/db/20240502102550_easyengine_check_and_update_docker_one.php index 9a25e20e4..d16b04f69 100644 --- a/migrations/db/20240502102550_easyengine_check_and_update_docker_one.php +++ b/migrations/db/20240502102550_easyengine_check_and_update_docker_one.php @@ -4,12 +4,12 @@ use EE; use EE\Migration\Base; -use Symfony\Component\Filesystem\Filesystem; class CheckAndUpdateDockerOne extends Base { public function __construct() { + parent::__construct(); if ( $this->is_first_execution ) { $this->skip_this_migration = true; } @@ -48,32 +48,14 @@ public function up() { } EE::debug( 'Checking docker-compose version' ); - $docker_compose_version = trim( EE::launch( 'docker-compose version --short' )->stdout ); - $docker_compose_path = EE::launch( 'command -v docker-compose' )->stdout; - $docker_compose_path = trim( $docker_compose_path ); - $docker_compose_backup_path = EE_BACKUP_DIR . '/docker-compose.backup'; - $docker_compose_new_path = EE_BACKUP_DIR . '/docker-compose'; - $fs = new Filesystem(); - if ( ! $fs->exists( EE_BACKUP_DIR ) ) { - $fs->mkdir( EE_BACKUP_DIR ); - } - $fs->copy( $docker_compose_path, $docker_compose_backup_path ); - - if ( version_compare( '1.29.2', $docker_compose_version, '!=' ) ) { - EE::exec( "curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m) -o $docker_compose_new_path && chmod +x $docker_compose_new_path" ); - EE::exec( "mv $docker_compose_new_path $docker_compose_path" ); - } + // Installs only when docker-compose is missing or too old; it never downgrades. + Containers::update_docker_compose(); // Check the version again post update. $docker_version = trim( EE::launch( 'docker version --format "{{.Server.Version}}"' )->stdout ); if ( version_compare( $docker_version, '20.10.10', '<' ) ) { EE::error( 'Docker version should be 20.10.10 or above. Please update Docker and try again.' ); } - - $docker_compose_version = trim( EE::launch( 'docker-compose version --short' )->stdout ); - if ( version_compare( '1.29.2', $docker_compose_version, '!=' ) ) { - EE::error( 'Docker-compose version should be 1.29.2. Please update Docker-compose and try again.' ); - } } /** diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index d52aae148..a02d76ab2 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -11,9 +11,18 @@ */ class Containers { + /** EasyEngine needs at least this docker-compose (see Runner::check_requirements()). */ + const DOCKER_COMPOSE_MIN_VERSION = '2.26.1'; + + /** Installed when docker-compose is missing or older than the minimum. */ + const DOCKER_COMPOSE_INSTALL_VERSION = '2.27.0'; + /** @var RevertableStepProcessor $rsp Keeps track of migration state. Reverts on error */ private static $rsp; + /** @var bool Whether update_docker_compose() replaced the binary in this process. */ + private static $replaced_docker_compose = false; + /** * Main entry point for the script. Migrates all containers * @@ -123,38 +132,133 @@ public static function docker_cleanup() { } /** - * Update docker-compose to v2.27.0 if lower version is installed. + * Install docker-compose DOCKER_COMPOSE_INSTALL_VERSION when it is missing or older than DOCKER_COMPOSE_MIN_VERSION. + * A newer one is never replaced. A failed download or check only warns and keeps the installed binary. */ public static function update_docker_compose() { - $docker_compose_version = trim( EE::launch( 'docker-compose version --short' )->stdout ); + $installed = self::get_docker_compose_version(); + if ( '' !== $installed && version_compare( $installed, self::DOCKER_COMPOSE_MIN_VERSION, '>=' ) ) { + EE::debug( "docker-compose $installed is recent enough, not replacing it." ); + + return; + } + + $version = self::DOCKER_COMPOSE_INSTALL_VERSION; + $url = self::get_docker_compose_download_url( $version, php_uname( 's' ), php_uname( 'm' ) ); + if ( '' === $url ) { + EE::warning( sprintf( 'EasyEngine requires docker-compose %s or newer, and there is no docker-compose %s build for %s %s. Please install it manually.', self::DOCKER_COMPOSE_MIN_VERSION, $version, php_uname( 's' ), php_uname( 'm' ) ) ); + + return; + } + $docker_compose_path = trim( EE::launch( 'command -v docker-compose' )->stdout ); + $docker_compose_path = '' === $docker_compose_path ? '/usr/local/bin/docker-compose' : $docker_compose_path; $docker_compose_backup_path = EE_BACKUP_DIR . '/docker-compose.backup'; $docker_compose_new_path = EE_BACKUP_DIR . '/docker-compose'; - if ( version_compare( '2.27.0', $docker_compose_version, '>' ) ) { + $fs = new Filesystem(); + $fs->mkdir( EE_BACKUP_DIR ); + $fs->remove( $docker_compose_new_path ); + + $failure = ''; + if ( ! EE::exec( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 -o %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $url ) ) ) ) { + $failure = "the download of $url failed"; + } else { + $checksum = strtok( trim( EE::launch( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 %s', escapeshellarg( "$url.sha256" ) ) )->stdout ), " \t" ); + $actual = is_file( $docker_compose_new_path ) ? hash_file( 'sha256', $docker_compose_new_path ) : ''; + if ( ! preg_match( '/^[0-9a-f]{64}$/', (string) $checksum ) || ! hash_equals( (string) $checksum, (string) $actual ) ) { + $failure = "the checksum of $url doesn't match"; + } else { + $fs->chmod( $docker_compose_new_path, 0755 ); + $new_version = self::normalize_version( EE::launch( escapeshellarg( $docker_compose_new_path ) . ' version --short' )->stdout ); + if ( ! version_compare( $new_version, self::DOCKER_COMPOSE_MIN_VERSION, '>=' ) ) { + $failure = 'the downloaded binary does not run'; + } + } + } - $fs = new Filesystem(); - if ( ! $fs->exists( EE_BACKUP_DIR ) ) { - $fs->mkdir( EE_BACKUP_DIR ); + if ( '' === $failure ) { + if ( $fs->exists( $docker_compose_path ) ) { + $fs->copy( $docker_compose_path, $docker_compose_backup_path, true ); } - $fs->copy( $docker_compose_path, $docker_compose_backup_path ); + if ( EE::exec( sprintf( 'mv %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $docker_compose_path ) ) ) ) { + self::$replaced_docker_compose = true; + EE::debug( "Installed docker-compose $version at $docker_compose_path" ); - EE::exec( "curl -L https://github.com/docker/compose/releases/download/v2.27.0/docker-compose-$(uname -s)-$(uname -m) -o $docker_compose_new_path && chmod +x $docker_compose_new_path" ); - EE::exec( "mv $docker_compose_new_path $docker_compose_path" ); + return; + } + $failure = "it could not be moved to $docker_compose_path"; } + + $fs->remove( $docker_compose_new_path ); + EE::warning( sprintf( 'Could not install docker-compose %s (%s). EasyEngine requires docker-compose %s or newer, found %s.', $version, $failure, self::DOCKER_COMPOSE_MIN_VERSION, '' === $installed ? 'none' : $installed ) ); } /** - * Revert docker-compose to previous version. + * Revert docker-compose to the binary update_docker_compose() replaced in this run, if it did. + * An older backup from an earlier run is never restored. */ public static function revert_docker_compose() { - $docker_compose_path = EE::launch( 'command -v docker-compose' )->stdout; - $docker_compose_path = trim( $docker_compose_path ); + if ( ! self::$replaced_docker_compose ) { + return; + } + + $docker_compose_path = trim( EE::launch( 'command -v docker-compose' )->stdout ); $docker_compose_backup_path = EE_BACKUP_DIR . '/docker-compose.backup'; $fs = new Filesystem(); - $fs->copy( $docker_compose_backup_path, $docker_compose_path ); + if ( '' !== $docker_compose_path && $fs->exists( $docker_compose_backup_path ) ) { + $fs->copy( $docker_compose_backup_path, $docker_compose_path, true ); + } + self::$replaced_docker_compose = false; + } + + /** + * @return string Installed docker-compose version as x.y.z, or '' if there is none. + */ + public static function get_docker_compose_version() { + return self::normalize_version( EE::launch( 'docker-compose version --short' )->stdout ); + } + + /** + * Release asset URL of a docker-compose v2 build. + * + * @param string $version Version without `v`. + * @param string $os `uname -s`, e.g. Linux or Darwin. + * @param string $arch `uname -m`. + * + * @return string '' when there is no build for this platform. + */ + public static function get_docker_compose_download_url( $version, $os, $arch ) { + $os = strtolower( $os ); + $arches = [ + 'x86_64' => 'x86_64', + 'amd64' => 'x86_64', + 'aarch64' => 'aarch64', + 'arm64' => 'aarch64', + 'armv7l' => 'armv7', + 'armv6l' => 'armv6', + 'ppc64le' => 'ppc64le', + 's390x' => 's390x', + 'riscv64' => 'riscv64', + ]; + $arch = $arches[ strtolower( $arch ) ] ?? ''; + + if ( '' === $arch || ! ( 'linux' === $os || ( 'darwin' === $os && in_array( $arch, [ 'x86_64', 'aarch64' ], true ) ) ) ) { + return ''; + } + + return sprintf( 'https://github.com/docker/compose/releases/download/v%s/docker-compose-%s-%s', $version, $os, $arch ); + } + + /** + * @param string $output Output of `docker-compose version --short`, e.g. `2.27.0`, `v2.40.3` or `5.5.1-desktop.1`. + * + * @return string x.y.z, or ''. + */ + private static function normalize_version( $output ) { + return preg_match( '/(\d+\.\d+\.\d+)/', (string) $output, $matches ) ? $matches[1] : ''; } /** From 026461fed746024b418ff8fa457264f814223ca8 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:00:41 +0530 Subject: [PATCH 05/27] fix(db): create the schema of an empty EasyEngine database EE_DB decided whether the schema existed by whether the file existed, so a 0-byte ee.sqlite (left by running the sqlite3 CLI on the not-yet-created DB, or by a first run that died after PDO created the file) made every command that reaches the migration check die with an uncaught 'no such table: options' until the file was deleted. A missing, empty or table-less database now gets the base tables, created with IF NOT EXISTS in one transaction. A database that has tables but lacks migrations or options, or isn't a SQLite file, is left untouched and ee stops with a message naming the file and how to recover, instead of an uncaught exception. --- php/class-ee-db.php | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/php/class-ee-db.php b/php/class-ee-db.php index 3d361198a..4c67656b1 100644 --- a/php/class-ee-db.php +++ b/php/class-ee-db.php @@ -32,17 +32,23 @@ public function __construct() { */ private static function init_db() { - if ( ! ( file_exists( DB ) ) ) { - self::create_required_tables(); - - return; - } + // PDO creates a missing file; an empty one (e.g. left by the sqlite3 CLI or an interrupted first run) has no schema either. + clearstatcache( true, DB ); + $is_new = ! file_exists( DB ) || 0 === filesize( DB ); try { self::$pdo = new PDO( 'sqlite:' . DB ); self::$pdo->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); + $tables = $is_new ? [] : self::$pdo->query( "SELECT name FROM sqlite_master WHERE type = 'table'" )->fetchAll( PDO::FETCH_COLUMN ); } catch ( PDOException $exception ) { - EE::error( 'Unable to initialize connection to EE sqlite database: ' . $exception->getMessage() ); + EE::error( sprintf( 'Unable to open the EasyEngine database %s: %s. If it is damaged, restore it from a backup, or move it away to start with an empty database.', DB, $exception->getMessage() ) ); + } + + if ( empty( $tables ) ) { + self::create_required_tables(); + } elseif ( array_diff( [ 'migrations', 'options' ], $tables ) ) { + // Tables but no base schema: not ours to initialise over. + EE::error( sprintf( 'The EasyEngine database %s lacks the %s table(s). Restore it from a backup, or move it away to start with an empty database.', DB, implode( ', ', array_diff( [ 'migrations', 'options' ], $tables ) ) ) ); } } @@ -50,30 +56,32 @@ private static function init_db() { * Sqlite database creation. */ private static function create_required_tables() { - try { - self::$pdo = new PDO( 'sqlite:' . DB ); - self::$pdo->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); - } catch ( PDOException $exception ) { - EE::error( $exception->getMessage() ); - } - $query = 'CREATE TABLE migrations ( + $queries = [ + 'CREATE TABLE IF NOT EXISTS migrations ( id INTEGER, migration VARCHAR, type VARCHAR, timestamp DATETIME, PRIMARY KEY (id) - );'; - - $query .= 'CREATE TABLE options ( + );', + 'CREATE TABLE IF NOT EXISTS options ( key VARCHAR NOT NULL, value VARCHAR NOT NULL, PRIMARY KEY (key) - );'; + );', + ]; try { - self::$pdo->exec( $query ); + self::$pdo->beginTransaction(); + foreach ( $queries as $query ) { + self::$pdo->exec( $query ); + } + self::$pdo->commit(); } catch ( PDOException $exception ) { + if ( self::$pdo->inTransaction() ) { + self::$pdo->rollBack(); + } EE::error( 'Encountered Error while creating table: ' . $exception->getMessage() ); } } From ee371241f878f2082cbb6a9ab8a6a52857465aeb Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:03:10 +0530 Subject: [PATCH 06/27] fix(migration): never restore a docker-compose backup from an earlier run When no docker-compose was installed, update_docker_compose() made no backup but still marked the binary as replaced, so a failed image migration restored whatever docker-compose.backup an earlier run had left over the new install. The old backup is now removed before a new one is taken. --- php/EE/Migration/Containers.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index a02d76ab2..757ffc3a5 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -179,6 +179,8 @@ public static function update_docker_compose() { } if ( '' === $failure ) { + // A backup from an earlier run must never be restored over this install. + $fs->remove( $docker_compose_backup_path ); if ( $fs->exists( $docker_compose_path ) ) { $fs->copy( $docker_compose_path, $docker_compose_backup_path, true ); } From a935f0ac933a4b41d7aefcc909b62ff3cd283342 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:05:25 +0530 Subject: [PATCH 07/27] fix(migration): accept a password-protected redis and fractional timeouts in the readiness gate redis-cli ping answers NOAUTH when redis.conf sets requirepass, so a working redis failed the gate and every upgrade rolled back; NOAUTH now counts as alive, as 'Access denied' already does for the db. A fractional global-service-ready-timeout (e.g. 0.5) was truncated to 0 and failed the first probe at once; it is now rounded up. The comment on the upgrade order no longer claims the db is only reverted when the proxy fails: a failing site step after the globals reverts all of them, as before. --- php/EE/Migration/GlobalContainers.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 5c9ba6b97..ad605aa05 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -12,7 +12,8 @@ class GlobalContainers { /** * Upgrade order. The proxy goes last so that a failing backend is rolled back before every site loses its proxy, - * and the database just before it, so its datadir is only reverted onto the old image if the proxy fails. + * and the database just before it, so of the globals only a failing proxy reverts its datadir onto the old image + * (a failure in a later site step still reverts every global). */ const UPGRADE_ORDER = [ 'easyengine/cron', @@ -142,7 +143,7 @@ public static function wait_until_ready( $service, $container, $timeout = null ) private static function get_ready_timeout( $service ) { $configured = \EE\Utils\get_config_value( 'global-service-ready-timeout' ); if ( is_numeric( $configured ) && $configured > 0 ) { - return (int) $configured; + return (int) ceil( $configured ); } return 'global-db' === $service ? self::DB_READY_TIMEOUT : self::READY_TIMEOUT; @@ -215,7 +216,8 @@ private static function probe( $service, $container ) { case 'global-redis': $pong = trim( EE::launch( $exec . 'redis-cli ping' )->stdout ); - return 'PONG' === $pong ? '' : 'redis-cli ping failed'; + // NOAUTH (a requirepass in redis.conf) still means redis answers. + return 'PONG' === $pong || 0 === strpos( $pong, 'NOAUTH' ) ? '' : 'redis-cli ping failed'; } return ''; From 498d20e45252aeaec45f28f174940bc61611d808 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:07:49 +0530 Subject: [PATCH 08/27] fix(migration): report a missing migration file instead of a class-not-found error Both executors threw `new Exception()` inside the EE\Migration namespace, which is an Error for the missing class EE\Migration\Exception. That used to end in exit(1); now that the executors rethrow, the step processor would report 'Class "EE\Migration\Exception" not found' as the reason. They throw \Exception with the missing path instead. --- php/EE/Migration/CustomContainerMigrations.php | 2 +- php/EE/Migration/Executor.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/php/EE/Migration/CustomContainerMigrations.php b/php/EE/Migration/CustomContainerMigrations.php index 984ec0e9e..1f9b50981 100644 --- a/php/EE/Migration/CustomContainerMigrations.php +++ b/php/EE/Migration/CustomContainerMigrations.php @@ -161,7 +161,7 @@ private static function execute_migration_stack( $migrations ) { if ( ! file_exists( $migration_path ) ) { EE::error( "Unable to find migration file at $migration_path", false ); - throw new Exception(); + throw new \Exception( "Unable to find migration file at $migration_path" ); } require( $migration_path ); diff --git a/php/EE/Migration/Executor.php b/php/EE/Migration/Executor.php index 176158665..5749e3cf5 100644 --- a/php/EE/Migration/Executor.php +++ b/php/EE/Migration/Executor.php @@ -101,7 +101,7 @@ private static function execute_migration_stack( $migrations ) { if ( ! file_exists( $migration_path ) ) { EE::error( "Unable to find migration file at $migration_path", false ); - throw new Exception(); + throw new \Exception( "Unable to find migration file at $migration_path" ); } require( $migration_path ); From 40e64642618ca0b200a6f8fa57fa9bb06afd5d97 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:10:38 +0530 Subject: [PATCH 09/27] fix(migration): revert only the docker-compose replacement of the image migration's own step The replaced flag was process-wide, so when the CheckAndUpdateDockerOne DB migration (which has no undo) installed 2.27.0 and the image migration's update-compose step then had nothing to do, a later failure in the image migration still restored the old binary, possibly compose v1, before the site and global reverts ran against containers the v2 binary had created. Each update_docker_compose() call now starts with the flag cleared, so the step's revert undoes only its own replacement. --- php/EE/Migration/Containers.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 757ffc3a5..1a2ef731e 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -20,7 +20,7 @@ class Containers { /** @var RevertableStepProcessor $rsp Keeps track of migration state. Reverts on error */ private static $rsp; - /** @var bool Whether update_docker_compose() replaced the binary in this process. */ + /** @var bool Whether the last update_docker_compose() call replaced the binary. */ private static $replaced_docker_compose = false; /** @@ -137,7 +137,9 @@ public static function docker_cleanup() { */ public static function update_docker_compose() { - $installed = self::get_docker_compose_version(); + // Only this call's replacement may be reverted, not one made earlier in the run (e.g. by a DB migration, which has no undo). + self::$replaced_docker_compose = false; + $installed = self::get_docker_compose_version(); if ( '' !== $installed && version_compare( $installed, self::DOCKER_COMPOSE_MIN_VERSION, '>=' ) ) { EE::debug( "docker-compose $installed is recent enough, not replacing it." ); @@ -198,7 +200,7 @@ public static function update_docker_compose() { } /** - * Revert docker-compose to the binary update_docker_compose() replaced in this run, if it did. + * Revert docker-compose to the binary the last update_docker_compose() call replaced, if it did. * An older backup from an earlier run is never restored. */ public static function revert_docker_compose() { From 8634458fbf4d60151a7b1b405f2a0261240d7abe Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:13:05 +0530 Subject: [PATCH 10/27] fix(ee): let a migration step capture exits from EE::runcommand() With exit_error on (the default), runcommand() ended the process with a raw exit() when a launched command failed, and an in-process command's EE::error() was swallowed into a return code whenever the caller was already capturing exits. Inside a migration step (e.g. auth-command's global auth update reached from a site's admin-tools enable) the first skipped the rollback and the second hid the failure. A failed launched command now goes through EE::halt(), which exits as before unless exits are captured, and an in-process exit with a non-zero code is rethrown to a capturing caller. --- php/class-ee.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/php/class-ee.php b/php/class-ee.php index 390f2c263..6156f9dde 100644 --- a/php/class-ee.php +++ b/php/class-ee.php @@ -1238,7 +1238,8 @@ public static function runcommand( $command, $options = array() ) { if ( -1 == $return_code ) { self::warning( 'Spawned process returned exit code -1, which could be caused by a custom compiled version of PHP that uses the --enable-sigchild option.' ); } elseif ( $return_code && $exit_error ) { - exit( $return_code ); + // halt(), not exit(): a migration step capturing exits must still roll back. + self::halt( $return_code ); } if ( true === $return || 'stdout' === $return ) { $retval = trim( $stdout ); @@ -1263,6 +1264,7 @@ public static function runcommand( $command, $options = array() ) { self::$logger->ob_start(); } $previous_capture_exit = self::$capture_exit; + $exit_exception = null; if ( ! $exit_error ) { self::$capture_exit = true; } @@ -1275,6 +1277,10 @@ public static function runcommand( $command, $options = array() ) { $return_code = 0; } catch ( ExitException $e ) { $return_code = $e->getCode(); + // With exit_error the caller would have exited; a caller that captures exits gets the exit instead. + if ( $exit_error && $return_code ) { + $exit_exception = $e; + } } if ( $return ) { $execution_logger = self::$logger; @@ -1298,6 +1304,9 @@ public static function runcommand( $command, $options = array() ) { } // Restore rather than clear: a caller (e.g. a migration step) may be capturing exits itself. self::$capture_exit = $previous_capture_exit; + if ( $exit_exception ) { + throw $exit_exception; + } } if ( ( true === $return || 'stdout' === $return ) && 'json' === $parse ) { From 019901b70aeec6d1d11446834cf97275fffae244 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:15:46 +0530 Subject: [PATCH 11/27] fix(migration): update docker-compose before recreating global services Recreating a service in place with `up -d ` fails with docker-compose v1 on Docker 25 and newer: v1 stops and renames the old container, then crashes with KeyError: 'ContainerConfig', and a retry or the rollback's `up -d` fails the same way, so the service stays down. The old flow stopped and removed the services first, which v1 still handles, and only updated docker-compose at the end, so a v1 host would lose its first recreated global and never reach the update. The update-compose step now runs first in the image migration (and so is reverted last), and when docker-compose is still older than 2.26.1 (the download failed) the global recreates and the rollback stop and remove the service before `up -d`. --- php/EE/Migration/Containers.php | 7 ++++--- php/EE/Migration/GlobalContainers.php | 23 +++++++++++++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 1a2ef731e..334806e15 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -69,9 +69,10 @@ public static function start_container_migration() { return; } + // First: compose v1 can't recreate containers on Docker 25+, and on a rollback it is restored last. + self::maybe_update_docker_compose(); self::migrate_global_containers( $updated_images ); self::migrate_site_containers( $updated_images ); - self::maybe_update_docker_compose(); self::save_upgraded_image_versions( $current_versions, $img_versions, $updated_images ); if ( ! self::$rsp->execute() ) { @@ -82,8 +83,8 @@ public static function start_container_migration() { } /** - * Maybe update docker-compose at the end of migration. - * Need to update to latest docker-compose version for new template changes. + * Maybe update docker-compose before any container is recreated, for the new templates and because v1 fails to + * recreate containers on current Docker. */ public static function maybe_update_docker_compose() { diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index ad605aa05..0f9cfa133 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -72,7 +72,7 @@ public static function get_service_name( $container ) { public static function upgrade_global_service( $service, $container ) { EE::debug( "Start upgrading $container" ); - if ( ! chdir( EE_SERVICE_DIR ) || ! EE::exec( \EE_DOCKER::docker_compose_with_custom() . ' up -d ' . $service ) ) { + if ( ! chdir( EE_SERVICE_DIR ) || ! self::compose_up( $service ) ) { throw new \Exception( "Unable to recreate $container with its new image. Please check logs for more details." ); } @@ -80,6 +80,25 @@ public static function upgrade_global_service( $service, $container ) { EE::debug( "Complete upgrading $container" ); } + /** + * `up -d` global services in the current directory. docker-compose v1 can't recreate a container on Docker 25+ + * (KeyError: 'ContainerConfig', leaving it stopped), so below the minimum version they are stopped and removed first. + * + * @param string $services Space-separated compose services. + * + * @return bool + */ + private static function compose_up( $services ) { + $compose = \EE_DOCKER::docker_compose_with_custom(); + $up = "$compose up -d $services"; + $version = Containers::get_docker_compose_version(); + if ( '' !== $version && version_compare( $version, Containers::DOCKER_COMPOSE_MIN_VERSION, '<' ) ) { + return EE::exec( "$compose stop $services && $compose rm -f $services && $up" ); + } + + return EE::exec( $up ); + } + /** * Wait until a global container runs, isn't restarting, is healthy if it has a healthcheck, * and passes its service's readiness probe, then stays so for a moment. Unprobed services only need to have started. @@ -280,7 +299,7 @@ public static function revert_global_containers( $source_path, $dest_path, $upda chdir( EE_ROOT_DIR . '/services' ); - if ( ! EE::exec( \EE_DOCKER::docker_compose_with_custom() . ' up -d ' . $services_to_regenerate ) ) { + if ( ! self::compose_up( trim( $services_to_regenerate ) ) ) { throw new \Exception( 'Unable to downgrade global containers. Please check logs for more details.' ); } foreach ( $updated_images as $image_name ) { From 7b7e265c3a683e9dd77729de27727c2b454344ba Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:18:36 +0530 Subject: [PATCH 12/27] fix(migration): check the running proxy config first, bound the compose download, bypass http_proxy in the probe A vhost.d/conf.d/htpasswd file broken after the running proxy loaded it made the recreated proxy fail, and the rollback's proxy on the old image failed the same way, taking every site down; the proxy step now runs nginx -t in the running container first and fails before touching it. The docker-compose download and its .sha256 had only a connect timeout, so a stalled transfer hung the upgrade; they now have --max-time. The proxy probe's curl followed an exported http_proxy, so a corporate proxy could time the probe out or answer for it; it now uses --noproxy '*'. --- php/EE/Migration/Containers.php | 5 +++-- php/EE/Migration/GlobalContainers.php | 9 ++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 334806e15..ad7dce0ea 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -165,10 +165,11 @@ public static function update_docker_compose() { $fs->remove( $docker_compose_new_path ); $failure = ''; - if ( ! EE::exec( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 -o %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $url ) ) ) ) { + // --max-time: a stalled download must not hang the upgrade. + if ( ! EE::exec( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 --max-time 300 -o %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $url ) ) ) ) { $failure = "the download of $url failed"; } else { - $checksum = strtok( trim( EE::launch( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 %s', escapeshellarg( "$url.sha256" ) ) )->stdout ), " \t" ); + $checksum = strtok( trim( EE::launch( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 --max-time 60 %s', escapeshellarg( "$url.sha256" ) ) )->stdout ), " \t" ); $actual = is_file( $docker_compose_new_path ) ? hash_file( 'sha256', $docker_compose_new_path ) : ''; if ( ! preg_match( '/^[0-9a-f]{64}$/', (string) $checksum ) || ! hash_equals( (string) $checksum, (string) $actual ) ) { $failure = "the checksum of $url doesn't match"; diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 0f9cfa133..742826d8b 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -72,6 +72,12 @@ public static function get_service_name( $container ) { public static function upgrade_global_service( $service, $container ) { EE::debug( "Start upgrading $container" ); + // A config already broken on disk would stop the new proxy and the restored one alike: refuse before touching it. + if ( 'global-nginx-proxy' === $service && 'running' === self::get_container_state( $container )['status'] + && ! EE::exec( 'docker exec ' . escapeshellarg( $container ) . ' nginx -t' ) ) { + throw new \Exception( "The configuration of the running $container fails `nginx -t`, so a recreated proxy would not start. Fix it (see `docker exec $container nginx -t`) and run the upgrade again." ); + } + if ( ! chdir( EE_SERVICE_DIR ) || ! self::compose_up( $service ) ) { throw new \Exception( "Unable to recreate $container with its new image. Please check logs for more details." ); } @@ -222,7 +228,8 @@ private static function probe( $service, $container ) { $port = (string) \EE\Utils\get_config_value( 'proxy_80_port', '80' ); $host = false === strpos( $port, ':' ) ? '127.0.0.1' : substr( $port, 0, strrpos( $port, ':' ) ); $port = false === strpos( $port, ':' ) ? $port : substr( $port, strrpos( $port, ':' ) + 1 ); - $status = trim( EE::launch( sprintf( "curl -s -o /dev/null -w '%%{http_code}' --max-time 5 %s", escapeshellarg( "http://$host:$port/" ) ) )->stdout ); + // --noproxy: an http_proxy exported for pulls must not answer for the local proxy. + $status = trim( EE::launch( sprintf( "curl -s --noproxy '*' -o /dev/null -w '%%{http_code}' --max-time 5 %s", escapeshellarg( "http://$host:$port/" ) ) )->stdout ); return preg_match( '/^[1-5]\d\d$/', $status ) ? '' : "no HTTP response on port $port"; From e495dcf46f730206dd1151f6efaaed4ad0b0961f Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:21:21 +0530 Subject: [PATCH 13/27] fix(migration): check the running proxy config before any global service is recreated The nginx -t check of the running proxy ran in the proxy's own upgrade step, which comes last, so a config already broken on disk only stopped the upgrade after cron, redis and the database were recreated on their new images, and the rollback then started the old database image on a datadir the new one may have upgraded. The check now runs as its own step before the global compose file is backed up and regenerated, so a broken proxy config stops the upgrade before any service is touched. --- php/EE/Migration/Containers.php | 13 ++++++++++++- php/EE/Migration/GlobalContainers.php | 20 ++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index ad7dce0ea..102466456 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -355,6 +355,18 @@ private static function migrate_global_containers( $updated_images ) { $global_compose_file_path = EE_ROOT_DIR . '/services/docker-compose.yml'; $global_compose_file_backup_path = EE_BACKUP_DIR . '/services/docker-compose.yml.backup'; + $all_global_images = GlobalContainers::get_all_global_images_with_service_name(); + + if ( in_array( 'easyengine/nginx-proxy', $updated_global_images, true ) ) { + // Before anything is recreated: a rollback after the DB moved to a new image may not get its datadir back. + self::$rsp->add_step( + 'check-running-proxy-config', + 'EE\Migration\GlobalContainers::check_running_proxy_config', + null, + [ $all_global_images['easyengine/nginx-proxy'] ], + null + ); + } self::$rsp->add_step( 'backup-global-docker-compose-file', @@ -374,7 +386,6 @@ private static function migrate_global_containers( $updated_images ) { ); // No down steps: the backup step's down restores the old compose file and recreates these services from it. - $all_global_images = GlobalContainers::get_all_global_images_with_service_name(); foreach ( GlobalContainers::sort_for_upgrade( $updated_global_images ) as $image_name ) { $global_container_name = $all_global_images[ $image_name ]; self::$rsp->add_step( diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 742826d8b..715da1f65 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -60,6 +60,20 @@ public static function get_service_name( $container ) { return preg_replace( '/^services_(.+)_1$/', '$1', $container ); } + /** + * Refuse to upgrade when the running proxy's config fails `nginx -t`: it lives on volumes, so the recreated proxy and + * the restored one would both fail to start. Runs before any global service is recreated. + * + * @param string $container Proxy container name. + * + * @throws \Exception + */ + public static function check_running_proxy_config( $container ) { + if ( 'running' === self::get_container_state( $container )['status'] && ! EE::exec( 'docker exec ' . escapeshellarg( $container ) . ' nginx -t' ) ) { + throw new \Exception( "The configuration of the running $container fails `nginx -t`, so a recreated proxy would not start. Fix it (see `docker exec $container nginx -t`) and run the upgrade again." ); + } + } + /** * Recreate one global service with the image in the (already regenerated) global compose file, * then wait until it is ready. Throws on failure, so the image migration rolls back. @@ -72,12 +86,6 @@ public static function get_service_name( $container ) { public static function upgrade_global_service( $service, $container ) { EE::debug( "Start upgrading $container" ); - // A config already broken on disk would stop the new proxy and the restored one alike: refuse before touching it. - if ( 'global-nginx-proxy' === $service && 'running' === self::get_container_state( $container )['status'] - && ! EE::exec( 'docker exec ' . escapeshellarg( $container ) . ' nginx -t' ) ) { - throw new \Exception( "The configuration of the running $container fails `nginx -t`, so a recreated proxy would not start. Fix it (see `docker exec $container nginx -t`) and run the upgrade again." ); - } - if ( ! chdir( EE_SERVICE_DIR ) || ! self::compose_up( $service ) ) { throw new \Exception( "Unable to recreate $container with its new image. Please check logs for more details." ); } From 853379a2e85c658c3b341916ae3473784b9bd13d Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:23:45 +0530 Subject: [PATCH 14/27] fix(migration): stop and remove first only the global services docker-compose v1 must recreate Every recreate and the rollback stopped and removed the listed services before up -d whenever docker-compose was older than 2.26.1. Only v1 needs that (it fails with KeyError: 'ContainerConfig' when it recreates a container on Docker 25+); v2 builds below 2.26.1 recreate in place. And because the rollback listed every updated service, a failure early in the upgrade also stopped and removed the database and the proxy that were still running untouched on their old images, taking every site down while the rollback ran. Now only v1 takes the stop and remove path, and only for the services this run has actually recreated; up -d leaves the others alone on both versions. The rollback also derives the service names with the same helper as the upgrade steps. --- php/EE/Migration/Containers.php | 1 + php/EE/Migration/GlobalContainers.php | 33 +++++++++++++++++++-------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 102466456..0c75cca2f 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -356,6 +356,7 @@ private static function migrate_global_containers( $updated_images ) { $global_compose_file_path = EE_ROOT_DIR . '/services/docker-compose.yml'; $global_compose_file_backup_path = EE_BACKUP_DIR . '/services/docker-compose.yml.backup'; $all_global_images = GlobalContainers::get_all_global_images_with_service_name(); + GlobalContainers::reset_recreated(); if ( in_array( 'easyengine/nginx-proxy', $updated_global_images, true ) ) { // Before anything is recreated: a rollback after the DB moved to a new image may not get its datadir back. diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 715da1f65..4d0f61b86 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -38,6 +38,9 @@ class GlobalContainers { /** @var callable|null Test seam: replaces sleep( $seconds ) while waiting for a service. */ public static $sleeper; + /** @var string[] Services this run has recreated, the only ones a rollback must recreate again. */ + private static $recreated = []; + /** * Sort updated global images into the order their services are upgraded in. * @@ -60,6 +63,13 @@ public static function get_service_name( $container ) { return preg_replace( '/^services_(.+)_1$/', '$1', $container ); } + /** + * Start a new image migration with no service recorded as recreated. + */ + public static function reset_recreated() { + self::$recreated = []; + } + /** * Refuse to upgrade when the running proxy's config fails `nginx -t`: it lives on volumes, so the recreated proxy and * the restored one would both fail to start. Runs before any global service is recreated. @@ -86,7 +96,8 @@ public static function check_running_proxy_config( $container ) { public static function upgrade_global_service( $service, $container ) { EE::debug( "Start upgrading $container" ); - if ( ! chdir( EE_SERVICE_DIR ) || ! self::compose_up( $service ) ) { + self::$recreated[] = $service; + if ( ! chdir( EE_SERVICE_DIR ) || ! self::compose_up( $service, [ $service ] ) ) { throw new \Exception( "Unable to recreate $container with its new image. Please check logs for more details." ); } @@ -96,18 +107,21 @@ public static function upgrade_global_service( $service, $container ) { /** * `up -d` global services in the current directory. docker-compose v1 can't recreate a container on Docker 25+ - * (KeyError: 'ContainerConfig', leaving it stopped), so below the minimum version they are stopped and removed first. + * (KeyError: 'ContainerConfig', leaving it stopped), so with v1 the services to recreate are stopped and removed first. * - * @param string $services Space-separated compose services. + * @param string $services Space-separated compose services. + * @param string[] $recreate Those whose container must be replaced; v1's `up -d` leaves the others alone. * * @return bool */ - private static function compose_up( $services ) { + private static function compose_up( $services, $recreate ) { $compose = \EE_DOCKER::docker_compose_with_custom(); $up = "$compose up -d $services"; $version = Containers::get_docker_compose_version(); - if ( '' !== $version && version_compare( $version, Containers::DOCKER_COMPOSE_MIN_VERSION, '<' ) ) { - return EE::exec( "$compose stop $services && $compose rm -f $services && $up" ); + if ( $recreate && '' !== $version && version_compare( $version, '2.0.0', '<' ) ) { + $recreate = implode( ' ', $recreate ); + + return EE::exec( "$compose stop $recreate && $compose rm -f $recreate && $up" ); } return EE::exec( $up ); @@ -302,9 +316,10 @@ public static function revert_global_containers( $source_path, $dest_path, $upda $services_to_regenerate = ''; $all_global_images = self::get_all_global_images_with_service_name(); foreach ( $updated_images as $image_name ) { - $global_container_name = $all_global_images[ $image_name ]; - $services_to_regenerate .= ltrim( rtrim( $global_container_name, '_1' ), 'services_' ) . ' '; + $services_to_regenerate .= self::get_service_name( $all_global_images[ $image_name ] ) . ' '; } + $recreated = self::$recreated; + self::$recreated = []; if ( empty( trim( $services_to_regenerate ) ) ) { return; } @@ -314,7 +329,7 @@ public static function revert_global_containers( $source_path, $dest_path, $upda chdir( EE_ROOT_DIR . '/services' ); - if ( ! self::compose_up( trim( $services_to_regenerate ) ) ) { + if ( ! self::compose_up( trim( $services_to_regenerate ), array_values( array_intersect( explode( ' ', trim( $services_to_regenerate ) ), $recreated ) ) ) ) { throw new \Exception( 'Unable to downgrade global containers. Please check logs for more details.' ); } foreach ( $updated_images as $image_name ) { From 95b8944b4abad59af89770bdde3c4ab4c7c891fa Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:26:16 +0530 Subject: [PATCH 15/27] fix(migration): bound the retries of the docker-compose download curl retries a transfer that hit --max-time, so with --retry 3 a stalled download could hold the upgrade for about 20 minutes, and the upgrade tries the install up to twice per run when it fails. --retry-max-time stops new attempts after two minutes (one for the checksum). --- php/EE/Migration/Containers.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 0c75cca2f..4f7d10586 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -165,11 +165,11 @@ public static function update_docker_compose() { $fs->remove( $docker_compose_new_path ); $failure = ''; - // --max-time: a stalled download must not hang the upgrade. - if ( ! EE::exec( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 --max-time 300 -o %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $url ) ) ) ) { + // --max-time and --retry-max-time: a stalled download must not hang the upgrade (curl retries timeouts too). + if ( ! EE::exec( sprintf( 'curl -fsSL --retry 3 --retry-max-time 120 --connect-timeout 10 --max-time 300 -o %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $url ) ) ) ) { $failure = "the download of $url failed"; } else { - $checksum = strtok( trim( EE::launch( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 --max-time 60 %s', escapeshellarg( "$url.sha256" ) ) )->stdout ), " \t" ); + $checksum = strtok( trim( EE::launch( sprintf( 'curl -fsSL --retry 3 --retry-max-time 60 --connect-timeout 10 --max-time 60 %s', escapeshellarg( "$url.sha256" ) ) )->stdout ), " \t" ); $actual = is_file( $docker_compose_new_path ) ? hash_file( 'sha256', $docker_compose_new_path ) : ''; if ( ! preg_match( '/^[0-9a-f]{64}$/', (string) $checksum ) || ! hash_equals( (string) $checksum, (string) $actual ) ) { $failure = "the checksum of $url doesn't match"; From 2b6a4c7c4a18f14be5aa90154fc23f6346b1e8a5 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:36:29 +0530 Subject: [PATCH 16/27] fix(migration): keep the global compose snapshot of an upgrade that did not complete An upgrade killed after it recreated some global services leaves their new tags in the global compose file. The next attempt copied that half-upgraded file over the snapshot, so if it then failed, its rollback restored the new tags and recreated the services it had not touched yet, the database and the proxy included, on their new images instead of putting them back. The snapshot is now marked pending until the upgrade completes or is rolled back, and a run that finds a pending snapshot keeps it instead of taking a new one. --- php/EE/Migration/Containers.php | 11 ++++++++++- php/EE/Migration/GlobalContainers.php | 28 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 4f7d10586..608482f6f 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -114,6 +114,15 @@ public static function save_upgraded_image_versions( $current_versions, $new_ver [ $current_versions, $updated_images ] ); + // The upgrade is committed: a later run takes a fresh snapshot of the global compose file. + self::$rsp->add_step( + 'clear-pending-global-compose-backup', + 'EE\Migration\GlobalContainers::clear_pending_compose_backup', + null, + [ EE_BACKUP_DIR . '/services/docker-compose.yml.backup' ], + null + ); + self::$rsp->add_step( 'prune-old-docker-images', 'EE\Migration\Containers::docker_cleanup', @@ -371,7 +380,7 @@ private static function migrate_global_containers( $updated_images ) { self::$rsp->add_step( 'backup-global-docker-compose-file', - 'EE\Migration\SiteContainers::backup_restore', + 'EE\Migration\GlobalContainers::backup_compose_file', 'EE\Migration\GlobalContainers::revert_global_containers', [ $global_compose_file_path, $global_compose_file_backup_path ], [ $global_compose_file_backup_path, $global_compose_file_path, $updated_global_images ] diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 4d0f61b86..4f2a25d03 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -63,6 +63,33 @@ public static function get_service_name( $container ) { return preg_replace( '/^services_(.+)_1$/', '$1', $container ); } + /** + * Snapshot the global compose file before it is regenerated. A run that was interrupted after its snapshot left the + * services half upgraded, so its snapshot (marked pending until a run completes) is kept instead of the current file. + * + * @param string $source Global compose file. + * @param string $backup Snapshot path. + */ + public static function backup_compose_file( $source, $backup ) { + $fs = new Filesystem(); + if ( $fs->exists( [ $backup, $backup . '.pending' ] ) ) { + EE::debug( "Keeping $backup from an upgrade that did not complete" ); + + return; + } + $fs->copy( $source, $backup, true ); + $fs->touch( $backup . '.pending' ); + } + + /** + * Mark the compose snapshot as no longer needed by a later run: the upgrade completed or was rolled back. + * + * @param string $backup Snapshot path. + */ + public static function clear_pending_compose_backup( $backup ) { + ( new Filesystem() )->remove( $backup . '.pending' ); + } + /** * Start a new image migration with no service recorded as recreated. */ @@ -340,6 +367,7 @@ public static function revert_global_containers( $source_path, $dest_path, $upda EE::warning( 'After restoring the previous image: ' . $e->getMessage() ); } } + self::clear_pending_compose_backup( $source_path ); EE::debug( 'Complete restoring global docker-compose.yml file from backup' ); } From 8aaa57c5edc73a633845e671acb1e6b5547ba169 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 22:02:59 +0530 Subject: [PATCH 17/27] fix(migration): roll back the global services an interrupted attempt already upgraded A service that a killed upgrade had already recreated runs its new image, so the next attempt doesn't count it as updated. When that attempt failed, the rollback restored the old compose file but left the service on the new image, out of step with its compose configuration and recorded tag. The rollback now also recreates every global service whose running image differs from the one in the restored compose file, and checks its readiness like the others. --- php/EE/Migration/GlobalContainers.php | 47 ++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 4f2a25d03..c219489b8 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -4,6 +4,7 @@ use EE; use Symfony\Component\Filesystem\Filesystem; +use Symfony\Component\Yaml\Yaml; /** * Upgrade existing global containers to new docker-image @@ -340,29 +341,34 @@ public static function get_updated_global_images( $updated_images ) { */ public static function revert_global_containers( $source_path, $dest_path, $updated_images ) { - $services_to_regenerate = ''; - $all_global_images = self::get_all_global_images_with_service_name(); + $all_global_images = self::get_all_global_images_with_service_name(); + $services = []; foreach ( $updated_images as $image_name ) { - $services_to_regenerate .= self::get_service_name( $all_global_images[ $image_name ] ) . ' '; + $services[] = self::get_service_name( $all_global_images[ $image_name ] ); } $recreated = self::$recreated; self::$recreated = []; - if ( empty( trim( $services_to_regenerate ) ) ) { + if ( empty( $services ) ) { return; } EE::debug( 'Start restoring global docker-compose.yml file from backup' ); $fs = new Filesystem(); $fs->copy( $source_path, $dest_path, true ); + // Services an earlier, interrupted attempt moved to a new image aren't among this run's updates: put them back too. + $moved = array_diff( self::get_services_off_compose_file( $dest_path ), $services ); + $services = array_merge( $services, $moved ); + $recreate = array_values( array_merge( array_intersect( $services, $recreated ), $moved ) ); + chdir( EE_ROOT_DIR . '/services' ); - if ( ! self::compose_up( trim( $services_to_regenerate ), array_values( array_intersect( explode( ' ', trim( $services_to_regenerate ) ), $recreated ) ) ) ) { + if ( ! self::compose_up( implode( ' ', $services ), $recreate ) ) { throw new \Exception( 'Unable to downgrade global containers. Please check logs for more details.' ); } - foreach ( $updated_images as $image_name ) { - $container = $all_global_images[ $image_name ]; + $containers = array_combine( array_map( [ self::class, 'get_service_name' ], $all_global_images ), $all_global_images ); + foreach ( $services as $service ) { try { - self::wait_until_ready( self::get_service_name( $container ), $container ); + self::wait_until_ready( $service, $containers[ $service ] ); } catch ( \Exception $e ) { EE::warning( 'After restoring the previous image: ' . $e->getMessage() ); } @@ -371,6 +377,31 @@ public static function revert_global_containers( $source_path, $dest_path, $upda EE::debug( 'Complete restoring global docker-compose.yml file from backup' ); } + /** + * @param string $compose_file Global compose file. + * + * @return string[] Global services that run an image other than the one the compose file sets for them. + */ + private static function get_services_off_compose_file( $compose_file ) { + try { + $config = Yaml::parseFile( $compose_file ); + } catch ( \Exception $e ) { + return []; + } + $services = []; + foreach ( self::get_all_global_images_with_service_name() as $container ) { + $service = self::get_service_name( $container ); + $image = is_array( $config ) && isset( $config['services'][ $service ]['image'] ) ? $config['services'][ $service ]['image'] : ''; + $running = EE::launch( sprintf( "docker inspect --format '{{.State.Status}} {{.Config.Image}}' %s", escapeshellarg( $container ) ) ); + $parts = 0 === $running->return_code ? preg_split( '/\s+/', trim( $running->stdout ) ) : []; + if ( '' !== $image && 2 === count( $parts ) && 'running' === $parts[0] && $parts[1] !== $image ) { + $services[] = $service; + } + } + + return $services; + } + /** * Stop global container and remove them. * From 73d60d42d37c2582851253b2aef3e2adf5ac8d28 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 22:14:47 +0530 Subject: [PATCH 18/27] fix(migration): read the global compose snapshot with Spyc, which the phar ships The phar leaves out symfony/yaml, so the rollback stopped with a class-not-found error before it recreated anything. --- php/EE/Migration/GlobalContainers.php | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index c219489b8..70ab407a0 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -4,7 +4,7 @@ use EE; use Symfony\Component\Filesystem\Filesystem; -use Symfony\Component\Yaml\Yaml; +use Mustangostang\Spyc; /** * Upgrade existing global containers to new docker-image @@ -383,11 +383,8 @@ public static function revert_global_containers( $source_path, $dest_path, $upda * @return string[] Global services that run an image other than the one the compose file sets for them. */ private static function get_services_off_compose_file( $compose_file ) { - try { - $config = Yaml::parseFile( $compose_file ); - } catch ( \Exception $e ) { - return []; - } + // Spyc, not symfony/yaml: the phar doesn't ship symfony/yaml. + $config = is_file( $compose_file ) ? Spyc::YAMLLoad( $compose_file ) : []; $services = []; foreach ( self::get_all_global_images_with_service_name() as $container ) { $service = self::get_service_name( $container ); From ef097e193b21557f744f733b8e13a7669a89a3a9 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 22:35:13 +0530 Subject: [PATCH 19/27] fix(migration): recheck global services an interrupted upgrade recreated A killed upgrade leaves the services it recreated on their new tags without recording them, and running containers take precedence over the recorded tags. The next run therefore skipped them: one stuck in a restart loop on a broken new image was never gated, and with nothing else to update the run reported success and left the pending snapshot behind. While a pending snapshot exists, the global services now count by their recorded tags, so the next run recreates (a no-op when nothing changed) and gates them, and can roll them back to the snapshot. A restart-looping global service counts as present, both when choosing the services to upgrade and when a rollback looks for services off the snapshot. --- php/EE/Migration/Containers.php | 11 +++++++++++ php/EE/Migration/GlobalContainers.php | 17 ++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 608482f6f..b2ce883c6 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -37,6 +37,17 @@ public static function start_container_migration() { $current_versions = self::get_current_docker_images_versions(); $updated_images = []; + // An interrupted upgrade recreated some global services without recording their tags. Count those by their + // recorded tags, so this run checks their readiness again and can roll them back to the pending snapshot. + if ( GlobalContainers::has_pending_compose_backup( EE_BACKUP_DIR . '/services/docker-compose.yml.backup' ) ) { + foreach ( array_keys( GlobalContainers::get_all_global_images_with_service_name() ) as $image ) { + $recorded = EE\Model\Option::get( $image ); + if ( $recorded ) { + $current_versions[ $image ] = $recorded; + } + } + } + $skip_download = [ 'easyengine/php5.6', 'easyengine/php7.0', diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 70ab407a0..88d9a1998 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -73,7 +73,7 @@ public static function get_service_name( $container ) { */ public static function backup_compose_file( $source, $backup ) { $fs = new Filesystem(); - if ( $fs->exists( [ $backup, $backup . '.pending' ] ) ) { + if ( self::has_pending_compose_backup( $backup ) ) { EE::debug( "Keeping $backup from an upgrade that did not complete" ); return; @@ -82,6 +82,15 @@ public static function backup_compose_file( $source, $backup ) { $fs->touch( $backup . '.pending' ); } + /** + * @param string $backup Snapshot path. + * + * @return bool Whether an upgrade that did not complete left its snapshot. + */ + public static function has_pending_compose_backup( $backup ) { + return ( new Filesystem() )->exists( [ $backup, $backup . '.pending' ] ); + } + /** * Mark the compose snapshot as no longer needed by a later run: the upgrade completed or was rolled back. * @@ -323,7 +332,8 @@ public static function get_updated_global_images( $updated_images ) { $global_images = self::get_all_global_images_with_service_name(); $running_global_services = []; foreach ( $global_images as $image => $container_name ) { - if ( 'running' === \EE_DOCKER::container_status( $container_name ) ) { + // A restart-looping service is upgraded too (State.Running is false between its restarts). + if ( in_array( self::get_container_state( $container_name )['status'], [ 'running', 'restarting' ], true ) ) { $running_global_services[] = $image; } } @@ -391,7 +401,8 @@ private static function get_services_off_compose_file( $compose_file ) { $image = is_array( $config ) && isset( $config['services'][ $service ]['image'] ) ? $config['services'][ $service ]['image'] : ''; $running = EE::launch( sprintf( "docker inspect --format '{{.State.Status}} {{.Config.Image}}' %s", escapeshellarg( $container ) ) ); $parts = 0 === $running->return_code ? preg_split( '/\s+/', trim( $running->stdout ) ) : []; - if ( '' !== $image && 2 === count( $parts ) && 'running' === $parts[0] && $parts[1] !== $image ) { + // A restart-looping container counts: it is the failed new image to put back. One never created doesn't. + if ( '' !== $image && 2 === count( $parts ) && in_array( $parts[0], [ 'running', 'restarting' ], true ) && $parts[1] !== $image ) { $services[] = $service; } } From b0f11903ab225923f753308c557daa0884ab1eda Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 24 Oct 2026 00:05:26 +0530 Subject: [PATCH 20/27] fix(migration): record the new tag of a stopped global service only once it is upgraded The image migration recreated only the global services whose container was running, but saved the new tag of every changed image. A stopped or exited service (e.g. after `ee service disable redis`) kept its old container while the database said it was upgraded, so nothing would ever move it, and `ee service enable` started it on the old image. When no running global service changed, the global compose file wasn't regenerated either, so a service without a container (e.g. global-redis before the first cache site) was later created from the old tag. Now the updated global services are split by the state of their container: - running (or restart-looping): recreated and gated as before, and their new tags recorded; - stopped (exited, created, paused, dead): left alone, and their old tags stay recorded, so the database again says what each container runs; - missing: nothing to recreate; their new tags are recorded and the regenerated compose file is what they are created from later. The global compose file is now regenerated (and restored on a rollback) whenever any global image changed, so it carries the new tags of the stopped and missing services too: `ee service enable` brings a stopped service up from it on the new image once it recreates rather than restarts the container (a separate service-command change). Why not recreate a stopped service on its new image without starting it (`up --no-start`)? Docker brings a manually stopped `restart: always` container back when the daemon starts, but never a created one, so the upgrade would silently change what a stopped proxy or database does after a reboot. Its old image would also be pruned, leaving no way back if the new one (e.g. a MariaDB major version) can't open the datadir, and the datadir upgrade would happen later with no readiness gate. Leaving the stopped container as it is keeps its reboot behaviour and its image, cannot fail or roll back the upgrade, and keeps every recreate behind the readiness gate: the next upgrade that finds it running moves it, gated. --- php/EE/Migration/Containers.php | 21 ++++++++++----- php/EE/Migration/GlobalContainers.php | 39 ++++++++++++++++++++------- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index b2ce883c6..427252f98 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -82,9 +82,9 @@ public static function start_container_migration() { // First: compose v1 can't recreate containers on Docker 25+, and on a rollback it is restored last. self::maybe_update_docker_compose(); - self::migrate_global_containers( $updated_images ); + $not_upgraded = self::migrate_global_containers( $updated_images ); self::migrate_site_containers( $updated_images ); - self::save_upgraded_image_versions( $current_versions, $img_versions, $updated_images ); + self::save_upgraded_image_versions( $current_versions, $img_versions, array_values( array_diff( $updated_images, $not_upgraded ) ) ); if ( ! self::$rsp->execute() ) { throw new \Exception( 'Unable to migrate sites to newer version' ); @@ -365,15 +365,22 @@ public static function get_current_docker_images_versions() { * Migrates global containers. These are container which are not created per site (i.e. ee-cron-scheduler). * * @param $updated_images array of updated images. + * + * @return array Updated images of stopped services, which keep their old container and recorded tag. */ private static function migrate_global_containers( $updated_images ) { - $updated_global_images = GlobalContainers::get_updated_global_images( $updated_images ); - if ( empty( $updated_global_images ) ) { - return; + $global_compose_file_path = EE_ROOT_DIR . '/services/docker-compose.yml'; + + // A stopped service keeps its container (Docker restarts it on boot) and the old image the prune then keeps, so its + // old tag stays recorded; the regenerated compose file still has the new one for `ee service enable`. A missing + // service is created from that file later. Without a file, one is generated with the new tags when needed. + $by_state = GlobalContainers::get_updated_global_images_by_state( $updated_images ); + $updated_global_images = $by_state['running']; + if ( empty( $updated_global_images ) && ( ! is_file( $global_compose_file_path ) || empty( $by_state['stopped'] ) && empty( $by_state['missing'] ) ) ) { + return $by_state['stopped']; } - $global_compose_file_path = EE_ROOT_DIR . '/services/docker-compose.yml'; $global_compose_file_backup_path = EE_BACKUP_DIR . '/services/docker-compose.yml.backup'; $all_global_images = GlobalContainers::get_all_global_images_with_service_name(); GlobalContainers::reset_recreated(); @@ -417,6 +424,8 @@ private static function migrate_global_containers( $updated_images ) { null ); } + + return $by_state['stopped']; } /** diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 88d9a1998..94118b359 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -328,17 +328,38 @@ private static function pause( $seconds ) { * @return array */ public static function get_updated_global_images( $updated_images ) { + return self::get_updated_global_images_by_state( $updated_images )['running']; + } - $global_images = self::get_all_global_images_with_service_name(); - $running_global_services = []; - foreach ( $global_images as $image => $container_name ) { + /** + * Updated global images by the state of their service's container. + * + * @param array $updated_images Updated docker images. + * + * @return array `running` (running or restart-looping), `stopped` (any other existing container) and `missing` images. + */ + public static function get_updated_global_images_by_state( $updated_images ) { + $by_state = [ + 'running' => [], + 'stopped' => [], + 'missing' => [], + ]; + foreach ( self::get_all_global_images_with_service_name() as $image => $container_name ) { + if ( ! in_array( $image, $updated_images, true ) ) { + continue; + } + $status = self::get_container_state( $container_name )['status']; // A restart-looping service is upgraded too (State.Running is false between its restarts). - if ( in_array( self::get_container_state( $container_name )['status'], [ 'running', 'restarting' ], true ) ) { - $running_global_services[] = $image; + if ( in_array( $status, [ 'running', 'restarting' ], true ) ) { + $by_state['running'][] = $image; + } elseif ( '' === $status ) { + $by_state['missing'][] = $image; + } else { + $by_state['stopped'][] = $image; } } - return array_intersect( $running_global_services, $updated_images ); + return $by_state; } /** @@ -358,11 +379,9 @@ public static function revert_global_containers( $source_path, $dest_path, $upda } $recreated = self::$recreated; self::$recreated = []; - if ( empty( $services ) ) { - return; - } EE::debug( 'Start restoring global docker-compose.yml file from backup' ); $fs = new Filesystem(); + // Also when no running service changed: the regenerated file alone carries the new tags of the others. $fs->copy( $source_path, $dest_path, true ); // Services an earlier, interrupted attempt moved to a new image aren't among this run's updates: put them back too. @@ -372,7 +391,7 @@ public static function revert_global_containers( $source_path, $dest_path, $upda chdir( EE_ROOT_DIR . '/services' ); - if ( ! self::compose_up( implode( ' ', $services ), $recreate ) ) { + if ( $services && ! self::compose_up( implode( ' ', $services ), $recreate ) ) { throw new \Exception( 'Unable to downgrade global containers. Please check logs for more details.' ); } $containers = array_combine( array_map( [ self::class, 'get_service_name' ], $all_global_images ), $all_global_images ); From e49c44aed9550fb3360f6b3a7b0ed25d09f66177 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 24 Oct 2026 00:05:26 +0530 Subject: [PATCH 21/27] fix(migration): keep the networks of stopped containers and the global networks in the cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the image migration, `docker network prune` removed every EasyEngine network with no running container. That includes the networks of stopped containers, which `docker start` (e.g. `ee service enable` of a stopped global service) then fails on with "network not found", and the global networks that disabled sites join as external networks, so `ee site enable` of a disabled WordPress site failed when only stopped services used ee-global-backend-network. The cleanup now removes only EasyEngine networks that no container, running or stopped, is attached to (`docker ps -a --filter network=…`), and never the two global networks. --- php/EE/Migration/Containers.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 427252f98..a62d429ea 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -149,7 +149,17 @@ public static function save_upgraded_image_versions( $current_versions, $new_ver */ public static function docker_cleanup() { EE::exec( 'docker image prune -af --filter=label=org.label-schema.vendor="EasyEngine"' ); - EE::exec( 'docker network prune -f --filter=label=org.label-schema.vendor="EasyEngine"' ); + + // Not `docker network prune`: it also removes the networks of stopped containers, which then can't start, and the + // global networks that disabled sites join as external networks on `ee site enable`. + $keep = [ defined( 'GLOBAL_FRONTEND_NETWORK' ) ? GLOBAL_FRONTEND_NETWORK : 'ee-global-frontend-network', defined( 'GLOBAL_BACKEND_NETWORK' ) ? GLOBAL_BACKEND_NETWORK : 'ee-global-backend-network' ]; + $networks = array_filter( explode( "\n", trim( EE::launch( 'docker network ls --filter=label=org.label-schema.vendor="EasyEngine" --format "{{.Name}}"' )->stdout ) ) ); + foreach ( array_diff( $networks, $keep ) as $network ) { + $users = EE::launch( 'docker ps -aq --filter network=' . escapeshellarg( $network ) ); + if ( 0 === $users->return_code && '' === trim( $users->stdout ) ) { + EE::exec( 'docker network rm ' . escapeshellarg( $network ) ); + } + } } /** From e8b6c7a77b7f07171d4c7f76061e3b8955dd3b7e Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 24 Oct 2026 00:06:06 +0530 Subject: [PATCH 22/27] fix(migration): move disabled sites' compose files to the new images The image migration picked the sites to upgrade from `docker-compose images`, which lists nothing for a disabled site (`ee site disable` removes its containers). Disabled sites were therefore skipped while their images were recorded as upgraded and the old ones pruned, so a later `ee site enable` (without `--refresh`) re-pulled the old tags and ran them next to config a container migration may already have moved to the new format. A disabled site is now selected from its own docker-compose.yml, when it sets one of the changed images to a tag other than this release's (the same rule as for enabled sites, which are selected from their containers). Its file is backed up and regenerated the way `ee site enable --refresh` does it (`dump_docker_compose_yml()`), and nothing is started, so `ee site enable` then starts the site on the current images and site-command needs no change. Only the main docker-compose.yml is regenerated, as for enabled sites; `docker-compose-custom.yml` and `user-docker-compose/` stay as they are. The backup and the regeneration are one step that only warns when it fails (with the `ee site enable --refresh` hint) and puts the old file back: a site that serves nothing must not block or roll back the upgrade of the live ones. A regenerated file with an image that has no tag (what a missing `img-versions.json` entry produces, with only a PHP warning) counts as a failure. A rollback of the run restores the file only if this run backed it up, never a backup left by an earlier run. The new tags of a disabled site's images may still be pruned when no container uses them; `ee site enable` pulls them like any site create, and no longer needs the old ones. --- php/EE/Migration/Containers.php | 19 +++++- php/EE/Migration/SiteContainers.php | 94 +++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index a62d429ea..3442ba739 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -447,14 +447,29 @@ private static function migrate_global_containers( $updated_images ) { */ public static function migrate_site_containers( $updated_images ) { - $db = new \EE_DB(); - $sites = ( $db->table( 'sites' )->all() ); + $db = new \EE_DB(); + $sites = ( $db->table( 'sites' )->all() ); + $img_versions = EE\Utils\get_image_versions(); foreach ( $sites as $site ) { $docker_yml = $site['site_fs_path'] . '/docker-compose.yml'; $docker_yml_backup = EE_BACKUP_DIR . '/' . $site['site_url'] . '/docker-compose.yml.backup'; + // A disabled site has no containers, only its compose file, which nothing else would move to the new images. + if ( ! $site['site_enabled'] ) { + if ( SiteContainers::is_disabled_site_image_changed( $updated_images, $site, $img_versions ) ) { + self::$rsp->add_step( + sprintf( 'update-disabled-%s-docker-compose', $site['site_url'] ), + 'EE\Migration\SiteContainers::update_disabled_site_compose_file', + 'EE\Migration\SiteContainers::restore_disabled_site_compose_file', + [ $site, $docker_yml_backup ], + [ $site ] + ); + } + continue; + } + if ( ! SiteContainers::is_site_service_image_changed( $updated_images, $site ) ) { continue; } diff --git a/php/EE/Migration/SiteContainers.php b/php/EE/Migration/SiteContainers.php index e3a9d3bea..041a062e6 100644 --- a/php/EE/Migration/SiteContainers.php +++ b/php/EE/Migration/SiteContainers.php @@ -4,6 +4,7 @@ use EE; use Symfony\Component\Filesystem\Filesystem; +use Mustangostang\Spyc; /** * Migrate site specific containers to new images. @@ -93,6 +94,99 @@ public static function is_site_service_image_changed( $updated_images, $site_inf return false; } + /** @var string[] Disabled sites whose compose file this run backed up, the only ones a rollback restores. */ + private static $backed_up = []; + + /** + * EasyEngine images a site's docker-compose.yml sets, as image => tag. + * + * @param string $compose_file Path of the file. + * + * @return array + */ + public static function get_compose_file_images( $compose_file ) { + try { + // Spyc, not symfony/yaml: the phar doesn't ship symfony/yaml. + $config = is_file( $compose_file ) ? Spyc::YAMLLoad( $compose_file ) : []; + } catch ( \Exception $e ) { + $config = []; + } + $images = []; + if ( ! is_array( $config ) || empty( $config['services'] ) || ! is_array( $config['services'] ) ) { + return $images; + } + foreach ( $config['services'] as $service ) { + if ( is_array( $service ) && ! empty( $service['image'] ) && is_string( $service['image'] ) && 0 === strpos( $service['image'], 'easyengine/' ) ) { + list( $image, $tag ) = explode( ':', $service['image'], 2 ) + [ 1 => '' ]; + $images[ $image ] = $tag; + } + } + + return $images; + } + + /** + * Whether a disabled site's docker-compose.yml sets one of the updated images to a tag other than this release's. + * A disabled site has no containers for `docker-compose images` to list. + * + * @param array $updated_images Updated images. + * @param array $site_info Site row. + * @param array $img_versions This release's image tags. + * + * @return bool + */ + public static function is_disabled_site_image_changed( $updated_images, $site_info, $img_versions ) { + foreach ( self::get_compose_file_images( $site_info['site_fs_path'] . '/docker-compose.yml' ) as $image => $tag ) { + if ( in_array( $image, $updated_images, true ) && isset( $img_versions[ $image ] ) && $img_versions[ $image ] !== $tag ) { + return true; + } + } + + return false; + } + + /** + * Back up and regenerate a disabled site's docker-compose.yml with this release's images, without starting anything, + * so that `ee site enable` starts it on them. A failure only warns and keeps the old file: a disabled site must not + * block or roll back the upgrade of the others. + * + * @param array $site_info Site row. + * @param string $backup_path Where to back up its docker-compose.yml. + */ + public static function update_disabled_site_compose_file( $site_info, $backup_path ) { + $compose_file = $site_info['site_fs_path'] . '/docker-compose.yml'; + $fs = new Filesystem(); + unset( self::$backed_up[ $site_info['site_url'] ] ); + try { + $fs->copy( $compose_file, $backup_path, true ); + self::$backed_up[ $site_info['site_url'] ] = $backup_path; + self::generate_site_docker_compose_file( $site_info, self::get_site_object( $site_info['site_type'] ) ); + // A missing img-versions.json entry only raises a PHP warning and writes an image without a tag. + if ( preg_match( '/^\s*image:\s*\S+:\s*$/m', (string) file_get_contents( $compose_file ) ) ) { + throw new \Exception( 'an image has no tag' ); + } + } catch ( \Throwable $e ) { + if ( isset( self::$backed_up[ $site_info['site_url'] ] ) ) { + $fs->copy( $backup_path, $compose_file, true ); + } + unset( self::$backed_up[ $site_info['site_url'] ] ); + $reason = '' === $e->getMessage() ? 'see the error above' : $e->getMessage(); + EE::warning( sprintf( 'Could not update docker-compose.yml of the disabled site %1$s to the new images (%2$s). Run `ee site enable %1$s --refresh` to enable it on them.', $site_info['site_url'], $reason ) ); + } + } + + /** + * Rollback of update_disabled_site_compose_file(): restore the file it backed up in this run, if any. + * + * @param array $site_info Site row. + */ + public static function restore_disabled_site_compose_file( $site_info ) { + if ( isset( self::$backed_up[ $site_info['site_url'] ] ) ) { + ( new Filesystem() )->copy( self::$backed_up[ $site_info['site_url'] ], $site_info['site_fs_path'] . '/docker-compose.yml', true ); + unset( self::$backed_up[ $site_info['site_url'] ] ); + } + } + /** * Generate docker-compose.yml for specific site. * From c9d7b35ffc0c0bf42e4bca64b9ab357848dfa2af Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 24 Oct 2026 00:41:42 +0530 Subject: [PATCH 23/27] fix(migration): neither pull nor redo the upgrade for images only stopped global services use A stopped global service keeps its old tag recorded, so its image counts as changed on every later image migration. Each run pulled it, and the cleanup then pruned the pulled image again because no container uses it: one wasted pull per upgrade on stable, and on nightly (where the migration runs on every command) a full pull on every command, which failed every command while Docker Hub was unreachable or rate-limited. Now: - an image only stopped global services use, and nothing running, is not pulled; Compose pulls it when something first uses it (`ee service enable`), as for untracked images. A stopped service already on its new tag has it locally; - when the only changed images are those, the global compose file already has their new tags and no disabled site needs updating, the migration ends there instead of regenerating files and inspecting every site again; - a stopped service whose container already runs the new tag (recreated by an interrupted run) has that tag recorded, so the database says what it runs and the cleanup may prune the old image. Images of missing services are still pulled and then recorded, as before, so they don't come back on the next run. --- php/EE/Migration/Containers.php | 85 +++++++++++++++++++++------ php/EE/Migration/GlobalContainers.php | 25 ++++++-- php/EE/Migration/SiteContainers.php | 22 +------ 3 files changed, 88 insertions(+), 44 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 3442ba739..b35a994e0 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -23,6 +23,9 @@ class Containers { /** @var bool Whether the last update_docker_compose() call replaced the binary. */ private static $replaced_docker_compose = false; + /** @var string[] Images (without tag) the last get_current_docker_images_versions() found running. */ + private static $running_images = []; + /** * Main entry point for the script. Migrates all containers * @@ -64,14 +67,9 @@ public static function start_container_migration() { 'easyengine/newrelic-daemon', ]; - // An image no container runs and the DB doesn't track (e.g. mailhog, the amd64-only easyengine/php) has nothing - // to upgrade: pulling it only fails on hosts it isn't built for. Compose pulls it when something first uses it. foreach ( $img_versions as $img => $version ) { if ( array_key_exists( $img, $current_versions ) && $current_versions[ $img ] !== $version ) { $updated_images[] = $img; - if ( ! in_array( $img, $skip_download, true ) ) { - self::pull_or_error( $img, $version ); - } } } @@ -80,11 +78,28 @@ public static function start_container_migration() { return; } + // Stopped global services keep their old image and tag (see migrate_global_containers()), so their images stay + // "updated" on every later run: when nothing running uses them and nothing else is left to do, stop here. + $by_state = GlobalContainers::get_updated_global_images_by_state( $updated_images, $img_versions ); + $idle = array_diff( $by_state['stopped'], self::$running_images ); + if ( ! array_diff( $updated_images, $idle ) && ! self::has_old_tags( EE_SERVICE_DIR . '/docker-compose.yml', $idle, $img_versions ) && ! self::get_disabled_sites_to_update( $updated_images, $img_versions ) ) { + EE::debug( 'Only stopped global services have new images, and their compose files have them already.' ); + + return; + } + + // An image no container runs and the DB doesn't track (e.g. mailhog, the amd64-only easyengine/php) has nothing + // to upgrade: pulling it only fails on hosts it isn't built for. Nor has one only stopped services use, which keep + // their old image. Compose pulls these when something first uses them. A stopped service on its new tag has it. + foreach ( array_diff( $updated_images, $idle, $by_state['current'], $skip_download ) as $img ) { + self::pull_or_error( $img, $img_versions[ $img ] ); + } + // First: compose v1 can't recreate containers on Docker 25+, and on a rollback it is restored last. self::maybe_update_docker_compose(); - $not_upgraded = self::migrate_global_containers( $updated_images ); + self::migrate_global_containers( $by_state ); self::migrate_site_containers( $updated_images ); - self::save_upgraded_image_versions( $current_versions, $img_versions, array_values( array_diff( $updated_images, $not_upgraded ) ) ); + self::save_upgraded_image_versions( $current_versions, $img_versions, array_values( array_diff( $updated_images, $by_state['stopped'] ) ) ); if ( ! self::$rsp->execute() ) { throw new \Exception( 'Unable to migrate sites to newer version' ); @@ -357,7 +372,7 @@ public static function get_current_docker_images_versions() { $dockerImages = EE::launch( 'docker ps --format "{{.Image}}" | grep "^easyengine/" | sort -u' )->stdout; $dockerImages = explode( "\n", trim( $dockerImages ) ); - $dockerImages = array_reduce( $dockerImages, function ( $result, $image ) { + $dockerImages = array_reduce( array_filter( $dockerImages ), function ( $result, $image ) { [ $imageName, $tag ] = explode( ':', $image, 2 ) + [ 1 => null ]; $result[ $imageName ] = $tag; @@ -365,7 +380,8 @@ public static function get_current_docker_images_versions() { return $result; }, [] ); - $mergedImages = $dockerImages + $dbImages; + self::$running_images = array_keys( $dockerImages ); + $mergedImages = $dockerImages + $dbImages; $mergedImages = array_filter( $mergedImages ); return $mergedImages; @@ -374,21 +390,18 @@ public static function get_current_docker_images_versions() { /** * Migrates global containers. These are container which are not created per site (i.e. ee-cron-scheduler). * - * @param $updated_images array of updated images. - * - * @return array Updated images of stopped services, which keep their old container and recorded tag. + * @param array $by_state Updated global images by state, from GlobalContainers::get_updated_global_images_by_state(). */ - private static function migrate_global_containers( $updated_images ) { + private static function migrate_global_containers( $by_state ) { $global_compose_file_path = EE_ROOT_DIR . '/services/docker-compose.yml'; - // A stopped service keeps its container (Docker restarts it on boot) and the old image the prune then keeps, so its - // old tag stays recorded; the regenerated compose file still has the new one for `ee service enable`. A missing + // A stopped service keeps its container (Docker restarts it on boot) and the old image the cleanup then keeps, so + // its old tag stays recorded; the regenerated compose file has the new one for `ee service enable`. A missing // service is created from that file later. Without a file, one is generated with the new tags when needed. - $by_state = GlobalContainers::get_updated_global_images_by_state( $updated_images ); $updated_global_images = $by_state['running']; if ( empty( $updated_global_images ) && ( ! is_file( $global_compose_file_path ) || empty( $by_state['stopped'] ) && empty( $by_state['missing'] ) ) ) { - return $by_state['stopped']; + return; } $global_compose_file_backup_path = EE_BACKUP_DIR . '/services/docker-compose.yml.backup'; @@ -434,8 +447,42 @@ private static function migrate_global_containers( $updated_images ) { null ); } + } - return $by_state['stopped']; + /** + * @param string $compose_file A docker-compose.yml. + * @param array $images Images to check. + * @param array $img_versions This release's tags. + * + * @return bool Whether the file sets one of $images to a tag other than this release's. + */ + private static function has_old_tags( $compose_file, $images, $img_versions ) { + foreach ( SiteContainers::get_compose_file_images( $compose_file ) as $image => $tag ) { + if ( in_array( $image, $images, true ) && isset( $img_versions[ $image ] ) && $img_versions[ $image ] !== $tag ) { + return true; + } + } + + return false; + } + + /** + * @param array $updated_images Updated images. + * @param array $img_versions This release's tags. + * + * @return array Disabled sites whose docker-compose.yml sets an updated image to an old tag. + */ + private static function get_disabled_sites_to_update( $updated_images, $img_versions ) { + $sites = ( new \EE_DB() )->table( 'sites' )->all(); + + return array_values( + array_filter( + $sites, + function ( $site ) use ( $updated_images, $img_versions ) { + return ! $site['site_enabled'] && self::has_old_tags( $site['site_fs_path'] . '/docker-compose.yml', $updated_images, $img_versions ); + } + ) + ); } /** @@ -458,7 +505,7 @@ public static function migrate_site_containers( $updated_images ) { // A disabled site has no containers, only its compose file, which nothing else would move to the new images. if ( ! $site['site_enabled'] ) { - if ( SiteContainers::is_disabled_site_image_changed( $updated_images, $site, $img_versions ) ) { + if ( self::has_old_tags( $docker_yml, $updated_images, $img_versions ) ) { self::$rsp->add_step( sprintf( 'update-disabled-%s-docker-compose', $site['site_url'] ), 'EE\Migration\SiteContainers::update_disabled_site_compose_file', diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 94118b359..a8846e038 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -246,6 +246,17 @@ private static function get_container_state( $container ) { return array_combine( [ 'status', 'restarting', 'restart_count', 'health' ], array_pad( array_slice( $parts, 0, 4 ), 4, '' ) ); } + /** + * @param string $container Container name. + * + * @return string The image it was created from, e.g. `easyengine/redis:v4.13.1`, or ''. + */ + private static function get_container_image( $container ) { + $result = EE::launch( sprintf( "docker inspect --format '{{.Config.Image}}' %s", escapeshellarg( $container ) ) ); + + return 0 === $result->return_code ? trim( $result->stdout ) : ''; + } + /** * @param string $service Compose service. * @param string $container Container name. @@ -334,14 +345,18 @@ public static function get_updated_global_images( $updated_images ) { /** * Updated global images by the state of their service's container. * - * @param array $updated_images Updated docker images. + * @param array $updated_images Updated docker images. + * @param array|null $img_versions This release's tags; defaults to img-versions.json. * - * @return array `running` (running or restart-looping), `stopped` (any other existing container) and `missing` images. + * @return array `running` (running or restart-looping), `stopped` (any other existing container, still on an old tag), + * `current` (stopped, but already on the new tag, e.g. recreated by an interrupted run) and `missing` images. */ - public static function get_updated_global_images_by_state( $updated_images ) { - $by_state = [ + public static function get_updated_global_images_by_state( $updated_images, $img_versions = null ) { + $img_versions = null === $img_versions ? EE\Utils\get_image_versions() : $img_versions; + $by_state = [ 'running' => [], 'stopped' => [], + 'current' => [], 'missing' => [], ]; foreach ( self::get_all_global_images_with_service_name() as $image => $container_name ) { @@ -354,6 +369,8 @@ public static function get_updated_global_images_by_state( $updated_images ) { $by_state['running'][] = $image; } elseif ( '' === $status ) { $by_state['missing'][] = $image; + } elseif ( isset( $img_versions[ $image ] ) && self::get_container_image( $container_name ) === "$image:{$img_versions[ $image ]}" ) { + $by_state['current'][] = $image; } else { $by_state['stopped'][] = $image; } diff --git a/php/EE/Migration/SiteContainers.php b/php/EE/Migration/SiteContainers.php index 041a062e6..60483582c 100644 --- a/php/EE/Migration/SiteContainers.php +++ b/php/EE/Migration/SiteContainers.php @@ -98,7 +98,7 @@ public static function is_site_service_image_changed( $updated_images, $site_inf private static $backed_up = []; /** - * EasyEngine images a site's docker-compose.yml sets, as image => tag. + * EasyEngine images a docker-compose.yml sets, as image => tag. * * @param string $compose_file Path of the file. * @@ -125,26 +125,6 @@ public static function get_compose_file_images( $compose_file ) { return $images; } - /** - * Whether a disabled site's docker-compose.yml sets one of the updated images to a tag other than this release's. - * A disabled site has no containers for `docker-compose images` to list. - * - * @param array $updated_images Updated images. - * @param array $site_info Site row. - * @param array $img_versions This release's image tags. - * - * @return bool - */ - public static function is_disabled_site_image_changed( $updated_images, $site_info, $img_versions ) { - foreach ( self::get_compose_file_images( $site_info['site_fs_path'] . '/docker-compose.yml' ) as $image => $tag ) { - if ( in_array( $image, $updated_images, true ) && isset( $img_versions[ $image ] ) && $img_versions[ $image ] !== $tag ) { - return true; - } - } - - return false; - } - /** * Back up and regenerate a disabled site's docker-compose.yml with this release's images, without starting anything, * so that `ee site enable` starts it on them. A failure only warns and keeps the old file: a disabled site must not From 8d5107bc5d3f2323cd6d83af6825ed2c33260a0d Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 24 Oct 2026 00:41:42 +0530 Subject: [PATCH 24/27] fix(migration): keep a disabled site's failed compose update best-effort when its file can't be restored --- php/EE/Migration/SiteContainers.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/php/EE/Migration/SiteContainers.php b/php/EE/Migration/SiteContainers.php index 60483582c..203dff56b 100644 --- a/php/EE/Migration/SiteContainers.php +++ b/php/EE/Migration/SiteContainers.php @@ -146,8 +146,12 @@ public static function update_disabled_site_compose_file( $site_info, $backup_pa throw new \Exception( 'an image has no tag' ); } } catch ( \Throwable $e ) { - if ( isset( self::$backed_up[ $site_info['site_url'] ] ) ) { - $fs->copy( $backup_path, $compose_file, true ); + try { + if ( isset( self::$backed_up[ $site_info['site_url'] ] ) ) { + $fs->copy( $backup_path, $compose_file, true ); + } + } catch ( \Exception $restore_error ) { + EE::warning( sprintf( 'Could not restore %s from %s: %s', $compose_file, $backup_path, $restore_error->getMessage() ) ); } unset( self::$backed_up[ $site_info['site_url'] ] ); $reason = '' === $e->getMessage() ? 'see the error above' : $e->getMessage(); From 6281803480ca92c5e829c42662bef2c8061a7f36 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 24 Oct 2026 01:08:57 +0530 Subject: [PATCH 25/27] fix(migration): update stale disabled sites on every run, pin the rollback snapshot to running images, clear a finished snapshot Three gaps in the handling of stopped services and disabled sites: - A disabled site was only checked for the images that changed in this run. An interrupted run that had already recreated the enabled sites' nginx, or an earlier upgrade that recorded nginx without updating the disabled sites, left nginx out of that list, so those sites kept the old tag (and with no other change the migration stopped before looking at sites at all). Disabled sites are now compared with this release's tags, and checked even when no image changed; nothing else runs then (no pull, no recreate, no cleanup). - A stopped service an earlier upgrade deferred has its new tag in the global compose file while its container keeps the old image. Once Docker restarted it (e.g. on boot), the next upgrade's snapshot took that untested tag, so a failed readiness gate rolled the service back onto it instead of the image that worked. A fresh snapshot now sets each global service to the image its container was created from; a pending snapshot from an interrupted run is kept as it is. - When only deferred stopped services were left, the early end didn't clear a pending snapshot left by an interrupted run, so a later upgrade would have rolled back to that outdated snapshot. Nothing running needs a gate in that case, so the snapshot is now marked done. --- php/EE/Migration/Containers.php | 28 ++++++++++++++++++--------- php/EE/Migration/GlobalContainers.php | 22 +++++++++++++++++++++ 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index b35a994e0..117ff47fe 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -74,6 +74,13 @@ public static function start_container_migration() { } if ( empty( $updated_images ) ) { + // Disabled sites an earlier upgrade skipped, or an interrupted one left behind, have no image in $updated_images. + if ( self::get_disabled_sites_to_update( $img_versions ) ) { + self::migrate_site_containers( [] ); + if ( ! self::$rsp->execute() ) { + throw new \Exception( 'Unable to migrate sites to newer version' ); + } + } return; } @@ -82,8 +89,10 @@ public static function start_container_migration() { // "updated" on every later run: when nothing running uses them and nothing else is left to do, stop here. $by_state = GlobalContainers::get_updated_global_images_by_state( $updated_images, $img_versions ); $idle = array_diff( $by_state['stopped'], self::$running_images ); - if ( ! array_diff( $updated_images, $idle ) && ! self::has_old_tags( EE_SERVICE_DIR . '/docker-compose.yml', $idle, $img_versions ) && ! self::get_disabled_sites_to_update( $updated_images, $img_versions ) ) { + if ( ! array_diff( $updated_images, $idle ) && ! self::has_old_tags( EE_SERVICE_DIR . '/docker-compose.yml', $idle, $img_versions ) && ! self::get_disabled_sites_to_update( $img_versions ) ) { EE::debug( 'Only stopped global services have new images, and their compose files have them already.' ); + // Nothing running needs a gate (that would count as updated), so an interrupted run's snapshot is done with. + GlobalContainers::clear_pending_compose_backup( EE_BACKUP_DIR . '/services/docker-compose.yml.backup' ); return; } @@ -467,19 +476,18 @@ private static function has_old_tags( $compose_file, $images, $img_versions ) { } /** - * @param array $updated_images Updated images. - * @param array $img_versions This release's tags. + * @param array $img_versions This release's tags. * - * @return array Disabled sites whose docker-compose.yml sets an updated image to an old tag. + * @return array Disabled sites whose docker-compose.yml sets an EasyEngine image to a tag other than this release's. */ - private static function get_disabled_sites_to_update( $updated_images, $img_versions ) { + private static function get_disabled_sites_to_update( $img_versions ) { $sites = ( new \EE_DB() )->table( 'sites' )->all(); return array_values( array_filter( $sites, - function ( $site ) use ( $updated_images, $img_versions ) { - return ! $site['site_enabled'] && self::has_old_tags( $site['site_fs_path'] . '/docker-compose.yml', $updated_images, $img_versions ); + function ( $site ) use ( $img_versions ) { + return ! $site['site_enabled'] && self::has_old_tags( $site['site_fs_path'] . '/docker-compose.yml', array_keys( $img_versions ), $img_versions ); } ) ); @@ -504,8 +512,10 @@ public static function migrate_site_containers( $updated_images ) { $docker_yml_backup = EE_BACKUP_DIR . '/' . $site['site_url'] . '/docker-compose.yml.backup'; // A disabled site has no containers, only its compose file, which nothing else would move to the new images. + // It is compared with this release's tags, not with this run's changed images: those can miss an image an + // interrupted run already moved for the enabled sites, or an earlier upgrade recorded without updating it. if ( ! $site['site_enabled'] ) { - if ( self::has_old_tags( $docker_yml, $updated_images, $img_versions ) ) { + if ( self::has_old_tags( $docker_yml, array_keys( $img_versions ), $img_versions ) ) { self::$rsp->add_step( sprintf( 'update-disabled-%s-docker-compose', $site['site_url'] ), 'EE\Migration\SiteContainers::update_disabled_site_compose_file', @@ -517,7 +527,7 @@ public static function migrate_site_containers( $updated_images ) { continue; } - if ( ! SiteContainers::is_site_service_image_changed( $updated_images, $site ) ) { + if ( ! $updated_images || ! SiteContainers::is_site_service_image_changed( $updated_images, $site ) ) { continue; } diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index a8846e038..7d1889077 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -79,9 +79,31 @@ public static function backup_compose_file( $source, $backup ) { return; } $fs->copy( $source, $backup, true ); + self::pin_running_images( $backup ); $fs->touch( $backup . '.pending' ); } + /** + * Set each global service in a compose snapshot to the image its container was created from. A stopped service an + * earlier upgrade left on its old image already has its new tag in the compose file, and a rollback must bring back + * the image that worked, not that untested one. + * + * @param string $compose_file Snapshot of the global compose file. + */ + private static function pin_running_images( $compose_file ) { + $config = (string) file_get_contents( $compose_file ); + $pinned = $config; + foreach ( self::get_all_global_images_with_service_name() as $image => $container ) { + $current = self::get_container_image( $container ); + if ( 0 === strpos( $current, "$image:" ) ) { + $pinned = preg_replace( '/^(\s*container_name:\s*' . preg_quote( $container, '/' ) . '\s*\n\s*image:\s*)' . preg_quote( $image, '/' ) . ':\S+[ \t]*$/m', '${1}' . $current, $pinned, 1 ); + } + } + if ( $pinned !== $config ) { + file_put_contents( $compose_file, $pinned ); + } + } + /** * @param string $backup Snapshot path. * From bd7eb476c514647ca28050cf42ad10e1f1f4251b Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Sat, 24 Oct 2026 01:57:18 +0530 Subject: [PATCH 26/27] fix(migration): clear a finished rollback snapshot when no image changed A run interrupted after it saved the new tags but before it marked the global compose snapshot done left the snapshot pending. The retry found no changed image and returned early, so the marker stayed, and the next release's upgrade reused that outdated snapshot: a failed upgrade there would have restored images and configuration from before an already completed release. With every tag recorded and nothing changed there is nothing left to roll back, so the early return now marks the snapshot done. --- php/EE/Migration/Containers.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 117ff47fe..c8898a93c 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -81,6 +81,8 @@ public static function start_container_migration() { throw new \Exception( 'Unable to migrate sites to newer version' ); } } + // Every tag is recorded (e.g. a run interrupted just after saving them): a pending snapshot is done with. + GlobalContainers::clear_pending_compose_backup( EE_BACKUP_DIR . '/services/docker-compose.yml.backup' ); return; } From 625d170145c71a2e333cc5b65d39fde00f620114 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Mon, 26 Oct 2026 08:26:28 +0530 Subject: [PATCH 27/27] fix(migration): bound every readiness probe with a timeout inside the container The readiness probes ran `docker exec` with no time limit, so a redis or database that accepts connections but never answers hung the upgrade forever. Each probe (redis-cli ping, the database ping and nginx -t, including the check of the running proxy config) now runs under `timeout 10` inside the container, and a probe killed by it counts as not ready, so the existing wait and rollback handle it. Images whose `timeout` doesn't work (busybox before 1.30) run the probe as before. --- php/EE/Migration/GlobalContainers.php | 62 +++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 88d9a1998..6320e387b 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -33,6 +33,9 @@ class GlobalContainers { /** Seconds a service must stay up without restarting after its first successful probe. */ const SETTLE_SECONDS = 2; + /** Seconds one readiness probe may run inside the container before it counts as failed. */ + const PROBE_TIMEOUT = 10; + /** Services with a readiness probe. The others (cron, newrelic-daemon) only need to have started. */ const PROBED_SERVICES = [ 'global-nginx-proxy', 'global-db', 'global-redis' ]; @@ -116,7 +119,14 @@ public static function reset_recreated() { * @throws \Exception */ public static function check_running_proxy_config( $container ) { - if ( 'running' === self::get_container_state( $container )['status'] && ! EE::exec( 'docker exec ' . escapeshellarg( $container ) . ' nginx -t' ) ) { + if ( 'running' !== self::get_container_state( $container )['status'] ) { + return; + } + $result = self::probe_exec( $container, 'nginx', '-t' ); + if ( self::timed_out( $result ) ) { + throw new \Exception( sprintf( '`nginx -t` in the running %s did not finish within %d s, so the upgrade cannot check its configuration. Check `docker exec %s nginx -t` and run the upgrade again.', $container, self::PROBE_TIMEOUT, $container ) ); + } + if ( $result->return_code ) { throw new \Exception( "The configuration of the running $container fails `nginx -t`, so a recreated proxy would not start. Fix it (see `docker exec $container nginx -t`) and run the upgrade again." ); } } @@ -276,12 +286,11 @@ private static function not_ready_reason( $service, $container, $state ) { * @return string Why it failed, or '' when it passed. */ private static function probe( $service, $container ) { - $exec = 'docker exec ' . escapeshellarg( $container ) . ' '; - switch ( $service ) { case 'global-nginx-proxy': - if ( ! EE::exec( $exec . 'nginx -t' ) ) { - return 'nginx -t failed'; + $result = self::probe_exec( $container, 'nginx', '-t' ); + if ( $result->return_code ) { + return self::probe_failure( 'nginx -t', $result ); } // Any HTTP status proves nginx answers through the published port; 443 is the same process. $port = (string) \EE\Utils\get_config_value( 'proxy_80_port', '80' ); @@ -296,18 +305,55 @@ private static function probe( $service, $container ) { // Over TCP: the entrypoint's temporary init/upgrade server listens on the socket only. Access denied still means alive. $ping = 'for c in mariadb-admin mysqladmin; do command -v $c >/dev/null 2>&1 && exec $c ping -h 127.0.0.1 --protocol=tcp --connect-timeout=2 --silent; done; exit 1'; - return EE::exec( $exec . 'sh -c ' . escapeshellarg( $ping ) ) ? '' : 'mysqladmin ping failed'; + $result = self::probe_exec( $container, 'sh', '-c', $ping ); + + return $result->return_code ? self::probe_failure( 'mysqladmin ping', $result ) : ''; case 'global-redis': - $pong = trim( EE::launch( $exec . 'redis-cli ping' )->stdout ); + $result = self::probe_exec( $container, 'redis-cli', 'ping' ); + $pong = trim( $result->stdout ); // NOAUTH (a requirepass in redis.conf) still means redis answers. - return 'PONG' === $pong || 0 === strpos( $pong, 'NOAUTH' ) ? '' : 'redis-cli ping failed'; + return 'PONG' === $pong || 0 === strpos( $pong, 'NOAUTH' ) ? '' : self::probe_failure( 'redis-cli ping', $result ); } return ''; } + /** + * Run a probe command in a container under `timeout`, so a service that accepts connections but never answers + * can't hang the upgrade. Where `timeout` doesn't work (busybox before 1.30 wants `-t`), the command runs unbounded. + * + * @param string $container Container name. + * @param string ...$args Command and its arguments. + * + * @return \EE\ProcessRun + */ + private static function probe_exec( $container, ...$args ) { + $bounded = sprintf( 'if timeout 1 true >/dev/null 2>&1; then exec timeout %d "$@"; fi; exec "$@"', self::PROBE_TIMEOUT ); + + return EE::launch( 'docker exec ' . escapeshellarg( $container ) . ' sh -c ' . escapeshellarg( $bounded ) . ' probe ' . implode( ' ', array_map( 'escapeshellarg', $args ) ) ); + } + + /** + * @param \EE\ProcessRun $result From probe_exec(). + * + * @return bool Whether `timeout` killed the probe: 124 from coreutils, 143 (SIGTERM) from busybox. + */ + private static function timed_out( $result ) { + return in_array( (int) $result->return_code, [ 124, 143 ], true ); + } + + /** + * @param string $probe Probe name. + * @param \EE\ProcessRun $result From probe_exec(). + * + * @return string Why the probe failed. + */ + private static function probe_failure( $probe, $result ) { + return self::timed_out( $result ) ? sprintf( '%s timed out after %d s', $probe, self::PROBE_TIMEOUT ) : "$probe failed"; + } + /** * @param int|float $seconds Seconds to sleep. */