diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index f8fce960..4c562a1e 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -12,6 +12,8 @@ jobs: test: #---------------------------------------------------------------------- runs-on: ubuntu-latest name: Behat Tests - PHP ${{ matrix.php }} + env: + RCLONE_CONFIG: /etc/rclone/rclone.conf strategy: fail-fast: false matrix: @@ -57,6 +59,14 @@ jobs: sudo curl -L "https://github.com/docker/compose/releases/download/$VERSION/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose + - name: Set up rclone for backup and restore + run: | + sudo apt-get update -q + sudo apt-get install -y -q rclone + sudo mkdir -p "$(dirname "$RCLONE_CONFIG")" /var/tmp/ee-backups + # EasyEngine's default rclone-path is easyengine:easyengine. + printf '[easyengine]\ntype = alias\nremote = /var/tmp/ee-backups\n' | sudo tee "$RCLONE_CONFIG" >/dev/null + - name: Install dependencies run: | cd "$GITHUB_WORKSPACE/.." @@ -69,6 +79,8 @@ jobs: composer install --prefer-dist --no-progress --no-interaction else sed -i 's/\(easyengine\/.*\):\ \".*\"/\1:\ \"dev-develop\"/' composer.json + # Test this checkout of site-command, not its develop branch. + composer config repositories.site-command '{"type": "path", "url": "'"$GITHUB_WORKSPACE"'", "options": {"symlink": false, "versions": {"easyengine/site-command": "dev-develop"}}}' composer update --prefer-dist --no-progress --no-interaction fi composer dump-autoload --no-dev diff --git a/features/upload-progress.feature b/features/upload-progress.feature new file mode 100644 index 00000000..0ab45c68 --- /dev/null +++ b/features/upload-progress.feature @@ -0,0 +1,170 @@ +@remote @destructive +Feature: Session upload progress stays off + Re-runs the upgrade migrations on the host (destructive) and backs up to the rclone remote from config.yml. + Those steps run only on CI or with EE_BEHAT_DESTRUCTIVE=1. + + # "STDOUT should return exactly" checks a scenario's first command, so each check is its own scenario. + # The scenarios share their sites and run in order. + + Scenario: Sites with a CRLF PATH-only setting, an explicit global On and no custom.ini + When I run 'bin/ee site create bh-site-command-upload-crlf.test --type=php' + And I run 'bash features/upload-progress.sh php-ini-on bh-site-command-upload-crlf.test && bash features/upload-progress.sh crlf-path bh-site-command-upload-crlf.test' + And I run 'bash features/upload-progress.sh probe bh-site-command-upload-crlf.test' + And I run 'bin/ee site reload bh-site-command-upload-crlf.test' + And I run 'bin/ee site create bh-site-command-upload-on.test --type=php' + And I run 'bash features/upload-progress.sh append-on bh-site-command-upload-on.test' + And I run 'bash features/upload-progress.sh probe bh-site-command-upload-on.test' + And I run 'bin/ee site reload bh-site-command-upload-on.test' + And I run 'bash features/upload-progress.sh save bh-site-command-upload-on.test' + And I run 'bin/ee site create bh-site-command-upload-missing.test --type=php' + And I run 'bash features/upload-progress.sh remove bh-site-command-upload-missing.test' + And I run 'bin/ee site reload bh-site-command-upload-missing.test' + Then return value of command 1 should be 0 + And return value of command 2 should be 0 + And return value of command 3 should be 0 + And return value of command 4 should be 0 + And return value of command 5 should be 0 + And return value of command 6 should be 0 + And return value of command 7 should be 0 + And return value of command 8 should be 0 + And return value of command 9 should be 0 + And return value of command 10 should be 0 + And return value of command 11 should be 0 + And return value of command 12 should be 0 + + Scenario: Before the upgrade the PATH-only site runs with upload progress On + When I run 'curl -s http://bh-site-command-upload-crlf.test/upe.php' + Then STDOUT should return exactly + """ + string(1) "1" + """ + + Scenario: Upgrade + When I run 'bash features/upload-progress.sh upgrade' + Then return value should be 0 + And STDERR should return something like + """ + Skipped disabling session.upload_progress for bh-site-command-upload-missing.test: custom.ini is missing or not a regular file. + """ + + Scenario: The PATH-only site runs with upload progress Off + When I run 'sleep 3; curl -s http://bh-site-command-upload-crlf.test/upe.php' + Then STDOUT should return exactly + """ + string(0) "" + """ + + Scenario: The Off line is at the top + When I run 'bash features/upload-progress.sh line2 bh-site-command-upload-crlf.test' + Then STDOUT should return exactly + """ + session.upload_progress.enabled = Off + """ + + Scenario: The explicit On is kept + When I run 'curl -s http://bh-site-command-upload-on.test/upe.php' + Then STDOUT should return exactly + """ + string(1) "1" + """ + + Scenario: The file with the explicit On is unchanged + When I run 'bash features/upload-progress.sh same bh-site-command-upload-on.test' + Then STDOUT should return exactly + """ + same + """ + + Scenario: A missing custom.ini is not created by the upgrade + When I run 'bash features/upload-progress.sh exists bh-site-command-upload-missing.test' + Then STDOUT should return exactly + """ + missing + """ + + Scenario: Run the migration again + When I run 'bash features/upload-progress.sh save bh-site-command-upload-crlf.test && bash features/upload-progress.sh upgrade' + Then return value should be 0 + And STDERR should return something like + """ + Skipped disabling session.upload_progress for bh-site-command-upload-missing.test: custom.ini is missing or not a regular file. + """ + + Scenario: A second run changes nothing + When I run 'bash features/upload-progress.sh same bh-site-command-upload-crlf.test' + Then STDOUT should return exactly + """ + same + """ + + Scenario: Back up a site whose custom.ini lacks the Off line + When I run 'bin/ee site create bh-site-command-upload-restore.test --type=php' + And I run 'bash features/upload-progress.sh php-ini-on bh-site-command-upload-restore.test && bash features/upload-progress.sh strip bh-site-command-upload-restore.test' + And I run 'bash features/upload-progress.sh probe bh-site-command-upload-restore.test' + And I run 'bin/ee site reload bh-site-command-upload-restore.test' + And I run 'bash features/upload-progress.sh backup bh-site-command-upload-restore.test' + Then return value of command 1 should be 0 + And return value of command 2 should be 0 + And return value of command 3 should be 0 + And return value of command 4 should be 0 + And return value of command 5 should be 0 + + Scenario: Without the Off line the site runs with upload progress On + When I run 'curl -s http://bh-site-command-upload-restore.test/upe.php' + Then STDOUT should return exactly + """ + string(1) "1" + """ + + Scenario: Restoring that backup turns upload progress off again + When I run 'bash features/upload-progress.sh restore bh-site-command-upload-restore.test' + Then return value should be 0 + And STDOUT should return something like + """ + The restored custom.ini did not set session.upload_progress.enabled, so it was set to Off again. + """ + + Scenario: The restored file has the Off line at the top + When I run 'bash features/upload-progress.sh line2 bh-site-command-upload-restore.test' + Then STDOUT should return exactly + """ + session.upload_progress.enabled = Off + """ + + Scenario: The restored site runs with upload progress Off + When I run 'sleep 3; curl -s http://bh-site-command-upload-restore.test/upe.php' + Then STDOUT should return exactly + """ + string(0) "" + """ + + Scenario: Back up the site without custom.ini + When I run 'bash features/upload-progress.sh backup bh-site-command-upload-missing.test' + Then return value should be 0 + + Scenario: Restoring a backup without custom.ini warns instead of creating it + When I run 'bash features/upload-progress.sh restore bh-site-command-upload-missing.test' + Then return value should be 0 + And STDERR should return something like + """ + Could not check session.upload_progress in the restored custom.ini: it is missing or not a regular file. + """ + + Scenario: The restore did not create custom.ini + When I run 'bash features/upload-progress.sh exists bh-site-command-upload-missing.test' + Then STDOUT should return exactly + """ + missing + """ + + Scenario: Clean up + When I run 'bin/ee site delete bh-site-command-upload-crlf.test --yes' + And I run 'bin/ee site delete bh-site-command-upload-on.test --yes' + And I run 'bin/ee site delete bh-site-command-upload-missing.test --yes' + And I run 'bin/ee site delete bh-site-command-upload-restore.test --yes' + And I run 'bash features/upload-progress.sh clean' + Then return value of command 1 should be 0 + And return value of command 2 should be 0 + And return value of command 3 should be 0 + And return value of command 4 should be 0 + And return value of command 5 should be 0 diff --git a/features/upload-progress.sh b/features/upload-progress.sh new file mode 100755 index 00000000..ce9f0e4b --- /dev/null +++ b/features/upload-progress.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# upload-progress.sh []: helpers for upload-progress.feature, whose steps can't nest quotes. +set -e +ini=/opt/easyengine/sites/$2/config/php/php/conf.d/custom.ini +guard() { + if [ -z "$CI" ] && [ "$EE_BEHAT_DESTRUCTIVE" != 1 ]; then + echo "$1 re-runs the upgrade migrations or uses the rclone remote: only on CI or with EE_BEHAT_DESTRUCTIVE=1" >&2 + exit 1 + fi +} +case $1 in +# Before v4.13.3 neither php.ini nor custom.ini set it; On is set explicitly so newer templates and images don't hide the gap. +php-ini-on) sed -i '/session\.upload_progress\.enabled/d' "${ini%/conf.d/custom.ini}/php.ini"; printf '\nsession.upload_progress.enabled = On\n' >> "${ini%/conf.d/custom.ini}/php.ini" ;; +crlf-path) printf 'upload_max_filesize = 100M\r\n[PATH=/var/www/htdocs/x]\r\nsession.upload_progress.enabled = Off\r\n' > "$ini" ;; +append-on) printf '\nsession.upload_progress.enabled = On\n' >> "$ini" ;; +strip) sed -i '/session\.upload_progress\.enabled/d; /Session upload progress is off/d' "$ini" ;; +remove) rm "$ini" ;; +exists) if [ -e "$ini" ] || [ -L "$ini" ]; then echo present; else echo missing; fi ;; +probe) printf '%s' ' "/opt/easyengine/sites/$2/app/htdocs/upe.php" ;; +# Runs the upgrade migrations again, the re-check included. +upgrade) + guard upgrade + php <<'PHP' +setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); +$db->exec( "DELETE FROM migrations WHERE migration LIKE '%recheck_session_upload_progress%'" ); +$db->exec( "UPDATE options SET value = '4.13.2' WHERE key = 'version'" ); +PHP + bin/ee cli info ;; +backup | restore) guard "$1"; bin/ee site "$1" "$2" ;; +save) cp "$ini" "/tmp/upload-progress-$2.before" ;; +same) cmp "/tmp/upload-progress-$2.before" "$ini" && echo same ;; +line2) tr -s '\r\n' '\n' < "$ini" | sed -n 2p ;; +clean) rm -f /tmp/upload-progress-bh-site-command-upload-*.before ;; +*) echo "unknown action $1" >&2; exit 1 ;; +esac diff --git a/migrations/container/20261023193000_site-command_drop_deprecated_session_sid_settings.php b/migrations/container/20261023193000_site-command_drop_deprecated_session_sid_settings.php new file mode 100644 index 00000000..2f673f6f --- /dev/null +++ b/migrations/container/20261023193000_site-command_drop_deprecated_session_sid_settings.php @@ -0,0 +1,262 @@ + '26', + 'session.sid_bits_per_character' => '5', + ]; + + /** Comments shipped right above each directive, removed together with it. */ + const SHIPPED_COMMENTS = [ + 'session.sid_length' => [ + '; Set session ID character length. This value could be between 22 to 256.', + '; Shorter length than default is supported only for compatibility reason.', + '; Users should use 32 or more chars.', + '; http://php.net/session.sid-length', + '; Default Value: 32', + '; Development Value: 26', + '; Production Value: 26', + ], + 'session.sid_bits_per_character' => [ + '; Define how many bits are stored in each character when converting', + '; the binary hash data to something readable.', + '; Possible values:', + '; 4 (4 bits: 0-9, a-f)', + '; 5 (5 bits: 0-9, a-v)', + '; 6 (6 bits: 0-9, a-z, A-Z, "-", ",")', + '; Default Value: 4', + '; Development Value: 5', + '; Production Value: 5', + '; http://php.net/session.hash-bits-per-character', + ], + ]; + + /** Entry of the quick reference at the top of the shipped php.ini. */ + const SHIPPED_QUICK_REFERENCE = [ + '; session.sid_bits_per_character', + '; Default Value: 4', + '; Development Value: 5', + '; Production Value: 5', + ]; + + private $sites; + + /** @var array Files changed by up(), restored by down(). */ + private $changed = []; + + public function __construct() { + + parent::__construct(); + $this->sites = Site::all(); + if ( $this->is_first_execution || ! $this->sites ) { + $this->skip_this_migration = true; + } + } + + /** + * @throws EE\ExitException + */ + public function up() { + + if ( $this->skip_this_migration ) { + EE::debug( 'Skipping drop-deprecated-session-sid-settings migration as it is not needed.' ); + + return; + } + + foreach ( $this->sites as $site ) { + if ( ! in_array( $site->site_type, [ 'php', 'wp' ], true ) || ! in_array( (string) $site->php_version, self::PHP_VERSIONS, true ) ) { + continue; + } + + // One broken site must not block the upgrade of the others. + try { + $this->migrate_site( $site ); + } catch ( \Throwable $e ) { + EE::warning( sprintf( 'Could not remove the deprecated session ID settings from the php.ini of %s: %s', $site->site_url, $e->getMessage() ) ); + } + } + } + + /** + * Restores the files changed by up(). + * + * @throws EE\ExitException + */ + public function down() { + + foreach ( array_reverse( $this->changed ) as $change ) { + try { + $old = file_get_contents( $change['backup'] ); + if ( false === $old ) { + throw new \Exception( 'Could not read ' . $change['backup'] ); + } + put_container_file_contents( $change['ini'], $change['dir'], $old, null, self::MAX_SIZE ); + $this->fs->remove( $change['backup'] ); + self::reload_fpm( $change['site'] ); + } catch ( \Throwable $e ) { + EE::warning( sprintf( 'Could not restore the php.ini of %s from %s: %s', $change['site']->site_url, $change['backup'], $e->getMessage() ) ); + } + } + $this->changed = []; + } + + /** + * @param Site $site PHP 8.4 or 8.5 site. + * + * @throws \Exception + */ + private function migrate_site( $site ) { + + $ini = $site->site_fs_path . '/config/php/php/php.ini'; + + // Missing on macOS, where php.ini stays in the container. + if ( ! file_exists( $ini ) && ! is_link( $ini ) ) { + EE::debug( "Skipping $site->site_url: no php.ini at $ini." ); + + return; + } + + // The site's php container can swap anything in its config volume, so php.ini is read and rewritten in place through a checked handle. + $dir = site_php_conf_dir( $site->site_fs_path, 'php' ); + $content = get_container_file_contents( $ini, $dir, self::MAX_SIZE ); + + list( $new_content, $removed ) = self::strip_settings( $content ); + if ( $new_content === $content ) { + EE::debug( "$ini has no deprecated session ID settings." ); + + return; + } + + $backup = EE_BACKUP_DIR . '/' . $site->site_url . '/php.ini.with-session-sid-settings'; + $this->fs->dumpFile( $backup, $content ); + + try { + put_container_file_contents( $ini, $dir, $new_content, $content, self::MAX_SIZE ); + } catch ( \Exception $e ) { + // A failed write may have left php.ini partly written; any other failure left it as it was. + if ( 0 === strpos( $e->getMessage(), 'could not write' ) ) { + throw new \Exception( $e->getMessage() . "; the previous file is $backup" ); + } + $this->fs->remove( $backup ); + throw $e; + } + + $this->changed[] = [ + 'site' => $site, + 'ini' => $ini, + 'dir' => $dir, + 'backup' => $backup, + ]; + EE::debug( "Removed the deprecated session ID settings from $ini; the previous file is $backup." ); + + foreach ( $removed as $line => $shipped ) { + if ( ! $shipped ) { + EE::warning( sprintf( 'Removed `%s` from the php.ini of %s: it is deprecated as of PHP 8.4, and PHP now uses its default. The previous file is saved as %s.', $line, $site->site_url, $backup ) ); + } + } + + self::reload_fpm( $site ); + } + + /** + * Removes the two directives, wherever they are set, and the shipped comments that document them. + * Every other line is kept byte for byte. + * + * @param string $content php.ini content. + * + * @return array New content, and the removed directive lines => whether the value is the shipped one. + */ + public static function strip_settings( $content ) { + + // After every LF, and after a CR not followed by LF, so a CR-only file isn't read as one line. + $lines = preg_split( '/(?<=\n)|(?<=\r)(?!\n)/', $content, -1, PREG_SPLIT_NO_EMPTY ); + $norm = array_map( + function ( $line ) { + return rtrim( $line, " \t\r\n" ); + }, + $lines + ); + $drop = []; + $removed = []; + + $quick_count = count( self::SHIPPED_QUICK_REFERENCE ); + foreach ( array_keys( $norm ) as $i ) { + if ( array_slice( $norm, $i, $quick_count ) === self::SHIPPED_QUICK_REFERENCE ) { + $drop += array_fill( $i, $quick_count, true ); + if ( isset( $norm[ $i + $quick_count ] ) && '' === $norm[ $i + $quick_count ] ) { + $drop[ $i + $quick_count ] = true; + } + } + } + + foreach ( $norm as $i => $line ) { + if ( ! preg_match( '/^\s*(session\.sid_length|session\.sid_bits_per_character)\s*=(.*)$/', $line, $matches ) ) { + continue; + } + $name = $matches[1]; + $drop[ $i ] = true; + + // Inline comment and quotes don't change the value. + $value = trim( preg_replace( '/;.*$/', '', $matches[2] ), " \t\"'" ); + $removed[ trim( $line ) ] = self::SHIPPED_VALUES[ $name ] === $value; + + $comments = self::SHIPPED_COMMENTS[ $name ]; + $count = count( $comments ); + if ( $i >= $count && array_slice( $norm, $i - $count, $count ) === $comments ) { + $drop += array_fill( $i - $count, $count, true ); + if ( isset( $norm[ $i + 1 ] ) && '' === $norm[ $i + 1 ] ) { + $drop[ $i + 1 ] = true; + } + } + } + + return [ implode( '', array_diff_key( $lines, $drop ) ), $removed ]; + } + + /** + * PHP-FPM reads php.ini only when it starts; a graceful reload (as `ee site reload --php` does) applies it. + * Stopped containers read it when they start. + * + * @param Site $site Site. + */ + private static function reload_fpm( $site ) { + + if ( ! $site->site_enabled || ! chdir( $site->site_fs_path ) ) { + return; + } + + $compose = \EE_DOCKER::docker_compose_with_custom(); + $id = trim( EE::launch( "$compose ps -q php" )->stdout ); + if ( '' === $id || 'running' !== \EE_DOCKER::container_status( $id ) ) { + EE::debug( "The php container of $site->site_url is not running." ); + + return; + } + + if ( EE::launch( "$compose exec -T php bash -c 'kill -USR2 1'" )->return_code ) { + EE::warning( sprintf( 'Could not reload PHP-FPM of %1$s. Run `ee site restart %1$s --php` to apply its new php.ini.', $site->site_url ) ); + } + } +} diff --git a/migrations/container/20261023233000_site-command_recheck_session_upload_progress.php b/migrations/container/20261023233000_site-command_recheck_session_upload_progress.php new file mode 100644 index 00000000..8d18a1b8 --- /dev/null +++ b/migrations/container/20261023233000_site-command_recheck_session_upload_progress.php @@ -0,0 +1,414 @@ + [ 'site' => Site, 'file' => string, 'dir' => string, 'backup' => string, 'written' => bool ] for every custom.ini changed by up(). */ + private $changed = []; + + /** @var array site_url => php container ID seen by up(), for running sites. */ + private $containers = []; + + /** @var string|null */ + private $backup_root; + + public function __construct() { + + parent::__construct(); + if ( $this->is_first_execution ) { + $this->skip_this_migration = true; + + return; + } + + $this->sites = array_filter( + Site::all() ?: [], + function ( $site ) { + return in_array( $site->site_type, self::SITE_TYPES, true ); + } + ); + if ( empty( $this->sites ) ) { + $this->skip_this_migration = true; + } + } + + /** + * Add the setting to every custom.ini that doesn't set it globally. + */ + public function up() { + + if ( $this->skip_this_migration ) { + EE::debug( 'Skipping recheck-session-upload-progress migration as it is not needed.' ); + + return; + } + + $this->backup_root = EE_BACKUP_DIR . '/session-upload-progress-recheck-' . date( 'Ymd-His' ); + if ( file_exists( $this->backup_root ) ) { + $this->backup_root .= '-' . getmypid(); + } + + foreach ( $this->sites as $site ) { + // One broken site must not block the upgrade of the others. + try { + $this->update_site( $site ); + } catch ( \Throwable $e ) { + if ( isset( $this->changed[ $site->site_url ] ) ) { + $this->restore_file( $site->site_url ); + } + unset( $this->containers[ $site->site_url ] ); + EE::warning( "Could not check session.upload_progress in the custom.ini of $site->site_url: {$e->getMessage()}. " . self::manual_fix( $site ) ); + } + } + + if ( empty( $this->changed ) ) { + $this->remove_empty_backup_root(); + + return; + } + + $written = array_filter( array_column( $this->changed, 'written' ) ); + if ( ! $written ) { + return; + } + EE::log( 'Disabled session.upload_progress in custom.ini of ' . count( $written ) . ' more site(s) where it was not set globally. The original files are in ' . $this->backup_root ); + + // The image migration may recreate php containers, which then read the new file anyway; reload only the others, once it is done. + if ( method_exists( '\EE\Migration\CustomContainerMigrations', 'keep_executed_migrations' ) ) { + EE::add_hook( 'after_docker_image_migration', [ $this, 'reload_unchanged_containers' ] ); + } else { + $this->reload_unchanged_containers(); + } + } + + /** + * Restore the original custom.ini files and reload the running sites. + */ + public function down() { + + if ( empty( $this->changed ) ) { + return; + } + + $sites = array_column( $this->changed, 'site' ); + $this->restore_files(); + + // A rolled-back image migration may have restarted them on the new file. + foreach ( $sites as $site ) { + if ( $site->site_enabled ) { + $id = $this->get_php_container( $site ); + if ( null !== $id && '' !== $id ) { + $this->reload_php( $site->site_url, $id ); + } + } + } + } + + /** + * Gracefully reload php-fpm of the sites whose php container is still the one up() saw. + */ + public function reload_unchanged_containers() { + + foreach ( $this->containers as $site_url => $id ) { + if ( ! isset( $this->changed[ $site_url ] ) ) { + continue; + } + $now = $this->get_php_container( $this->changed[ $site_url ]['site'] ); + if ( $now !== $id ) { + if ( null !== $now ) { + EE::debug( "$site_url: php container was recreated or stopped, no reload needed." ); + } + continue; + } + $this->reload_php( $site_url, $id ); + } + $this->containers = []; + } + + /** + * Whether PHP reads an ini value as true, like zend_ini_parse_bool(), which falls back to atoi(). + * + * @param string $value Value as the ini scanner gives it. + * + * @return bool + */ + public static function is_on( string $value ) { + + return in_array( strtolower( $value ), [ 'true', 'yes', 'on' ], true ) || 1 === preg_match( '/^\s*[+-]?0*[1-9]/', $value ); + } + + /** + * @param Site $site Site. + * + * @return string The manual fix, for warnings. + */ + private static function manual_fix( $site ) { + + return "Add 'session.upload_progress.enabled = Off' to the top of $site->site_fs_path/config/php/php/conf.d/custom.ini and run: ee site reload $site->site_url"; + } + + /** + * Add the setting to one site's custom.ini. Problems with one site are warnings, so they don't block the upgrade. + * + * @param Site $site Site to check. + * + * @throws \Exception + */ + private function update_site( $site ) { + + $file = $site->site_fs_path . '/config/php/php/conf.d/custom.ini'; + $manual = self::manual_fix( $site ); + + // Not created when missing: PHP can't create a file safely in a directory the site's container can swap. The v4.13.3 migration created it wherever conf.d existed. + clearstatcache( true ); + if ( is_link( $file ) || ! is_file( $file ) ) { + EE::warning( "Skipped disabling session.upload_progress for $site->site_url: custom.ini is missing or not a regular file. $manual" ); + + return; + } + + // Read-only first: a file that needs no change may be read-only or hard-linked. + $dir = site_php_conf_dir( $site->site_fs_path ); + $handle = open_container_file( $file, 'r', $dir ); + try { + $content = read_container_file( $handle, self::MAX_SIZE ); + } finally { + fclose( $handle ); + } + + $new_content = add_session_upload_progress_off( $content ); + if ( null === $new_content ) { + $value = ini_global_value( preg_replace( '/^\xEF\xBB\xBF/', '', $content ), self::DIRECTIVE ); + if ( null !== $value && self::is_on( $value ) ) { + EE::log( "Kept session.upload_progress On for $site->site_url: its custom.ini turns it on. Set it to Off there unless your app needs it." ); + } else { + EE::debug( "$site->site_url: custom.ini already turns off session.upload_progress." ); + } + + return; + } + + if ( ! ini_sets_globally( preg_replace( '/^\xEF\xBB\xBF/', '', $new_content ), self::DIRECTIVE ) ) { + EE::warning( "Skipped disabling session.upload_progress for $site->site_url: could not add it to custom.ini safely. $manual" ); + + return; + } + + $handle = open_container_file( $file, 'r+', $dir ); + try { + if ( read_container_file( $handle, self::MAX_SIZE ) !== $content ) { + throw new \Exception( 'custom.ini changed while it was checked' ); + } + $written = $this->write_ini( $site, $handle, $file, $dir, $content, $new_content, $manual ); + } finally { + fclose( $handle ); + } + + if ( ! $written ) { + return; + } + + $id = $site->site_enabled ? $this->get_php_container( $site ) : ''; + if ( null !== $id && '' !== $id ) { + $this->containers[ $site->site_url ] = $id; + } + EE::debug( "$site->site_url: disabled session.upload_progress in custom.ini." ); + } + + /** + * Back up and rewrite custom.ini through the handle from open_container_file(). + * + * @param Site $site Site. + * @param resource $handle custom.ini handle. + * @param string $file custom.ini path. + * @param string $dir Real path of conf.d. + * @param string $content Current content. + * @param string $new_content New content. + * @param string $manual Manual fix, for warnings. + * + * @return bool Whether the file now holds the new content. + */ + private function write_ini( $site, $handle, string $file, string $dir, string $content, string $new_content, string $manual ) { + + $stat = fstat( $handle ); + $backup = $this->backup_root . '/' . $site->site_url . '/custom.ini'; + try { + $this->fs->mkdir( dirname( $backup ), 0700 ); + $this->fs->dumpFile( $backup, $content ); + $this->fs->chmod( $backup, $stat['mode'] & 0777 ); + $this->fs->touch( $backup, $stat['mtime'] ); + } catch ( \Throwable $e ) { + EE::warning( "Skipped disabling session.upload_progress for $site->site_url: could not back up custom.ini ({$e->getMessage()}). $manual" ); + + return false; + } + + $entry = [ + 'site' => $site, + 'file' => $file, + 'dir' => $dir, + 'backup' => $backup, + 'written' => true, + ]; + + try { + rewrite_container_file( $handle, $new_content ); + } catch ( \Exception $e ) { + EE::warning( "Could not disable session.upload_progress for $site->site_url: could not write custom.ini. $manual" ); + try { + rewrite_container_file( $handle, $content ); + } catch ( \Exception $e2 ) { + // Left for down() to retry from the backup. + $entry['written'] = false; + $this->changed[ $site->site_url ] = $entry; + EE::warning( "Could not restore $file; its original is $backup" ); + + return false; + } + $this->remove_backup( $backup ); + + return false; + } + + $this->changed[ $site->site_url ] = $entry; + + return true; + } + + /** + * Restore every file changed by up(). + */ + private function restore_files() { + + foreach ( array_keys( $this->changed ) as $site_url ) { + $this->restore_file( $site_url ); + } + $this->containers = []; + $this->remove_empty_backup_root(); + } + + /** + * Restore one file changed by up(); its backup is removed once restored. Never throws. + * + * @param string $site_url Site. + */ + private function restore_file( string $site_url ) { + + $entry = $this->changed[ $site_url ]; + unset( $this->changed[ $site_url ] ); + + try { + $content = @file_get_contents( $entry['backup'] ); + if ( false === $content ) { + throw new \Exception( 'could not read the backup' ); + } + $handle = open_container_file( $entry['file'], 'r+', $entry['dir'] ); + try { + // Skip the write when it never happened; read no more than needed to tell. + if ( stream_get_contents( $handle, strlen( $content ) + 1 ) !== $content ) { + rewrite_container_file( $handle, $content ); + } + } finally { + fclose( $handle ); + } + } catch ( \Throwable $e ) { + EE::warning( "Could not restore {$entry['file']} ({$e->getMessage()}). Its original is {$entry['backup']}" ); + + return; + } + $this->remove_backup( $entry['backup'] ); + EE::debug( "$site_url: restored custom.ini." ); + } + + /** + * @param string|null $backup Backup file; its directory is removed. + */ + private function remove_backup( $backup ) { + + if ( null === $backup ) { + return; + } + try { + $this->fs->remove( dirname( $backup ) ); + } catch ( \Throwable $e ) { + EE::debug( "Could not remove $backup: {$e->getMessage()}" ); + } + } + + private function remove_empty_backup_root() { + + try { + if ( ! empty( $this->backup_root ) && is_dir( $this->backup_root ) && ! ( new \FilesystemIterator( $this->backup_root ) )->valid() ) { + $this->fs->remove( $this->backup_root ); + } + } catch ( \Throwable $e ) { + EE::debug( "Could not remove the empty $this->backup_root: {$e->getMessage()}" ); + } + } + + /** + * @param Site $site Site. + * + * @return string|null ID of the site's running php container, '' when none runs, null when the lookup failed (already warned). + */ + private function get_php_container( $site ) { + + $cwd = getcwd(); + $result = @chdir( $site->site_fs_path ) ? EE::launch( \EE_DOCKER::docker_compose_with_custom() . ' ps -q php' ) : null; + if ( false !== $cwd ) { + chdir( $cwd ); + } + + if ( null === $result || 0 !== $result->return_code ) { + EE::warning( "Could not check the php container of $site->site_url, so php-fpm may still run with the old session.upload_progress setting. Fix the error, then run: ee site reload $site->site_url" ); + + return null; + } + + return trim( $result->stdout ); + } + + /** + * Same graceful reload as `ee site reload`: php-fpm re-reads its ini files. + * + * @param string $site_url Site. + * @param string $id php container ID. + */ + private function reload_php( string $site_url, string $id ) { + + $result = EE::launch( 'docker exec --user root ' . escapeshellarg( $id ) . " sh -c 'kill -USR2 1'" ); + if ( 0 === $result->return_code ) { + EE::debug( "$site_url: reloaded php-fpm." ); + } else { + EE::warning( "Could not reload php-fpm of $site_url. Run: ee site reload $site_url" ); + } + } +} diff --git a/src/helper/Site_Backup_Restore.php b/src/helper/Site_Backup_Restore.php index d2620cc4..7583930c 100644 --- a/src/helper/Site_Backup_Restore.php +++ b/src/helper/Site_Backup_Restore.php @@ -53,6 +53,9 @@ class Site_Backup_Restore { // Global backup lock handle for serializing backups private $global_backup_lock_handle = null; + // ID of the site's php container while its config is restored; '' when it isn't running. + private $php_container_id; + public function __construct() { $this->fs = new Filesystem(); } @@ -1684,15 +1687,294 @@ private function restore_php_conf( $backup_dir ) { } if ( $this->fs->exists( sprintf( '%s/php', $backup_dir ) ) ) { - $restore_command = sprintf( 'rsync -a %s/php/php-fpm.d/ %s/config/php/php-fpm.d/', $backup_dir, EE_ROOT_DIR . '/sites/' . $this->site_data['site_url'] ); - EE::exec( $restore_command ); + $this->restore_php_conf_files( $backup_dir . '/php' ); + } - $restore_command = sprintf( 'rsync -a %s/php/php/php.ini %s/config/php/php/php.ini', $backup_dir, EE_ROOT_DIR . '/sites/' . $this->site_data['site_url'] ); - EE::exec( $restore_command ); + $this->keep_session_upload_progress_off(); + } - $restore_command = sprintf( 'rsync -a %s/php/php/conf.d/custom.ini %s/config/php/php/conf.d/custom.ini', $backup_dir, EE_ROOT_DIR . '/sites/' . $this->site_data['site_url'] ); - EE::exec( $restore_command ); + /** + * Copy the backed-up php-fpm.d tree, php.ini and custom.ini over the site's ones. + * + * The container can swap anything in its config volume, so existing files are written in place through checked handles and missing entries are created inside the container; the backup came from that volume too, so it may hold only directories and regular files. + * Permission bits come from the backup; owners don't (the backup has none), so a created entry belongs to its parent's owner and an existing one keeps its own. + * + * @param string $source Extracted `php` directory of the backup. + */ + private function restore_php_conf_files( $source ) { + + $site_url = $this->site_data['site_url']; + $site_dir = EE_ROOT_DIR . '/sites/' . $site_url; + clearstatcache( true ); + // Some 7-Zip versions store config/php itself as a link, which points back at the live volume. + if ( is_link( $source ) ) { + EE::warning( 'Could not restore the php config: the backup holds a symlink instead of it.' ); + + return; } + + $entries = []; + $refused = []; + foreach ( [ 'php', 'php/php.ini', 'php/conf.d', 'php/conf.d/custom.ini' ] as $path ) { + $this->list_php_conf_backup( $source, $path, false, $entries, $refused ); + } + $this->list_php_conf_backup( $source, 'php-fpm.d', true, $entries, $refused ); + foreach ( $refused as $path => $reason ) { + EE::warning( "Could not restore config/php/$path: $reason." ); + } + + $this->php_container_id = null; + $kept = EE_BACKUP_DIR . '/' . $site_url . '-php-conf-not-restored-' . date( 'Ymd-His' ); + $not_restored = []; + $skipped = []; + $mode_not_set = []; + foreach ( $entries as $path => $stat ) { + $is_dir = 0040000 === ( $stat['mode'] & 0170000 ); + $mode = sprintf( '%04o', $stat['mode'] & 0777 ); + $parent = dirname( $path ); + $content = null; + if ( isset( $skipped[ $parent ] ) ) { + $skipped[ $path ] = true; + continue; + } + try { + if ( ! $is_dir ) { + $content = \EE\Site\Utils\get_container_file_contents( "$source/$path", realpath( $source ) . ( '.' === $parent ? '' : "/$parent" ) ); + } + if ( isset( $not_restored[ $parent ] ) ) { + $not_restored[ $path ] = $not_restored[ $parent ]; + if ( $is_dir ) { + continue; + } + throw new \Exception( "config/php/{$not_restored[ $parent ]} was not restored" ); + } + $dir = rtrim( \EE\Site\Utils\site_php_conf_dir( $site_dir, '.' === $parent ? '' : $parent ), '/' ); + $dest = "$dir/" . basename( $path ); + clearstatcache( true ); + $current = @lstat( $dest ); + if ( realpath( $dir ) !== $dir ) { + throw new \Exception( "config/php/$parent is not a directory, or a directory above it is a symlink" ); + } + if ( ! $current ) { + // On macOS config/php isn't mounted (only custom.ini is), so an entry missing there has no effect, as before. + if ( 'Darwin' === PHP_OS_FAMILY ) { + EE::debug( "Not restoring config/php/$path: missing in the site." ); + $skipped[ $path ] = true; + continue; + } + if ( ! $this->php_container_op( $is_dir ? 'dir' : 'file', $path, $mode, $content ) ) { + $mode_not_set[] = "config/php/$path (could not change it in its php container)"; + } + } elseif ( $is_dir ) { + if ( 0040000 !== ( $current['mode'] & 0170000 ) || realpath( $dest ) !== $dest ) { + throw new \Exception( 'it is not a directory in the site' ); + } + } else { + \EE\Site\Utils\put_container_file_contents( $dest, $dir, $content ); + } + // Not on macOS, where the container doesn't see config/php. + if ( $current && 'Darwin' !== PHP_OS_FAMILY && $mode !== sprintf( '%04o', $current['mode'] & 0777 ) ) { + try { + if ( ! $this->php_container_op( 'mode', $path, $mode ) ) { + throw new \Exception( 'could not change it in its php container' ); + } + } catch ( \Exception $e ) { + $mode_not_set[] = "config/php/$path ({$e->getMessage()})"; + } + } + } catch ( \Exception $e ) { + $not_restored[ $path ] = $not_restored[ $path ] ?? $path; + $message = "Could not restore config/php/$path ({$e->getMessage()})."; + if ( null !== $content ) { + try { + $this->fs->dumpFile( "$kept/$path", $content ); + $fix = isset( $not_restored[ $parent ] ) ? "once config/php/{$not_restored[ $parent ]} is a plain directory again" : 'once the cause is fixed'; + $message .= " The backed-up file is kept as $kept/$path; put it back $fix, then run: ee site reload $site_url"; + } catch ( \Exception $dump_error ) { + $message .= ' Could not keep the backed-up file either: ' . $dump_error->getMessage(); + } + } + EE::warning( $message ); + } + } + if ( $mode_not_set ) { + EE::warning( 'Restored, but could not set the backed-up mode of: ' . implode( ', ', $mode_not_set ) . '.' ); + } + } + + /** + * Collect a backed-up entry below the extracted `php` directory, and with $recurse everything below it, parents first. + * + * @param string $source Extracted `php` directory of the backup. + * @param string $path Path below it. + * @param bool $recurse Whether to collect a directory's entries. + * @param array $entries Path => lstat() of each directory and regular file. + * @param array $refused Path => reason for each entry that is skipped. + * @param int $depth Directory depth. + * + * @return bool False once the entry limit is reached. + */ + private function list_php_conf_backup( $source, $path, $recurse, array &$entries, array &$refused, $depth = 0 ) { + + $parent = dirname( $path ); + if ( '.' !== $parent && ! isset( $entries[ $parent ] ) ) { + return true; + } + // Refused entries count too, so a crafted tree can't produce unbounded warnings. + if ( count( $entries ) + count( $refused ) >= 1000 ) { + $refused[ $path ] = 'the backup holds more than 1000 entries, so this and the rest were not restored'; + + return false; + } + $stat = @lstat( "$source/$path" ); + if ( ! $stat ) { + return true; + } + $type = $stat['mode'] & 0170000; + if ( 0040000 !== $type && 0100000 !== $type ) { + $refused[ $path ] = 'the backup holds a symlink or special file there'; + + return true; + } + if ( $depth > 16 ) { + $refused[ $path ] = 'it is nested too deep'; + + return true; + } + $entries[ $path ] = $stat; + if ( ! $recurse || 0040000 !== $type ) { + return true; + } + $handle = @opendir( "$source/$path" ); + if ( false === $handle ) { + return true; + } + $more = true; + while ( $more && false !== ( $name = readdir( $handle ) ) ) { + if ( '.' === $name || '..' === $name ) { + continue; + } + if ( preg_match( '/[\x00-\x1f\x7f]/', $name ) ) { + $refused[ $path . '/' . addcslashes( $name, "\0..\37\177" ) ] = 'its name holds control characters'; + continue; + } + $more = $this->list_php_conf_backup( $source, "$path/$name", true, $entries, $refused, $depth + 1 ); + } + closedir( $handle ); + + return $more; + } + + /** + * Create a directory or file the site's php config lacks, or set the mode of one, from inside its running php container, where a symlink the container made can't point at the host. + * + * The step runs as the owner of the parent directory, which must be a real directory (no symlink in its path), so it can't change what that owner couldn't, and a created entry belongs to that owner. + * + * @param string $op 'dir', 'file' or 'mode'. + * @param string $path Path below config/php (/usr/local/etc in the container). + * @param string $mode Octal permission bits. + * @param string|null $content Content of a created file. + * + * @return bool Whether the mode was set; false when only that failed. + * @throws \Exception + */ + private function php_container_op( $op, $path, $mode, $content = null ) { + + if ( ! preg_match( '#^(php-fpm\.d|php)(/[^/]+)*$#', $path ) || preg_match( '#(^|/)\.\.?(/|$)|[\x00-\x1f\x7f]#', $path ) || ! preg_match( '/^0[0-7]{3}$/', $mode ) ) { + throw new \Exception( 'unexpected path or mode' ); + } + + if ( null === $this->php_container_id ) { + $cwd = getcwd(); + chdir( EE_ROOT_DIR . '/sites/' . $this->site_data['site_url'] ); + $id = trim( EE::launch( \EE_DOCKER::docker_compose_with_custom() . ' ps -q php' )->stdout ); + if ( false !== $cwd ) { + chdir( $cwd ); + } + $this->php_container_id = ( '' !== $id && 'running' === \EE_DOCKER::container_status( $id ) ) ? $id : ''; + } + if ( '' === $this->php_container_id ) { + throw new \Exception( 'its php container is not running' ); + } + + // Changes into the parent and acts relative to it, so a directory swapped above it afterwards can't redirect the step. + $script = 'set -e; op=$2; m=$3; p=$1/$4; d=${p%/*}; n=./${p##*/} +cd -P -- "$d" 2>/dev/null && [ "$(pwd -P)" = "$d" ] || exit 3 +if [ "$op" = owner ]; then stat -c %u:%g .; exit 0; fi +if [ "$op" = mode ]; then [ ! -L "$n" ] || exit 4; chmod -- "$m" "$n" || exit 5; exit 0; fi +[ ! -e "$n" ] && [ ! -L "$n" ] || exit 4 +if [ "$op" = dir ]; then mkdir -m "$m" -- "$n"; exit 0; fi +(umask 077; set -C; cat > "$n") +chmod -- "$m" "$n" || exit 5'; + $run = function ( $user, $op, $stdin = null ) use ( $path, $mode, $script ) { + return EE::launch( sprintf( 'docker exec %s-u %s %s sh -c %s sh /usr/local/etc %s %s %s%s', $stdin ? '-i ' : '', escapeshellarg( $user ), escapeshellarg( $this->php_container_id ), escapeshellarg( $script ), escapeshellarg( $op ), escapeshellarg( $mode ), escapeshellarg( $path ), $stdin ? ' < ' . escapeshellarg( $stdin ) : '' ) ); + }; + + $owner = $run( 'root', 'owner' ); + $user = trim( $owner->stdout ); + if ( 3 === $owner->return_code || ! preg_match( '/^\d+:\d+$/', $user ) ) { + throw new \Exception( 'a directory above it is not a real directory in its php container' ); + } + + $tmp = null; + if ( 'file' === $op ) { + $tmp = tempnam( \EE\Utils\get_temp_dir(), 'ee-php-conf-' ); + if ( false === $tmp || strlen( $content ) !== file_put_contents( $tmp, $content ) ) { + throw new \Exception( 'could not write a temporary file' ); + } + } + $result = $run( $user, $op, $tmp ); + if ( $tmp ) { + unlink( $tmp ); + } + switch ( $result->return_code ) { + case 0: + return true; + case 5: + return false; + case 3: + throw new \Exception( 'a directory above it is not a real directory in its php container' ); + case 4: + throw new \Exception( 'mode' === $op ? 'it is a symlink in its php container' : 'it appeared in its php container while it was restored' ); + default: + throw new \Exception( 'mode' === $op ? 'could not change it in its php container' : 'it is missing in the site and could not be created in its php container' ); + } + } + + /** + * A backup taken before v4.13.3 brings back a custom.ini without `session.upload_progress.enabled = Off`, so add it + * again unless the restored file sets the directive itself. The site reload at the end of the restore applies it. + */ + private function keep_session_upload_progress_off() { + $site_url = $this->site_data['site_url']; + $file = EE_ROOT_DIR . '/sites/' . $site_url . '/config/php/php/conf.d/custom.ini'; + $manual = "Add 'session.upload_progress.enabled = Off' to $file and run: ee site reload $site_url"; + + // Not created when missing: PHP can't create a file safely in a directory the site's container can swap. + if ( is_link( $file ) || ! is_file( $file ) ) { + EE::warning( "Could not check session.upload_progress in the restored custom.ini: it is missing or not a regular file. $manual" ); + + return; + } + + // The directory is in the site's config volume, which its php container can change: every read and write goes through a checked handle. + try { + $dir = \EE\Site\Utils\site_php_conf_dir( EE_ROOT_DIR . '/sites/' . $site_url ); + $content = \EE\Site\Utils\get_container_file_contents( $file, $dir, 65536 ); + + $new_content = \EE\Site\Utils\add_session_upload_progress_off( $content ); + if ( null === $new_content ) { + return; + } + + \EE\Site\Utils\put_container_file_contents( $file, $dir, $new_content, $content, 65536 ); + } catch ( \Exception $e ) { + EE::warning( "Could not turn off session.upload_progress in the restored custom.ini ({$e->getMessage()}). $manual" ); + + return; + } + + EE::log( 'The restored custom.ini did not set session.upload_progress.enabled, so it was set to Off again. Set it to On there only if your app needs it.' ); } /** diff --git a/src/helper/class-ee-site.php b/src/helper/class-ee-site.php index f9dd4ff2..b9e4b3e2 100644 --- a/src/helper/class-ee-site.php +++ b/src/helper/class-ee-site.php @@ -934,19 +934,38 @@ protected function update_php( $args, $assoc_args ) { $array_data = ( array ) $this->site_data; $this->site_data = reset( $array_data ); - EE::log( 'Taking backup of old php config.' ); $site_backup_dir = $this->site_data['site_fs_path'] . '/.backup'; $php_conf_backup_dir = $site_backup_dir . '/config/php-' . $old_php_version; $php_conf_dir = $this->site_data['site_fs_path'] . '/config/php'; $php_confd_dir = $this->site_data['site_fs_path'] . '/config/php/php/conf.d'; - $this->fs->mkdir( $php_conf_backup_dir ); - $this->fs->mirror( $php_conf_dir, $php_conf_backup_dir ); + $rollback = [ + 'site' => $this->site_data, + 'backup' => $php_conf_backup_dir, + 'stopped' => false, + 'switched' => false, + ]; + + // Pull the new image before php stops, so the site is down only while its config is switched. $this->dump_docker_compose_yml( [ 'nohttps' => $no_https ] ); - EE::log( 'Starting site with new PHP version: ' . $php_version . '. This may take sometime.' ); - $this->enable( $args, [ 'force' => true ] ); + chdir( $this->site_data['site_fs_path'] ); + if ( ! EE::exec( \EE_DOCKER::docker_compose_with_custom() . ' pull php' ) ) { + throw new \Exception( "Could not pull the image of PHP $php_version." ); + } + + // The php container can swap anything in its config volume for a symlink, so it stays stopped until the volume is replaced. + $rollback['stopped'] = $this->stop_php_for_config_change( $php_conf_dir ); + + EE::log( 'Taking backup of old php config.' ); + // A fresh copy, so nothing is written through a symlink an older backup holds; that one is kept aside. + if ( $this->fs->exists( $php_conf_backup_dir ) || is_link( $php_conf_backup_dir ) ) { + $this->fs->rename( $php_conf_backup_dir, $php_conf_backup_dir . '.' . date( 'YmdHis' ) ); + } + $this->fs->mkdir( $php_conf_backup_dir ); + $this->fs->mirror( $php_conf_dir, $php_conf_backup_dir ); EE::log( 'Updating php config.' ); + $rollback['switched'] = true; $temp_dir = EE\Utils\get_temp_dir(); $zip_path = $temp_dir . "phpconf-$php_version.zip"; $unzip_folder = $temp_dir . "php-$php_version"; @@ -978,7 +997,14 @@ protected function update_php( $args, $assoc_args ) { EE::exec( "chown -R www-data: $php_conf_dir" ); + EE::log( 'Starting site with new PHP version: ' . $php_version . '. This may take sometime.' ); + $rollback['stopped'] = true; + $this->enable( $args, [ 'force' => true ], false ); + } catch ( \Exception $e ) { + if ( ! empty( $rollback ) ) { + $this->roll_back_php_update( $rollback, $old_php_version, $no_https ); + } EE::error( $e->getMessage() ); } $site->save(); @@ -987,6 +1013,103 @@ protected function update_php( $args, $assoc_args ) { delem_log( 'site php version update end' ); } + /** + * Put a failed `update --php` back: the old config (with php stopped again), the old compose file, and a running php container. + * + * @param array $rollback Site data as an array, the config backup dir, whether php was stopped and whether the config was switched. + * @param string $old_version PHP version to go back to. + * @param bool $no_https Whether the site has no SSL. + */ + private function roll_back_php_update( $rollback, $old_version, $no_https ) { + + $site_fs_path = $rollback['site']['site_fs_path']; + $manual = 'Run: ee site enable ' . $rollback['site']['site_url'] . ' --force'; + chdir( $site_fs_path ); + $compose = \EE_DOCKER::docker_compose_with_custom(); + + if ( $rollback['switched'] ) { + try { + // enable() may have started the new container already; the volume is only safe to touch with it stopped. + if ( ! EE::exec( "$compose stop php" ) ) { + throw new \Exception( 'could not stop the php container' ); + } + $volume = realpath( $site_fs_path . '/config/php' ); + self::check_config_volume_unused( $volume ); + \EE\Site\Utils\check_plain_tree( $volume ); + $this->fs->remove( array_map( function ( $name ) use ( $volume ) { + return "$volume/$name"; + }, array_diff( scandir( $volume ), [ '.', '..' ] ) ) ); + $this->fs->mirror( $rollback['backup'], $volume ); + \EE\Site\Utils\chown_container_tree( "$volume/php/conf.d", 'www-data' ); + EE::log( 'Restored the previous php config.' ); + } catch ( \Exception $e ) { + EE::warning( "Could not restore the previous php config from {$rollback['backup']} ({$e->getMessage()}). $manual" ); + } + } + + $this->site_data = $rollback['site']; + $this->site_data['php_version'] = $old_version; + $this->dump_docker_compose_yml( [ 'nohttps' => $no_https ] ); + + if ( $rollback['stopped'] && ! \EE_DOCKER::docker_compose_up( $site_fs_path, [ 'php' ] ) ) { + EE::warning( "Could not start the php container again. $manual" ); + } + } + + /** + * Stop the site's php container, the only one that mounts its config volume, and check the volume holds only directories and regular files. + * + * @param string $php_conf_dir The site's config/php. + * + * @return bool Whether a running container was stopped. + * @throws \Exception When it can't be stopped, another running container mounts the volume, or the volume holds anything else. + */ + private function stop_php_for_config_change( $php_conf_dir ) { + + chdir( $this->site_data['site_fs_path'] ); + $compose = \EE_DOCKER::docker_compose_with_custom(); + $id = trim( EE::launch( "$compose ps -q php" )->stdout ); + $running = '' !== $id && 'running' === \EE_DOCKER::container_status( $id ); + if ( $running && ! EE::exec( "$compose stop php" ) ) { + throw new \Exception( 'Could not stop the php container to update its config.' ); + } + + try { + $volume = realpath( $php_conf_dir ); + if ( false === $volume ) { + throw new \Exception( "Could not resolve $php_conf_dir." ); + } + self::check_config_volume_unused( $volume ); + \EE\Site\Utils\check_plain_tree( $volume ); + } catch ( \Exception $e ) { + $message = 'Could not update the php config: ' . rtrim( $e->getMessage(), '.' ) . '. Nothing was changed.'; + if ( $running && ! EE::exec( "$compose start php" ) ) { + $message .= ' Could not start the php container again. Run: ee site enable ' . $this->site_data['site_url'] . ' --force'; + } + throw new \Exception( $message ); + } + + return $running; + } + + /** + * On Linux config/php links to the volume's _data; a custom compose file may mount it in another service too. + * + * @param string $volume Real path of config/php. + * + * @throws \Exception When a running container still mounts it. + */ + private static function check_config_volume_unused( $volume ) { + + if ( '_data' !== basename( $volume ) ) { + return; + } + $users = EE::launch( 'docker ps -q --filter ' . escapeshellarg( 'volume=' . basename( dirname( $volume ) ) ) ); + if ( $users->return_code || '' !== trim( $users->stdout ) ) { + throw new \Exception( 'The php config volume is still mounted by a running container (' . implode( ', ', preg_split( '/\s+/', trim( $users->stdout ) ) ) . '); stop it and re-run the update.' ); + } + } + protected function sendmail_path_update( $msmtp ) { $custom_ini_path = $this->site_data['site_fs_path'] . '/config/php/php/conf.d/custom.ini'; @@ -1242,7 +1365,12 @@ public function enable( $args, $assoc_args, $exit_on_error = true ) { # Required when newrelic is enabled on site. Newrelic ini is updated via docker-entrypoint. $php_confd_dir = $this->site_data->site_fs_path . '/config/php/php/conf.d'; if ( $this->fs->exists( $php_confd_dir ) ) { - $this->fs->chown( $php_confd_dir, 'www-data', true ); + // The php container may be running and can swap anything in conf.d, so no symlink is followed. + try { + \EE\Site\Utils\chown_container_tree( \EE\Site\Utils\site_php_conf_dir( $this->site_data->site_fs_path ), 'www-data' ); + } catch ( \Exception $e ) { + \EE::warning( sprintf( 'Could not give www-data the php conf.d of %s: %s', $this->site_data->site_url, $e->getMessage() ) ); + } } if ( \EE_DOCKER::docker_compose_up( $this->site_data->site_fs_path, $containers_to_start ) ) { diff --git a/src/helper/site-utils.php b/src/helper/site-utils.php index a4766b19..a05e8987 100644 --- a/src/helper/site-utils.php +++ b/src/helper/site-utils.php @@ -1623,3 +1623,392 @@ function get_wp_core_download_command( string $path, array $args = [] ) { // docker_compose_exec() wraps the command in bash -c "...". return addcslashes( $script, '"$`\\' ); } + +/** + * Add `session.upload_progress.enabled = Off` to a site's custom.ini content, unless the content already sets it globally. + * + * Same rule as the v4.13.3 upload-progress migrations: an effective global line, On or Off, is the user's choice and is kept. + * + * @param string $content custom.ini content. + * + * @return string|null The new content, or null when it already sets the directive. + */ +function add_session_upload_progress_off( string $content ) { + + $bom = ''; + if ( 0 === strpos( $content, "\xEF\xBB\xBF" ) ) { + $bom = "\xEF\xBB\xBF"; + $content = substr( $content, 3 ); + } + + if ( ini_sets_globally( $content, 'session.upload_progress.enabled' ) ) { + return null; + } + + $eol = false !== strpos( $content, "\r\n" ) ? "\r\n" : ( false === strpos( $content, "\n" ) && false !== strpos( $content, "\r" ) ? "\r" : "\n" ); + $lines = '; Session upload progress is off for security; set it to On only if your app needs it.' . $eol . 'session.upload_progress.enabled = Off' . $eol; + + if ( '' === trim( $content ) ) { + return $bom . $lines; + } + + // At the very top, so neither a [PATH=]/[HOST=] section nor a multi-line quoted value can capture it. + return $bom . $lines . $eol . $content; +} + +/** + * Whether ini content sets a directive outside any [PATH=]/[HOST=] section, read the way PHP reads php.ini. + * + * @param string $content ini content without BOM. + * @param string $key Directive name (case-sensitive, like PHP). + * + * @return bool + */ +function ini_sets_globally( string $content, string $key ) { + + return null !== ini_global_value( $content, $key ); +} + +/** + * The value a directive gets outside any [PATH=]/[HOST=] section, read the way PHP reads php.ini. + * + * @param string $content ini content without BOM. + * @param string $key Directive name (case-sensitive, like PHP). + * + * @return string|null The value as PHP's ini scanner gives it (On is "1", Off is ""), or null when it isn't set there. + */ +function ini_global_value( string $content, string $key ) { + + $global = ini_global_part( $content ); + // Without sections, entries of ordinary sections are global and `foo[key] = x` stays an array, so it can't match. + $ini = @parse_ini_string( $global, false, INI_SCANNER_NORMAL ); + // PHP applies the lines above a syntax error, so only that part counts. + for ( $lines = preg_split( '/(?<=\n)|(?<=\r)(?!\n)/', $global ), $n = count( $lines ) - 1; false === $ini && $n > 0; $n-- ) { + $ini = @parse_ini_string( implode( '', array_slice( $lines, 0, $n ) ), false, INI_SCANNER_NORMAL ); + } + + return false !== $ini && array_key_exists( $key, $ini ) && ! is_array( $ini[ $key ] ) ? (string) $ini[ $key ] : null; +} + +/** + * The part of ini content before the first section named PATH or HOST plus at least one more character. + * + * PHP scopes every entry after such a header to it, even under later ordinary headers. The parser decides where each + * header ends and what it is called, so any line ending, quoted names and several headers on one line are handled. + * + * @param string $content ini content without BOM. + * + * @return string + */ +function ini_global_part( string $content ) { + + $probe = '__ee_ini_probe_' . md5( $content ); + for ( $open = strpos( $content, '[' ); false !== $open; $open = strpos( $content, '[', $open + 1 ) ) { + // Only where the parser expects a new entry can '[' start a header: not in a quoted value, a comment or a value. + $probed = @parse_ini_string( substr( $content, 0, $open ) . $probe . ' = 1', true, INI_SCANNER_NORMAL ); + if ( false === $probed || ! array_key_exists( $probe, $probed ) && ! array_filter( $probed, function ( $entry ) use ( $probe ) { + return is_array( $entry ) && array_key_exists( $probe, $entry ); + } ) ) { + continue; + } + // A quoted name may contain ']', so try each one until the text up to it parses. + for ( $close = strpos( $content, ']', $open ); false !== $close; $close = strpos( $content, ']', $close + 1 ) ) { + if ( false === @parse_ini_string( substr( $content, 0, $close + 1 ), true, INI_SCANNER_NORMAL ) ) { + continue; + } + $header = @parse_ini_string( substr( $content, $open, $close - $open + 1 ), true, INI_SCANNER_NORMAL ); + // PHP scopes only a name with something after PATH or HOST; entries under a bare [PATH] stay global. + if ( is_array( $header ) && 1 === count( $header ) && preg_match( '/^(PATH|HOST)./is', (string) key( $header ) ) ) { + return substr( $content, 0, $open ); + } + break; + } + } + + return $content; +} + +/** + * Real path of a directory in a site's php config. config/php is EasyEngine's own link to the site's config volume; everything below it belongs to the site's php container. + * + * @param string $site_fs_path Site root. + * @param string $sub Directory below config/php. + * + * @return string + * @throws \Exception + */ +function site_php_conf_dir( string $site_fs_path, string $sub = 'php/conf.d' ) { + + $base = realpath( $site_fs_path . '/config/php' ); + if ( false === $base ) { + throw new \Exception( 'could not resolve config/php' ); + } + + return $base . '/' . $sub; +} + +/** + * Open a file in a directory the site's php container can change, as a handle for every later read and write, so nothing swapped in at that path is followed afterwards. + * + * Host root must not write through a symlink, a hard link or a swapped directory, so the handle has to be the regular file in $dir. On Linux the kernel's own path of the open file decides; elsewhere the path is checked before and after opening. + * + * Left open: a symlink, FIFO or device renamed into place between the lstat() check and the open is opened (non-blocking) and closed unread and unwritten, and opening some devices has side effects. Only an open with O_NOFOLLOW/openat, which PHP lacks, or one inside the container closes that. + * + * @param string $file Path. + * @param string $mode 'r' or 'r+'. Nothing is created: PHP expands every path it creates or links to an absolute one, so it can't be confined to a directory the container can swap. + * @param string $dir Real path of the directory the file must be in. + * + * @return resource + * @throws \Exception + */ +function open_container_file( string $file, string $mode, string $dir ) { + + if ( 'r' !== $mode && 'r+' !== $mode ) { + throw new \InvalidArgumentException( "Unsupported mode $mode." ); + } + + $name = basename( $file ); + clearstatcache( true ); + if ( realpath( dirname( $file ) ) !== $dir ) { + throw new \Exception( "a directory above $name is a symlink" ); + } + + // PHP can't open without following a symlink (no O_NOFOLLOW, O_PATH or openat), so refuse anything but a regular file before opening it. + $before = @lstat( $file ); + if ( ! $before ) { + throw new \Exception( "$name is missing" ); + } + if ( 0100000 !== ( $before['mode'] & 0170000 ) ) { + throw new \Exception( "$name is not a regular file" ); + } + + // Non-blocking ('n', O_NONBLOCK): a FIFO or device swapped in after the check above is still opened, but can't stall, and is closed below before any read or write. + $handle = @fopen( $file, $mode . 'n' ); + if ( false === $handle ) { + throw new \Exception( "could not open $name" ); + } + + $opened = fstat( $handle ); + clearstatcache( true ); + $linked = @lstat( $file ); + $real = opened_file_path( $handle ); + $error = null; + if ( ! $opened || ! $linked || $opened['dev'] !== $before['dev'] || $opened['ino'] !== $before['ino'] || $opened['dev'] !== $linked['dev'] || $opened['ino'] !== $linked['ino'] || 0100000 !== ( $opened['mode'] & 0170000 ) || 0100000 !== ( $linked['mode'] & 0170000 ) ) { + $error = "$name is not a regular file"; + } elseif ( realpath( dirname( $file ) ) !== $dir || ( null !== $real && "$dir/$name" !== $real ) ) { + $error = "a directory above $name is a symlink"; + } elseif ( 'r' !== $mode && 1 !== $opened['nlink'] ) { + $error = "$name has other hard links"; + } + if ( null !== $error ) { + fclose( $handle ); + throw new \Exception( $error ); + } + stream_set_blocking( $handle, true ); + + return $handle; +} + +/** + * The kernel's path of an open file, from /proc/self/fd, which no later swap of a directory changes. + * + * @param resource $handle Open file. + * + * @return string|null Null without /proc (macOS); '' when the file isn't found there. + */ +function opened_file_path( $handle ) { + + $want = fstat( $handle ); + $fds = $want ? @scandir( '/proc/self/fd' ) : false; + if ( ! $fds ) { + return null; + } + + clearstatcache( true ); + foreach ( $fds as $fd ) { + if ( ! preg_match( '/^\d+$/', $fd ) ) { + continue; + } + $stat = @stat( "/proc/self/fd/$fd" ); + if ( $stat && $stat['dev'] === $want['dev'] && $stat['ino'] === $want['ino'] ) { + $path = @readlink( "/proc/self/fd/$fd" ); + + return false === $path ? '' : $path; + } + } + + return ''; +} + +/** + * Overwrite an open file through its handle: owner, mode and inode stay (macOS bind-mounts single config files). + * + * @param resource $handle Handle from open_container_file(). + * @param string $content New content. + * + * @throws \Exception When the file doesn't hold exactly $content afterwards. + */ +function rewrite_container_file( $handle, string $content ) { + + // The read-back stops one byte past the content, so data the container appends can't make it unbounded. + if ( ! rewind( $handle ) || strlen( $content ) !== fwrite( $handle, $content ) || ! fflush( $handle ) || ! ftruncate( $handle, strlen( $content ) ) || ! rewind( $handle ) || stream_get_contents( $handle, strlen( $content ) + 1 ) !== $content ) { + throw new \Exception( 'could not write ' . basename( stream_get_meta_data( $handle )['uri'] ) ); + } +} + +/** + * Read an open file, at most $max bytes, so a file the container keeps growing can't exhaust memory. + * + * @param resource $handle Handle from open_container_file(). + * @param int $max Largest size accepted. + * + * @return string + * @throws \Exception When it can't be read or is larger. + */ +function read_container_file( $handle, int $max = 65536 ) { + + $name = basename( stream_get_meta_data( $handle )['uri'] ); + $content = stream_get_contents( $handle, $max + 1 ); + if ( false === $content ) { + throw new \Exception( "could not read $name" ); + } + if ( strlen( $content ) > $max ) { + throw new \Exception( "$name is larger than $max bytes" ); + } + + return $content; +} + +/** + * Read a file in a directory the site's php container can change. + * + * @param string $file Path. + * @param string $dir Real path of the directory the file must be in. + * @param int $max Largest size accepted. + * + * @return string + * @throws \Exception When it is missing, not a regular file in $dir, unreadable or larger than $max. + */ +function get_container_file_contents( string $file, string $dir, int $max = 1048576 ) { + + $handle = open_container_file( $file, 'r', $dir ); + try { + return read_container_file( $handle, $max ); + } finally { + fclose( $handle ); + } +} + +/** + * Overwrite an existing file in a directory the site's php container can change, in place, so owner, mode and inode stay. + * + * @param string $file Path. + * @param string $dir Real path of the directory the file must be in. + * @param string $content New content. + * @param string|null $expected Content the file must still hold, e.g. the one the change was made from; null for any. + * @param int $max Largest current size accepted. + * + * @return bool Whether it was written; false when it already held $content. + * @throws \Exception When it isn't a regular file in $dir, holds something else than $expected, or can't be written (the old content is put back). + */ +function put_container_file_contents( string $file, string $dir, string $content, ?string $expected = null, int $max = 1048576 ) { + + // Compared read-only first, so a file that needs no change isn't opened for writing (a hard link or an immutable file would refuse that). + $current = get_container_file_contents( $file, $dir, $max ); + if ( null !== $expected && $current !== $expected ) { + throw new \Exception( basename( $file ) . ' changed while it was checked' ); + } + if ( $current === $content ) { + return false; + } + + $handle = open_container_file( $file, 'r+', $dir ); + try { + if ( read_container_file( $handle, $max ) !== $current ) { + throw new \Exception( basename( $file ) . ' changed while it was checked' ); + } + try { + rewrite_container_file( $handle, $content ); + } catch ( \Exception $e ) { + // A short write leaves part of the file; put the old one back. + rewrite_container_file( $handle, $current ); + throw $e; + } + } finally { + fclose( $handle ); + } + + return true; +} + +/** + * Refuse a directory tree that holds anything but directories and regular files. + * + * Only conclusive while nothing can change the tree, e.g. with the site's php container stopped. + * + * @param string $dir Real path of the directory. + * + * @throws \Exception Naming the first other entry. + */ +function check_plain_tree( string $dir ) { + + clearstatcache( true ); + $names = @scandir( $dir ); + if ( false === $names ) { + throw new \Exception( "could not read $dir" ); + } + + foreach ( array_diff( $names, [ '.', '..' ] ) as $name ) { + $path = "$dir/$name"; + $stat = @lstat( $path ); + $type = $stat ? $stat['mode'] & 0170000 : 0; + if ( 0040000 === $type ) { + check_plain_tree( $path ); + } elseif ( 0100000 !== $type ) { + throw new \Exception( "$path is not a regular file or directory" ); + } + } +} + +/** + * Give a user a directory in a site's config volume and every directory and regular file below it, without following symlinks. + * + * Symlinks, FIFOs and devices are left alone, and a directory must still be at its real path before its entries are changed. + * + * @param string $dir Real path of the directory. + * @param string|int $user User name or ID. + * + * @throws \Exception + */ +function chown_container_tree( string $dir, $user ) { + + clearstatcache( true ); + $stat = @lstat( $dir ); + if ( ! $stat || 0040000 !== ( $stat['mode'] & 0170000 ) || realpath( $dir ) !== $dir ) { + throw new \Exception( "$dir is not a directory, or a directory above it is a symlink" ); + } + if ( ! @lchown( $dir, $user ) ) { + throw new \Exception( "could not change the owner of $dir" ); + } + + $names = @scandir( $dir ); + if ( false === $names ) { + throw new \Exception( "could not read $dir" ); + } + foreach ( array_diff( $names, [ '.', '..' ] ) as $name ) { + $path = "$dir/$name"; + clearstatcache( true ); + $entry = @lstat( $path ); + $type = $entry ? $entry['mode'] & 0170000 : 0; + if ( 0040000 === $type ) { + chown_container_tree( $path, $user ); + } elseif ( 0100000 === $type ) { + if ( realpath( $dir ) !== $dir ) { + throw new \Exception( "a directory above $path is a symlink" ); + } + if ( ! @lchown( $path, $user ) ) { + throw new \Exception( "could not change the owner of $path" ); + } + } + } +} diff --git a/templates/config/php-fpm/php81.zip b/templates/config/php-fpm/php81.zip index 49402070..bd3414fb 100644 Binary files a/templates/config/php-fpm/php81.zip and b/templates/config/php-fpm/php81.zip differ diff --git a/templates/config/php-fpm/php82.zip b/templates/config/php-fpm/php82.zip index 4facee46..c8749f3a 100644 Binary files a/templates/config/php-fpm/php82.zip and b/templates/config/php-fpm/php82.zip differ diff --git a/templates/config/php-fpm/php83.zip b/templates/config/php-fpm/php83.zip index 13aaceb1..00fd4b7b 100644 Binary files a/templates/config/php-fpm/php83.zip and b/templates/config/php-fpm/php83.zip differ diff --git a/templates/config/php-fpm/php84.zip b/templates/config/php-fpm/php84.zip index 9bed38b6..2201c3a8 100644 Binary files a/templates/config/php-fpm/php84.zip and b/templates/config/php-fpm/php84.zip differ diff --git a/templates/config/php-fpm/php85.zip b/templates/config/php-fpm/php85.zip index 31ce07cd..2f1a2996 100644 Binary files a/templates/config/php-fpm/php85.zip and b/templates/config/php-fpm/php85.zip differ diff --git a/templates/config/php-fpm/phplatest.zip b/templates/config/php-fpm/phplatest.zip deleted file mode 100644 index 4facee46..00000000 Binary files a/templates/config/php-fpm/phplatest.zip and /dev/null differ