feat(migration): add the migration runner and schema differ

bin/cli.php

@@ -69,6 +69,7 @@
 include 'tasks/query.php';
 include 'tasks/relationships.php';
 include 'tasks/operators.php';
+include 'tasks/migrate.php';
 
 $cli
     ->error()

bin/tasks/migrate.php

@@ -0,0 +1,184 @@
+<?php
+
+/**
+ * @var CLI $cli
+ */
+global $cli;
+
+use Utopia\CLI\CLI;
+use Utopia\Console;
+use Utopia\Database\Database;
+use Utopia\Database\Migration\Generator;
+use Utopia\Database\Migration\Migration;
+use Utopia\Database\Migration\Runner;
+use Utopia\Validator\Boolean;
+use Utopia\Validator\Integer;
+use Utopia\Validator\Text;
+
+/**
+ * @Example
+ * docker compose exec tests bin/cli migrate --adapter=mysql --name=testing --path=migrations
+ * docker compose exec tests bin/cli migrate:rollback --adapter=mysql --name=testing --path=migrations --steps=1
+ * docker compose exec tests bin/cli migrate:status --adapter=mysql --name=testing --path=migrations
+ * docker compose exec tests bin/cli migrate:fresh --adapter=mysql --name=testing --path=migrations
+ * docker compose exec tests bin/cli migrate:generate --name=add_users_table
+ */
+
+$cli
+    ->task('migrate')
+    ->desc('Run pending database migrations')
+    ->param('path', 'migrations', new Text(0), 'Path to migration files', true)
+    ->param('adapter', '', new Text(0), 'Database adapter')
+    ->param('name', '', new Text(0), 'Database name')
+    ->param('namespace', '_ns', new Text(0), 'Database namespace', true)
+    ->param('sharedTables', false, new Boolean(true), 'Whether to use shared tables', true)
+    ->inject('database')
+    ->action(function (string $path, string $adapter, string $name, string $namespace, bool $sharedTables, callable $database) {
+        $migrations = loadMigrations($path);
+
+        if ($migrations === []) {
+            Console::warning('No migration files found in: ' . $path);
+
+            return;
+        }
+
+        Console::info('Running migrations...');
+
+        $db = $database($adapter, $name, $namespace, $sharedTables);
+        if (! $db instanceof Database) {
+            throw new \RuntimeException('The database resource must return a Database instance.');
+        }
+        $runner = new Runner($db);
+        $count = $runner->migrate($migrations);
+
+        Console::success("Ran {$count} migration(s).");
+    });
+
+$cli
+    ->task('migrate:rollback')
+    ->desc('Rollback the last batch of migrations')
+    ->param('path', 'migrations', new Text(0), 'Path to migration files', true)
+    ->param('steps', 1, new Integer(true), 'Number of batches to rollback', true)
+    ->param('adapter', '', new Text(0), 'Database adapter')
+    ->param('name', '', new Text(0), 'Database name')
+    ->param('namespace', '_ns', new Text(0), 'Database namespace', true)
+    ->param('sharedTables', false, new Boolean(true), 'Whether to use shared tables', true)
+    ->inject('database')
+    ->action(function (string $path, int $steps, string $adapter, string $name, string $namespace, bool $sharedTables, callable $database) {
+        $migrations = loadMigrations($path);
+        $db = $database($adapter, $name, $namespace, $sharedTables);
+        if (! $db instanceof Database) {
+            throw new \RuntimeException('The database resource must return a Database instance.');
+        }
+        $runner = new Runner($db);
+        $count = $runner->rollback($migrations, $steps);
+
+        Console::success("Rolled back {$count} migration(s).");
+    });
+
+$cli
+    ->task('migrate:status')
+    ->desc('Show the status of all migrations')
+    ->param('path', 'migrations', new Text(0), 'Path to migration files', true)
+    ->param('adapter', '', new Text(0), 'Database adapter')
+    ->param('name', '', new Text(0), 'Database name')
+    ->param('namespace', '_ns', new Text(0), 'Database namespace', true)
+    ->param('sharedTables', false, new Boolean(true), 'Whether to use shared tables', true)
+    ->inject('database')
+    ->action(function (string $path, string $adapter, string $name, string $namespace, bool $sharedTables, callable $database) {
+        $migrations = loadMigrations($path);
+        $db = $database($adapter, $name, $namespace, $sharedTables);
+        if (! $db instanceof Database) {
+            throw new \RuntimeException('The database resource must return a Database instance.');
+        }
+        $runner = new Runner($db);
+        $status = $runner->status($migrations);
+
+        Console::info(\str_pad('Version', 20) . \str_pad('Name', 40) . 'Applied');
+        Console::info(\str_repeat('-', 70));
+
+        foreach ($status as $entry) {
+            $applied = $entry['applied'] ? 'Yes' : 'No';
+            Console::log(\str_pad($entry['version'], 20) . \str_pad($entry['name'], 40) . $applied);
+        }
+    });
+
+$cli
+    ->task('migrate:fresh')
+    ->desc('Drop all collections and re-run all migrations')
+    ->param('path', 'migrations', new Text(0), 'Path to migration files', true)
+    ->param('adapter', '', new Text(0), 'Database adapter')
+    ->param('name', '', new Text(0), 'Database name')
+    ->param('namespace', '_ns', new Text(0), 'Database namespace', true)
+    ->param('sharedTables', false, new Boolean(true), 'Whether to use shared tables', true)
+    ->inject('database')
+    ->action(function (string $path, string $adapter, string $name, string $namespace, bool $sharedTables, callable $database) {
+        $migrations = loadMigrations($path);
+        $db = $database($adapter, $name, $namespace, $sharedTables);
+        if (! $db instanceof Database) {
+            throw new \RuntimeException('The database resource must return a Database instance.');
+        }
+        $runner = new Runner($db);
+
+        Console::warning('Dropping all collections and re-migrating...');
+        $count = $runner->fresh($migrations);
+
+        Console::success("Fresh migration complete. Ran {$count} migration(s).");
+    });
+
+$cli
+    ->task('migrate:generate')
+    ->desc('Generate an empty migration file')
+    ->param('name', '', new Text(0), 'Migration name (e.g. add_users_table)')
+    ->param('path', 'migrations', new Text(0), 'Output directory', true)
+    ->action(function (string $name, string $path) {
+        $timestamp = \date('YmdHis');
+        $className = 'V' . $timestamp . '_' . \str_replace(' ', '', \ucwords(\str_replace('_', ' ', $name)));
+
+        $generator = new Generator();
+        $content = $generator->generateEmpty($className);
+
+        if (! \is_dir($path)) {
+            \mkdir($path, 0755, true);
+        }
+
+        $filePath = $path . '/' . $className . '.php';
+        \file_put_contents($filePath, $content);
+
+        Console::success("Created migration: {$filePath}");
+    });
+
+/**
+ * @return array<Migration>
+ */
+function loadMigrations(string $path): array
+{
+    if (! \is_dir($path)) {
+        return [];
+    }
+
+    $migrations = [];
+    $files = \glob($path . '/*.php');
+
+    if ($files === false) {
+        return [];
+    }
+
+    foreach ($files as $file) {
+        $before = \get_declared_classes();
+
+        require_once $file;
+
+        // migrate:generate writes the class under a namespace, so the file name
+        // is not the class name and looking it up that way finds nothing --
+        // silently, leaving the run reporting success having skipped it. Take
+        // whatever the file declared instead of guessing at it.
+        foreach (\array_diff(\get_declared_classes(), $before) as $className) {
+            if (\is_subclass_of($className, Migration::class)) {
+                $migrations[] = new $className();
+            }
+        }
+    }
+
+    return $migrations;
+}

phpstan.neon

@@ -7,6 +7,8 @@ parameters:
         - src
         - tests
     scanFiles:
+        # Declares loadMigrations(), which tests/unit/CLITasksTest.php calls.
+        - bin/tasks/migrate.php
         - stubs/Swoole/Database/DetectsLostConnections.stub.php
         - stubs/Swoole/Database/PDOProxy.stub.php
         - stubs/Swoole/Database/PDOStatementProxy.stub.php

src/Database/Migration/Generator.php

@@ -0,0 +1,154 @@
+<?php
+
+namespace Utopia\Database\Migration;
+
+use Utopia\Database\Schema\Change;
+use Utopia\Database\Schema\ChangeType;
+use Utopia\Database\Schema\DiffResult;
+
+class Generator
+{
+    public function generate(DiffResult $diff, string $className, string $namespace = 'App\\Migration'): string
+    {
+        $version = $this->extractVersion($className);
+        $upLines = [];
+        $downLines = [];
+
+        foreach ($diff->changes as $change) {
+            $up = $this->generateUpStatement($change);
+            $down = $this->generateDownStatement($change);
+
+            if ($up !== null) {
+                $upLines[] = "        {$up}";
+            }
+
+            if ($down !== null) {
+                $downLines[] = "        {$down}";
+            }
+        }
+
+        $upBody = $upLines !== [] ? \implode("\n", $upLines) : '        // No changes';
+        $downBody = $downLines !== [] ? \implode("\n", \array_reverse($downLines)) : '        // No changes';
+
+        return <<<PHP
+        <?php
+
+        namespace {$namespace};
+
+        use Utopia\Database\Database;
+        use Utopia\Database\Migration\Migration;
+
+        class {$className} extends Migration
+        {
+            public function version(): string
+            {
+                return '{$version}';
+            }
+
+            public function up(Database \$db): void
+            {
+        {$upBody}
+            }
+
+            public function down(Database \$db): void
+            {
+        {$downBody}
+            }
+        }
+
+        PHP;
+    }
+
+    public function generateEmpty(string $className, string $namespace = 'App\\Migration'): string
+    {
+        $version = $this->extractVersion($className);
+
+        return <<<PHP
+        <?php
+
+        namespace {$namespace};
+
+        use Utopia\Database\Database;
+        use Utopia\Database\Migration\Migration;
+
+        class {$className} extends Migration
+        {
+            public function version(): string
+            {
+                return '{$version}';
+            }
+
+            public function up(Database \$db): void
+            {
+                //
+            }
+
+            public function down(Database \$db): void
+            {
+                //
+            }
+        }
+
+        PHP;
+    }
+
+    private function extractVersion(string $className): string
+    {
+        if (\preg_match('/^V(\d+)_/', $className, $matches)) {
+            return $matches[1];
+        }
+
+        return $className;
+    }
+
+    private function generateUpStatement(Change $change): ?string
+    {
+        $collectionId = $this->collectionId($change);
+
+        return match ($change->type) {
+            ChangeType::AddAttribute => $change->attribute !== null
+                ? "\$db->createAttribute('{$collectionId}', new \\Utopia\\Database\\Attribute(key: '{$change->attribute->key}', type: \\Utopia\\Query\\Schema\\ColumnType::" . \ucfirst($change->attribute->type->value) . ", size: {$change->attribute->size}));"
+                : null,
+            ChangeType::DropAttribute => $change->attribute !== null
+                ? "\$db->deleteAttribute('{$collectionId}', '{$change->attribute->key}');"
+                : null,
+            ChangeType::AddIndex => $change->index !== null
+                ? "\$db->createIndex('{$collectionId}', new \\Utopia\\Database\\Index(key: '{$change->index->key}', type: \\Utopia\\Query\\Schema\\IndexType::" . \ucfirst($change->index->type->value) . ", attributes: " . \var_export($change->index->attributes, true) . '));'
+                : null,
+            ChangeType::DropIndex => $change->index !== null
+                ? "\$db->deleteIndex('{$collectionId}', '{$change->index->key}');"
+                : null,
+            default => null,
+        };
+    }
+
+    private function generateDownStatement(Change $change): ?string
+    {
+        $collectionId = $this->collectionId($change);
+
+        return match ($change->type) {
+            ChangeType::AddAttribute => $change->attribute !== null
+                ? "\$db->deleteAttribute('{$collectionId}', '{$change->attribute->key}');"
+                : null,
+            ChangeType::DropAttribute => $change->attribute !== null
+                ? "\$db->createAttribute('{$collectionId}', new \\Utopia\\Database\\Attribute(key: '{$change->attribute->key}', type: \\Utopia\\Query\\Schema\\ColumnType::" . \ucfirst($change->attribute->type->value) . ", size: {$change->attribute->size}));"
+                : null,
+            ChangeType::AddIndex => $change->index !== null
+                ? "\$db->deleteIndex('{$collectionId}', '{$change->index->key}');"
+                : null,
+            ChangeType::DropIndex => $change->index !== null
+                ? "\$db->createIndex('{$collectionId}', new \\Utopia\\Database\\Index(key: '{$change->index->key}', type: \\Utopia\\Query\\Schema\\IndexType::" . \ucfirst($change->index->type->value) . ", attributes: " . \var_export($change->index->attributes, true) . '));'
+                : null,
+            default => null,
+        };
+    }
+
+    private function collectionId(Change $change): string
+    {
+        if ($change->collectionId === null || $change->collectionId === '') {
+            return '{collectionId}';
+        }
+
+        return $change->collectionId;
+    }
+}

src/Database/Migration/Migration.php

@@ -0,0 +1,19 @@
+<?php
+
+namespace Utopia\Database\Migration;
+
+use Utopia\Database\Database;
+
+abstract class Migration
+{
+    abstract public function version(): string;
+
+    abstract public function up(Database $db): void;
+
+    abstract public function down(Database $db): void;
+
+    public function name(): string
+    {
+        return static::class;
+    }
+}