Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,7 @@ ui/src/**/*.test.ts linguist-documentation
ui/scripts/** linguist-documentation
clients/typescript/tests/** linguist-documentation
clients/typescript/scripts/** linguist-documentation

# The control-store SQL is embedded in the binaries and checksummed; keep it LF on every checkout.
crates/sqlbuild-orchestration/src/store/*.sql text eol=lf
crates/sqlbuild-orchestration/tests/store/schema/baseline/*.sql text eol=lf
59 changes: 59 additions & 0 deletions crates/sqlbuild-cli/src/argv/command_table.json
Original file line number Diff line number Diff line change
Expand Up @@ -1182,6 +1182,65 @@
"default": false,
"help": "print the OpenAPI contract as JSON without opening a database",
"group": 1
},
{
"kind": "parsers",
"dest": "server_command",
"nargs": "A...",
"required": false,
"group": 0,
"parsers": [
{
"name": "migrate",
"parser": {
"prog": "sqb server migrate",
"description": "Apply the control store's pending schema migrations and exit.",
"groups": [
{
"title": "positional arguments"
},
{
"title": "options"
}
],
"actions": [
{
"kind": "help",
"options": [
"-h",
"--help"
],
"dest": "help",
"nargs": 0,
"default": {
"suppress": true
},
"help": "show this help message and exit",
"group": 1
},
{
"kind": "store_true",
"options": [
"--dry-run"
],
"dest": "control_dry_run",
"nargs": 0,
"const": true,
"default": false,
"help": "list the migrations that would run without changing the store",
"group": 1
}
]
}
}
],
"choices_help": [
{
"dest": "migrate",
"metavar": "migrate",
"help": "apply pending control-store migrations and exit"
}
]
}
]
}
Expand Down
86 changes: 86 additions & 0 deletions crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ use sqlbuild_orchestration::store::main::migrate_store::migrate_store;
use sqlbuild_orchestration::store::main::open_store::open_store;
use sqlbuild_orchestration::store::main::set_release_version::set_release_version;
use sqlbuild_orchestration::store::models::ControlStore;
use sqlbuild_orchestration::store_migrations::main::plan_store_migration::plan_store_migration;
use sqlbuild_orchestration::store_migrations::models::StoreMigrationPlan;
use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS;
use sqlbuild_orchestration::warehouse_metadata::models::{
MetadataBatch, MetadataScan, StatisticsDueRead, StatisticsRead, WarehouseHistoryFailure,
Expand Down Expand Up @@ -117,6 +119,7 @@ use crate::native_commands::models::{
MetadataReply, NativeCommand, NativeInvocation, ProjectLocation, ScanToken,
};
use crate::native_commands::types::LocationPublication;
use crate::server_config::main::read_server_config::read_server_config;
use crate::server_config::models::ServerSection;
use crate::workers::main::docker_image_sources::docker_image_sources;
use crate::workers::main::open_development_workers::open_development_workers;
Expand Down Expand Up @@ -1886,3 +1889,86 @@ pub(crate) async fn compile_next_generation(
.await?;
Ok(compiled.map(|compiled| (Some((compiled, generation)), metadata, compiled_inputs)))
}

/// Run `sqb server migrate` on `runtime` and return its exit code.
pub(crate) fn run_store_migration(namespace: &Namespace, runtime: &tokio::runtime::Runtime) -> i32 {
drop(write_output(&[(
Stream::Stderr,
"Opening control store...\n",
)]));
match runtime.block_on(store_migration(
namespace,
flag(namespace, "control_dry_run"),
)) {
Ok(report) => {
let lost = write_output(&[(Stream::Stdout, &report)]);
let exit_code = finish_lost_stdout(lost.as_ref(), 0);
if exit_code == 0 {
drop(write_output(&[(
Stream::Stderr,
"Control store migration completed successfully\n",
)]));
}
exit_code
}
Err(error) => {
drop(write_output(&[(
Stream::Stderr,
&format!("Control store migration failed: {error}\n"),
)]));
1
}
}
}

/// Plan and apply the store's migrations; a dry run never creates a missing SQLite file.
async fn store_migration(namespace: &Namespace, dry_run: bool) -> Result<String, ControlError> {
let config = read_server_config(namespace)?;
let url = control_database(
namespace,
&control_database_variable(namespace, &config.server),
)?;
if dry_run
&& let Some(path) = url
.strip_prefix("sqlite://")
.or((!url.contains("://")).then_some(url.as_str()))
&& !matches!(path, "" | ":memory:")
&& !std::path::Path::new(path).exists()
{
return Ok(format!("No control store at {path}: would create one.\n"));
}
let store = open_store(&url).await?;
let plan = plan_store_migration(&store).await?;
if !dry_run {
migrate_store(&store).await?;
}
Ok(describe(&plan, dry_run))
}

/// The plan as lines for a person: what would happen with `--dry-run`, what happened without it.
fn describe(plan: &StoreMigrationPlan, dry_run: bool) -> String {
let (create, upgrade, record) = if dry_run {
("would create", "would migrate", "would record")
} else {
("created", "migrated", "recorded")
};
match plan {
StoreMigrationPlan::Create { version } => {
format!("The control store is empty: {create} schema version {version}.\n")
}
StoreMigrationPlan::Current { version } => {
format!("The control store is at schema version {version}: nothing to migrate.\n")
}
StoreMigrationPlan::Upgrade { from, to, names } if names.is_empty() => format!(
"The control store's schema is version {from} without a recorded version: \
{record} version {to}.\n"
),
StoreMigrationPlan::Upgrade { from, to, names } => format!(
"The control store {upgrade} from schema version {from} to {to}:\n{}",
names
.iter()
.map(|name| format!(" {name}\n"))
.collect::<String>()
),
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use sqlbuild_orchestration::api::main::openapi_document::openapi_document;
use crate::argv::models::{Namespace, Stream};
use crate::native_commands::_helpers::decision::{flag, text};
use crate::native_commands::_helpers::execution::run_control_action;
use crate::native_commands::_helpers::execution::run_store_migration;
use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output};
use crate::native_commands::constants::{ADMIN_COMMAND, SCHEDULER_COMMAND, SERVER_COMMAND};

Expand Down Expand Up @@ -45,6 +46,9 @@ pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option<i32> {
return Some(1);
}
};
if text(namespace, "server_command").as_deref() == Some("migrate") {
return Some(run_store_migration(namespace, &runtime));
}
let admin = command == ADMIN_COMMAND;
let role = if admin { "administration" } else { "service" };
let result = runtime.block_on(run_control_action(namespace, &command));
Expand Down
55 changes: 54 additions & 1 deletion crates/sqlbuild-cli/tests/native_commands/test_control.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
use std::process::Command;

use crate::helpers::{api_ready, isolated_binary, start_control, wait_for_health};
use crate::test_types::{CommandTestCase, OpenApiTestCase, ServiceTestCase, TriggerCursorTestCase};
use crate::test_types::{
CommandTestCase, OpenApiTestCase, ServiceTestCase, StoreMigrationCommandTestCase,
TriggerCursorTestCase,
};

#[test]
fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_python_and_database_access_are_not_required()
Expand Down Expand Up @@ -364,6 +367,56 @@ fn given_a_standalone_server_when_exporting_openapi_then_the_worker_contract_is_
}
}

#[test]
fn given_a_control_store_when_running_server_migrate_then_dry_run_lists_and_migrate_applies() {
let test_cases = [
StoreMigrationCommandTestCase {
description: "dry run without a store creates nothing",
arguments: &["server", "migrate", "--dry-run"],
expected_stdout: "No control store at ./.sqlbuild/control.sqlite3: would create one.\n",
expected_store: false,
},
StoreMigrationCommandTestCase {
description: "migrate an empty store",
arguments: &["server", "migrate"],
expected_stdout: "The control store is empty: created schema version 1.\n",
expected_store: true,
},
StoreMigrationCommandTestCase {
description: "dry run on a current store",
arguments: &["server", "migrate", "--dry-run"],
expected_stdout: "The control store is at schema version 1: nothing to migrate.\n",
expected_store: true,
},
];
let directory = tempfile::tempdir().expect("isolated CLI directory");
let binary = isolated_binary(directory.path());
for test_case in test_cases {
let output = Command::new(&binary)
.args(test_case.arguments)
.current_dir(directory.path())
.env("PATH", "")
.env_remove("SQLBUILD_CONTROL_DATABASE_URL")
.output()
.expect("run sqb server migrate");
assert_eq!(
(
output.status.success(),
String::from_utf8_lossy(&output.stdout).into_owned(),
directory.path().join(".sqlbuild/control.sqlite3").exists(),
),
(
true,
test_case.expected_stdout.to_owned(),
test_case.expected_store
),
"{}: {}",
test_case.description,
String::from_utf8_lossy(&output.stderr)
);
}
}

#[test]
fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then_migrations_auth_and_shutdown_are_native()
{
Expand Down
8 changes: 8 additions & 0 deletions crates/sqlbuild-cli/tests/native_commands/test_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,3 +81,11 @@ pub(crate) struct LocationHealthTestCase {
pub(crate) expected_state: &'static str,
pub(crate) expected_exit: i32,
}

pub(crate) struct StoreMigrationCommandTestCase {
pub(crate) description: &'static str,
pub(crate) arguments: &'static [&'static str],
pub(crate) expected_stdout: &'static str,
/// Whether the SQLite store file exists after the command.
pub(crate) expected_store: bool,
}
1 change: 1 addition & 0 deletions crates/sqlbuild-orchestration/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ pub mod scheduler;
pub mod sensors;
pub mod snapshots;
pub mod store;
pub mod store_migrations;
pub mod warehouse_history;
pub mod warehouse_metadata;
pub mod webhooks;
Expand Down
82 changes: 0 additions & 82 deletions crates/sqlbuild-orchestration/src/store/_helpers/migration.rs

This file was deleted.

2 changes: 0 additions & 2 deletions crates/sqlbuild-orchestration/src/store/_helpers/mod.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
pub(crate) mod changes;
pub(crate) mod database;
pub(crate) mod migration;
pub(crate) mod open_sqlite;
pub(crate) mod recall_rows;
pub(crate) mod recall_schema;
pub(crate) mod release_version;
#[cfg(test)]
pub(crate) mod statement_count;
8 changes: 7 additions & 1 deletion crates/sqlbuild-orchestration/src/store/constants.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
//! Control-store limits.
//! Control-store limits and schema.

/// A SQLite WAL past this size is reset right after the commit that grew it.
pub(crate) const LARGE_WAL_BYTES: u64 = 64 * 1024 * 1024;

/// The control-store schema template, rendered per database type.
pub(crate) const SCHEMA: &str = include_str!("schema.sql");

/// Indexes, applied on every open; like `SCHEMA`, a change needs a numbered migration.
pub(crate) const INDEXES: &str = include_str!("indexes.sql");
Loading
Loading