From 143f858a10ca8af3fed1f748d39d27fae2fa6eb3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:59:06 +0000 Subject: [PATCH 1/3] feat(server): migrate the control store with numbered migrations The control store records a schema version next to its checksum. A server creates a new store at the current version and upgrades an older one by applying the numbered migrations after it, in one transaction under the store lock. On SQLite, the transaction runs with foreign keys off so a migration can rebuild a referenced table, then checks them before committing. Stores from releases before versioning that match the version 1 baseline are upgraded in place. The server refuses a store written by a newer release, a store older than the baseline, and a store at the current version whose schema differs, each with the fix. `sqb server migrate` applies the pending migrations and exits; `--dry-run` lists them without changing the store. Contract tests on SQLite and PostgreSQL check that the baseline plus every migration matches a new store, that a populated store keeps its rows, that a migration can rebuild sqb_runs, the refusals, and the dry run. --- .../sqlbuild-cli/src/argv/command_table.json | 59 + .../src/native_commands/_helpers/execution.rs | 76 + .../main/run_control_invocation.rs | 4 + .../tests/native_commands/test_control.rs | 47 +- .../tests/native_commands/test_types.rs | 6 + crates/sqlbuild-orchestration/src/lib.rs | 1 + .../src/store/_helpers/migration.rs | 82 - .../src/store/_helpers/mod.rs | 2 - .../src/store/constants.rs | 8 +- .../src/store/main/begin.rs | 16 +- .../src/store/main/migrate_store.rs | 51 +- .../src/store/models.rs | 2 + .../store_migrations/_helpers/migration.rs | 193 +++ .../_helpers/migration_run.rs | 108 ++ .../src/store_migrations/_helpers/mod.rs | 3 + .../_helpers/recall_schema.rs | 0 .../src/store_migrations/constants.rs | 12 + .../main/apply_store_migrations.rs | 11 + .../src/store_migrations/main/mod.rs | 2 + .../main/plan_store_migration.rs | 24 + .../src/store_migrations/mod.rs | 6 + .../src/store_migrations/models.rs | 27 + .../tests/store/schema/baseline/postgres.sql | 1431 +++++++++++++++++ .../tests/store/schema/baseline/rows.sql | 17 + .../tests/store/schema/baseline/sqlite.sql | 1431 +++++++++++++++++ .../tests/store/schema/helpers.rs | 209 ++- .../tests/store/schema/test_contracts.rs | 2 + .../store/schema/test_store_migrations.rs | 412 +++++ .../tests/store/schema/test_types.rs | 52 + .../sqlbuild/references/docs/cli/server.md | 34 +- website/src/content/docs/docs/cli/server.mdx | 34 +- 31 files changed, 4217 insertions(+), 145 deletions(-) delete mode 100644 crates/sqlbuild-orchestration/src/store/_helpers/migration.rs create mode 100644 crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs create mode 100644 crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration_run.rs create mode 100644 crates/sqlbuild-orchestration/src/store_migrations/_helpers/mod.rs rename crates/sqlbuild-orchestration/src/{store => store_migrations}/_helpers/recall_schema.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/store_migrations/constants.rs create mode 100644 crates/sqlbuild-orchestration/src/store_migrations/main/apply_store_migrations.rs create mode 100644 crates/sqlbuild-orchestration/src/store_migrations/main/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/store_migrations/main/plan_store_migration.rs create mode 100644 crates/sqlbuild-orchestration/src/store_migrations/mod.rs create mode 100644 crates/sqlbuild-orchestration/src/store_migrations/models.rs create mode 100644 crates/sqlbuild-orchestration/tests/store/schema/baseline/postgres.sql create mode 100644 crates/sqlbuild-orchestration/tests/store/schema/baseline/rows.sql create mode 100644 crates/sqlbuild-orchestration/tests/store/schema/baseline/sqlite.sql create mode 100644 crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index cc1c12d0d6..19ba19cb80 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1182,6 +1182,65 @@ "default": false, "help": "print the OpenAPI contract as JSON without opening a database", "group": 1 + }, + { + "kind": "parsers", + "dest": "server_command", + "nargs": "A...", + "required": false, + "group": 0, + "parsers": [ + { + "name": "migrate", + "parser": { + "prog": "sqb server migrate", + "description": "Apply the control store's pending schema migrations and exit.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--dry-run" + ], + "dest": "control_dry_run", + "nargs": 0, + "const": true, + "default": false, + "help": "list the migrations that would run without changing the store", + "group": 1 + } + ] + } + } + ], + "choices_help": [ + { + "dest": "migrate", + "metavar": "migrate", + "help": "apply pending control-store migrations and exit" + } + ] } ] } diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 556186ab90..59da49a414 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -59,6 +59,8 @@ use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::main::set_release_version::set_release_version; use sqlbuild_orchestration::store::models::ControlStore; +use sqlbuild_orchestration::store_migrations::main::plan_store_migration::plan_store_migration; +use sqlbuild_orchestration::store_migrations::models::StoreMigrationPlan; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ MetadataBatch, MetadataScan, StatisticsDueRead, StatisticsRead, WarehouseHistoryFailure, @@ -117,6 +119,7 @@ use crate::native_commands::models::{ MetadataReply, NativeCommand, NativeInvocation, ProjectLocation, ScanToken, }; use crate::native_commands::types::LocationPublication; +use crate::server_config::main::read_server_config::read_server_config; use crate::server_config::models::ServerSection; use crate::workers::main::docker_image_sources::docker_image_sources; use crate::workers::main::open_development_workers::open_development_workers; @@ -1886,3 +1889,76 @@ pub(crate) async fn compile_next_generation( .await?; Ok(compiled.map(|compiled| (Some((compiled, generation)), metadata, compiled_inputs))) } + +/// Run `sqb server migrate` on `runtime` and return its exit code. +pub(crate) fn run_store_migration(namespace: &Namespace, runtime: &tokio::runtime::Runtime) -> i32 { + drop(write_output(&[( + Stream::Stderr, + "Opening control store...\n", + )])); + match runtime.block_on(store_migration( + namespace, + flag(namespace, "control_dry_run"), + )) { + Ok(report) => { + let lost = write_output(&[(Stream::Stdout, &report)]); + let exit_code = finish_lost_stdout(lost.as_ref(), 0); + if exit_code == 0 { + drop(write_output(&[( + Stream::Stderr, + "Control store migration completed successfully\n", + )])); + } + exit_code + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Control store migration failed: {error}\n"), + )])); + 1 + } + } +} + +async fn store_migration(namespace: &Namespace, dry_run: bool) -> Result { + let config = read_server_config(namespace)?; + let url = control_database( + namespace, + &control_database_variable(namespace, &config.server), + )?; + let store = open_store(&url).await?; + let plan = plan_store_migration(&store).await?; + if !dry_run { + migrate_store(&store).await?; + } + Ok(describe(&plan, dry_run)) +} + +/// The plan as lines for a person: what would happen with `--dry-run`, what happened without it. +fn describe(plan: &StoreMigrationPlan, dry_run: bool) -> String { + let (create, upgrade, record) = if dry_run { + ("would create", "would migrate", "would record") + } else { + ("created", "migrated", "recorded") + }; + match plan { + StoreMigrationPlan::Create { version } => { + format!("The control store is empty: {create} schema version {version}.\n") + } + StoreMigrationPlan::Current { version } => { + format!("The control store is at schema version {version}: nothing to migrate.\n") + } + StoreMigrationPlan::Upgrade { from, to, names } if names.is_empty() => format!( + "The control store's schema is version {from} without a recorded version: \ + {record} version {to}.\n" + ), + StoreMigrationPlan::Upgrade { from, to, names } => format!( + "The control store {upgrade} from schema version {from} to {to}:\n{}", + names + .iter() + .map(|name| format!(" {name}\n")) + .collect::() + ), + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs index a77a31147f..30b95fce50 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs @@ -5,6 +5,7 @@ use sqlbuild_orchestration::api::main::openapi_document::openapi_document; use crate::argv::models::{Namespace, Stream}; use crate::native_commands::_helpers::decision::{flag, text}; use crate::native_commands::_helpers::execution::run_control_action; +use crate::native_commands::_helpers::execution::run_store_migration; use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; use crate::native_commands::constants::{ADMIN_COMMAND, SCHEDULER_COMMAND, SERVER_COMMAND}; @@ -45,6 +46,9 @@ pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { return Some(1); } }; + if text(namespace, "server_command").as_deref() == Some("migrate") { + return Some(run_store_migration(namespace, &runtime)); + } let admin = command == ADMIN_COMMAND; let role = if admin { "administration" } else { "service" }; let result = runtime.block_on(run_control_action(namespace, &command)); diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 59bea6e574..5db1dbd2aa 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -1,7 +1,10 @@ use std::process::Command; use crate::helpers::{api_ready, isolated_binary, start_control, wait_for_health}; -use crate::test_types::{CommandTestCase, OpenApiTestCase, ServiceTestCase, TriggerCursorTestCase}; +use crate::test_types::{ + CommandTestCase, OpenApiTestCase, ServiceTestCase, StoreMigrationCommandTestCase, + TriggerCursorTestCase, +}; #[test] fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_python_and_database_access_are_not_required() @@ -364,6 +367,48 @@ fn given_a_standalone_server_when_exporting_openapi_then_the_worker_contract_is_ } } +#[test] +fn given_a_control_store_when_running_server_migrate_then_dry_run_lists_and_migrate_applies() { + let test_cases = [ + StoreMigrationCommandTestCase { + description: "dry run on an empty store", + arguments: &["server", "migrate", "--dry-run"], + expected_stdout: "The control store is empty: would create schema version 1.\n", + }, + StoreMigrationCommandTestCase { + description: "migrate an empty store", + arguments: &["server", "migrate"], + expected_stdout: "The control store is empty: created schema version 1.\n", + }, + StoreMigrationCommandTestCase { + description: "dry run on a current store", + arguments: &["server", "migrate", "--dry-run"], + expected_stdout: "The control store is at schema version 1: nothing to migrate.\n", + }, + ]; + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let binary = isolated_binary(directory.path()); + for test_case in test_cases { + let output = Command::new(&binary) + .args(test_case.arguments) + .current_dir(directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .output() + .expect("run sqb server migrate"); + assert_eq!( + ( + output.status.success(), + String::from_utf8_lossy(&output.stdout).into_owned() + ), + (true, test_case.expected_stdout.to_owned()), + "{}: {}", + test_case.description, + String::from_utf8_lossy(&output.stderr) + ); + } +} + #[test] fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then_migrations_auth_and_shutdown_are_native() { diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs index 689f38a303..f502be6f1d 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -81,3 +81,9 @@ pub(crate) struct LocationHealthTestCase { pub(crate) expected_state: &'static str, pub(crate) expected_exit: i32, } + +pub(crate) struct StoreMigrationCommandTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_stdout: &'static str, +} diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs index 05119775f3..1f9ff273a7 100644 --- a/crates/sqlbuild-orchestration/src/lib.rs +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -14,6 +14,7 @@ pub mod scheduler; pub mod sensors; pub mod snapshots; pub mod store; +pub mod store_migrations; pub mod warehouse_history; pub mod warehouse_metadata; pub mod webhooks; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs deleted file mode 100644 index 8a2ed7baa2..0000000000 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! One schema template, rendered only at the database type boundary. - -use sqlbuild_recall::records::main::sha256_hex::sha256_hex; - -use crate::store::_helpers::recall_schema::recall_schema; -use crate::store::errors::ControlError; -use crate::store::models::ControlStore; -use crate::store::types::Row; - -pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); - -/// Additive indexes, created on every open and kept out of the digest so old stores gain them. -pub(crate) const INDEXES: &str = include_str!("../indexes.sql"); - -pub(crate) fn statements(sql: &str) -> impl Iterator { - sql.split(';') - .map(str::trim) - .filter(|statement| !statement.is_empty()) -} - -pub(crate) fn schema_digest(store: &ControlStore) -> String { - sha256_hex(format!("{}\n{SCHEMA}", recall_schema(store).join(";\n")).as_bytes()) -} - -/// Accept only the exact schema this server creates, or say how to start a new store. -pub(crate) fn verify_schema( - store: &ControlStore, - rows: &[Row], - digest: &str, -) -> Result<(), ControlError> { - if rows - .iter() - .all(|row| row.get("checksum").and_then(|value| value.as_str()) == Some(digest)) - { - return Ok(()); - } - let quote = |text: &str| format!("'{}'", text.replace('\'', r"'\''")); - let (scope, command) = match store { - ControlStore::Sqlite(path) => { - let path = path.to_string_lossy(); - let command = format!( - "rm -f {} {} {}", - quote(&path), - quote(&format!("{path}-wal")), - quote(&format!("{path}-shm")) - ); - ("", command) - } - ControlStore::Postgres(pool) => { - let options = pool.connect_options(); - let host = options.get_socket().map_or_else( - || options.get_host().to_owned(), - |socket| socket.to_string_lossy().into_owned(), - ); - let target = format!( - "-h {} -p {} -U {} {}", - quote(&host), - options.get_port(), - quote(options.get_username()), - quote(options.get_database().unwrap_or(options.get_username())) - ); - ( - " and everything else in that database", - format!("dropdb {target} && createdb {target}"), - ) - } - }; - Err(ControlError::Schema(format!( - "this control store was created by a different sqb release and cannot be upgraded; \ - if a newer release created it, upgrade this server instead. To start a new, empty \ - store, which discards its run history{scope}, stop every server using it and run: \ - {command}" - ))) -} - -pub(crate) fn render(store: &ControlStore, sql: &str) -> String { - let postgres = matches!(store, ControlStore::Postgres(_)); - sql.replace("{uuid}", if postgres { "UUID" } else { "TEXT" }) - .replace("{time}", if postgres { "TIMESTAMPTZ" } else { "TEXT" }) - .replace("{binary}", if postgres { "BYTEA" } else { "BLOB" }) - .replace("{json}", if postgres { "JSONB" } else { "JSON" }) -} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs index 420775f72e..d225ac992f 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs @@ -1,9 +1,7 @@ pub(crate) mod changes; pub(crate) mod database; -pub(crate) mod migration; pub(crate) mod open_sqlite; pub(crate) mod recall_rows; -pub(crate) mod recall_schema; pub(crate) mod release_version; #[cfg(test)] pub(crate) mod statement_count; diff --git a/crates/sqlbuild-orchestration/src/store/constants.rs b/crates/sqlbuild-orchestration/src/store/constants.rs index 61d1f56a56..601bf6e64d 100644 --- a/crates/sqlbuild-orchestration/src/store/constants.rs +++ b/crates/sqlbuild-orchestration/src/store/constants.rs @@ -1,4 +1,10 @@ -//! Control-store limits. +//! Control-store limits and schema. /// A SQLite WAL past this size is reset right after the commit that grew it. pub(crate) const LARGE_WAL_BYTES: u64 = 64 * 1024 * 1024; + +/// The control-store schema template, rendered per database type. +pub(crate) const SCHEMA: &str = include_str!("schema.sql"); + +/// Indexes, applied on every open; like `SCHEMA`, a change needs a numbered migration. +pub(crate) const INDEXES: &str = include_str!("indexes.sql"); diff --git a/crates/sqlbuild-orchestration/src/store/main/begin.rs b/crates/sqlbuild-orchestration/src/store/main/begin.rs index 7deced8adc..fa9f823a87 100644 --- a/crates/sqlbuild-orchestration/src/store/main/begin.rs +++ b/crates/sqlbuild-orchestration/src/store/main/begin.rs @@ -27,12 +27,16 @@ pub(crate) async fn begin( )?; connection.busy_timeout(Duration::from_secs(30))?; connection.set_db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, true)?; - connection.execute_batch(if mode == TransactionMode::Write { - "PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" - } else { - "PRAGMA foreign_keys=ON; PRAGMA query_only=ON; BEGIN" + connection.execute_batch(match mode { + TransactionMode::Write => { + "PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" + } + TransactionMode::Migrate => { + "PRAGMA synchronous=FULL; PRAGMA foreign_keys=OFF; BEGIN IMMEDIATE" + } + TransactionMode::Read => "PRAGMA foreign_keys=ON; PRAGMA query_only=ON; BEGIN", })?; - let notifications = if mode == TransactionMode::Write { + let notifications = if mode != TransactionMode::Read { Some(sqlite_changes(&path)?) } else { None @@ -43,7 +47,7 @@ pub(crate) async fn begin( } ControlStore::Postgres(pool) => { let mut transaction = pool.begin().await?; - if mode == TransactionMode::Write { + if mode != TransactionMode::Read { sqlx_core::query::query("SELECT pg_advisory_xact_lock(1936810572, 1)") .execute(&mut *transaction) .await?; diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index e82180367e..e4360a83a3 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -1,51 +1,10 @@ -//! Schema creation under the same lock used by all other control-plane writes. +//! Schema creation and numbered migrations under the same lock used by all other control-plane writes. -use uuid::Uuid; - -use crate::store::_helpers::migration::{ - INDEXES, SCHEMA, render, schema_digest, statements, verify_schema, -}; -use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::ControlStore; +use crate::store_migrations::main::apply_store_migrations::apply_store_migrations; -/// Create or verify the schema, then add any missing additive indexes. +/// Create the schema or apply its pending migrations in one transaction, then add any missing additive indexes. pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { - let mut transaction = begin(store, TransactionMode::Write).await?; - (transaction, _) = query(transaction, &render(store, - "CREATE TABLE IF NOT EXISTS sqb_schema_version (singleton BIGINT PRIMARY KEY CHECK (singleton = 1), \ - checksum TEXT NOT NULL, applied_at {time} NOT NULL)"), vec![]).await?; - let (mut transaction, rows) = query( - transaction, - "SELECT checksum FROM sqb_schema_version", - vec![], - ) - .await?; - let digest = schema_digest(store); - verify_schema(store, &rows, &digest)?; - if rows.is_empty() { - for statement in recall_schema(store) { - (transaction, _) = query(transaction, &statement, vec![]).await?; - } - for statement in statements(SCHEMA) { - (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; - } - let now = transaction.now; - (transaction, _) = query(transaction, - "INSERT INTO sqb_installation (singleton, installation_id, created_at) VALUES (1, $1, $2)", - vec![Parameter::Id(Uuid::now_v7()), Parameter::Time(now)]).await?; - (transaction, _) = query( - transaction, - "INSERT INTO sqb_schema_version (singleton, checksum, applied_at) VALUES (1, $1, $2)", - vec![Parameter::Text(digest), Parameter::Time(now)], - ) - .await?; - } - for statement in statements(INDEXES) { - (transaction, _) = query(transaction, statement, vec![]).await?; - } - commit(transaction).await + apply_store_migrations(store).await } diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs index fc9a7371b2..7b39d52ef4 100644 --- a/crates/sqlbuild-orchestration/src/store/models.rs +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -70,6 +70,8 @@ pub(crate) enum TargetRow { pub(crate) enum TransactionMode { Read, Write, + /// A write with SQLite foreign keys off, so a migration can rebuild a referenced table; it checks them before commit. + Migrate, } /// An owned transaction; abandoning it rolls back, including when an async task is cancelled. diff --git a/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs new file mode 100644 index 0000000000..7521738ec0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs @@ -0,0 +1,193 @@ +//! One schema template, rendered only at the database type boundary, and the numbered migrations to it. + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, ControlTransaction}; +use crate::store_migrations::_helpers::recall_schema::recall_schema; +use crate::store_migrations::constants::{ + BASELINE_VERSION, POSTGRES_BASELINE_CHECKSUM, SQLITE_BASELINE_CHECKSUM, +}; +use crate::store_migrations::models::{StoreMigration, StoreMigrationPlan}; + +use crate::store::constants::{INDEXES, SCHEMA}; + +/// The migrations after the baseline, in order: entry `i` includes `store_migrations/sql/{NNNN}_{name}.{sqlite,postgres}.sql` and brings a store to version `NNNN = BASELINE_VERSION + i + 1`. +pub(crate) const MIGRATIONS: &[StoreMigration] = &[]; + +/// The schema version a store created by this build has. +pub(crate) fn current_version(migrations: &[StoreMigration]) -> i64 { + BASELINE_VERSION + i64::try_from(migrations.len()).unwrap_or(i64::MAX - BASELINE_VERSION) +} + +pub(crate) fn statements(sql: &str) -> impl Iterator { + sql.split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) +} + +/// The checksum of the schema this build creates, recorded with its version. +pub(crate) fn schema_digest(store: &ControlStore) -> String { + sha256_hex(format!("{}\n{SCHEMA}\n{INDEXES}", recall_schema(store).join(";\n")).as_bytes()) +} + +/// The statements of `migration` for `store`'s database type. +pub(crate) fn migration_sql(store: &ControlStore, migration: &StoreMigration) -> String { + render( + store, + match store { + ControlStore::Sqlite(_) => migration.sqlite, + ControlStore::Postgres(_) => migration.postgres, + }, + ) +} + +/// The store's recorded `(checksum, version)`; the version is `None` for stores from before versioning. +pub(crate) async fn recorded_schema( + store: &ControlStore, + transaction: ControlTransaction, +) -> Result<(ControlTransaction, Option<(String, Option)>), ControlError> { + let columns = match store { + ControlStore::Sqlite(_) => "SELECT name FROM pragma_table_info('sqb_schema_version')", + ControlStore::Postgres(_) => { + "SELECT column_name AS name FROM information_schema.columns \ + WHERE table_schema = current_schema() AND table_name = 'sqb_schema_version'" + } + }; + let (transaction, columns) = query(transaction, columns, vec![]).await?; + let has = |name: &str| columns.iter().any(|row| row["name"].as_str() == Some(name)); + if !has("checksum") { + return Ok((transaction, None)); + } + let select = if has("version") { + "SELECT checksum, version FROM sqb_schema_version" + } else { + "SELECT checksum, NULL AS version FROM sqb_schema_version" + }; + let (transaction, rows) = query(transaction, select, vec![]).await?; + let recorded = rows.first().map(|row| { + ( + row["checksum"].as_str().unwrap_or_default().to_owned(), + row["version"].as_i64(), + ) + }); + Ok((transaction, recorded)) +} + +/// What opening a store with `recorded` schema does, or why this build refuses it. +pub(crate) fn migration_plan( + store: &ControlStore, + recorded: Option<(&str, Option)>, + migrations: &[StoreMigration], +) -> Result { + let current = current_version(migrations); + let Some((checksum, version)) = recorded else { + return Ok(StoreMigrationPlan::Create { version: current }); + }; + let baseline_checksum = match store { + ControlStore::Sqlite(_) => SQLITE_BASELINE_CHECKSUM, + ControlStore::Postgres(_) => POSTGRES_BASELINE_CHECKSUM, + }; + let version = match version { + Some(version) => version, + None if checksum == baseline_checksum => BASELINE_VERSION, + None => return Err(refusal(store, Refusal::OlderThanBaseline)), + }; + if version > current { + return Err(refusal(store, Refusal::Newer { version, current })); + } + if version < BASELINE_VERSION { + return Err(refusal(store, Refusal::OlderThanBaseline)); + } + if version == current && !matches!(recorded, Some((_, None))) { + return if checksum == schema_digest(store) { + Ok(StoreMigrationPlan::Current { version }) + } else { + Err(refusal(store, Refusal::Mismatch { version })) + }; + } + let pending = &migrations[usize::try_from(version - BASELINE_VERSION).unwrap_or(0)..]; + Ok(StoreMigrationPlan::Upgrade { + from: version, + to: current, + names: pending + .iter() + .zip(version + 1..) + .map(|(migration, number)| format!("{number:04}_{}", migration.name)) + .collect(), + }) +} + +enum Refusal { + Newer { version: i64, current: i64 }, + OlderThanBaseline, + Mismatch { version: i64 }, +} + +/// Why this build refuses a store, and how to start a new one. +fn refusal(store: &ControlStore, refusal: Refusal) -> ControlError { + let (scope, command) = recreate_command(store); + let recreate = format!( + "To start a new, empty store, which discards its run history{scope}, stop every \ + server using it and run: {command}" + ); + ControlError::Schema(match refusal { + Refusal::Newer { version, current } => format!( + "this control store's schema is version {version}, newer than this build's \ + version {current}; upgrade this server to the release that last opened it or a later one" + ), + Refusal::OlderThanBaseline => format!( + "this control store's schema is older than version {BASELINE_VERSION}, the oldest \ + this build upgrades; open it once with an earlier release that still has the \ + migrations it needs, then with this one. {recreate}" + ), + Refusal::Mismatch { version } => format!( + "this control store's schema doesn't match this build's, although both are version \ + {version}; the build's schema changed without a migration. {recreate}" + ), + }) +} + +/// The scope of what recreating `store` discards, and the command that does it. +fn recreate_command(store: &ControlStore) -> (&'static str, String) { + let quote = |text: &str| format!("'{}'", text.replace('\'', r"'\''")); + match store { + ControlStore::Sqlite(path) => { + let path = path.to_string_lossy(); + let command = format!( + "rm -f {} {} {}", + quote(&path), + quote(&format!("{path}-wal")), + quote(&format!("{path}-shm")) + ); + ("", command) + } + ControlStore::Postgres(pool) => { + let options = pool.connect_options(); + let host = options.get_socket().map_or_else( + || options.get_host().to_owned(), + |socket| socket.to_string_lossy().into_owned(), + ); + let target = format!( + "-h {} -p {} -U {} {}", + quote(&host), + options.get_port(), + quote(options.get_username()), + quote(options.get_database().unwrap_or(options.get_username())) + ); + ( + " and everything else in that database", + format!("dropdb {target} && createdb {target}"), + ) + } + } +} + +pub(crate) fn render(store: &ControlStore, sql: &str) -> String { + let postgres = matches!(store, ControlStore::Postgres(_)); + sql.replace("{uuid}", if postgres { "UUID" } else { "TEXT" }) + .replace("{time}", if postgres { "TIMESTAMPTZ" } else { "TEXT" }) + .replace("{binary}", if postgres { "BYTEA" } else { "BLOB" }) + .replace("{json}", if postgres { "JSONB" } else { "JSON" }) +} diff --git a/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration_run.rs b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration_run.rs new file mode 100644 index 0000000000..117177f693 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration_run.rs @@ -0,0 +1,108 @@ +//! Creating a control store or applying its pending migrations, with the migration list as a parameter. + +use uuid::Uuid; + +use crate::store::constants::{INDEXES, SCHEMA}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store_migrations::_helpers::migration::{ + migration_plan, migration_sql, recorded_schema, render, schema_digest, statements, +}; +use crate::store_migrations::_helpers::recall_schema::recall_schema; +use crate::store_migrations::constants::BASELINE_VERSION; +use crate::store_migrations::models::{StoreMigration, StoreMigrationPlan}; + +/// `migrate_store` with the migrations `migrations` in place of this build's. +pub(crate) async fn migrate_with( + store: &ControlStore, + migrations: &[StoreMigration], +) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Migrate).await?; + let (mut transaction, recorded) = recorded_schema(store, transaction).await?; + let plan = migration_plan( + store, + recorded + .as_ref() + .map(|(checksum, version)| (checksum.as_str(), *version)), + migrations, + )?; + let digest = schema_digest(store); + let now = transaction.now; + match plan { + StoreMigrationPlan::Current { .. } => {} + StoreMigrationPlan::Create { version } => { + (transaction, _) = query(transaction, &render(store, + "CREATE TABLE sqb_schema_version (singleton BIGINT PRIMARY KEY CHECK (singleton = 1), \ + checksum TEXT NOT NULL, applied_at {time} NOT NULL, version BIGINT)"), vec![]).await?; + for statement in recall_schema(store) { + (transaction, _) = query(transaction, &statement, vec![]).await?; + } + for statement in statements(SCHEMA) { + (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; + } + (transaction, _) = query(transaction, + "INSERT INTO sqb_installation (singleton, installation_id, created_at) VALUES (1, $1, $2)", + vec![Parameter::Id(Uuid::now_v7()), Parameter::Time(now)]).await?; + (transaction, _) = query( + transaction, + "INSERT INTO sqb_schema_version (singleton, checksum, applied_at, version) \ + VALUES (1, $1, $2, $3)", + vec![ + Parameter::Text(digest), + Parameter::Time(now), + Parameter::Integer(version), + ], + ) + .await?; + } + StoreMigrationPlan::Upgrade { from, to, .. } => { + if recorded + .as_ref() + .is_some_and(|(_, version)| version.is_none()) + { + (transaction, _) = query( + transaction, + "ALTER TABLE sqb_schema_version ADD COLUMN version BIGINT", + vec![], + ) + .await?; + } + let pending = &migrations[usize::try_from(from - BASELINE_VERSION).unwrap_or(0)..]; + for migration in pending { + let sql = migration_sql(store, migration); + for statement in statements(&sql) { + (transaction, _) = query(transaction, statement, vec![]).await?; + } + } + if !pending.is_empty() && matches!(store, ControlStore::Sqlite(_)) { + let violations; + (transaction, violations) = + query(transaction, "PRAGMA foreign_key_check", vec![]).await?; + if !violations.is_empty() { + return Err(ControlError::Schema(format!( + "migrating the control store to version {to} would break {} foreign key \ + references; the store is unchanged", + violations.len() + ))); + } + } + (transaction, _) = query( + transaction, + "UPDATE sqb_schema_version SET checksum = $1, applied_at = $2, version = $3", + vec![ + Parameter::Text(digest), + Parameter::Time(now), + Parameter::Integer(to), + ], + ) + .await?; + } + } + for statement in statements(INDEXES) { + (transaction, _) = query(transaction, statement, vec![]).await?; + } + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/store_migrations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/mod.rs new file mode 100644 index 0000000000..83115c1e5a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod migration; +pub(crate) mod migration_run; +pub(crate) mod recall_schema; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/recall_schema.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs rename to crates/sqlbuild-orchestration/src/store_migrations/_helpers/recall_schema.rs diff --git a/crates/sqlbuild-orchestration/src/store_migrations/constants.rs b/crates/sqlbuild-orchestration/src/store_migrations/constants.rs new file mode 100644 index 0000000000..9ba1ddd11c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/constants.rs @@ -0,0 +1,12 @@ +//! The control-store schema baseline. + +/// The oldest schema version this build upgrades; migrations at or below it are squashed away. +pub(crate) const BASELINE_VERSION: i64 = 1; + +/// The checksum unversioned SQLite stores record for the baseline schema (releases before versioning). +pub(crate) const SQLITE_BASELINE_CHECKSUM: &str = + "228a951fe04d75eedcefdb44095afe652b178d9b2f32f3a5287c143877d39ce2"; + +/// The checksum unversioned PostgreSQL stores record for the baseline schema. +pub(crate) const POSTGRES_BASELINE_CHECKSUM: &str = + "163045e32da43a8b0ddf17ea22f6dd41d1ff7bffe81836f3cbb35a8754915c3e"; diff --git a/crates/sqlbuild-orchestration/src/store_migrations/main/apply_store_migrations.rs b/crates/sqlbuild-orchestration/src/store_migrations/main/apply_store_migrations.rs new file mode 100644 index 0000000000..56b96f3dd2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/main/apply_store_migrations.rs @@ -0,0 +1,11 @@ +//! Bring a control store to this build's schema version. + +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::store_migrations::_helpers::migration::MIGRATIONS; +use crate::store_migrations::_helpers::migration_run::migrate_with; + +/// Create the schema or apply this build's pending migrations in one transaction, then add any missing additive indexes. +pub(crate) async fn apply_store_migrations(store: &ControlStore) -> Result<(), ControlError> { + migrate_with(store, MIGRATIONS).await +} diff --git a/crates/sqlbuild-orchestration/src/store_migrations/main/mod.rs b/crates/sqlbuild-orchestration/src/store_migrations/main/mod.rs new file mode 100644 index 0000000000..492ce6198b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/main/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod apply_store_migrations; +pub mod plan_store_migration; diff --git a/crates/sqlbuild-orchestration/src/store_migrations/main/plan_store_migration.rs b/crates/sqlbuild-orchestration/src/store_migrations/main/plan_store_migration.rs new file mode 100644 index 0000000000..67d47985e3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/main/plan_store_migration.rs @@ -0,0 +1,24 @@ +//! What opening a control store would do to its schema, without changing it. + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::store_migrations::_helpers::migration::{MIGRATIONS, migration_plan, recorded_schema}; +use crate::store_migrations::models::StoreMigrationPlan; + +/// The schema change `migrate_store` would make to `store`, or why it would refuse the store. +pub async fn plan_store_migration( + store: &ControlStore, +) -> Result { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, recorded) = recorded_schema(store, transaction).await?; + commit(transaction).await?; + migration_plan( + store, + recorded + .as_ref() + .map(|(checksum, version)| (checksum.as_str(), *version)), + MIGRATIONS, + ) +} diff --git a/crates/sqlbuild-orchestration/src/store_migrations/mod.rs b/crates/sqlbuild-orchestration/src/store_migrations/mod.rs new file mode 100644 index 0000000000..5229b9811f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/mod.rs @@ -0,0 +1,6 @@ +//! Control-store schema versions: creating a store, numbered migrations, and the dry-run plan. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/store_migrations/models.rs b/crates/sqlbuild-orchestration/src/store_migrations/models.rs new file mode 100644 index 0000000000..9fdfc2cb3e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/models.rs @@ -0,0 +1,27 @@ +//! Numbered control-store migrations and what opening a store would do with them. + +/// One numbered control-store migration, with its SQL for each store type. +#[derive(Clone, Copy, Debug)] +pub struct StoreMigration { + /// The file name without its number, such as `add_skipped_runs`. + pub name: &'static str, + /// Statements for SQLite, separated by `;`. + pub sqlite: &'static str, + /// Statements for PostgreSQL, separated by `;`. + pub postgres: &'static str, +} + +/// What opening a control store would do to its schema. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StoreMigrationPlan { + /// No store yet: create one from the current schema at `version`. + Create { version: i64 }, + /// The store is at the current `version` already. + Current { version: i64 }, + /// Apply the numbered migrations `names`, in order, from `from` to `to`. + Upgrade { + from: i64, + to: i64, + names: Vec, + }, +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/baseline/postgres.sql b/crates/sqlbuild-orchestration/tests/store/schema/baseline/postgres.sql new file mode 100644 index 0000000000..77bde76f7b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/baseline/postgres.sql @@ -0,0 +1,1431 @@ +CREATE TABLE IF NOT EXISTS "recall_meta" ( + "component" TEXT NOT NULL, + "key" TEXT NOT NULL, + "value" TEXT NOT NULL, + "recorded_at" TIMESTAMPTZ NOT NULL, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("component", "key", "recorded_at") +); + +CREATE TABLE IF NOT EXISTS "recall_invocations" ( + "command" TEXT, + "subcommand" TEXT, + "selection" JSONB, + "exclusion" JSONB, + "project_name" TEXT, + "project_dir_hash" TEXT, + "target" TEXT, + "adapter" TEXT, + "destination_database" TEXT, + "destination_schema" TEXT, + "destination_fingerprint" TEXT, + "sqlbuild_version" TEXT, + "git_revision" TEXT, + "git_dirty" BOOLEAN, + "host_token" TEXT, + "pid" BIGINT, + "external_context" JSONB, + "recall_filter_active" BOOLEAN, + "argv_digest" TEXT, + "origin" TEXT, + "command_mode" TEXT, + "selection_count" BIGINT, + "selection_truncated" BOOLEAN, + "selector_files" JSONB, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id") +); + +CREATE INDEX IF NOT EXISTS "recall_invocations_started" ON "recall_invocations" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_output" ( + "record_id" TEXT NOT NULL, + "sequence" BIGINT NOT NULL, + "stream" TEXT NOT NULL, + "occurred_at" TIMESTAMPTZ NOT NULL, + "data" BYTEA NOT NULL, + "byte_offset" BIGINT NOT NULL, + "byte_count" BIGINT NOT NULL, + "priority" TEXT, + "dropped_before" BIGINT NOT NULL, + "omitted_bytes" BIGINT, + "omitted_sequences" BIGINT, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "sequence") +); + +CREATE INDEX IF NOT EXISTS "recall_output_expiry" ON "recall_output" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_events" ( + "event_id" TEXT NOT NULL, + "invocation_sequence" BIGINT, + "occurred_at" TIMESTAMPTZ NOT NULL, + "event_type" TEXT NOT NULL, + "event_kind" TEXT NOT NULL, + "severity" TEXT NOT NULL, + "producer" TEXT NOT NULL, + "event_schema_version" BIGINT NOT NULL, + "run_id" TEXT, + "resource_id" TEXT, + "resource_attempt_id" TEXT, + "operation_id" TEXT, + "statement_id" TEXT, + "command" TEXT, + "resource_kind" TEXT, + "resource_name" TEXT, + "error_code" TEXT, + "error_type" TEXT, + "duration_ms" DOUBLE PRECISION, + "exit_code" BIGINT, + "query_id" TEXT, + "sql_digest" TEXT, + "intent" TEXT, + "statement_kind" TEXT, + "purpose" TEXT, + "outcome" TEXT, + "row_count" BIGINT, + "affected_rows" BIGINT, + "payload" JSONB NOT NULL, + "started_at" TIMESTAMPTZ, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + storage_id BIGINT GENERATED ALWAYS AS IDENTITY, + UNIQUE ("event_id") +); + +CREATE INDEX IF NOT EXISTS "recall_events_invocation" ON "recall_events" ("invocation_id", "invocation_sequence"); + +CREATE INDEX IF NOT EXISTS "recall_events_type" ON "recall_events" ("event_type", "occurred_at"); + +CREATE INDEX IF NOT EXISTS "recall_events_statement" ON "recall_events" ("statement_id"); + +CREATE INDEX IF NOT EXISTS "recall_events_expiry" ON "recall_events" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_diagnostics" ( + "record_id" TEXT NOT NULL, + "sequence" BIGINT NOT NULL, + "occurred_at" TIMESTAMPTZ NOT NULL, + "severity" TEXT NOT NULL, + "logger" TEXT NOT NULL, + "source" TEXT NOT NULL, + "message" TEXT NOT NULL, + "fields" JSONB NOT NULL, + "log_stream_id" TEXT, + "run_id" TEXT, + "resource_id" TEXT, + "resource_attempt_id" TEXT, + "operation_id" TEXT, + "statement_id" TEXT, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "sequence") +); + +CREATE INDEX IF NOT EXISTS "recall_diagnostics_severity" ON "recall_diagnostics" ("severity"); + +CREATE INDEX IF NOT EXISTS "recall_diagnostics_expiry" ON "recall_diagnostics" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_artifacts" ( + "digest" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "media_type" TEXT NOT NULL, + "size" BIGINT NOT NULL, + "content" BYTEA NOT NULL, + "seen_at" TIMESTAMPTZ NOT NULL, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("digest") +); + +CREATE INDEX IF NOT EXISTS "recall_artifacts_seen" ON "recall_artifacts" ("seen_at"); + +CREATE TABLE IF NOT EXISTS "recall_artifact_refs" ( + "digest" TEXT NOT NULL, + "ref_id" TEXT NOT NULL, + "statement_id" TEXT, + "resource_id" TEXT, + "role" TEXT NOT NULL, + "referenced_at" TIMESTAMPTZ NOT NULL, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "digest", "ref_id") +); + +CREATE INDEX IF NOT EXISTS "recall_artifact_refs_digest" ON "recall_artifact_refs" ("digest"); + +CREATE TABLE IF NOT EXISTS "recall_heartbeats" ( + "beat_at" TIMESTAMPTZ NOT NULL, + "stale_at" TIMESTAMPTZ NOT NULL, + "host_token" TEXT, + "pid" BIGINT, + "command" TEXT, + "subcommand" TEXT, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "beat_at") +); + +CREATE TABLE IF NOT EXISTS "recall_delivery" ( + "stream" TEXT NOT NULL, + "accepted" BIGINT NOT NULL, + "delivered" BIGINT NOT NULL, + "filtered" BIGINT NOT NULL, + "dropped" BIGINT NOT NULL, + "failed" BIGINT NOT NULL, + "conflicts" BIGINT NOT NULL, + "flush_complete" BOOLEAN NOT NULL, + "written_at" TIMESTAMPTZ NOT NULL, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "stream") +); + +CREATE TABLE IF NOT EXISTS "recall_conflicts" ( + "dataset" TEXT NOT NULL, + "record_key" TEXT NOT NULL, + "existing_sha256" TEXT NOT NULL, + "rejected_sha256" TEXT NOT NULL, + "detected_at" TIMESTAMPTZ NOT NULL, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("dataset", "record_key", "rejected_sha256") +); + +CREATE INDEX IF NOT EXISTS "recall_conflicts_invocation" ON "recall_conflicts" ("invocation_id"); + +CREATE OR REPLACE VIEW "invocations" AS +WITH ids AS ( + SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at + FROM ( + SELECT invocation_id, invocation_started_at FROM recall_invocations + UNION ALL SELECT invocation_id, invocation_started_at FROM recall_heartbeats + UNION ALL SELECT invocation_id, invocation_started_at FROM recall_events + WHERE event_type IN ('invocation_started', 'invocation_completed', 'invocation_failed') + ) AS source_rows + GROUP BY invocation_id +), +started AS ( + SELECT invocation_id, MIN(occurred_at) AS started_at, MIN(command) AS command + FROM recall_events + WHERE event_type = 'invocation_started' + GROUP BY invocation_id +), +terminals AS ( + SELECT invocation_id, event_type, occurred_at, exit_code, duration_ms, error_type, error_code, + (payload #>> '{error_message}') AS error_message, + (payload #>> '{signal_number}')::BIGINT AS signal_number, + (payload #>> '{cost_status}') AS cost_status, + ROW_NUMBER() OVER ( + PARTITION BY invocation_id + ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC + ) AS position + FROM recall_events + WHERE event_type IN ('invocation_completed', 'invocation_failed') +), +beats AS ( + SELECT invocation_id, MAX(beat_at) AS last_heartbeat_at, MAX(stale_at) AS stale_at, + MIN(command) AS command, MIN(subcommand) AS subcommand + FROM recall_heartbeats + GROUP BY invocation_id +), +projected AS ( + SELECT + ids.invocation_id, + ids.invocation_started_at, + COALESCE(started.started_at, ids.invocation_started_at) AS started_at, + CASE + WHEN terminals.event_type = 'invocation_completed' THEN 'completed' + WHEN terminals.event_type = 'invocation_failed' + AND terminals.signal_number IS NOT NULL THEN 'cancelled' + WHEN terminals.event_type = 'invocation_failed' THEN 'failed' + WHEN beats.stale_at IS NULL THEN 'unknown' + WHEN beats.stale_at >= CURRENT_TIMESTAMP THEN 'running' + ELSE 'abandoned' + END AS status, + terminals.event_type AS terminal_event, + terminals.occurred_at AS terminal_at, + terminals.duration_ms AS terminal_duration_ms, + terminals.exit_code, + terminals.error_type, + terminals.error_code, + terminals.error_message, + terminals.signal_number, + terminals.cost_status, + COALESCE(context.command, started.command, beats.command) AS command, + COALESCE(context.subcommand, beats.subcommand) AS subcommand, + context.origin, + context.command_mode, + context.target, + context.adapter, + context.project_name, + context.selection, + COALESCE(context.selection_count, jsonb_array_length(context.selection)) + AS selection_count, + context.selection_truncated, + context.selector_files, + context.exclusion, + context.sqlbuild_version, + context.git_revision, + context.git_dirty, + context.host_token, + context.pid, + context.external_context, + CASE WHEN context.external_context IS NOT NULL + THEN (context.external_context #>> '{integration,name}') END + AS integration, + CASE WHEN context.external_context IS NOT NULL + THEN (context.external_context #>> '{integration,job_name}') END + AS integration_job, + CASE WHEN context.external_context IS NOT NULL + THEN (context.external_context #>> '{integration,run_id}') END + AS integration_run_id, + CASE WHEN context.external_context IS NOT NULL + THEN (context.external_context #>> '{integration,retry_number}')::BIGINT END + AS integration_retry, + context.recall_filter_active, + beats.last_heartbeat_at + FROM ids + LEFT JOIN recall_invocations AS context ON context.invocation_id = ids.invocation_id + LEFT JOIN started ON started.invocation_id = ids.invocation_id + LEFT JOIN terminals + ON terminals.invocation_id = ids.invocation_id AND terminals.position = 1 + LEFT JOIN beats ON beats.invocation_id = ids.invocation_id +) +SELECT + invocation_id, + invocation_started_at, + started_at, + status, + CASE + WHEN terminal_event IS NOT NULL THEN terminal_at + WHEN status = 'abandoned' THEN last_heartbeat_at + END AS ended_at, + CASE + WHEN terminal_event IS NOT NULL THEN terminal_duration_ms + WHEN status = 'abandoned' THEN CAST(ROUND(CAST(EXTRACT(EPOCH FROM (last_heartbeat_at - started_at)) * 1000 AS NUMERIC), 3) AS DOUBLE PRECISION) + END AS duration_ms, + CASE + WHEN terminal_event IS NOT NULL THEN 0 + WHEN status = 'abandoned' THEN 1 + END AS ended_at_estimated, + exit_code, + error_type, + error_code, + error_message, + signal_number, + cost_status, + command, + subcommand, + origin, + command_mode, + target, + adapter, + project_name, + selection, + selection_count, + selection_truncated, + selector_files, + exclusion, + sqlbuild_version, + git_revision, + git_dirty, + host_token, + pid, + external_context, + integration, + integration_job, + integration_run_id, + integration_retry, + recall_filter_active, + last_heartbeat_at +FROM projected; + +CREATE OR REPLACE VIEW "statements" AS +WITH facts AS ( + SELECT * + FROM recall_events + WHERE statement_id IS NOT NULL AND event_type LIKE 'statement!_%' ESCAPE '!' +), +firsts AS ( + SELECT invocation_id, invocation_started_at, statement_id, run_id, resource_id, resource_attempt_id, operation_id, + intent, statement_kind, purpose, sql_digest, + ROW_NUMBER() OVER ( + PARTITION BY statement_id ORDER BY invocation_sequence, occurred_at, event_id + ) AS position + FROM facts +), +started AS ( + SELECT statement_id, MIN(started_at) AS started_at + FROM ( + SELECT statement_id, occurred_at AS started_at FROM facts WHERE event_type = 'statement_started' + UNION ALL + SELECT statement_id, started_at FROM facts WHERE started_at IS NOT NULL + ) AS starts + GROUP BY statement_id +), +submitted AS ( + SELECT statement_id, MIN(occurred_at) AS submitted_at + FROM facts WHERE event_type = 'statement_submitted' GROUP BY statement_id +), +query_ids AS ( + SELECT statement_id, MAX(query_id) AS query_id FROM facts GROUP BY statement_id +), +terminals AS ( + SELECT statement_id, event_type, occurred_at, query_id, error_type, error_code, duration_ms, + row_count, affected_rows, outcome, + ROW_NUMBER() OVER ( + PARTITION BY statement_id + ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC + ) AS position + FROM facts + WHERE event_type IN ('statement_completed', 'statement_failed') +), +captures AS ( + SELECT firsts.statement_id, ( + SELECT MAX(CAST(artifact.digest IS NOT NULL AS INTEGER)) + FROM recall_artifact_refs AS ref + LEFT JOIN recall_artifacts AS artifact ON artifact.digest = ref.digest + WHERE ref.invocation_id = firsts.invocation_id + AND ref.digest = 'sha256:' || firsts.sql_digest + ) AS stored + FROM firsts + WHERE firsts.position = 1 +), +resources AS ( + SELECT invocation_id, resource_id, MIN(resource_name) AS resource_name, + MIN(resource_kind) AS resource_kind + FROM recall_events + WHERE resource_id IS NOT NULL AND resource_name IS NOT NULL + GROUP BY invocation_id, resource_id +) +SELECT + firsts.invocation_id, + firsts.invocation_started_at, + firsts.statement_id, + firsts.run_id, + firsts.resource_id, + resources.resource_name, + resources.resource_kind, + firsts.resource_attempt_id, + firsts.operation_id, + firsts.intent, + firsts.statement_kind, + firsts.purpose, + COALESCE(terminals.query_id, query_ids.query_id) AS query_id, + started.started_at, + submitted.submitted_at, + terminals.occurred_at AS ended_at, + CASE terminals.event_type + WHEN 'statement_completed' THEN 'completed' + WHEN 'statement_failed' THEN + CASE WHEN terminals.outcome = 'handled' THEN 'handled' ELSE 'failed' END + ELSE 'no_terminal' + END AS outcome, + terminals.error_type, + terminals.error_code, + terminals.duration_ms, + terminals.row_count, + terminals.affected_rows, + firsts.sql_digest, + COALESCE(captures.stored, 0) AS sql_available, + CASE captures.stored + WHEN 1 THEN NULL + WHEN 0 THEN 'expired_or_not_captured' + ELSE 'not_captured' + END AS sql_unavailable_reason +FROM firsts +LEFT JOIN started ON started.statement_id = firsts.statement_id +LEFT JOIN submitted ON submitted.statement_id = firsts.statement_id +LEFT JOIN query_ids ON query_ids.statement_id = firsts.statement_id +LEFT JOIN terminals + ON terminals.statement_id = firsts.statement_id AND terminals.position = 1 +LEFT JOIN captures ON captures.statement_id = firsts.statement_id +LEFT JOIN resources + ON resources.invocation_id = firsts.invocation_id + AND resources.resource_id = firsts.resource_id +WHERE firsts.position = 1; + +CREATE OR REPLACE VIEW "audit_results" AS +SELECT + invocation_id, + invocation_started_at, + event_id, + occurred_at, + run_id, + resource_id, + resource_attempt_id, + (payload #>> '{audit_name}') AS audit_name, + (payload #>> '{audit_definition_name}') AS audit_definition_name, + CASE WHEN (payload #>> '{target_name}') IS NULL THEN NULL ELSE + COALESCE((payload #>> '{target_database}') || '.', '') + || COALESCE((payload #>> '{target_schema}') || '.', '') + || (payload #>> '{target_name}') + END AS target_relation, + (payload #>> '{attached_target_kind}') AS attached_target_kind, + (payload #>> '{attached_target_name}') AS attached_target_name, + (payload #>> '{attached_column_name}') AS "column", + outcome, + (payload #>> '{severity}') AS severity, + (payload #>> '{violation_count}')::BIGINT AS violation_count, + (payload #>> '{measured_value}')::DOUBLE PRECISION AS measured_value, + (payload #>> '{evaluation_mode}') AS evaluation_mode, + (payload #>> '{execution_error}') AS execution_error, + sql_digest, + (payload #>> '{evidence_count}')::BIGINT AS evidence_count, + (payload #>> '{evidence_truncated}')::BOOLEAN::INTEGER AS evidence_truncated, + (payload #>> '{sample_digest}') AS sample_digest +FROM recall_events +WHERE event_type = 'audit_completed'; + +CREATE OR REPLACE VIEW "output_invocations" AS +WITH received AS ( + SELECT invocation_id, + MIN(invocation_started_at) AS invocation_started_at, + COUNT(*) AS chunks, + CAST(SUM(byte_count) AS BIGINT) AS bytes, + CAST(SUM(CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END) AS BIGINT) AS stdout_bytes, + CAST(SUM(CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END) AS BIGINT) AS stderr_bytes, + MAX(sequence) AS max_sequence, + CAST(SUM(dropped_before) AS BIGINT) AS queue_drops, + CAST(SUM(COALESCE(omitted_sequences, 1)) AS BIGINT) AS spans, + CAST(SUM(COALESCE(omitted_bytes, 0)) AS BIGINT) AS truncated_bytes + FROM recall_output + GROUP BY invocation_id +), +delivery AS ( + SELECT * FROM recall_delivery WHERE stream = 'output' +), +ids AS ( + SELECT invocation_id, invocation_started_at FROM received + UNION SELECT invocation_id, invocation_started_at FROM delivery +), +counted AS ( + SELECT + ids.invocation_id, + ids.invocation_started_at, + COALESCE(received.chunks, 0) AS chunks, + COALESCE(received.bytes, 0) AS bytes, + COALESCE(received.stdout_bytes, 0) AS stdout_bytes, + COALESCE(received.stderr_bytes, 0) AS stderr_bytes, + COALESCE(received.truncated_bytes, 0) AS truncated_bytes, + received.max_sequence, + GREATEST( + COALESCE(received.max_sequence + 1 - received.spans, 0), + COALESCE(delivery.accepted, 0) - COALESCE(received.chunks, 0) + ) AS missing_sequences, + GREATEST(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, + COALESCE(delivery.filtered, 0) AS filtered, + COALESCE(delivery.failed, 0) AS failed, + COALESCE(CAST(context.recall_filter_active AS INTEGER), 0) AS filter_active, + COALESCE(CAST(delivery.flush_complete AS INTEGER), 0) AS flush_complete, + delivery.invocation_id IS NOT NULL AS delivery_reported, + COALESCE(delivery.delivered, 0) AS delivered + FROM ids + LEFT JOIN received ON received.invocation_id = ids.invocation_id + LEFT JOIN delivery ON delivery.invocation_id = ids.invocation_id + LEFT JOIN recall_invocations AS context ON context.invocation_id = ids.invocation_id +) +SELECT + invocation_id, + invocation_started_at, + chunks, + bytes, + stdout_bytes, + stderr_bytes, + truncated_bytes, + max_sequence, + missing_sequences, + reported_drops, + filtered, + failed, + filter_active, + flush_complete, + CASE + WHEN missing_sequences = 0 AND truncated_bytes > 0 THEN 'truncated' + WHEN missing_sequences = 0 THEN NULL + WHEN filtered >= missing_sequences THEN 'filtered' + WHEN filtered + reported_drops + failed >= missing_sequences THEN 'dropped' + ELSE 'unexplained' + END AS gap_reason, + CASE + WHEN delivery_reported AND flush_complete = 1 AND missing_sequences = 0 + AND reported_drops = 0 AND filtered = 0 AND failed = 0 AND truncated_bytes = 0 + AND chunks >= delivered + THEN 1 ELSE 0 + END AS complete +FROM counted; + +CREATE OR REPLACE VIEW "output_tail" AS +SELECT invocation_id, invocation_started_at, sequence, stream, occurred_at, byte_count, + data AS text +FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY invocation_id ORDER BY sequence DESC) AS position + FROM recall_output +) AS source_rows +WHERE position <= 200; + +CREATE OR REPLACE VIEW "output_transcripts" AS +SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at, + COALESCE(string_agg(CASE WHEN stream = 'stdout' THEN data END, ''::BYTEA ORDER BY sequence), ''::BYTEA) AS stdout, + COALESCE(string_agg(CASE WHEN stream = 'stderr' THEN data END, ''::BYTEA ORDER BY sequence), ''::BYTEA) AS stderr, + COALESCE(string_agg(data, ''::BYTEA ORDER BY sequence), ''::BYTEA) AS combined +FROM recall_output GROUP BY invocation_id; + +CREATE OR REPLACE VIEW "events" AS +SELECT + event_id, + invocation_sequence, + occurred_at, + event_type, + event_kind, + severity, + producer, + event_schema_version, + run_id, + resource_id, + resource_attempt_id, + operation_id, + statement_id, + command, + resource_kind, + resource_name, + error_code, + error_type, + duration_ms, + exit_code, + query_id, + sql_digest, + intent, + statement_kind, + purpose, + outcome, + row_count, + affected_rows, + CASE WHEN producer = 'sqlbuild' THEN payload || jsonb_strip_nulls(jsonb_build_object('command', command, 'resource_kind', resource_kind, 'resource_name', resource_name, 'error_code', error_code, 'error_type', error_type, 'query_id', query_id, 'sql_digest', sql_digest, 'intent', intent, 'statement_kind', statement_kind, 'purpose', purpose, 'outcome', outcome, 'exit_code', exit_code, 'row_count', row_count, 'affected_rows', affected_rows)) ELSE payload END AS payload, + started_at, + invocation_id, + invocation_started_at, + schema_version, + producer_version, + ingested_at, + content_hash, + storage_id +FROM recall_events; + +CREATE OR REPLACE VIEW "diagnostics" AS +SELECT * FROM recall_diagnostics; + +CREATE OR REPLACE VIEW "artifacts" AS +SELECT + refs.invocation_id, + refs.invocation_started_at, + refs.digest, + refs.ref_id, + refs.role, + refs.statement_id, + refs.resource_id, + artifact.kind, + artifact.media_type, + artifact.size, + CAST(artifact.digest IS NOT NULL AS INTEGER) AS available, + CASE WHEN artifact.digest IS NULL THEN 'expired_or_not_captured' END AS unavailable_reason, + CASE WHEN artifact.media_type LIKE 'text/%' OR artifact.media_type = 'application/json' + THEN artifact.content END AS text, + artifact.content +FROM recall_artifact_refs AS refs +LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest; + +CREATE OR REPLACE VIEW "conflicts" AS +SELECT * FROM recall_conflicts; + +CREATE TABLE sqb_installation ( + singleton BIGINT PRIMARY KEY CHECK (singleton = 1), + installation_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE sqb_locations ( + location_id UUID PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + archived_at TIMESTAMPTZ +); + +CREATE TABLE sqb_source_texts ( + text_hash TEXT PRIMARY KEY, + body BYTEA NOT NULL +); + +CREATE TABLE sqb_node_definitions ( + node_hash TEXT PRIMARY KEY, + body BYTEA NOT NULL +); + +CREATE TABLE sqb_documents ( + document_hash TEXT PRIMARY KEY, + body BYTEA NOT NULL +); + +CREATE TABLE sqb_snapshots ( + snapshot_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + generation BIGINT NOT NULL CHECK (generation > 0), + git_revision TEXT, + git_dirty BOOLEAN NOT NULL, + git_author TEXT, + image_id TEXT, + environment_hash TEXT NOT NULL, + sqb_version TEXT NOT NULL, + manifest_digest TEXT NOT NULL, + published_at TIMESTAMPTZ NOT NULL, + UNIQUE (location_id, target, generation), + UNIQUE (location_id, target, snapshot_id) +); + +CREATE INDEX sqb_snapshots_location_time ON sqb_snapshots(location_id, target, published_at); + +CREATE TABLE sqb_snapshot_nodes ( + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + node_key TEXT NOT NULL, + node_kind TEXT NOT NULL, + node_hash TEXT NOT NULL REFERENCES sqb_node_definitions(node_hash), + PRIMARY KEY (snapshot_id, node_key) +); + +CREATE TABLE sqb_snapshot_sources ( + snapshot_id UUID NOT NULL, + node_key TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('raw_sql', 'compiled_sql', 'python_source')), + text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + PRIMARY KEY (snapshot_id, node_key, kind), + FOREIGN KEY (snapshot_id, node_key) REFERENCES sqb_snapshot_nodes(snapshot_id, node_key) +); + +CREATE TABLE sqb_snapshot_documents ( + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('jobs', 'groups', 'triggers', 'roles', 'lineage', 'columns', 'providers')), + document_hash TEXT NOT NULL REFERENCES sqb_documents(document_hash), + PRIMARY KEY (snapshot_id, kind) +); + +CREATE TABLE sqb_current_snapshots ( + location_id UUID NOT NULL, + target TEXT NOT NULL, + snapshot_id UUID NOT NULL, + PRIMARY KEY (location_id, target), + FOREIGN KEY (location_id, target, snapshot_id) + REFERENCES sqb_snapshots(location_id, target, snapshot_id) +); + +CREATE TABLE sqb_snapshot_changes ( + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + change_seq BIGINT NOT NULL, + change_kind TEXT NOT NULL CHECK (change_kind IN ('added', 'changed', 'removed')), + subject_key TEXT NOT NULL, + before_hash TEXT, + after_hash TEXT, + PRIMARY KEY (snapshot_id, change_seq) +); + +CREATE TABLE sqb_scheduler_lease ( + name TEXT PRIMARY KEY CHECK (name = 'scheduler'), + holder UUID NOT NULL, + epoch BIGINT NOT NULL CHECK (epoch > 0), + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE sqb_jobs ( + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + job_key TEXT NOT NULL, + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('job', 'group')), + definition_json JSONB NOT NULL, + PRIMARY KEY (location_id, target, job_key) +); + +CREATE TABLE sqb_triggers ( + trigger_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('schedule', 'sensor')), + name TEXT NOT NULL, + job_key TEXT NOT NULL, + definition_json JSONB NOT NULL, + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + removed_in_snapshot_id UUID REFERENCES sqb_snapshots(snapshot_id), + UNIQUE (location_id, target, kind, name) +); + +CREATE TABLE sqb_trigger_state ( + trigger_id UUID PRIMARY KEY REFERENCES sqb_triggers(trigger_id), + status TEXT NOT NULL CHECK (status IN ('running', 'paused')), + status_override TEXT CHECK (status_override IN ('running', 'paused')), + started_at TIMESTAMPTZ NOT NULL, + cursor_json JSONB, + last_tick_for TIMESTAMPTZ, + next_due_at TIMESTAMPTZ +); + +CREATE INDEX sqb_trigger_state_due ON sqb_trigger_state(next_due_at) WHERE status = 'running'; + +CREATE TABLE sqb_ticks ( + tick_id UUID PRIMARY KEY, + trigger_id UUID NOT NULL REFERENCES sqb_triggers(trigger_id), + scheduled_for TIMESTAMPTZ NOT NULL, + evaluated_at TIMESTAMPTZ NOT NULL, + epoch BIGINT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('requested', 'covered', 'skipped')), + run_id UUID, + covered_by_run_id UUID, + active_run_id UUID, + UNIQUE (trigger_id, scheduled_for) +); + +CREATE INDEX sqb_ticks_trigger_time ON sqb_ticks(trigger_id, scheduled_for); + +CREATE TABLE sqb_runs ( + run_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + image_id TEXT, + kind TEXT NOT NULL CHECK (kind IN ('job', 'group', 'adhoc')), + origin TEXT NOT NULL CHECK (origin IN ('manual', 'schedule', 'sensor', 'rerun', 'backfill')), + job_key TEXT, + parent_run_id UUID REFERENCES sqb_runs(run_id), + root_run_id UUID NOT NULL, + trigger_id UUID REFERENCES sqb_triggers(trigger_id), + tick_id UUID REFERENCES sqb_ticks(tick_id), + rerun_of_run_id UUID REFERENCES sqb_runs(run_id), + requested_by TEXT NOT NULL, + priority BIGINT NOT NULL DEFAULT 0, + selection_json JSONB NOT NULL, + request_json JSONB NOT NULL, + authored_json JSONB, + write_set_digest TEXT NOT NULL, + rewrite_history BOOLEAN NOT NULL DEFAULT FALSE, + status TEXT NOT NULL CHECK (status IN ('queued', 'launched', 'running', 'unresponsive', 'success', 'warn', 'partial', 'failed', 'cancelled')), + status_reason TEXT, + queued_at TIMESTAMPTZ NOT NULL, + launched_at TIMESTAMPTZ, + started_at TIMESTAMPTZ, + ended_at TIMESTAMPTZ, + cancel_requested_at TIMESTAMPTZ, + node_counts_json JSONB NOT NULL, + last_event_storage_id BIGINT NOT NULL, + worker_instance UUID, + container_id TEXT, + claimed_at TIMESTAMPTZ, + last_heartbeat_at TIMESTAMPTZ +); + +CREATE INDEX sqb_runs_location_time ON sqb_runs(location_id, target, queued_at); + +CREATE INDEX sqb_runs_location_ended ON sqb_runs(location_id, target, ended_at); + +CREATE INDEX sqb_runs_queue ON sqb_runs(priority DESC, queued_at) WHERE status = 'queued'; + +CREATE INDEX sqb_runs_active ON sqb_runs(location_id, target, job_key) WHERE status IN ('queued', 'launched', 'running', 'unresponsive'); + +CREATE TABLE sqb_run_nodes ( + run_id UUID NOT NULL REFERENCES sqb_runs(run_id), + node_key TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'success', 'warn', 'failed', 'skipped', 'cancelled')), + writes BOOLEAN NOT NULL, + started_at TIMESTAMPTZ, + ended_at TIMESTAMPTZ, + rows_affected BIGINT, + error_code TEXT, + skipped_because_of TEXT, + kind TEXT NOT NULL DEFAULT 'node' CHECK (kind IN ('node', 'check')), + finished_storage_id BIGINT, + PRIMARY KEY (run_id, node_key) +); + +CREATE INDEX sqb_run_nodes_node ON sqb_run_nodes(node_key, run_id); + +CREATE INDEX sqb_run_nodes_finished ON sqb_run_nodes(finished_storage_id) +WHERE finished_storage_id IS NOT NULL; + +CREATE INDEX sqb_run_nodes_decisive ON sqb_run_nodes(node_key, finished_storage_id) +WHERE finished_storage_id IS NOT NULL AND status IN ('success', 'failed'); + +CREATE TABLE sqb_write_claims ( + target TEXT NOT NULL, + node_key TEXT NOT NULL, + run_id UUID NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (target, node_key) +); + +CREATE INDEX sqb_write_claims_run ON sqb_write_claims(run_id); + +CREATE TABLE sqb_concurrency_slots ( + group_key TEXT NOT NULL, + slot BIGINT NOT NULL CHECK (slot > 0), + run_id UUID NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (group_key, slot), + UNIQUE (group_key, run_id) +); + +CREATE INDEX sqb_concurrency_slots_run ON sqb_concurrency_slots(run_id); + +CREATE TABLE sqb_principals ( + principal_id UUID PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL, + disabled_at TIMESTAMPTZ +); + +CREATE TABLE sqb_access_tokens ( + token_id UUID PRIMARY KEY, + secret_hash TEXT NOT NULL, + location_id UUID REFERENCES sqb_locations(location_id), + run_id UUID REFERENCES sqb_runs(run_id), + scope_json JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ, + CHECK ( + (location_id IS NOT NULL AND run_id IS NULL) + OR (location_id IS NULL AND run_id IS NOT NULL) + ) +); + +CREATE INDEX sqb_access_tokens_location ON sqb_access_tokens(location_id); + +CREATE INDEX sqb_access_tokens_run ON sqb_access_tokens(run_id); + +CREATE TABLE sqb_security_audit ( + event_id UUID PRIMARY KEY, + occurred_at TIMESTAMPTZ NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + granting_role TEXT NOT NULL, + request_id UUID NOT NULL, + ip_address TEXT, + scope_json JSONB NOT NULL, + before_json JSONB NOT NULL, + after_json JSONB NOT NULL, + storage_id BIGINT +); + +CREATE INDEX sqb_security_audit_time ON sqb_security_audit(occurred_at, event_id); + +CREATE INDEX sqb_security_audit_trigger ON sqb_security_audit((scope_json ->> 'trigger_id'), occurred_at, event_id) +WHERE action = 'trigger.changed'; + +CREATE TABLE sqb_location_instances ( + instance_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + last_seen_at TIMESTAMPTZ NOT NULL, + health_json JSONB NOT NULL +); + +CREATE INDEX sqb_location_instances_location_seen ON sqb_location_instances(location_id, last_seen_at); + +CREATE TABLE sqb_location_work ( + request_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + generation BIGINT NOT NULL, + request_json JSONB NOT NULL, + requested_by TEXT NOT NULL, + requested_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + claim_id UUID, + session_id UUID REFERENCES sqb_location_instances(instance_id), + claimed_until TIMESTAMPTZ, + result_json JSONB, + completed_at TIMESTAMPTZ +); + +CREATE INDEX sqb_location_work_pending ON sqb_location_work(location_id, target, requested_at); + +CREATE TABLE sqb_platform_sink_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); + +CREATE TABLE sqb_sensor_evaluation_events ( + event_id UUID PRIMARY KEY, + evaluation_id UUID NOT NULL, + trigger_id UUID NOT NULL REFERENCES sqb_triggers(trigger_id), + phase TEXT NOT NULL CHECK (phase IN ('requested', 'completed', 'failed', 'timed_out', 'abandoned')), + occurred_at TIMESTAMPTZ NOT NULL, + fact_json JSONB NOT NULL, + UNIQUE (evaluation_id, phase) +); + +CREATE INDEX sqb_sensor_evaluation_history ON sqb_sensor_evaluation_events(trigger_id, phase, occurred_at, evaluation_id); + +CREATE INDEX sqb_sensor_evaluation_expiry ON sqb_sensor_evaluation_events(occurred_at); + +CREATE TABLE sqb_run_requests ( + trigger_id UUID NOT NULL REFERENCES sqb_triggers(trigger_id), + run_key TEXT NOT NULL, + evaluation_id UUID NOT NULL, + run_id UUID NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (trigger_id, run_key) +); + +CREATE INDEX sqb_runs_terminal_cursor ON sqb_runs(location_id, target, job_key, last_event_storage_id) +WHERE status IN ('success', 'warn', 'partial', 'failed', 'cancelled'); + +CREATE INDEX sqb_runs_parent ON sqb_runs(parent_run_id) WHERE parent_run_id IS NOT NULL; + +CREATE TABLE sqb_warehouse_freshness ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT, + scan_id UUID +); + +CREATE INDEX sqb_warehouse_freshness_scope ON sqb_warehouse_freshness (location_id, target, observed_at, event_id); + +CREATE TABLE sqb_warehouse_fingerprints ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + definition_text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT, + node_type TEXT, + node_name TEXT, + target_database TEXT, + target_schema TEXT, + target_name TEXT, + scan_id UUID +); + +CREATE INDEX sqb_warehouse_fingerprints_scope ON sqb_warehouse_fingerprints (location_id, target, observed_at, event_id); + +CREATE TABLE sqb_development_locations ( + project_path TEXT NOT NULL, + target TEXT NOT NULL, + location_id UUID NOT NULL UNIQUE REFERENCES sqb_locations(location_id), + PRIMARY KEY (project_path, target) +); + +CREATE TABLE sqb_password_credentials ( + principal_id UUID PRIMARY KEY REFERENCES sqb_principals(principal_id), + credential_id UUID NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + changed_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE sqb_sessions ( + session_id UUID PRIMARY KEY, + principal_id UUID NOT NULL REFERENCES sqb_principals(principal_id), + credential_id UUID, + secret_hash TEXT NOT NULL, + csrf_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ +); + +CREATE INDEX sqb_sessions_person ON sqb_sessions(principal_id); + +CREATE TABLE sqb_password_login_attempts ( + attempt_id UUID PRIMARY KEY, + identity_key TEXT NOT NULL, + peer_address TEXT NOT NULL, + attempted_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX sqb_password_login_user_time ON sqb_password_login_attempts(identity_key, attempted_at); + +CREATE INDEX sqb_password_login_peer_time ON sqb_password_login_attempts(peer_address, attempted_at); + +CREATE INDEX sqb_password_login_time ON sqb_password_login_attempts(attempted_at); + +CREATE TABLE sqb_oidc_login_flows ( + flow_id UUID PRIMARY KEY, + configuration_id TEXT NOT NULL, + state_hash TEXT NOT NULL UNIQUE, + browser_hash TEXT NOT NULL, + nonce TEXT NOT NULL, + pkce_verifier TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX sqb_oidc_login_flows_expiry ON sqb_oidc_login_flows(expires_at); + +CREATE TABLE sqb_role_assignments ( + assignment_id UUID PRIMARY KEY, + principal_id UUID NOT NULL REFERENCES sqb_principals(principal_id), + role_key TEXT NOT NULL, + location_id UUID REFERENCES sqb_locations(location_id), + target TEXT, + active_key TEXT NOT NULL UNIQUE +); + +CREATE INDEX sqb_role_assignments_principal ON sqb_role_assignments(principal_id); + +CREATE TABLE sqb_person_groups ( + principal_id UUID NOT NULL REFERENCES sqb_principals(principal_id), + group_name TEXT NOT NULL, + PRIMARY KEY (principal_id, group_name) +); + +CREATE TABLE sqb_group_roles ( + group_name TEXT NOT NULL, + role_key TEXT NOT NULL, + location_id UUID, + target TEXT +); + +CREATE INDEX sqb_fingerprints_namespace ON sqb_warehouse_fingerprints (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_freshness_namespace ON sqb_warehouse_freshness (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE UNIQUE INDEX sqb_security_audit_storage_id ON sqb_security_audit(storage_id); + +CREATE TABLE sqb_warehouse_audit_results ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + audit_name TEXT NOT NULL, + outcome TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID, + binding_key TEXT +); + +CREATE INDEX sqb_audit_results_namespace ON sqb_warehouse_audit_results (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_audit_results_binding ON sqb_warehouse_audit_results (location_id,target,binding_key,observed_at,event_id); + +CREATE TABLE sqb_warehouse_janitor_events ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + event_type TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_janitor_events_namespace ON sqb_warehouse_janitor_events (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_migrations ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + target_name TEXT, + destination_model TEXT NOT NULL, + decision TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_migrations_namespace ON sqb_warehouse_migrations (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_migrations_destination ON sqb_warehouse_migrations (location_id,target,destination_model,observed_at,event_id); + +CREATE TABLE sqb_warehouse_old_name_views ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + target_name TEXT, + migration_event_id TEXT NOT NULL, + event_type TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_old_name_views_namespace ON sqb_warehouse_old_name_views (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_old_name_views_migration ON sqb_warehouse_old_name_views (location_id,target,migration_event_id,observed_at,event_id); + +CREATE TABLE sqb_warehouse_microbatches ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + model_name TEXT NOT NULL, + target_name TEXT NOT NULL, + scope_kind TEXT NOT NULL, + scope_key TEXT NOT NULL, + record_type TEXT NOT NULL, + execution_run_id TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_microbatches_namespace ON sqb_warehouse_microbatches (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_microbatches_model ON sqb_warehouse_microbatches (location_id,target,model_name); + +CREATE TABLE sqb_warehouse_column_migrations ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_column_migrations_namespace ON sqb_warehouse_column_migrations (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_node_results ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_node_results_namespace ON sqb_warehouse_node_results (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_consumption ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_consumption_namespace ON sqb_warehouse_consumption (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_observations ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + kind TEXT NOT NULL, + node TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_observations_namespace ON sqb_warehouse_observations (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_observations_latest ON sqb_warehouse_observations (location_id,target,kind,node,observed_at,event_id); + +CREATE TABLE sqb_warehouse_resource_costs ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + run_id TEXT NOT NULL, + orchestration_run_id TEXT, + attempt BIGINT NOT NULL, + resource_type TEXT NOT NULL, + resource_name TEXT NOT NULL, + run_started_at TIMESTAMPTZ NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_resource_costs_namespace ON sqb_warehouse_resource_costs (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_resource_costs_run ON sqb_warehouse_resource_costs (location_id,target,run_id,attempt); + +CREATE INDEX sqb_resource_costs_orchestration_run ON sqb_warehouse_resource_costs (orchestration_run_id); + +CREATE INDEX sqb_resource_costs_started ON sqb_warehouse_resource_costs (location_id,target,run_started_at); + +CREATE INDEX sqb_freshness_source_latest ON sqb_warehouse_freshness ( + location_id, target, (record_json ->> 'source_name'), + (record_json ->> 'target_database'), (record_json ->> 'target_schema'), + (record_json ->> 'target_name'), observed_at DESC, event_id DESC +); + +CREATE INDEX sqb_freshness_source_received ON sqb_warehouse_freshness ( + location_id, target, (record_json ->> 'source_name'), + (record_json ->> 'target_database'), (record_json ->> 'target_schema'), + (record_json ->> 'target_name'), received_at DESC, observed_at DESC, event_id DESC +); + +CREATE TABLE sqb_security_audit_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); + +CREATE TABLE sqb_warehouse_catalog ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_warehouse_catalog_namespace ON sqb_warehouse_catalog + (location_id, target, state_schema, state_database, observed_at, event_id); + +CREATE INDEX sqb_fingerprint_node_latest ON sqb_warehouse_fingerprints + (location_id,target,node_type,node_name,target_database,target_schema,target_name,observed_at DESC,event_id DESC); + +CREATE TABLE sqb_warehouse_statistics ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + relation_name TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX sqb_statistics_relation_time ON sqb_warehouse_statistics + (location_id,target,state_schema,state_database,relation_name,observed_at,event_id); + +CREATE INDEX sqb_statistics_expiration ON sqb_warehouse_statistics (observed_at,event_id); + +CREATE INDEX sqb_catalog_statistics_scan ON sqb_warehouse_catalog + (location_id,target,state_schema,state_database,event_id); + +CREATE TABLE sqb_cli_login_flows ( + flow_id UUID PRIMARY KEY, + client_hash TEXT NOT NULL, + browser_hash TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + principal_id UUID, + session_id UUID +); + +CREATE INDEX sqb_cli_login_expiry ON sqb_cli_login_flows(expires_at); + +CREATE INDEX IF NOT EXISTS sqb_runs_tick ON sqb_runs(tick_id); + +CREATE INDEX IF NOT EXISTS sqb_ticks_expiry ON sqb_ticks(scheduled_for) WHERE outcome = 'covered'; + +CREATE INDEX IF NOT EXISTS sqb_runs_snapshot ON sqb_runs(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_snapshot_nodes_hash ON sqb_snapshot_nodes(node_hash); + +CREATE INDEX IF NOT EXISTS sqb_snapshot_sources_hash ON sqb_snapshot_sources(text_hash); + +CREATE INDEX IF NOT EXISTS sqb_snapshot_documents_hash ON sqb_snapshot_documents(document_hash); + +CREATE TABLE sqb_effective_roles ( + location_id UUID NOT NULL, + target TEXT NOT NULL, + role_key TEXT NOT NULL, + resolved_grants_json TEXT NOT NULL, + PRIMARY KEY (location_id, target, role_key) +); + +CREATE INDEX IF NOT EXISTS sqb_jobs_snapshot ON sqb_jobs(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_triggers_snapshot ON sqb_triggers(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_triggers_removed ON sqb_triggers(removed_in_snapshot_id) WHERE removed_in_snapshot_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_runs_trigger ON sqb_runs(trigger_id) WHERE trigger_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_runs_rerun_of ON sqb_runs(rerun_of_run_id) WHERE rerun_of_run_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_location_work_session ON sqb_location_work(session_id) WHERE session_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_location_work_snapshot ON sqb_location_work(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_run_requests_run ON sqb_run_requests(run_id); + +CREATE INDEX IF NOT EXISTS sqb_warehouse_fingerprints_text ON sqb_warehouse_fingerprints(definition_text_hash); + +CREATE INDEX IF NOT EXISTS sqb_role_assignments_location ON sqb_role_assignments(location_id); diff --git a/crates/sqlbuild-orchestration/tests/store/schema/baseline/rows.sql b/crates/sqlbuild-orchestration/tests/store/schema/baseline/rows.sql new file mode 100644 index 0000000000..47958cc18b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/baseline/rows.sql @@ -0,0 +1,17 @@ +INSERT INTO sqb_locations (location_id, name, created_by, created_at) VALUES ('018f0000-0000-7000-8000-000000000001', 'orders', 'admin', '2026-11-01T09:00:00Z'); + +INSERT INTO sqb_snapshots (snapshot_id, location_id, target, generation, git_dirty, environment_hash, sqb_version, manifest_digest, published_at) VALUES ('018f0000-0000-7000-8000-000000000002', '018f0000-0000-7000-8000-000000000001', 'prod', 1, FALSE, 'env-1', '0.241.0', 'manifest-1', '2026-11-01T09:05:00Z'); + +INSERT INTO sqb_principals (principal_id, name, created_at) VALUES ('018f0000-0000-7000-8000-000000000003', 'ana', '2026-11-01T09:01:00Z'); + +INSERT INTO sqb_role_assignments (assignment_id, principal_id, role_key, location_id, target, active_key) VALUES ('018f0000-0000-7000-8000-000000000004', '018f0000-0000-7000-8000-000000000003', 'operator', '018f0000-0000-7000-8000-000000000001', 'prod', 'ana:operator:orders:prod'); + +INSERT INTO sqb_access_tokens (token_id, secret_hash, location_id, scope_json, created_at, expires_at) VALUES ('018f0000-0000-7000-8000-000000000005', 'hash-1', '018f0000-0000-7000-8000-000000000001', '{"kind": "location"}', '2026-11-01T09:02:00Z', '2027-11-01T09:02:00Z'); + +INSERT INTO sqb_triggers (trigger_id, location_id, target, kind, name, job_key, definition_json, snapshot_id) VALUES ('018f0000-0000-7000-8000-000000000006', '018f0000-0000-7000-8000-000000000001', 'prod', 'schedule', 'nightly', 'daily_orders', '{"cron": "0 2 * * *"}', '018f0000-0000-7000-8000-000000000002'); + +INSERT INTO sqb_trigger_state (trigger_id, status, status_override, started_at, next_due_at) VALUES ('018f0000-0000-7000-8000-000000000006', 'paused', 'paused', '2026-11-01T09:06:00Z', '2026-11-02T02:00:00Z'); + +INSERT INTO sqb_runs (run_id, location_id, target, snapshot_id, kind, origin, job_key, root_run_id, requested_by, selection_json, request_json, write_set_digest, status, queued_at, ended_at, node_counts_json, last_event_storage_id) VALUES ('018f0000-0000-7000-8000-000000000007', '018f0000-0000-7000-8000-000000000001', 'prod', '018f0000-0000-7000-8000-000000000002', 'job', 'manual', 'daily_orders', '018f0000-0000-7000-8000-000000000007', 'principal:018f0000-0000-7000-8000-000000000003', '[]', '{}', 'digest-1', 'success', '2026-11-01T10:00:00Z', '2026-11-01T10:05:00Z', '{"success": 3}', 12); + +INSERT INTO sqb_run_nodes (run_id, node_key, status, writes, started_at, ended_at, rows_affected, finished_storage_id) VALUES ('018f0000-0000-7000-8000-000000000007', 'model.orders', 'success', TRUE, '2026-11-01T10:01:00Z', '2026-11-01T10:02:00Z', 3, 11); diff --git a/crates/sqlbuild-orchestration/tests/store/schema/baseline/sqlite.sql b/crates/sqlbuild-orchestration/tests/store/schema/baseline/sqlite.sql new file mode 100644 index 0000000000..beaab06a8f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/baseline/sqlite.sql @@ -0,0 +1,1431 @@ +CREATE TABLE IF NOT EXISTS "recall_meta" ( + "component" TEXT NOT NULL, + "key" TEXT NOT NULL, + "value" TEXT NOT NULL, + "recorded_at" TEXT NOT NULL, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("component", "key", "recorded_at") +); + +CREATE TABLE IF NOT EXISTS "recall_invocations" ( + "command" TEXT, + "subcommand" TEXT, + "selection" JSON, + "exclusion" JSON, + "project_name" TEXT, + "project_dir_hash" TEXT, + "target" TEXT, + "adapter" TEXT, + "destination_database" TEXT, + "destination_schema" TEXT, + "destination_fingerprint" TEXT, + "sqlbuild_version" TEXT, + "git_revision" TEXT, + "git_dirty" INTEGER, + "host_token" TEXT, + "pid" INTEGER, + "external_context" JSON, + "recall_filter_active" INTEGER, + "argv_digest" TEXT, + "origin" TEXT, + "command_mode" TEXT, + "selection_count" INTEGER, + "selection_truncated" INTEGER, + "selector_files" JSON, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id") +); + +CREATE INDEX IF NOT EXISTS "recall_invocations_started" ON "recall_invocations" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_output" ( + "record_id" TEXT NOT NULL, + "sequence" INTEGER NOT NULL, + "stream" TEXT NOT NULL, + "occurred_at" TEXT NOT NULL, + "data" BLOB NOT NULL, + "byte_offset" INTEGER NOT NULL, + "byte_count" INTEGER NOT NULL, + "priority" TEXT, + "dropped_before" INTEGER NOT NULL, + "omitted_bytes" INTEGER, + "omitted_sequences" INTEGER, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "sequence") +); + +CREATE INDEX IF NOT EXISTS "recall_output_expiry" ON "recall_output" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_events" ( + "event_id" TEXT NOT NULL, + "invocation_sequence" INTEGER, + "occurred_at" TEXT NOT NULL, + "event_type" TEXT NOT NULL, + "event_kind" TEXT NOT NULL, + "severity" TEXT NOT NULL, + "producer" TEXT NOT NULL, + "event_schema_version" INTEGER NOT NULL, + "run_id" TEXT, + "resource_id" TEXT, + "resource_attempt_id" TEXT, + "operation_id" TEXT, + "statement_id" TEXT, + "command" TEXT, + "resource_kind" TEXT, + "resource_name" TEXT, + "error_code" TEXT, + "error_type" TEXT, + "duration_ms" REAL, + "exit_code" INTEGER, + "query_id" TEXT, + "sql_digest" TEXT, + "intent" TEXT, + "statement_kind" TEXT, + "purpose" TEXT, + "outcome" TEXT, + "row_count" INTEGER, + "affected_rows" INTEGER, + "payload" JSON NOT NULL, + "started_at" TEXT, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + storage_id INTEGER PRIMARY KEY AUTOINCREMENT, + UNIQUE ("event_id") +); + +CREATE INDEX IF NOT EXISTS "recall_events_invocation" ON "recall_events" ("invocation_id", "invocation_sequence"); + +CREATE INDEX IF NOT EXISTS "recall_events_type" ON "recall_events" ("event_type", "occurred_at"); + +CREATE INDEX IF NOT EXISTS "recall_events_statement" ON "recall_events" ("statement_id"); + +CREATE INDEX IF NOT EXISTS "recall_events_expiry" ON "recall_events" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_diagnostics" ( + "record_id" TEXT NOT NULL, + "sequence" INTEGER NOT NULL, + "occurred_at" TEXT NOT NULL, + "severity" TEXT NOT NULL, + "logger" TEXT NOT NULL, + "source" TEXT NOT NULL, + "message" TEXT NOT NULL, + "fields" JSON NOT NULL, + "log_stream_id" TEXT, + "run_id" TEXT, + "resource_id" TEXT, + "resource_attempt_id" TEXT, + "operation_id" TEXT, + "statement_id" TEXT, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "sequence") +); + +CREATE INDEX IF NOT EXISTS "recall_diagnostics_severity" ON "recall_diagnostics" ("severity"); + +CREATE INDEX IF NOT EXISTS "recall_diagnostics_expiry" ON "recall_diagnostics" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_artifacts" ( + "digest" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "media_type" TEXT NOT NULL, + "size" INTEGER NOT NULL, + "content" BLOB NOT NULL, + "seen_at" TEXT NOT NULL, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("digest") +); + +CREATE INDEX IF NOT EXISTS "recall_artifacts_seen" ON "recall_artifacts" ("seen_at"); + +CREATE TABLE IF NOT EXISTS "recall_artifact_refs" ( + "digest" TEXT NOT NULL, + "ref_id" TEXT NOT NULL, + "statement_id" TEXT, + "resource_id" TEXT, + "role" TEXT NOT NULL, + "referenced_at" TEXT NOT NULL, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "digest", "ref_id") +); + +CREATE INDEX IF NOT EXISTS "recall_artifact_refs_digest" ON "recall_artifact_refs" ("digest"); + +CREATE TABLE IF NOT EXISTS "recall_heartbeats" ( + "beat_at" TEXT NOT NULL, + "stale_at" TEXT NOT NULL, + "host_token" TEXT, + "pid" INTEGER, + "command" TEXT, + "subcommand" TEXT, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "beat_at") +); + +CREATE TABLE IF NOT EXISTS "recall_delivery" ( + "stream" TEXT NOT NULL, + "accepted" INTEGER NOT NULL, + "delivered" INTEGER NOT NULL, + "filtered" INTEGER NOT NULL, + "dropped" INTEGER NOT NULL, + "failed" INTEGER NOT NULL, + "conflicts" INTEGER NOT NULL, + "flush_complete" INTEGER NOT NULL, + "written_at" TEXT NOT NULL, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "stream") +); + +CREATE TABLE IF NOT EXISTS "recall_conflicts" ( + "dataset" TEXT NOT NULL, + "record_key" TEXT NOT NULL, + "existing_sha256" TEXT NOT NULL, + "rejected_sha256" TEXT NOT NULL, + "detected_at" TEXT NOT NULL, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("dataset", "record_key", "rejected_sha256") +); + +CREATE INDEX IF NOT EXISTS "recall_conflicts_invocation" ON "recall_conflicts" ("invocation_id"); + +CREATE VIEW IF NOT EXISTS "invocations" AS +WITH ids AS ( + SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at + FROM ( + SELECT invocation_id, invocation_started_at FROM recall_invocations + UNION ALL SELECT invocation_id, invocation_started_at FROM recall_heartbeats + UNION ALL SELECT invocation_id, invocation_started_at FROM recall_events + WHERE event_type IN ('invocation_started', 'invocation_completed', 'invocation_failed') + ) AS source_rows + GROUP BY invocation_id +), +started AS ( + SELECT invocation_id, MIN(occurred_at) AS started_at, MIN(command) AS command + FROM recall_events + WHERE event_type = 'invocation_started' + GROUP BY invocation_id +), +terminals AS ( + SELECT invocation_id, event_type, occurred_at, exit_code, duration_ms, error_type, error_code, + json_extract(payload, '$.error_message') AS error_message, + json_extract(payload, '$.signal_number') AS signal_number, + json_extract(payload, '$.cost_status') AS cost_status, + ROW_NUMBER() OVER ( + PARTITION BY invocation_id + ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC + ) AS position + FROM recall_events + WHERE event_type IN ('invocation_completed', 'invocation_failed') +), +beats AS ( + SELECT invocation_id, MAX(beat_at) AS last_heartbeat_at, MAX(stale_at) AS stale_at, + MIN(command) AS command, MIN(subcommand) AS subcommand + FROM recall_heartbeats + GROUP BY invocation_id +), +projected AS ( + SELECT + ids.invocation_id, + ids.invocation_started_at, + COALESCE(started.started_at, ids.invocation_started_at) AS started_at, + CASE + WHEN terminals.event_type = 'invocation_completed' THEN 'completed' + WHEN terminals.event_type = 'invocation_failed' + AND terminals.signal_number IS NOT NULL THEN 'cancelled' + WHEN terminals.event_type = 'invocation_failed' THEN 'failed' + WHEN beats.stale_at IS NULL THEN 'unknown' + WHEN beats.stale_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now') THEN 'running' + ELSE 'abandoned' + END AS status, + terminals.event_type AS terminal_event, + terminals.occurred_at AS terminal_at, + terminals.duration_ms AS terminal_duration_ms, + terminals.exit_code, + terminals.error_type, + terminals.error_code, + terminals.error_message, + terminals.signal_number, + terminals.cost_status, + COALESCE(context.command, started.command, beats.command) AS command, + COALESCE(context.subcommand, beats.subcommand) AS subcommand, + context.origin, + context.command_mode, + context.target, + context.adapter, + context.project_name, + context.selection, + COALESCE(context.selection_count, json_array_length(context.selection)) + AS selection_count, + context.selection_truncated, + context.selector_files, + context.exclusion, + context.sqlbuild_version, + context.git_revision, + context.git_dirty, + context.host_token, + context.pid, + context.external_context, + CASE WHEN json_valid(context.external_context) + THEN json_extract(context.external_context, '$.integration.name') END + AS integration, + CASE WHEN json_valid(context.external_context) + THEN json_extract(context.external_context, '$.integration.job_name') END + AS integration_job, + CASE WHEN json_valid(context.external_context) + THEN json_extract(context.external_context, '$.integration.run_id') END + AS integration_run_id, + CASE WHEN json_valid(context.external_context) + THEN json_extract(context.external_context, '$.integration.retry_number') END + AS integration_retry, + context.recall_filter_active, + beats.last_heartbeat_at + FROM ids + LEFT JOIN recall_invocations AS context ON context.invocation_id = ids.invocation_id + LEFT JOIN started ON started.invocation_id = ids.invocation_id + LEFT JOIN terminals + ON terminals.invocation_id = ids.invocation_id AND terminals.position = 1 + LEFT JOIN beats ON beats.invocation_id = ids.invocation_id +) +SELECT + invocation_id, + invocation_started_at, + started_at, + status, + CASE + WHEN terminal_event IS NOT NULL THEN terminal_at + WHEN status = 'abandoned' THEN last_heartbeat_at + END AS ended_at, + CASE + WHEN terminal_event IS NOT NULL THEN terminal_duration_ms + WHEN status = 'abandoned' THEN ROUND((unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3) + END AS duration_ms, + CASE + WHEN terminal_event IS NOT NULL THEN 0 + WHEN status = 'abandoned' THEN 1 + END AS ended_at_estimated, + exit_code, + error_type, + error_code, + error_message, + signal_number, + cost_status, + command, + subcommand, + origin, + command_mode, + target, + adapter, + project_name, + selection, + selection_count, + selection_truncated, + selector_files, + exclusion, + sqlbuild_version, + git_revision, + git_dirty, + host_token, + pid, + external_context, + integration, + integration_job, + integration_run_id, + integration_retry, + recall_filter_active, + last_heartbeat_at +FROM projected; + +CREATE VIEW IF NOT EXISTS "statements" AS +WITH facts AS ( + SELECT * + FROM recall_events + WHERE statement_id IS NOT NULL AND event_type LIKE 'statement!_%' ESCAPE '!' +), +firsts AS ( + SELECT invocation_id, invocation_started_at, statement_id, run_id, resource_id, resource_attempt_id, operation_id, + intent, statement_kind, purpose, sql_digest, + ROW_NUMBER() OVER ( + PARTITION BY statement_id ORDER BY invocation_sequence, occurred_at, event_id + ) AS position + FROM facts +), +started AS ( + SELECT statement_id, MIN(started_at) AS started_at + FROM ( + SELECT statement_id, occurred_at AS started_at FROM facts WHERE event_type = 'statement_started' + UNION ALL + SELECT statement_id, started_at FROM facts WHERE started_at IS NOT NULL + ) AS starts + GROUP BY statement_id +), +submitted AS ( + SELECT statement_id, MIN(occurred_at) AS submitted_at + FROM facts WHERE event_type = 'statement_submitted' GROUP BY statement_id +), +query_ids AS ( + SELECT statement_id, MAX(query_id) AS query_id FROM facts GROUP BY statement_id +), +terminals AS ( + SELECT statement_id, event_type, occurred_at, query_id, error_type, error_code, duration_ms, + row_count, affected_rows, outcome, + ROW_NUMBER() OVER ( + PARTITION BY statement_id + ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC + ) AS position + FROM facts + WHERE event_type IN ('statement_completed', 'statement_failed') +), +captures AS ( + SELECT firsts.statement_id, ( + SELECT MAX(artifact.digest IS NOT NULL) + FROM recall_artifact_refs AS ref + LEFT JOIN recall_artifacts AS artifact ON artifact.digest = ref.digest + WHERE ref.invocation_id = firsts.invocation_id + AND ref.digest = 'sha256:' || firsts.sql_digest + ) AS stored + FROM firsts + WHERE firsts.position = 1 +), +resources AS ( + SELECT invocation_id, resource_id, MIN(resource_name) AS resource_name, + MIN(resource_kind) AS resource_kind + FROM recall_events + WHERE resource_id IS NOT NULL AND resource_name IS NOT NULL + GROUP BY invocation_id, resource_id +) +SELECT + firsts.invocation_id, + firsts.invocation_started_at, + firsts.statement_id, + firsts.run_id, + firsts.resource_id, + resources.resource_name, + resources.resource_kind, + firsts.resource_attempt_id, + firsts.operation_id, + firsts.intent, + firsts.statement_kind, + firsts.purpose, + COALESCE(terminals.query_id, query_ids.query_id) AS query_id, + started.started_at, + submitted.submitted_at, + terminals.occurred_at AS ended_at, + CASE terminals.event_type + WHEN 'statement_completed' THEN 'completed' + WHEN 'statement_failed' THEN + CASE WHEN terminals.outcome = 'handled' THEN 'handled' ELSE 'failed' END + ELSE 'no_terminal' + END AS outcome, + terminals.error_type, + terminals.error_code, + terminals.duration_ms, + terminals.row_count, + terminals.affected_rows, + firsts.sql_digest, + COALESCE(captures.stored, 0) AS sql_available, + CASE captures.stored + WHEN 1 THEN NULL + WHEN 0 THEN 'expired_or_not_captured' + ELSE 'not_captured' + END AS sql_unavailable_reason +FROM firsts +LEFT JOIN started ON started.statement_id = firsts.statement_id +LEFT JOIN submitted ON submitted.statement_id = firsts.statement_id +LEFT JOIN query_ids ON query_ids.statement_id = firsts.statement_id +LEFT JOIN terminals + ON terminals.statement_id = firsts.statement_id AND terminals.position = 1 +LEFT JOIN captures ON captures.statement_id = firsts.statement_id +LEFT JOIN resources + ON resources.invocation_id = firsts.invocation_id + AND resources.resource_id = firsts.resource_id +WHERE firsts.position = 1; + +CREATE VIEW IF NOT EXISTS "audit_results" AS +SELECT + invocation_id, + invocation_started_at, + event_id, + occurred_at, + run_id, + resource_id, + resource_attempt_id, + json_extract(payload, '$.audit_name') AS audit_name, + json_extract(payload, '$.audit_definition_name') AS audit_definition_name, + CASE WHEN json_extract(payload, '$.target_name') IS NULL THEN NULL ELSE + COALESCE(json_extract(payload, '$.target_database') || '.', '') + || COALESCE(json_extract(payload, '$.target_schema') || '.', '') + || json_extract(payload, '$.target_name') + END AS target_relation, + json_extract(payload, '$.attached_target_kind') AS attached_target_kind, + json_extract(payload, '$.attached_target_name') AS attached_target_name, + json_extract(payload, '$.attached_column_name') AS "column", + outcome, + json_extract(payload, '$.severity') AS severity, + json_extract(payload, '$.violation_count') AS violation_count, + json_extract(payload, '$.measured_value') AS measured_value, + json_extract(payload, '$.evaluation_mode') AS evaluation_mode, + json_extract(payload, '$.execution_error') AS execution_error, + sql_digest, + json_extract(payload, '$.evidence_count') AS evidence_count, + json_extract(payload, '$.evidence_truncated') AS evidence_truncated, + json_extract(payload, '$.sample_digest') AS sample_digest +FROM recall_events +WHERE event_type = 'audit_completed'; + +CREATE VIEW IF NOT EXISTS "output_invocations" AS +WITH received AS ( + SELECT invocation_id, + MIN(invocation_started_at) AS invocation_started_at, + COUNT(*) AS chunks, + SUM(byte_count) AS bytes, + SUM(CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END) AS stdout_bytes, + SUM(CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END) AS stderr_bytes, + MAX(sequence) AS max_sequence, + SUM(dropped_before) AS queue_drops, + SUM(COALESCE(omitted_sequences, 1)) AS spans, + SUM(COALESCE(omitted_bytes, 0)) AS truncated_bytes + FROM recall_output + GROUP BY invocation_id +), +delivery AS ( + SELECT * FROM recall_delivery WHERE stream = 'output' +), +ids AS ( + SELECT invocation_id, invocation_started_at FROM received + UNION SELECT invocation_id, invocation_started_at FROM delivery +), +counted AS ( + SELECT + ids.invocation_id, + ids.invocation_started_at, + COALESCE(received.chunks, 0) AS chunks, + COALESCE(received.bytes, 0) AS bytes, + COALESCE(received.stdout_bytes, 0) AS stdout_bytes, + COALESCE(received.stderr_bytes, 0) AS stderr_bytes, + COALESCE(received.truncated_bytes, 0) AS truncated_bytes, + received.max_sequence, + MAX( + COALESCE(received.max_sequence + 1 - received.spans, 0), + COALESCE(delivery.accepted, 0) - COALESCE(received.chunks, 0) + ) AS missing_sequences, + MAX(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, + COALESCE(delivery.filtered, 0) AS filtered, + COALESCE(delivery.failed, 0) AS failed, + COALESCE(context.recall_filter_active, 0) AS filter_active, + COALESCE(delivery.flush_complete, 0) AS flush_complete, + delivery.invocation_id IS NOT NULL AS delivery_reported, + COALESCE(delivery.delivered, 0) AS delivered + FROM ids + LEFT JOIN received ON received.invocation_id = ids.invocation_id + LEFT JOIN delivery ON delivery.invocation_id = ids.invocation_id + LEFT JOIN recall_invocations AS context ON context.invocation_id = ids.invocation_id +) +SELECT + invocation_id, + invocation_started_at, + chunks, + bytes, + stdout_bytes, + stderr_bytes, + truncated_bytes, + max_sequence, + missing_sequences, + reported_drops, + filtered, + failed, + filter_active, + flush_complete, + CASE + WHEN missing_sequences = 0 AND truncated_bytes > 0 THEN 'truncated' + WHEN missing_sequences = 0 THEN NULL + WHEN filtered >= missing_sequences THEN 'filtered' + WHEN filtered + reported_drops + failed >= missing_sequences THEN 'dropped' + ELSE 'unexplained' + END AS gap_reason, + CASE + WHEN delivery_reported AND flush_complete = 1 AND missing_sequences = 0 + AND reported_drops = 0 AND filtered = 0 AND failed = 0 AND truncated_bytes = 0 + AND chunks >= delivered + THEN 1 ELSE 0 + END AS complete +FROM counted; + +CREATE VIEW IF NOT EXISTS "output_tail" AS +SELECT invocation_id, invocation_started_at, sequence, stream, occurred_at, byte_count, + CAST(data AS TEXT) AS text +FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY invocation_id ORDER BY sequence DESC) AS position + FROM recall_output +) AS source_rows +WHERE position <= 200; + +CREATE VIEW IF NOT EXISTS "output_transcripts" AS +SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at, + CAST(COALESCE(group_concat(CASE WHEN stream = 'stdout' THEN data END, x'' ORDER BY sequence), x'') AS BLOB) AS stdout, + CAST(COALESCE(group_concat(CASE WHEN stream = 'stderr' THEN data END, x'' ORDER BY sequence), x'') AS BLOB) AS stderr, + CAST(COALESCE(group_concat(data, x'' ORDER BY sequence), x'') AS BLOB) AS combined +FROM recall_output GROUP BY invocation_id; + +CREATE VIEW IF NOT EXISTS "events" AS +SELECT + event_id, + invocation_sequence, + occurred_at, + event_type, + event_kind, + severity, + producer, + event_schema_version, + run_id, + resource_id, + resource_attempt_id, + operation_id, + statement_id, + command, + resource_kind, + resource_name, + error_code, + error_type, + duration_ms, + exit_code, + query_id, + sql_digest, + intent, + statement_kind, + purpose, + outcome, + row_count, + affected_rows, + CASE WHEN producer = 'sqlbuild' THEN json_patch(payload, json_remove(json_object('command', command, 'resource_kind', resource_kind, 'resource_name', resource_name, 'error_code', error_code, 'error_type', error_type, 'query_id', query_id, 'sql_digest', sql_digest, 'intent', intent, 'statement_kind', statement_kind, 'purpose', purpose, 'outcome', outcome, 'exit_code', exit_code, 'row_count', row_count, 'affected_rows', affected_rows), CASE WHEN command IS NULL THEN '$.command' ELSE '$.payload' END, CASE WHEN resource_kind IS NULL THEN '$.resource_kind' ELSE '$.payload' END, CASE WHEN resource_name IS NULL THEN '$.resource_name' ELSE '$.payload' END, CASE WHEN error_code IS NULL THEN '$.error_code' ELSE '$.payload' END, CASE WHEN error_type IS NULL THEN '$.error_type' ELSE '$.payload' END, CASE WHEN query_id IS NULL THEN '$.query_id' ELSE '$.payload' END, CASE WHEN sql_digest IS NULL THEN '$.sql_digest' ELSE '$.payload' END, CASE WHEN intent IS NULL THEN '$.intent' ELSE '$.payload' END, CASE WHEN statement_kind IS NULL THEN '$.statement_kind' ELSE '$.payload' END, CASE WHEN purpose IS NULL THEN '$.purpose' ELSE '$.payload' END, CASE WHEN outcome IS NULL THEN '$.outcome' ELSE '$.payload' END, CASE WHEN exit_code IS NULL THEN '$.exit_code' ELSE '$.payload' END, CASE WHEN row_count IS NULL THEN '$.row_count' ELSE '$.payload' END, CASE WHEN affected_rows IS NULL THEN '$.affected_rows' ELSE '$.payload' END)) ELSE payload END AS payload, + started_at, + invocation_id, + invocation_started_at, + schema_version, + producer_version, + ingested_at, + content_hash, + storage_id +FROM recall_events; + +CREATE VIEW IF NOT EXISTS "diagnostics" AS +SELECT * FROM recall_diagnostics; + +CREATE VIEW IF NOT EXISTS "artifacts" AS +SELECT + refs.invocation_id, + refs.invocation_started_at, + refs.digest, + refs.ref_id, + refs.role, + refs.statement_id, + refs.resource_id, + artifact.kind, + artifact.media_type, + artifact.size, + artifact.digest IS NOT NULL AS available, + CASE WHEN artifact.digest IS NULL THEN 'expired_or_not_captured' END AS unavailable_reason, + CASE WHEN artifact.media_type LIKE 'text/%' OR artifact.media_type = 'application/json' + THEN CAST(artifact.content AS TEXT) END AS text, + artifact.content +FROM recall_artifact_refs AS refs +LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest; + +CREATE VIEW IF NOT EXISTS "conflicts" AS +SELECT * FROM recall_conflicts; + +CREATE TABLE sqb_installation ( + singleton BIGINT PRIMARY KEY CHECK (singleton = 1), + installation_id TEXT NOT NULL, + created_at TEXT NOT NULL +); + +CREATE TABLE sqb_locations ( + location_id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_by TEXT NOT NULL, + created_at TEXT NOT NULL, + archived_at TEXT +); + +CREATE TABLE sqb_source_texts ( + text_hash TEXT PRIMARY KEY, + body BLOB NOT NULL +); + +CREATE TABLE sqb_node_definitions ( + node_hash TEXT PRIMARY KEY, + body BLOB NOT NULL +); + +CREATE TABLE sqb_documents ( + document_hash TEXT PRIMARY KEY, + body BLOB NOT NULL +); + +CREATE TABLE sqb_snapshots ( + snapshot_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + generation BIGINT NOT NULL CHECK (generation > 0), + git_revision TEXT, + git_dirty BOOLEAN NOT NULL, + git_author TEXT, + image_id TEXT, + environment_hash TEXT NOT NULL, + sqb_version TEXT NOT NULL, + manifest_digest TEXT NOT NULL, + published_at TEXT NOT NULL, + UNIQUE (location_id, target, generation), + UNIQUE (location_id, target, snapshot_id) +); + +CREATE INDEX sqb_snapshots_location_time ON sqb_snapshots(location_id, target, published_at); + +CREATE TABLE sqb_snapshot_nodes ( + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + node_key TEXT NOT NULL, + node_kind TEXT NOT NULL, + node_hash TEXT NOT NULL REFERENCES sqb_node_definitions(node_hash), + PRIMARY KEY (snapshot_id, node_key) +); + +CREATE TABLE sqb_snapshot_sources ( + snapshot_id TEXT NOT NULL, + node_key TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('raw_sql', 'compiled_sql', 'python_source')), + text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + PRIMARY KEY (snapshot_id, node_key, kind), + FOREIGN KEY (snapshot_id, node_key) REFERENCES sqb_snapshot_nodes(snapshot_id, node_key) +); + +CREATE TABLE sqb_snapshot_documents ( + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('jobs', 'groups', 'triggers', 'roles', 'lineage', 'columns', 'providers')), + document_hash TEXT NOT NULL REFERENCES sqb_documents(document_hash), + PRIMARY KEY (snapshot_id, kind) +); + +CREATE TABLE sqb_current_snapshots ( + location_id TEXT NOT NULL, + target TEXT NOT NULL, + snapshot_id TEXT NOT NULL, + PRIMARY KEY (location_id, target), + FOREIGN KEY (location_id, target, snapshot_id) + REFERENCES sqb_snapshots(location_id, target, snapshot_id) +); + +CREATE TABLE sqb_snapshot_changes ( + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + change_seq BIGINT NOT NULL, + change_kind TEXT NOT NULL CHECK (change_kind IN ('added', 'changed', 'removed')), + subject_key TEXT NOT NULL, + before_hash TEXT, + after_hash TEXT, + PRIMARY KEY (snapshot_id, change_seq) +); + +CREATE TABLE sqb_scheduler_lease ( + name TEXT PRIMARY KEY CHECK (name = 'scheduler'), + holder TEXT NOT NULL, + epoch BIGINT NOT NULL CHECK (epoch > 0), + expires_at TEXT NOT NULL +); + +CREATE TABLE sqb_jobs ( + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + job_key TEXT NOT NULL, + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('job', 'group')), + definition_json JSON NOT NULL, + PRIMARY KEY (location_id, target, job_key) +); + +CREATE TABLE sqb_triggers ( + trigger_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('schedule', 'sensor')), + name TEXT NOT NULL, + job_key TEXT NOT NULL, + definition_json JSON NOT NULL, + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + removed_in_snapshot_id TEXT REFERENCES sqb_snapshots(snapshot_id), + UNIQUE (location_id, target, kind, name) +); + +CREATE TABLE sqb_trigger_state ( + trigger_id TEXT PRIMARY KEY REFERENCES sqb_triggers(trigger_id), + status TEXT NOT NULL CHECK (status IN ('running', 'paused')), + status_override TEXT CHECK (status_override IN ('running', 'paused')), + started_at TEXT NOT NULL, + cursor_json JSON, + last_tick_for TEXT, + next_due_at TEXT +); + +CREATE INDEX sqb_trigger_state_due ON sqb_trigger_state(next_due_at) WHERE status = 'running'; + +CREATE TABLE sqb_ticks ( + tick_id TEXT PRIMARY KEY, + trigger_id TEXT NOT NULL REFERENCES sqb_triggers(trigger_id), + scheduled_for TEXT NOT NULL, + evaluated_at TEXT NOT NULL, + epoch BIGINT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('requested', 'covered', 'skipped')), + run_id TEXT, + covered_by_run_id TEXT, + active_run_id TEXT, + UNIQUE (trigger_id, scheduled_for) +); + +CREATE INDEX sqb_ticks_trigger_time ON sqb_ticks(trigger_id, scheduled_for); + +CREATE TABLE sqb_runs ( + run_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + image_id TEXT, + kind TEXT NOT NULL CHECK (kind IN ('job', 'group', 'adhoc')), + origin TEXT NOT NULL CHECK (origin IN ('manual', 'schedule', 'sensor', 'rerun', 'backfill')), + job_key TEXT, + parent_run_id TEXT REFERENCES sqb_runs(run_id), + root_run_id TEXT NOT NULL, + trigger_id TEXT REFERENCES sqb_triggers(trigger_id), + tick_id TEXT REFERENCES sqb_ticks(tick_id), + rerun_of_run_id TEXT REFERENCES sqb_runs(run_id), + requested_by TEXT NOT NULL, + priority BIGINT NOT NULL DEFAULT 0, + selection_json JSON NOT NULL, + request_json JSON NOT NULL, + authored_json JSON, + write_set_digest TEXT NOT NULL, + rewrite_history BOOLEAN NOT NULL DEFAULT FALSE, + status TEXT NOT NULL CHECK (status IN ('queued', 'launched', 'running', 'unresponsive', 'success', 'warn', 'partial', 'failed', 'cancelled')), + status_reason TEXT, + queued_at TEXT NOT NULL, + launched_at TEXT, + started_at TEXT, + ended_at TEXT, + cancel_requested_at TEXT, + node_counts_json JSON NOT NULL, + last_event_storage_id BIGINT NOT NULL, + worker_instance TEXT, + container_id TEXT, + claimed_at TEXT, + last_heartbeat_at TEXT +); + +CREATE INDEX sqb_runs_location_time ON sqb_runs(location_id, target, queued_at); + +CREATE INDEX sqb_runs_location_ended ON sqb_runs(location_id, target, ended_at); + +CREATE INDEX sqb_runs_queue ON sqb_runs(priority DESC, queued_at) WHERE status = 'queued'; + +CREATE INDEX sqb_runs_active ON sqb_runs(location_id, target, job_key) WHERE status IN ('queued', 'launched', 'running', 'unresponsive'); + +CREATE TABLE sqb_run_nodes ( + run_id TEXT NOT NULL REFERENCES sqb_runs(run_id), + node_key TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'success', 'warn', 'failed', 'skipped', 'cancelled')), + writes BOOLEAN NOT NULL, + started_at TEXT, + ended_at TEXT, + rows_affected BIGINT, + error_code TEXT, + skipped_because_of TEXT, + kind TEXT NOT NULL DEFAULT 'node' CHECK (kind IN ('node', 'check')), + finished_storage_id BIGINT, + PRIMARY KEY (run_id, node_key) +); + +CREATE INDEX sqb_run_nodes_node ON sqb_run_nodes(node_key, run_id); + +CREATE INDEX sqb_run_nodes_finished ON sqb_run_nodes(finished_storage_id) +WHERE finished_storage_id IS NOT NULL; + +CREATE INDEX sqb_run_nodes_decisive ON sqb_run_nodes(node_key, finished_storage_id) +WHERE finished_storage_id IS NOT NULL AND status IN ('success', 'failed'); + +CREATE TABLE sqb_write_claims ( + target TEXT NOT NULL, + node_key TEXT NOT NULL, + run_id TEXT NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (target, node_key) +); + +CREATE INDEX sqb_write_claims_run ON sqb_write_claims(run_id); + +CREATE TABLE sqb_concurrency_slots ( + group_key TEXT NOT NULL, + slot BIGINT NOT NULL CHECK (slot > 0), + run_id TEXT NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (group_key, slot), + UNIQUE (group_key, run_id) +); + +CREATE INDEX sqb_concurrency_slots_run ON sqb_concurrency_slots(run_id); + +CREATE TABLE sqb_principals ( + principal_id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL, + disabled_at TEXT +); + +CREATE TABLE sqb_access_tokens ( + token_id TEXT PRIMARY KEY, + secret_hash TEXT NOT NULL, + location_id TEXT REFERENCES sqb_locations(location_id), + run_id TEXT REFERENCES sqb_runs(run_id), + scope_json JSON NOT NULL, + created_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + revoked_at TEXT, + CHECK ( + (location_id IS NOT NULL AND run_id IS NULL) + OR (location_id IS NULL AND run_id IS NOT NULL) + ) +); + +CREATE INDEX sqb_access_tokens_location ON sqb_access_tokens(location_id); + +CREATE INDEX sqb_access_tokens_run ON sqb_access_tokens(run_id); + +CREATE TABLE sqb_security_audit ( + event_id TEXT PRIMARY KEY, + occurred_at TEXT NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + granting_role TEXT NOT NULL, + request_id TEXT NOT NULL, + ip_address TEXT, + scope_json JSON NOT NULL, + before_json JSON NOT NULL, + after_json JSON NOT NULL, + storage_id BIGINT +); + +CREATE INDEX sqb_security_audit_time ON sqb_security_audit(occurred_at, event_id); + +CREATE INDEX sqb_security_audit_trigger ON sqb_security_audit((scope_json ->> 'trigger_id'), occurred_at, event_id) +WHERE action = 'trigger.changed'; + +CREATE TABLE sqb_location_instances ( + instance_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + health_json JSON NOT NULL +); + +CREATE INDEX sqb_location_instances_location_seen ON sqb_location_instances(location_id, last_seen_at); + +CREATE TABLE sqb_location_work ( + request_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + generation BIGINT NOT NULL, + request_json JSON NOT NULL, + requested_by TEXT NOT NULL, + requested_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + claim_id TEXT, + session_id TEXT REFERENCES sqb_location_instances(instance_id), + claimed_until TEXT, + result_json JSON, + completed_at TEXT +); + +CREATE INDEX sqb_location_work_pending ON sqb_location_work(location_id, target, requested_at); + +CREATE TABLE sqb_platform_sink_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at TEXT NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); + +CREATE TABLE sqb_sensor_evaluation_events ( + event_id TEXT PRIMARY KEY, + evaluation_id TEXT NOT NULL, + trigger_id TEXT NOT NULL REFERENCES sqb_triggers(trigger_id), + phase TEXT NOT NULL CHECK (phase IN ('requested', 'completed', 'failed', 'timed_out', 'abandoned')), + occurred_at TEXT NOT NULL, + fact_json JSON NOT NULL, + UNIQUE (evaluation_id, phase) +); + +CREATE INDEX sqb_sensor_evaluation_history ON sqb_sensor_evaluation_events(trigger_id, phase, occurred_at, evaluation_id); + +CREATE INDEX sqb_sensor_evaluation_expiry ON sqb_sensor_evaluation_events(occurred_at); + +CREATE TABLE sqb_run_requests ( + trigger_id TEXT NOT NULL REFERENCES sqb_triggers(trigger_id), + run_key TEXT NOT NULL, + evaluation_id TEXT NOT NULL, + run_id TEXT NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (trigger_id, run_key) +); + +CREATE INDEX sqb_runs_terminal_cursor ON sqb_runs(location_id, target, job_key, last_event_storage_id) +WHERE status IN ('success', 'warn', 'partial', 'failed', 'cancelled'); + +CREATE INDEX sqb_runs_parent ON sqb_runs(parent_run_id) WHERE parent_run_id IS NOT NULL; + +CREATE TABLE sqb_warehouse_freshness ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT, + scan_id TEXT +); + +CREATE INDEX sqb_warehouse_freshness_scope ON sqb_warehouse_freshness (location_id, target, observed_at, event_id); + +CREATE TABLE sqb_warehouse_fingerprints ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at TEXT NOT NULL, + definition_text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT, + node_type TEXT, + node_name TEXT, + target_database TEXT, + target_schema TEXT, + target_name TEXT, + scan_id TEXT +); + +CREATE INDEX sqb_warehouse_fingerprints_scope ON sqb_warehouse_fingerprints (location_id, target, observed_at, event_id); + +CREATE TABLE sqb_development_locations ( + project_path TEXT NOT NULL, + target TEXT NOT NULL, + location_id TEXT NOT NULL UNIQUE REFERENCES sqb_locations(location_id), + PRIMARY KEY (project_path, target) +); + +CREATE TABLE sqb_password_credentials ( + principal_id TEXT PRIMARY KEY REFERENCES sqb_principals(principal_id), + credential_id TEXT NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + changed_at TEXT NOT NULL +); + +CREATE TABLE sqb_sessions ( + session_id TEXT PRIMARY KEY, + principal_id TEXT NOT NULL REFERENCES sqb_principals(principal_id), + credential_id TEXT, + secret_hash TEXT NOT NULL, + csrf_hash TEXT NOT NULL, + created_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + revoked_at TEXT +); + +CREATE INDEX sqb_sessions_person ON sqb_sessions(principal_id); + +CREATE TABLE sqb_password_login_attempts ( + attempt_id TEXT PRIMARY KEY, + identity_key TEXT NOT NULL, + peer_address TEXT NOT NULL, + attempted_at TEXT NOT NULL +); + +CREATE INDEX sqb_password_login_user_time ON sqb_password_login_attempts(identity_key, attempted_at); + +CREATE INDEX sqb_password_login_peer_time ON sqb_password_login_attempts(peer_address, attempted_at); + +CREATE INDEX sqb_password_login_time ON sqb_password_login_attempts(attempted_at); + +CREATE TABLE sqb_oidc_login_flows ( + flow_id TEXT PRIMARY KEY, + configuration_id TEXT NOT NULL, + state_hash TEXT NOT NULL UNIQUE, + browser_hash TEXT NOT NULL, + nonce TEXT NOT NULL, + pkce_verifier TEXT NOT NULL, + expires_at TEXT NOT NULL +); + +CREATE INDEX sqb_oidc_login_flows_expiry ON sqb_oidc_login_flows(expires_at); + +CREATE TABLE sqb_role_assignments ( + assignment_id TEXT PRIMARY KEY, + principal_id TEXT NOT NULL REFERENCES sqb_principals(principal_id), + role_key TEXT NOT NULL, + location_id TEXT REFERENCES sqb_locations(location_id), + target TEXT, + active_key TEXT NOT NULL UNIQUE +); + +CREATE INDEX sqb_role_assignments_principal ON sqb_role_assignments(principal_id); + +CREATE TABLE sqb_person_groups ( + principal_id TEXT NOT NULL REFERENCES sqb_principals(principal_id), + group_name TEXT NOT NULL, + PRIMARY KEY (principal_id, group_name) +); + +CREATE TABLE sqb_group_roles ( + group_name TEXT NOT NULL, + role_key TEXT NOT NULL, + location_id TEXT, + target TEXT +); + +CREATE INDEX sqb_fingerprints_namespace ON sqb_warehouse_fingerprints (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_freshness_namespace ON sqb_warehouse_freshness (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE UNIQUE INDEX sqb_security_audit_storage_id ON sqb_security_audit(storage_id); + +CREATE TABLE sqb_warehouse_audit_results ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + audit_name TEXT NOT NULL, + outcome TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT, + binding_key TEXT +); + +CREATE INDEX sqb_audit_results_namespace ON sqb_warehouse_audit_results (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_audit_results_binding ON sqb_warehouse_audit_results (location_id,target,binding_key,observed_at,event_id); + +CREATE TABLE sqb_warehouse_janitor_events ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + event_type TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_janitor_events_namespace ON sqb_warehouse_janitor_events (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_migrations ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + target_name TEXT, + destination_model TEXT NOT NULL, + decision TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_migrations_namespace ON sqb_warehouse_migrations (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_migrations_destination ON sqb_warehouse_migrations (location_id,target,destination_model,observed_at,event_id); + +CREATE TABLE sqb_warehouse_old_name_views ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + target_name TEXT, + migration_event_id TEXT NOT NULL, + event_type TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_old_name_views_namespace ON sqb_warehouse_old_name_views (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_old_name_views_migration ON sqb_warehouse_old_name_views (location_id,target,migration_event_id,observed_at,event_id); + +CREATE TABLE sqb_warehouse_microbatches ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + model_name TEXT NOT NULL, + target_name TEXT NOT NULL, + scope_kind TEXT NOT NULL, + scope_key TEXT NOT NULL, + record_type TEXT NOT NULL, + execution_run_id TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_microbatches_namespace ON sqb_warehouse_microbatches (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_microbatches_model ON sqb_warehouse_microbatches (location_id,target,model_name); + +CREATE TABLE sqb_warehouse_column_migrations ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_column_migrations_namespace ON sqb_warehouse_column_migrations (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_node_results ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_node_results_namespace ON sqb_warehouse_node_results (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_consumption ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_consumption_namespace ON sqb_warehouse_consumption (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_observations ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + kind TEXT NOT NULL, + node TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_observations_namespace ON sqb_warehouse_observations (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_observations_latest ON sqb_warehouse_observations (location_id,target,kind,node,observed_at,event_id); + +CREATE TABLE sqb_warehouse_resource_costs ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + run_id TEXT NOT NULL, + orchestration_run_id TEXT, + attempt BIGINT NOT NULL, + resource_type TEXT NOT NULL, + resource_name TEXT NOT NULL, + run_started_at TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_resource_costs_namespace ON sqb_warehouse_resource_costs (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_resource_costs_run ON sqb_warehouse_resource_costs (location_id,target,run_id,attempt); + +CREATE INDEX sqb_resource_costs_orchestration_run ON sqb_warehouse_resource_costs (orchestration_run_id); + +CREATE INDEX sqb_resource_costs_started ON sqb_warehouse_resource_costs (location_id,target,run_started_at); + +CREATE INDEX sqb_freshness_source_latest ON sqb_warehouse_freshness ( + location_id, target, (record_json ->> 'source_name'), + (record_json ->> 'target_database'), (record_json ->> 'target_schema'), + (record_json ->> 'target_name'), observed_at DESC, event_id DESC +); + +CREATE INDEX sqb_freshness_source_received ON sqb_warehouse_freshness ( + location_id, target, (record_json ->> 'source_name'), + (record_json ->> 'target_database'), (record_json ->> 'target_schema'), + (record_json ->> 'target_name'), received_at DESC, observed_at DESC, event_id DESC +); + +CREATE TABLE sqb_security_audit_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at TEXT NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); + +CREATE TABLE sqb_warehouse_catalog ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + observed_at TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_warehouse_catalog_namespace ON sqb_warehouse_catalog + (location_id, target, state_schema, state_database, observed_at, event_id); + +CREATE INDEX sqb_fingerprint_node_latest ON sqb_warehouse_fingerprints + (location_id,target,node_type,node_name,target_database,target_schema,target_name,observed_at DESC,event_id DESC); + +CREATE TABLE sqb_warehouse_statistics ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + relation_name TEXT NOT NULL, + observed_at TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL +); + +CREATE INDEX sqb_statistics_relation_time ON sqb_warehouse_statistics + (location_id,target,state_schema,state_database,relation_name,observed_at,event_id); + +CREATE INDEX sqb_statistics_expiration ON sqb_warehouse_statistics (observed_at,event_id); + +CREATE INDEX sqb_catalog_statistics_scan ON sqb_warehouse_catalog + (location_id,target,state_schema,state_database,event_id); + +CREATE TABLE sqb_cli_login_flows ( + flow_id TEXT PRIMARY KEY, + client_hash TEXT NOT NULL, + browser_hash TEXT NOT NULL, + expires_at TEXT NOT NULL, + principal_id TEXT, + session_id TEXT +); + +CREATE INDEX sqb_cli_login_expiry ON sqb_cli_login_flows(expires_at); + +CREATE INDEX IF NOT EXISTS sqb_runs_tick ON sqb_runs(tick_id); + +CREATE INDEX IF NOT EXISTS sqb_ticks_expiry ON sqb_ticks(scheduled_for) WHERE outcome = 'covered'; + +CREATE INDEX IF NOT EXISTS sqb_runs_snapshot ON sqb_runs(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_snapshot_nodes_hash ON sqb_snapshot_nodes(node_hash); + +CREATE INDEX IF NOT EXISTS sqb_snapshot_sources_hash ON sqb_snapshot_sources(text_hash); + +CREATE INDEX IF NOT EXISTS sqb_snapshot_documents_hash ON sqb_snapshot_documents(document_hash); + +CREATE TABLE sqb_effective_roles ( + location_id TEXT NOT NULL, + target TEXT NOT NULL, + role_key TEXT NOT NULL, + resolved_grants_json TEXT NOT NULL, + PRIMARY KEY (location_id, target, role_key) +); + +CREATE INDEX IF NOT EXISTS sqb_jobs_snapshot ON sqb_jobs(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_triggers_snapshot ON sqb_triggers(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_triggers_removed ON sqb_triggers(removed_in_snapshot_id) WHERE removed_in_snapshot_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_runs_trigger ON sqb_runs(trigger_id) WHERE trigger_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_runs_rerun_of ON sqb_runs(rerun_of_run_id) WHERE rerun_of_run_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_location_work_session ON sqb_location_work(session_id) WHERE session_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_location_work_snapshot ON sqb_location_work(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_run_requests_run ON sqb_run_requests(run_id); + +CREATE INDEX IF NOT EXISTS sqb_warehouse_fingerprints_text ON sqb_warehouse_fingerprints(definition_text_hash); + +CREATE INDEX IF NOT EXISTS sqb_role_assignments_location ON sqb_role_assignments(location_id); diff --git a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs index fa10659ee3..a29d3da5aa 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs @@ -10,9 +10,13 @@ use crate::store::main::commit::commit; use crate::store::main::migrate_store::migrate_store; use crate::store::main::open_store::open_store; use crate::store::main::query::query; -use crate::store::models::{Parameter, TransactionMode}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::tests::schema::test_types::StoreDialect; use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; use crate::store::types::Row; +use crate::store_migrations::constants::{POSTGRES_BASELINE_CHECKSUM, SQLITE_BASELINE_CHECKSUM}; +use crate::store_migrations::models::StoreMigration; const CHILD_STORE_VARIABLE: &str = "SQLBUILD_TEST_CRASH_STORE"; const ACKNOWLEDGED: &str = "committed "; @@ -119,3 +123,206 @@ pub(crate) fn wal_bytes_after_sequential_writes(path: &Path, commits: usize, byt std::fs::metadata(wal).map_or(0, |metadata| metadata.len()) }) } + +const BASELINE_SQLITE: &str = include_str!("baseline/sqlite.sql"); +const BASELINE_POSTGRES: &str = include_str!("baseline/postgres.sql"); +pub(crate) const BASELINE_ROWS: &str = include_str!("baseline/rows.sql"); +const SHAPE_SQLITE: &str = "SELECT type || ' ' || name || ': ' || COALESCE(sql, '') AS item \ + FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' AND tbl_name <> 'sqb_schema_version'"; +const SHAPE_POSTGRES: &str = "SELECT 'column ' || table_name || '.' || column_name || ' ' || data_type \ + || ' ' || is_nullable || ' ' || COALESCE(column_default, '') AS item \ + FROM information_schema.columns WHERE table_schema = current_schema() \ + AND table_name <> 'sqb_schema_version' \ + UNION ALL SELECT 'constraint ' || conrelid::regclass::text || ' ' || conname || ' ' \ + || pg_get_constraintdef(oid) FROM pg_constraint \ + WHERE connamespace = current_schema()::regnamespace \ + AND conrelid::regclass::text <> 'sqb_schema_version' \ + UNION ALL SELECT 'index ' || indexname || ' ' || replace(indexdef, current_schema() || '.', '') \ + FROM pg_indexes \ + WHERE schemaname = current_schema() AND tablename <> 'sqb_schema_version' \ + UNION ALL SELECT 'view ' || viewname || ' ' || definition FROM pg_views \ + WHERE schemaname = current_schema()"; +/// The SQLite facts the migration contracts use. +pub(crate) const SQLITE_STORE: StoreDialect = StoreDialect { + backend: Backend::Sqlite, + baseline_schema: BASELINE_SQLITE, + baseline_checksum: SQLITE_BASELINE_CHECKSUM, + time_type: "TEXT", + shape_sql: SHAPE_SQLITE, + recreate_command: "rm -f '", +}; +/// The PostgreSQL facts the migration contracts use. +pub(crate) const POSTGRES_STORE: StoreDialect = StoreDialect { + backend: Backend::Postgres, + baseline_schema: BASELINE_POSTGRES, + baseline_checksum: POSTGRES_BASELINE_CHECKSUM, + time_type: "TIMESTAMPTZ", + shape_sql: SHAPE_POSTGRES, + recreate_command: "dropdb -h '", +}; +/// The tables `rows.sql` fills: installation, locations, people, roles, tokens, triggers, run nodes and runs. +pub(crate) const POPULATED_TABLES: [&str; 10] = [ + "sqb_installation", + "sqb_locations", + "sqb_snapshots", + "sqb_principals", + "sqb_role_assignments", + "sqb_access_tokens", + "sqb_triggers", + "sqb_trigger_state", + "sqb_run_nodes", + "sqb_runs", +]; + +/// Run `statements` (separated by blank lines) in one write transaction. +async fn execute_all(store: &ControlStore, statements: &[&str]) { + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("begin write"); + for statement in statements { + (transaction, _) = query(transaction, statement, vec![]) + .await + .unwrap_or_else(|error| panic!("{statement}: {error}")); + } + commit(transaction).await.expect("commit"); +} + +fn blank_line_statements(sql: &str) -> Vec<&str> { + sql.split(";\n\n") + .map(|statement| statement.trim().trim_end_matches(';')) + .filter(|statement| !statement.is_empty()) + .collect() +} + +/// A store as a release before schema versions left it: `dialect`'s baseline schema and checksum, and `rows.sql`. +pub(crate) async fn create_unversioned_baseline(store: &ControlStore, dialect: &StoreDialect) { + let version_table = format!( + "CREATE TABLE sqb_schema_version (singleton BIGINT PRIMARY KEY CHECK (singleton = 1), \ + checksum TEXT NOT NULL, applied_at {} NOT NULL)", + dialect.time_type + ); + let stamp = format!( + "INSERT INTO sqb_schema_version (singleton, checksum, applied_at) \ + VALUES (1, '{}', '2026-11-01T08:00:00Z')", + dialect.baseline_checksum + ); + let installation = "INSERT INTO sqb_installation (singleton, installation_id, created_at) \ + VALUES (1, '018f0000-0000-7000-8000-000000000000', '2026-11-01T08:00:00Z')"; + let mut statements: Vec<&str> = vec![version_table.as_str()]; + statements.extend(blank_line_statements(dialect.baseline_schema)); + statements.extend([installation, stamp.as_str()]); + statements.extend(blank_line_statements(BASELINE_ROWS)); + execute_all(store, &statements).await; +} + +/// Run `sql`'s `;`-separated statements in one write transaction. +pub(crate) async fn execute(store: &ControlStore, sql: &str) { + let statements: Vec<&str> = sql + .split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) + .collect(); + execute_all(store, &statements).await; +} + +/// The store's tables, columns, constraints, indexes and views, normalised so equal schemas compare equal. +pub(crate) async fn schema_shape(store: &ControlStore, dialect: &StoreDialect) -> Vec { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("begin read"); + let (transaction, rows) = query(transaction, dialect.shape_sql, vec![]) + .await + .expect("read schema"); + commit(transaction).await.expect("finish read"); + let mut shape: Vec = rows + .iter() + .map(|row| { + let item: String = text(row, "item").replace('"', ""); + let words: Vec<&str> = item.split_whitespace().collect(); + words + .join(" ") + .replace("( ", "(") + .replace(" )", ")") + .replace(" ,", ",") + }) + .collect(); + shape.sort(); + shape +} + +/// Every row of `tables`, in primary-key order. +pub(crate) async fn table_rows(store: &ControlStore, tables: &[&str]) -> Vec> { + let mut transaction = begin(store, TransactionMode::Read) + .await + .expect("begin read"); + let mut all: Vec> = Vec::new(); + for table in tables { + let rows: Vec; + (transaction, rows) = query( + transaction, + &format!("SELECT * FROM {table} ORDER BY 1"), + vec![], + ) + .await + .expect("read table"); + all.push(rows); + } + commit(transaction).await.expect("finish read"); + all +} + +/// The store's recorded schema version. +pub(crate) async fn recorded_version(store: &ControlStore) -> Option { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("begin read"); + let (transaction, rows) = query(transaction, "SELECT * FROM sqb_schema_version", vec![]) + .await + .expect("read version"); + commit(transaction).await.expect("finish read"); + rows.first() + .and_then(|row| row.get("version")) + .and_then(serde_json::Value::as_i64) +} + +/// SQLite statements rebuilding `sqb_runs` with `skipped` added to its status check, and its indexes; empty on PostgreSQL. +pub(crate) async fn rebuilt_runs_table(store: &ControlStore) -> &'static str { + let ControlStore::Sqlite(_) = store else { + return ""; + }; + let transaction = begin(store, TransactionMode::Read) + .await + .expect("begin read"); + let (transaction, rows) = query( + transaction, + "SELECT type, sql FROM sqlite_master WHERE tbl_name = 'sqb_runs' AND sql IS NOT NULL \ + ORDER BY type DESC", + vec![], + ) + .await + .expect("read sqb_runs definition"); + commit(transaction).await.expect("finish read"); + let mut definitions = rows.iter().map(|row| text(row, "sql")); + let table = definitions + .next() + .expect("sqb_runs table") + .replacen("CREATE TABLE sqb_runs", "CREATE TABLE sqb_runs_rebuilt", 1) + .replace("'cancelled'))", "'cancelled', 'skipped'))"); + let indexes: Vec = definitions.collect(); + let sql = format!( + "{table};\nINSERT INTO sqb_runs_rebuilt SELECT * FROM sqb_runs;\nDROP TABLE sqb_runs;\n\ + ALTER TABLE sqb_runs_rebuilt RENAME TO sqb_runs;\n{}", + indexes.join(";\n") + ); + Box::leak(sql.into_boxed_str()) +} + +/// The numbered names a migration plan lists for `migrations` after version `from`. +pub(crate) fn migration_names(migrations: &[StoreMigration], from: i64) -> Vec { + migrations + .iter() + .skip(usize::try_from(from - 1).expect("version at or after the baseline")) + .zip(from + 1..) + .map(|(migration, number)| format!("{number:04}_{}", migration.name)) + .collect() +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs index 6a1a10b378..325eaa9c1e 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs @@ -6,6 +6,8 @@ pub(crate) mod helpers; mod test_crash_recovery; #[path = "test_foreign_key_indexes.rs"] mod test_foreign_key_indexes; +#[path = "test_store_migrations.rs"] +mod test_store_migrations; #[path = "test_store_upgrade.rs"] mod test_store_upgrade; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs new file mode 100644 index 0000000000..5c5c52d6f9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs @@ -0,0 +1,412 @@ +//! Numbered control-store migrations: baseline upgrades, table rebuilds, refusals and the dry run. + +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::schema::helpers::{ + BASELINE_ROWS, POPULATED_TABLES, POSTGRES_STORE, SQLITE_STORE, create_unversioned_baseline, + execute, migration_names, rebuilt_runs_table, recorded_version, schema_shape, table_rows, +}; +use crate::store::tests::schema::test_types::{ + BaselineUpgradeTestCase, BrokenMigrationTestCase, MigrationRefusalTestCase, + PinnedChecksumTestCase, StoreMigrationPlanTestCase, TableRebuildTestCase, +}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_migrations::_helpers::migration::{ + MIGRATIONS, current_version, migration_plan, schema_digest, +}; +use crate::store_migrations::_helpers::migration_run::migrate_with; +use crate::store_migrations::main::plan_store_migration::plan_store_migration; +use crate::store_migrations::models::{StoreMigration, StoreMigrationPlan}; + +const SKIPPED_RUN: &str = "INSERT INTO sqb_runs (run_id, location_id, target, snapshot_id, kind, \ + origin, root_run_id, parent_run_id, requested_by, selection_json, request_json, \ + write_set_digest, status, queued_at, node_counts_json, last_event_storage_id) VALUES \ + ('018f0000-0000-7000-8000-000000000008', '018f0000-0000-7000-8000-000000000001', 'prod', \ + '018f0000-0000-7000-8000-000000000002', 'job', 'manual', \ + '018f0000-0000-7000-8000-000000000007', '018f0000-0000-7000-8000-000000000007', 'admin', \ + '[]', '{}', 'digest-2', 'skipped', '2026-11-02T10:00:00Z', '{}', 13)"; +const SQLITE_CHECKSUMS: &[(i64, &str)] = &[( + 1, + "6d0e581f453370ec5156053d644dea3b22b19332cee588eb69ac3141ada5a76b", +)]; +const POSTGRES_CHECKSUMS: &[(i64, &str)] = &[( + 1, + "2a81e526eb1181eeab0b897e8b3f48ae2dd9425d85cb788874670d942bbc76c4", +)]; +const DANGLING_RUN_NODE: &str = "INSERT INTO sqb_run_nodes (run_id, node_key, status, writes) \ + VALUES ('018f0000-0000-7000-8000-0000000000ff', 'model.ghost', 'success', TRUE)"; +const WIDEN_RUN_STATUS_POSTGRES: &str = "ALTER TABLE sqb_runs DROP CONSTRAINT sqb_runs_status_check; \ + ALTER TABLE sqb_runs ADD CONSTRAINT sqb_runs_status_check CHECK (status IN ('queued', \ + 'launched', 'running', 'unresponsive', 'success', 'warn', 'partial', 'failed', 'cancelled', \ + 'skipped'))"; + +#[test] +fn given_store_at_the_baseline_when_migrating_then_it_matches_a_new_store_and_keeps_its_rows() { + let test_cases = [ + BaselineUpgradeTestCase { + description: "SQLite store", + dialect: &SQLITE_STORE, + expected_version: current_version(MIGRATIONS), + }, + BaselineUpgradeTestCase { + description: "PostgreSQL store", + dialect: &POSTGRES_STORE, + expected_version: current_version(MIGRATIONS), + }, + ]; + for test_case in test_cases { + run(async { + let (Some(upgraded), Some(created)) = ( + open_test_store(test_case.dialect.backend).await, + open_test_store(test_case.dialect.backend).await, + ) else { + return; + }; + create_unversioned_baseline(&upgraded.store, test_case.dialect).await; + let before = table_rows(&upgraded.store, &POPULATED_TABLES).await; + + migrate_store(&upgraded.store) + .await + .expect("upgrade the baseline store"); + migrate_store(&created.store).await.expect("create a store"); + + assert_eq!( + ( + schema_shape(&upgraded.store, test_case.dialect).await, + table_rows(&upgraded.store, &POPULATED_TABLES).await, + recorded_version(&upgraded.store).await, + ), + ( + schema_shape(&created.store, test_case.dialect).await, + before, + Some(test_case.expected_version), + ), + "{}: a schema change needs a numbered migration that brings the baseline to it, \ + and upgrading keeps every row", + test_case.description + ); + close_test_store(upgraded).await; + close_test_store(created).await; + }); + } +} + +#[test] +fn given_a_migration_rebuilding_a_referenced_table_when_migrating_then_rows_and_references_survive() +{ + let test_cases = [ + TableRebuildTestCase { + description: "SQLite store", + dialect: &SQLITE_STORE, + expected_runs: 2, + expected_version: 2, + }, + TableRebuildTestCase { + description: "PostgreSQL store", + dialect: &POSTGRES_STORE, + expected_runs: 2, + expected_version: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.dialect.backend).await else { + return; + }; + let store = &fixture.store; + migrate_with(store, &[]).await.expect("create the store"); + execute(store, BASELINE_ROWS).await; + let mut before = table_rows(store, &POPULATED_TABLES).await; + before.pop(); + let shape_before: Vec = schema_shape(store, test_case.dialect) + .await + .into_iter() + .filter(|item| !item.contains("'cancelled'")) + .collect(); + let migration = StoreMigration { + name: "widen_run_status", + sqlite: rebuilt_runs_table(store).await, + postgres: WIDEN_RUN_STATUS_POSTGRES, + }; + + migrate_with(store, &[migration]) + .await + .expect("rebuild sqb_runs"); + execute(store, SKIPPED_RUN).await; + + let shape_after: Vec = schema_shape(store, test_case.dialect) + .await + .into_iter() + .filter(|item| !item.contains("'cancelled'")) + .collect(); + let mut rows = table_rows(store, &POPULATED_TABLES).await; + let runs = rows.pop().expect("sqb_runs rows"); + assert_eq!( + (rows, runs.len(), shape_after, recorded_version(store).await), + ( + before, + test_case.expected_runs, + shape_before, + Some(test_case.expected_version) + ), + "{}: rows, other schema objects and version after the rebuild", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_store_this_build_cannot_upgrade_when_migrating_then_it_is_refused_unchanged() { + let test_cases = [ + MigrationRefusalTestCase { + description: "SQLite store from a newer build", + dialect: &SQLITE_STORE, + tamper: "UPDATE sqb_schema_version SET version = 99", + expected_fragments: vec!["schema is version 99, newer than this build's version"], + }, + MigrationRefusalTestCase { + description: "SQLite store older than the baseline", + dialect: &SQLITE_STORE, + tamper: "UPDATE sqb_schema_version SET checksum = 'unknown', version = NULL", + expected_fragments: vec!["older than version 1", SQLITE_STORE.recreate_command], + }, + MigrationRefusalTestCase { + description: "SQLite store whose schema changed without a migration", + dialect: &SQLITE_STORE, + tamper: "UPDATE sqb_schema_version SET checksum = 'unknown'", + expected_fragments: vec![ + "doesn't match this build's, although both are version", + SQLITE_STORE.recreate_command, + ], + }, + MigrationRefusalTestCase { + description: "PostgreSQL store from a newer build", + dialect: &POSTGRES_STORE, + tamper: "UPDATE sqb_schema_version SET version = 99", + expected_fragments: vec!["schema is version 99, newer than this build's version"], + }, + MigrationRefusalTestCase { + description: "PostgreSQL store older than the baseline", + dialect: &POSTGRES_STORE, + tamper: "UPDATE sqb_schema_version SET checksum = 'unknown', version = NULL", + expected_fragments: vec!["older than version 1", POSTGRES_STORE.recreate_command], + }, + MigrationRefusalTestCase { + description: "PostgreSQL store whose schema changed without a migration", + dialect: &POSTGRES_STORE, + tamper: "UPDATE sqb_schema_version SET checksum = 'unknown'", + expected_fragments: vec![ + "doesn't match this build's, although both are version", + POSTGRES_STORE.recreate_command, + ], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.dialect.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("create the store"); + execute(store, test_case.tamper).await; + let version_before = recorded_version(store).await; + + let error = migrate_store(store) + .await + .expect_err("refuse the store") + .to_string(); + + let found: Vec<&str> = test_case + .expected_fragments + .iter() + .copied() + .filter(|fragment| error.contains(fragment)) + .collect(); + assert_eq!( + (found, recorded_version(store).await), + (test_case.expected_fragments.clone(), version_before), + "{}: {error}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_store_when_planning_its_migration_then_the_plan_is_reported_without_changing_it() { + let current = current_version(MIGRATIONS); + let expected_plans = || { + ( + StoreMigrationPlan::Create { version: current }, + StoreMigrationPlan::Upgrade { + from: 1, + to: current, + names: migration_names(MIGRATIONS, 1), + }, + None, + ) + }; + let expected_after = || { + ( + StoreMigrationPlan::Current { version: current }, + StoreMigrationPlan::Upgrade { + from: current, + to: current + 1, + names: vec![format!("{:04}_widen_run_status", current + 1)], + }, + ) + }; + let test_cases = [ + StoreMigrationPlanTestCase { + description: "SQLite store", + dialect: &SQLITE_STORE, + expected_plans: expected_plans(), + expected_after: expected_after(), + }, + StoreMigrationPlanTestCase { + description: "PostgreSQL store", + dialect: &POSTGRES_STORE, + expected_plans: expected_plans(), + expected_after: expected_after(), + }, + ]; + for test_case in test_cases { + run(async { + let (Some(empty), Some(baseline)) = ( + open_test_store(test_case.dialect.backend).await, + open_test_store(test_case.dialect.backend).await, + ) else { + return; + }; + create_unversioned_baseline(&baseline.store, test_case.dialect).await; + + let plans = ( + plan_store_migration(&empty.store) + .await + .expect("plan an empty store"), + plan_store_migration(&baseline.store) + .await + .expect("plan the baseline store"), + recorded_version(&baseline.store).await, + ); + migrate_store(&baseline.store) + .await + .expect("upgrade the baseline store"); + let mut with_pending: Vec = MIGRATIONS.to_vec(); + with_pending.push(StoreMigration { + name: "widen_run_status", + sqlite: "", + postgres: "", + }); + let after = ( + plan_store_migration(&baseline.store) + .await + .expect("plan the upgraded store"), + migration_plan( + &baseline.store, + Some((schema_digest(&baseline.store).as_str(), Some(current))), + &with_pending, + ) + .expect("plan a pending migration"), + ); + + assert_eq!( + (plans, after), + (test_case.expected_plans, test_case.expected_after), + "{}", + test_case.description + ); + close_test_store(empty).await; + close_test_store(baseline).await; + }); + } +} + +#[test] +fn given_this_builds_schema_when_checksummed_then_it_matches_the_checksum_pinned_for_its_version() { + let test_cases = [ + PinnedChecksumTestCase { + description: "SQLite store", + dialect: &SQLITE_STORE, + expected_checksums: SQLITE_CHECKSUMS, + }, + PinnedChecksumTestCase { + description: "PostgreSQL store", + dialect: &POSTGRES_STORE, + expected_checksums: POSTGRES_CHECKSUMS, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.dialect.backend).await else { + return; + }; + let current = current_version(MIGRATIONS); + let checksum: String = schema_digest(&fixture.store); + + assert_eq!( + Some(checksum.as_str()), + test_case + .expected_checksums + .iter() + .find(|(version, _)| *version == current) + .map(|(_, pinned)| *pinned), + "{}: schema.sql, indexes.sql or the Recall tables changed without a numbered \ + migration; add one and pin the new version's checksum, never edit a pinned one", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_migration_that_breaks_a_reference_when_migrating_then_it_is_refused_unchanged() { + let test_cases = [ + BrokenMigrationTestCase { + description: "SQLite store", + dialect: &SQLITE_STORE, + expected_fragment: "foreign key references; the store is unchanged", + expected_version: Some(1), + }, + BrokenMigrationTestCase { + description: "PostgreSQL store", + dialect: &POSTGRES_STORE, + expected_fragment: "violates foreign key constraint", + expected_version: Some(1), + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.dialect.backend).await else { + return; + }; + let store = &fixture.store; + migrate_with(store, &[]).await.expect("create the store"); + let migration = StoreMigration { + name: "dangling_run_node", + sqlite: DANGLING_RUN_NODE, + postgres: DANGLING_RUN_NODE, + }; + + let error = migrate_with(store, &[migration]) + .await + .expect_err("refuse the migration") + .to_string(); + + assert_eq!( + ( + error.contains(test_case.expected_fragment), + recorded_version(store).await, + table_rows(store, &["sqb_run_nodes"]).await, + ), + (true, test_case.expected_version, vec![Vec::new()]), + "{}: {error}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs index 65f75293d7..69a97d13ab 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs @@ -3,6 +3,7 @@ use std::collections::BTreeSet; use crate::store::tests::store_test_types::Backend; +use crate::store_migrations::models::StoreMigrationPlan; pub(crate) struct CrashRecoveryTestCase { pub(crate) description: &'static str, @@ -34,3 +35,54 @@ pub(crate) struct StoreUpgradeTestCase { pub(crate) list_indexes: &'static str, pub(crate) expected_indexes: Vec<&'static str>, } + +/// What the migration contracts need to know about one store type. +pub(crate) struct StoreDialect { + pub(crate) backend: Backend, + pub(crate) baseline_schema: &'static str, + pub(crate) baseline_checksum: &'static str, + pub(crate) time_type: &'static str, + pub(crate) shape_sql: &'static str, + pub(crate) recreate_command: &'static str, +} + +pub(crate) struct BaselineUpgradeTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static StoreDialect, + pub(crate) expected_version: i64, +} + +pub(crate) struct TableRebuildTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static StoreDialect, + pub(crate) expected_runs: usize, + pub(crate) expected_version: i64, +} + +pub(crate) struct MigrationRefusalTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static StoreDialect, + pub(crate) tamper: &'static str, + pub(crate) expected_fragments: Vec<&'static str>, +} + +pub(crate) struct StoreMigrationPlanTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static StoreDialect, + pub(crate) expected_plans: (StoreMigrationPlan, StoreMigrationPlan, Option), + pub(crate) expected_after: (StoreMigrationPlan, StoreMigrationPlan), +} + +pub(crate) struct PinnedChecksumTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static StoreDialect, + /// One `(version, checksum)` per schema version: append one with each migration and never edit one. + pub(crate) expected_checksums: &'static [(i64, &'static str)], +} + +pub(crate) struct BrokenMigrationTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static StoreDialect, + pub(crate) expected_fragment: &'static str, + pub(crate) expected_version: Option, +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index f04ae8e5bb..2246f55bc7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -177,9 +177,37 @@ sqlite3 control.sqlite3 ".backup '/backups/control.sqlite3'" ## Upgrade -Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the -server after a location or worker has connected from a newer release; the server can't read what -they wrote. +Upgrade the server first, then the locations and workers that connect to it. On start, the server +migrates its control database to the schema of its release in one transaction, keeping people, +roles, tokens, triggers and run history. Releases with an older schema refuse the database +afterwards, so back it up before you upgrade (see +[Back up the control database](#back-up-the-control-database)). To roll back, restore that backup; +runs recorded since the upgrade are lost. + +To see what an upgrade changes before you start the server, run the new release with `--dry-run`: + +```bash +sqb server migrate --dry-run +``` + +```text +The control store would migrate from schema version 1 to 2: + 0002_add_skipped_runs +``` + +`sqb server migrate` without `--dry-run` applies the migrations and exits. It takes the same +`--database-env` and `--config` options as `sqb server`, given before `migrate`. + +The server refuses a control database it can't migrate and leaves it unchanged: + +| Control database | Fix | +| --- | --- | +| Written by a newer release | Run the release that last opened it, or a later one. | +| Older than the oldest schema this release migrates | Open it once with an earlier release that still migrates it, then with this one. Or start a new, empty store with the command in the message, which discards its history. | +| Same schema version but a different schema, as from a development build | Start a new, empty store with the command in the message. | + +Don't downgrade the server after a location or worker has connected from a newer release; the +server can't read what they wrote. | Newer than the server | What happens | Fix | | --- | --- | --- | diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 780a271d22..959a12b84b 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -159,9 +159,37 @@ sqlite3 control.sqlite3 ".backup '/backups/control.sqlite3'" ## Upgrade -Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the -server after a location or worker has connected from a newer release; the server can't read what -they wrote. +Upgrade the server first, then the locations and workers that connect to it. On start, the server +migrates its control database to the schema of its release in one transaction, keeping people, +roles, tokens, triggers and run history. Releases with an older schema refuse the database +afterwards, so back it up before you upgrade (see +[Back up the control database](#back-up-the-control-database)). To roll back, restore that backup; +runs recorded since the upgrade are lost. + +To see what an upgrade changes before you start the server, run the new release with `--dry-run`: + +```bash +sqb server migrate --dry-run +``` + +```text +The control store would migrate from schema version 1 to 2: + 0002_add_skipped_runs +``` + +`sqb server migrate` without `--dry-run` applies the migrations and exits. It takes the same +`--database-env` and `--config` options as `sqb server`, given before `migrate`. + +The server refuses a control database it can't migrate and leaves it unchanged: + +| Control database | Fix | +| --- | --- | +| Written by a newer release | Run the release that last opened it, or a later one. | +| Older than the oldest schema this release migrates | Open it once with an earlier release that still migrates it, then with this one. Or start a new, empty store with the command in the message, which discards its history. | +| Same schema version but a different schema, as from a development build | Start a new, empty store with the command in the message. | + +Don't downgrade the server after a location or worker has connected from a newer release; the +server can't read what they wrote. | Newer than the server | What happens | Fix | | --- | --- | --- | From 4c3c5721d6dd848eed6ae2fdd8a4bae4ddccf630 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:10:21 +0000 Subject: [PATCH 2/3] fix(server): make a dry run without a SQLite store create nothing `sqb server migrate --dry-run` opened the store to plan the migration, and opening a SQLite store creates its file and parent directories. A dry run on a missing file now reports that there is no store and that a migration would create one, without creating anything. --- .../src/native_commands/_helpers/execution.rs | 10 ++++++++++ .../tests/native_commands/test_control.rs | 16 ++++++++++++---- .../tests/native_commands/test_types.rs | 2 ++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 59da49a414..e3e44afd96 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -1921,12 +1921,22 @@ pub(crate) fn run_store_migration(namespace: &Namespace, runtime: &tokio::runtim } } +/// Plan and apply the store's migrations; a dry run never creates a missing SQLite file. async fn store_migration(namespace: &Namespace, dry_run: bool) -> Result { let config = read_server_config(namespace)?; let url = control_database( namespace, &control_database_variable(namespace, &config.server), )?; + if dry_run + && let Some(path) = url + .strip_prefix("sqlite://") + .or((!url.contains("://")).then_some(url.as_str())) + && !matches!(path, "" | ":memory:") + && !std::path::Path::new(path).exists() + { + return Ok(format!("No control store at {path}: would create one.\n")); + } let store = open_store(&url).await?; let plan = plan_store_migration(&store).await?; if !dry_run { diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 5db1dbd2aa..d299cb34da 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -371,19 +371,22 @@ fn given_a_standalone_server_when_exporting_openapi_then_the_worker_contract_is_ fn given_a_control_store_when_running_server_migrate_then_dry_run_lists_and_migrate_applies() { let test_cases = [ StoreMigrationCommandTestCase { - description: "dry run on an empty store", + description: "dry run without a store creates nothing", arguments: &["server", "migrate", "--dry-run"], - expected_stdout: "The control store is empty: would create schema version 1.\n", + expected_stdout: "No control store at ./.sqlbuild/control.sqlite3: would create one.\n", + expected_store: false, }, StoreMigrationCommandTestCase { description: "migrate an empty store", arguments: &["server", "migrate"], expected_stdout: "The control store is empty: created schema version 1.\n", + expected_store: true, }, StoreMigrationCommandTestCase { description: "dry run on a current store", arguments: &["server", "migrate", "--dry-run"], expected_stdout: "The control store is at schema version 1: nothing to migrate.\n", + expected_store: true, }, ]; let directory = tempfile::tempdir().expect("isolated CLI directory"); @@ -399,9 +402,14 @@ fn given_a_control_store_when_running_server_migrate_then_dry_run_lists_and_migr assert_eq!( ( output.status.success(), - String::from_utf8_lossy(&output.stdout).into_owned() + String::from_utf8_lossy(&output.stdout).into_owned(), + directory.path().join(".sqlbuild/control.sqlite3").exists(), + ), + ( + true, + test_case.expected_stdout.to_owned(), + test_case.expected_store ), - (true, test_case.expected_stdout.to_owned()), "{}: {}", test_case.description, String::from_utf8_lossy(&output.stderr) diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs index f502be6f1d..9d9d05ae93 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -86,4 +86,6 @@ pub(crate) struct StoreMigrationCommandTestCase { pub(crate) description: &'static str, pub(crate) arguments: &'static [&'static str], pub(crate) expected_stdout: &'static str, + /// Whether the SQLite store file exists after the command. + pub(crate) expected_store: bool, } From e86dcc98b4bbb5c4fa214f2ab880d8c7364a3fdb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:25:37 +0000 Subject: [PATCH 3/3] fix(server): checksum the control-store schema without its line endings Windows checkouts turn schema.sql and indexes.sql into CRLF, and the binaries embed them as checked out. A Windows build therefore recorded a different checksum than a Linux or macOS build for the same schema: it refused the baseline store from a Windows release as older than version 1, and Windows and Linux servers sharing a PostgreSQL store would disagree about whether its schema matches. The checksum is now taken over LF text, and the baseline accepts both checksums released stores recorded: from LF and from CRLF SQL. The SQL files are also checked out with LF everywhere. --- .gitattributes | 4 ++ .../store_migrations/_helpers/migration.rs | 21 +++++-- .../src/store_migrations/constants.rs | 16 +++-- .../tests/store/schema/helpers.rs | 23 ++++--- .../store/schema/test_store_migrations.rs | 60 +++++++++++++++++-- .../tests/store/schema/test_types.rs | 4 ++ 6 files changed, 102 insertions(+), 26 deletions(-) diff --git a/.gitattributes b/.gitattributes index 3dcbe083f9..94f8bf0a71 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,3 +9,7 @@ ui/src/**/*.test.ts linguist-documentation ui/scripts/** linguist-documentation clients/typescript/tests/** linguist-documentation clients/typescript/scripts/** linguist-documentation + +# The control-store SQL is embedded in the binaries and checksummed; keep it LF on every checkout. +crates/sqlbuild-orchestration/src/store/*.sql text eol=lf +crates/sqlbuild-orchestration/tests/store/schema/baseline/*.sql text eol=lf diff --git a/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs index 7521738ec0..087e8ff283 100644 --- a/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs @@ -7,7 +7,7 @@ use crate::store::main::query::query; use crate::store::models::{ControlStore, ControlTransaction}; use crate::store_migrations::_helpers::recall_schema::recall_schema; use crate::store_migrations::constants::{ - BASELINE_VERSION, POSTGRES_BASELINE_CHECKSUM, SQLITE_BASELINE_CHECKSUM, + BASELINE_VERSION, POSTGRES_BASELINE_CHECKSUMS, SQLITE_BASELINE_CHECKSUMS, }; use crate::store_migrations::models::{StoreMigration, StoreMigrationPlan}; @@ -29,7 +29,16 @@ pub(crate) fn statements(sql: &str) -> impl Iterator { /// The checksum of the schema this build creates, recorded with its version. pub(crate) fn schema_digest(store: &ControlStore) -> String { - sha256_hex(format!("{}\n{SCHEMA}\n{INDEXES}", recall_schema(store).join(";\n")).as_bytes()) + schema_text_digest(&recall_schema(store).join(";\n"), SCHEMA, INDEXES) +} + +/// The checksum of a schema's Recall DDL, `schema.sql` and `indexes.sql`, whatever line endings Git used. +pub(crate) fn schema_text_digest(recall: &str, schema: &str, indexes: &str) -> String { + sha256_hex( + format!("{recall}\n{schema}\n{indexes}") + .replace("\r\n", "\n") + .as_bytes(), + ) } /// The statements of `migration` for `store`'s database type. @@ -85,13 +94,13 @@ pub(crate) fn migration_plan( let Some((checksum, version)) = recorded else { return Ok(StoreMigrationPlan::Create { version: current }); }; - let baseline_checksum = match store { - ControlStore::Sqlite(_) => SQLITE_BASELINE_CHECKSUM, - ControlStore::Postgres(_) => POSTGRES_BASELINE_CHECKSUM, + let baseline_checksums = match store { + ControlStore::Sqlite(_) => SQLITE_BASELINE_CHECKSUMS, + ControlStore::Postgres(_) => POSTGRES_BASELINE_CHECKSUMS, }; let version = match version { Some(version) => version, - None if checksum == baseline_checksum => BASELINE_VERSION, + None if baseline_checksums.contains(&checksum) => BASELINE_VERSION, None => return Err(refusal(store, Refusal::OlderThanBaseline)), }; if version > current { diff --git a/crates/sqlbuild-orchestration/src/store_migrations/constants.rs b/crates/sqlbuild-orchestration/src/store_migrations/constants.rs index 9ba1ddd11c..56a601ec2b 100644 --- a/crates/sqlbuild-orchestration/src/store_migrations/constants.rs +++ b/crates/sqlbuild-orchestration/src/store_migrations/constants.rs @@ -3,10 +3,14 @@ /// The oldest schema version this build upgrades; migrations at or below it are squashed away. pub(crate) const BASELINE_VERSION: i64 = 1; -/// The checksum unversioned SQLite stores record for the baseline schema (releases before versioning). -pub(crate) const SQLITE_BASELINE_CHECKSUM: &str = - "228a951fe04d75eedcefdb44095afe652b178d9b2f32f3a5287c143877d39ce2"; +/// What unversioned SQLite stores record for the baseline: releases built from LF, then CRLF (Windows), SQL. +pub(crate) const SQLITE_BASELINE_CHECKSUMS: &[&str] = &[ + "228a951fe04d75eedcefdb44095afe652b178d9b2f32f3a5287c143877d39ce2", + "4cb5c0c4f392a2b3f4f2229e2ab8401870cd37b65346ada92cc70aa3ca2e0542", +]; -/// The checksum unversioned PostgreSQL stores record for the baseline schema. -pub(crate) const POSTGRES_BASELINE_CHECKSUM: &str = - "163045e32da43a8b0ddf17ea22f6dd41d1ff7bffe81836f3cbb35a8754915c3e"; +/// What unversioned PostgreSQL stores record for the baseline: releases built from LF, then CRLF, SQL. +pub(crate) const POSTGRES_BASELINE_CHECKSUMS: &[&str] = &[ + "163045e32da43a8b0ddf17ea22f6dd41d1ff7bffe81836f3cbb35a8754915c3e", + "90758add355967a65ddfcf233b5e66893f92449e17c84e8f9bab42fb703917ab", +]; diff --git a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs index a29d3da5aa..b38a5fef9c 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs @@ -15,7 +15,7 @@ use crate::store::tests::schema::test_types::StoreDialect; use crate::store::tests::store_helpers::run; use crate::store::tests::store_test_types::Backend; use crate::store::types::Row; -use crate::store_migrations::constants::{POSTGRES_BASELINE_CHECKSUM, SQLITE_BASELINE_CHECKSUM}; +use crate::store_migrations::constants::{POSTGRES_BASELINE_CHECKSUMS, SQLITE_BASELINE_CHECKSUMS}; use crate::store_migrations::models::StoreMigration; const CHILD_STORE_VARIABLE: &str = "SQLBUILD_TEST_CRASH_STORE"; @@ -146,7 +146,7 @@ const SHAPE_POSTGRES: &str = "SELECT 'column ' || table_name || '.' || column_na pub(crate) const SQLITE_STORE: StoreDialect = StoreDialect { backend: Backend::Sqlite, baseline_schema: BASELINE_SQLITE, - baseline_checksum: SQLITE_BASELINE_CHECKSUM, + baseline_checksum: SQLITE_BASELINE_CHECKSUMS[0], time_type: "TEXT", shape_sql: SHAPE_SQLITE, recreate_command: "rm -f '", @@ -155,7 +155,7 @@ pub(crate) const SQLITE_STORE: StoreDialect = StoreDialect { pub(crate) const POSTGRES_STORE: StoreDialect = StoreDialect { backend: Backend::Postgres, baseline_schema: BASELINE_POSTGRES, - baseline_checksum: POSTGRES_BASELINE_CHECKSUM, + baseline_checksum: POSTGRES_BASELINE_CHECKSUMS[0], time_type: "TIMESTAMPTZ", shape_sql: SHAPE_POSTGRES, recreate_command: "dropdb -h '", @@ -195,7 +195,11 @@ fn blank_line_statements(sql: &str) -> Vec<&str> { } /// A store as a release before schema versions left it: `dialect`'s baseline schema and checksum, and `rows.sql`. -pub(crate) async fn create_unversioned_baseline(store: &ControlStore, dialect: &StoreDialect) { +pub(crate) async fn create_unversioned_baseline( + store: &ControlStore, + dialect: &StoreDialect, + checksum: &str, +) { let version_table = format!( "CREATE TABLE sqb_schema_version (singleton BIGINT PRIMARY KEY CHECK (singleton = 1), \ checksum TEXT NOT NULL, applied_at {} NOT NULL)", @@ -203,15 +207,18 @@ pub(crate) async fn create_unversioned_baseline(store: &ControlStore, dialect: & ); let stamp = format!( "INSERT INTO sqb_schema_version (singleton, checksum, applied_at) \ - VALUES (1, '{}', '2026-11-01T08:00:00Z')", - dialect.baseline_checksum + VALUES (1, '{checksum}', '2026-11-01T08:00:00Z')" ); let installation = "INSERT INTO sqb_installation (singleton, installation_id, created_at) \ VALUES (1, '018f0000-0000-7000-8000-000000000000', '2026-11-01T08:00:00Z')"; + let (schema, rows) = ( + dialect.baseline_schema.replace("\r\n", "\n"), + BASELINE_ROWS.replace("\r\n", "\n"), + ); let mut statements: Vec<&str> = vec![version_table.as_str()]; - statements.extend(blank_line_statements(dialect.baseline_schema)); + statements.extend(blank_line_statements(&schema)); statements.extend([installation, stamp.as_str()]); - statements.extend(blank_line_statements(BASELINE_ROWS)); + statements.extend(blank_line_statements(&rows)); execute_all(store, &statements).await; } diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs index 5c5c52d6f9..fdcdeeccd5 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs @@ -1,5 +1,6 @@ //! Numbered control-store migrations: baseline upgrades, table rebuilds, refusals and the dry run. +use crate::store::constants::{INDEXES, SCHEMA}; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::schema::helpers::{ BASELINE_ROWS, POPULATED_TABLES, POSTGRES_STORE, SQLITE_STORE, create_unversioned_baseline, @@ -11,9 +12,10 @@ use crate::store::tests::schema::test_types::{ }; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store_migrations::_helpers::migration::{ - MIGRATIONS, current_version, migration_plan, schema_digest, + MIGRATIONS, current_version, migration_plan, schema_digest, schema_text_digest, }; use crate::store_migrations::_helpers::migration_run::migrate_with; +use crate::store_migrations::_helpers::recall_schema::recall_schema; use crate::store_migrations::main::plan_store_migration::plan_store_migration; use crate::store_migrations::models::{StoreMigration, StoreMigrationPlan}; @@ -43,13 +45,27 @@ const WIDEN_RUN_STATUS_POSTGRES: &str = "ALTER TABLE sqb_runs DROP CONSTRAINT sq fn given_store_at_the_baseline_when_migrating_then_it_matches_a_new_store_and_keeps_its_rows() { let test_cases = [ BaselineUpgradeTestCase { - description: "SQLite store", + description: "SQLite store from a Linux or macOS release", dialect: &SQLITE_STORE, + recorded_checksum: "228a951fe04d75eedcefdb44095afe652b178d9b2f32f3a5287c143877d39ce2", expected_version: current_version(MIGRATIONS), }, BaselineUpgradeTestCase { - description: "PostgreSQL store", + description: "SQLite store from a Windows release, built from CRLF SQL", + dialect: &SQLITE_STORE, + recorded_checksum: "4cb5c0c4f392a2b3f4f2229e2ab8401870cd37b65346ada92cc70aa3ca2e0542", + expected_version: current_version(MIGRATIONS), + }, + BaselineUpgradeTestCase { + description: "PostgreSQL store from a Linux or macOS release", + dialect: &POSTGRES_STORE, + recorded_checksum: "163045e32da43a8b0ddf17ea22f6dd41d1ff7bffe81836f3cbb35a8754915c3e", + expected_version: current_version(MIGRATIONS), + }, + BaselineUpgradeTestCase { + description: "PostgreSQL store from a Windows release, built from CRLF SQL", dialect: &POSTGRES_STORE, + recorded_checksum: "90758add355967a65ddfcf233b5e66893f92449e17c84e8f9bab42fb703917ab", expected_version: current_version(MIGRATIONS), }, ]; @@ -61,7 +77,12 @@ fn given_store_at_the_baseline_when_migrating_then_it_matches_a_new_store_and_ke ) else { return; }; - create_unversioned_baseline(&upgraded.store, test_case.dialect).await; + create_unversioned_baseline( + &upgraded.store, + test_case.dialect, + test_case.recorded_checksum, + ) + .await; let before = table_rows(&upgraded.store, &POPULATED_TABLES).await; migrate_store(&upgraded.store) @@ -280,7 +301,12 @@ fn given_a_store_when_planning_its_migration_then_the_plan_is_reported_without_c ) else { return; }; - create_unversioned_baseline(&baseline.store, test_case.dialect).await; + create_unversioned_baseline( + &baseline.store, + test_case.dialect, + test_case.dialect.baseline_checksum, + ) + .await; let plans = ( plan_store_migration(&empty.store) @@ -330,11 +356,25 @@ fn given_this_builds_schema_when_checksummed_then_it_matches_the_checksum_pinned PinnedChecksumTestCase { description: "SQLite store", dialect: &SQLITE_STORE, + line_ending: "\n", + expected_checksums: SQLITE_CHECKSUMS, + }, + PinnedChecksumTestCase { + description: "SQLite store built from a CRLF checkout", + dialect: &SQLITE_STORE, + line_ending: "\r\n", expected_checksums: SQLITE_CHECKSUMS, }, PinnedChecksumTestCase { description: "PostgreSQL store", dialect: &POSTGRES_STORE, + line_ending: "\n", + expected_checksums: POSTGRES_CHECKSUMS, + }, + PinnedChecksumTestCase { + description: "PostgreSQL store built from a CRLF checkout", + dialect: &POSTGRES_STORE, + line_ending: "\r\n", expected_checksums: POSTGRES_CHECKSUMS, }, ]; @@ -344,7 +384,15 @@ fn given_this_builds_schema_when_checksummed_then_it_matches_the_checksum_pinned return; }; let current = current_version(MIGRATIONS); - let checksum: String = schema_digest(&fixture.store); + let checked_out = |sql: &str| { + sql.replace("\r\n", "\n") + .replace('\n', test_case.line_ending) + }; + let checksum: String = schema_text_digest( + &recall_schema(&fixture.store).join(";\n"), + &checked_out(SCHEMA), + &checked_out(INDEXES), + ); assert_eq!( Some(checksum.as_str()), diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs index 69a97d13ab..a22fbbc5ab 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs @@ -49,6 +49,8 @@ pub(crate) struct StoreDialect { pub(crate) struct BaselineUpgradeTestCase { pub(crate) description: &'static str, pub(crate) dialect: &'static StoreDialect, + /// The checksum a release before versioning recorded for the baseline schema. + pub(crate) recorded_checksum: &'static str, pub(crate) expected_version: i64, } @@ -76,6 +78,8 @@ pub(crate) struct StoreMigrationPlanTestCase { pub(crate) struct PinnedChecksumTestCase { pub(crate) description: &'static str, pub(crate) dialect: &'static StoreDialect, + /// The line endings Git checked the SQL files out with. + pub(crate) line_ending: &'static str, /// One `(version, checksum)` per schema version: append one with each migration and never edit one. pub(crate) expected_checksums: &'static [(i64, &'static str)], }