diff --git a/.gitattributes b/.gitattributes index 3dcbe083f9..94f8bf0a71 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,3 +9,7 @@ ui/src/**/*.test.ts linguist-documentation ui/scripts/** linguist-documentation clients/typescript/tests/** linguist-documentation clients/typescript/scripts/** linguist-documentation + +# The control-store SQL is embedded in the binaries and checksummed; keep it LF on every checkout. +crates/sqlbuild-orchestration/src/store/*.sql text eol=lf +crates/sqlbuild-orchestration/tests/store/schema/baseline/*.sql text eol=lf diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index cc1c12d0d6..19ba19cb80 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1182,6 +1182,65 @@ "default": false, "help": "print the OpenAPI contract as JSON without opening a database", "group": 1 + }, + { + "kind": "parsers", + "dest": "server_command", + "nargs": "A...", + "required": false, + "group": 0, + "parsers": [ + { + "name": "migrate", + "parser": { + "prog": "sqb server migrate", + "description": "Apply the control store's pending schema migrations and exit.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--dry-run" + ], + "dest": "control_dry_run", + "nargs": 0, + "const": true, + "default": false, + "help": "list the migrations that would run without changing the store", + "group": 1 + } + ] + } + } + ], + "choices_help": [ + { + "dest": "migrate", + "metavar": "migrate", + "help": "apply pending control-store migrations and exit" + } + ] } ] } diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 556186ab90..e3e44afd96 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -59,6 +59,8 @@ use sqlbuild_orchestration::store::main::migrate_store::migrate_store; use sqlbuild_orchestration::store::main::open_store::open_store; use sqlbuild_orchestration::store::main::set_release_version::set_release_version; use sqlbuild_orchestration::store::models::ControlStore; +use sqlbuild_orchestration::store_migrations::main::plan_store_migration::plan_store_migration; +use sqlbuild_orchestration::store_migrations::models::StoreMigrationPlan; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ MetadataBatch, MetadataScan, StatisticsDueRead, StatisticsRead, WarehouseHistoryFailure, @@ -117,6 +119,7 @@ use crate::native_commands::models::{ MetadataReply, NativeCommand, NativeInvocation, ProjectLocation, ScanToken, }; use crate::native_commands::types::LocationPublication; +use crate::server_config::main::read_server_config::read_server_config; use crate::server_config::models::ServerSection; use crate::workers::main::docker_image_sources::docker_image_sources; use crate::workers::main::open_development_workers::open_development_workers; @@ -1886,3 +1889,86 @@ pub(crate) async fn compile_next_generation( .await?; Ok(compiled.map(|compiled| (Some((compiled, generation)), metadata, compiled_inputs))) } + +/// Run `sqb server migrate` on `runtime` and return its exit code. +pub(crate) fn run_store_migration(namespace: &Namespace, runtime: &tokio::runtime::Runtime) -> i32 { + drop(write_output(&[( + Stream::Stderr, + "Opening control store...\n", + )])); + match runtime.block_on(store_migration( + namespace, + flag(namespace, "control_dry_run"), + )) { + Ok(report) => { + let lost = write_output(&[(Stream::Stdout, &report)]); + let exit_code = finish_lost_stdout(lost.as_ref(), 0); + if exit_code == 0 { + drop(write_output(&[( + Stream::Stderr, + "Control store migration completed successfully\n", + )])); + } + exit_code + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Control store migration failed: {error}\n"), + )])); + 1 + } + } +} + +/// Plan and apply the store's migrations; a dry run never creates a missing SQLite file. +async fn store_migration(namespace: &Namespace, dry_run: bool) -> Result { + let config = read_server_config(namespace)?; + let url = control_database( + namespace, + &control_database_variable(namespace, &config.server), + )?; + if dry_run + && let Some(path) = url + .strip_prefix("sqlite://") + .or((!url.contains("://")).then_some(url.as_str())) + && !matches!(path, "" | ":memory:") + && !std::path::Path::new(path).exists() + { + return Ok(format!("No control store at {path}: would create one.\n")); + } + let store = open_store(&url).await?; + let plan = plan_store_migration(&store).await?; + if !dry_run { + migrate_store(&store).await?; + } + Ok(describe(&plan, dry_run)) +} + +/// The plan as lines for a person: what would happen with `--dry-run`, what happened without it. +fn describe(plan: &StoreMigrationPlan, dry_run: bool) -> String { + let (create, upgrade, record) = if dry_run { + ("would create", "would migrate", "would record") + } else { + ("created", "migrated", "recorded") + }; + match plan { + StoreMigrationPlan::Create { version } => { + format!("The control store is empty: {create} schema version {version}.\n") + } + StoreMigrationPlan::Current { version } => { + format!("The control store is at schema version {version}: nothing to migrate.\n") + } + StoreMigrationPlan::Upgrade { from, to, names } if names.is_empty() => format!( + "The control store's schema is version {from} without a recorded version: \ + {record} version {to}.\n" + ), + StoreMigrationPlan::Upgrade { from, to, names } => format!( + "The control store {upgrade} from schema version {from} to {to}:\n{}", + names + .iter() + .map(|name| format!(" {name}\n")) + .collect::() + ), + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs index a77a31147f..30b95fce50 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs @@ -5,6 +5,7 @@ use sqlbuild_orchestration::api::main::openapi_document::openapi_document; use crate::argv::models::{Namespace, Stream}; use crate::native_commands::_helpers::decision::{flag, text}; use crate::native_commands::_helpers::execution::run_control_action; +use crate::native_commands::_helpers::execution::run_store_migration; use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; use crate::native_commands::constants::{ADMIN_COMMAND, SCHEDULER_COMMAND, SERVER_COMMAND}; @@ -45,6 +46,9 @@ pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { return Some(1); } }; + if text(namespace, "server_command").as_deref() == Some("migrate") { + return Some(run_store_migration(namespace, &runtime)); + } let admin = command == ADMIN_COMMAND; let role = if admin { "administration" } else { "service" }; let result = runtime.block_on(run_control_action(namespace, &command)); diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index 59bea6e574..d299cb34da 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -1,7 +1,10 @@ use std::process::Command; use crate::helpers::{api_ready, isolated_binary, start_control, wait_for_health}; -use crate::test_types::{CommandTestCase, OpenApiTestCase, ServiceTestCase, TriggerCursorTestCase}; +use crate::test_types::{ + CommandTestCase, OpenApiTestCase, ServiceTestCase, StoreMigrationCommandTestCase, + TriggerCursorTestCase, +}; #[test] fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_python_and_database_access_are_not_required() @@ -364,6 +367,56 @@ fn given_a_standalone_server_when_exporting_openapi_then_the_worker_contract_is_ } } +#[test] +fn given_a_control_store_when_running_server_migrate_then_dry_run_lists_and_migrate_applies() { + let test_cases = [ + StoreMigrationCommandTestCase { + description: "dry run without a store creates nothing", + arguments: &["server", "migrate", "--dry-run"], + expected_stdout: "No control store at ./.sqlbuild/control.sqlite3: would create one.\n", + expected_store: false, + }, + StoreMigrationCommandTestCase { + description: "migrate an empty store", + arguments: &["server", "migrate"], + expected_stdout: "The control store is empty: created schema version 1.\n", + expected_store: true, + }, + StoreMigrationCommandTestCase { + description: "dry run on a current store", + arguments: &["server", "migrate", "--dry-run"], + expected_stdout: "The control store is at schema version 1: nothing to migrate.\n", + expected_store: true, + }, + ]; + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let binary = isolated_binary(directory.path()); + for test_case in test_cases { + let output = Command::new(&binary) + .args(test_case.arguments) + .current_dir(directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .output() + .expect("run sqb server migrate"); + assert_eq!( + ( + output.status.success(), + String::from_utf8_lossy(&output.stdout).into_owned(), + directory.path().join(".sqlbuild/control.sqlite3").exists(), + ), + ( + true, + test_case.expected_stdout.to_owned(), + test_case.expected_store + ), + "{}: {}", + test_case.description, + String::from_utf8_lossy(&output.stderr) + ); + } +} + #[test] fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then_migrations_auth_and_shutdown_are_native() { diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs index 689f38a303..9d9d05ae93 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_types.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -81,3 +81,11 @@ pub(crate) struct LocationHealthTestCase { pub(crate) expected_state: &'static str, pub(crate) expected_exit: i32, } + +pub(crate) struct StoreMigrationCommandTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_stdout: &'static str, + /// Whether the SQLite store file exists after the command. + pub(crate) expected_store: bool, +} diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs index 05119775f3..1f9ff273a7 100644 --- a/crates/sqlbuild-orchestration/src/lib.rs +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -14,6 +14,7 @@ pub mod scheduler; pub mod sensors; pub mod snapshots; pub mod store; +pub mod store_migrations; pub mod warehouse_history; pub mod warehouse_metadata; pub mod webhooks; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs deleted file mode 100644 index 8a2ed7baa2..0000000000 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! One schema template, rendered only at the database type boundary. - -use sqlbuild_recall::records::main::sha256_hex::sha256_hex; - -use crate::store::_helpers::recall_schema::recall_schema; -use crate::store::errors::ControlError; -use crate::store::models::ControlStore; -use crate::store::types::Row; - -pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); - -/// Additive indexes, created on every open and kept out of the digest so old stores gain them. -pub(crate) const INDEXES: &str = include_str!("../indexes.sql"); - -pub(crate) fn statements(sql: &str) -> impl Iterator { - sql.split(';') - .map(str::trim) - .filter(|statement| !statement.is_empty()) -} - -pub(crate) fn schema_digest(store: &ControlStore) -> String { - sha256_hex(format!("{}\n{SCHEMA}", recall_schema(store).join(";\n")).as_bytes()) -} - -/// Accept only the exact schema this server creates, or say how to start a new store. -pub(crate) fn verify_schema( - store: &ControlStore, - rows: &[Row], - digest: &str, -) -> Result<(), ControlError> { - if rows - .iter() - .all(|row| row.get("checksum").and_then(|value| value.as_str()) == Some(digest)) - { - return Ok(()); - } - let quote = |text: &str| format!("'{}'", text.replace('\'', r"'\''")); - let (scope, command) = match store { - ControlStore::Sqlite(path) => { - let path = path.to_string_lossy(); - let command = format!( - "rm -f {} {} {}", - quote(&path), - quote(&format!("{path}-wal")), - quote(&format!("{path}-shm")) - ); - ("", command) - } - ControlStore::Postgres(pool) => { - let options = pool.connect_options(); - let host = options.get_socket().map_or_else( - || options.get_host().to_owned(), - |socket| socket.to_string_lossy().into_owned(), - ); - let target = format!( - "-h {} -p {} -U {} {}", - quote(&host), - options.get_port(), - quote(options.get_username()), - quote(options.get_database().unwrap_or(options.get_username())) - ); - ( - " and everything else in that database", - format!("dropdb {target} && createdb {target}"), - ) - } - }; - Err(ControlError::Schema(format!( - "this control store was created by a different sqb release and cannot be upgraded; \ - if a newer release created it, upgrade this server instead. To start a new, empty \ - store, which discards its run history{scope}, stop every server using it and run: \ - {command}" - ))) -} - -pub(crate) fn render(store: &ControlStore, sql: &str) -> String { - let postgres = matches!(store, ControlStore::Postgres(_)); - sql.replace("{uuid}", if postgres { "UUID" } else { "TEXT" }) - .replace("{time}", if postgres { "TIMESTAMPTZ" } else { "TEXT" }) - .replace("{binary}", if postgres { "BYTEA" } else { "BLOB" }) - .replace("{json}", if postgres { "JSONB" } else { "JSON" }) -} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs index 420775f72e..d225ac992f 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs @@ -1,9 +1,7 @@ pub(crate) mod changes; pub(crate) mod database; -pub(crate) mod migration; pub(crate) mod open_sqlite; pub(crate) mod recall_rows; -pub(crate) mod recall_schema; pub(crate) mod release_version; #[cfg(test)] pub(crate) mod statement_count; diff --git a/crates/sqlbuild-orchestration/src/store/constants.rs b/crates/sqlbuild-orchestration/src/store/constants.rs index 61d1f56a56..601bf6e64d 100644 --- a/crates/sqlbuild-orchestration/src/store/constants.rs +++ b/crates/sqlbuild-orchestration/src/store/constants.rs @@ -1,4 +1,10 @@ -//! Control-store limits. +//! Control-store limits and schema. /// A SQLite WAL past this size is reset right after the commit that grew it. pub(crate) const LARGE_WAL_BYTES: u64 = 64 * 1024 * 1024; + +/// The control-store schema template, rendered per database type. +pub(crate) const SCHEMA: &str = include_str!("schema.sql"); + +/// Indexes, applied on every open; like `SCHEMA`, a change needs a numbered migration. +pub(crate) const INDEXES: &str = include_str!("indexes.sql"); diff --git a/crates/sqlbuild-orchestration/src/store/main/begin.rs b/crates/sqlbuild-orchestration/src/store/main/begin.rs index 7deced8adc..fa9f823a87 100644 --- a/crates/sqlbuild-orchestration/src/store/main/begin.rs +++ b/crates/sqlbuild-orchestration/src/store/main/begin.rs @@ -27,12 +27,16 @@ pub(crate) async fn begin( )?; connection.busy_timeout(Duration::from_secs(30))?; connection.set_db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, true)?; - connection.execute_batch(if mode == TransactionMode::Write { - "PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" - } else { - "PRAGMA foreign_keys=ON; PRAGMA query_only=ON; BEGIN" + connection.execute_batch(match mode { + TransactionMode::Write => { + "PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" + } + TransactionMode::Migrate => { + "PRAGMA synchronous=FULL; PRAGMA foreign_keys=OFF; BEGIN IMMEDIATE" + } + TransactionMode::Read => "PRAGMA foreign_keys=ON; PRAGMA query_only=ON; BEGIN", })?; - let notifications = if mode == TransactionMode::Write { + let notifications = if mode != TransactionMode::Read { Some(sqlite_changes(&path)?) } else { None @@ -43,7 +47,7 @@ pub(crate) async fn begin( } ControlStore::Postgres(pool) => { let mut transaction = pool.begin().await?; - if mode == TransactionMode::Write { + if mode != TransactionMode::Read { sqlx_core::query::query("SELECT pg_advisory_xact_lock(1936810572, 1)") .execute(&mut *transaction) .await?; diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs index e82180367e..e4360a83a3 100644 --- a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -1,51 +1,10 @@ -//! Schema creation under the same lock used by all other control-plane writes. +//! Schema creation and numbered migrations under the same lock used by all other control-plane writes. -use uuid::Uuid; - -use crate::store::_helpers::migration::{ - INDEXES, SCHEMA, render, schema_digest, statements, verify_schema, -}; -use crate::store::_helpers::recall_schema::recall_schema; use crate::store::errors::ControlError; -use crate::store::main::begin::begin; -use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::ControlStore; +use crate::store_migrations::main::apply_store_migrations::apply_store_migrations; -/// Create or verify the schema, then add any missing additive indexes. +/// Create the schema or apply its pending migrations in one transaction, then add any missing additive indexes. pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { - let mut transaction = begin(store, TransactionMode::Write).await?; - (transaction, _) = query(transaction, &render(store, - "CREATE TABLE IF NOT EXISTS sqb_schema_version (singleton BIGINT PRIMARY KEY CHECK (singleton = 1), \ - checksum TEXT NOT NULL, applied_at {time} NOT NULL)"), vec![]).await?; - let (mut transaction, rows) = query( - transaction, - "SELECT checksum FROM sqb_schema_version", - vec![], - ) - .await?; - let digest = schema_digest(store); - verify_schema(store, &rows, &digest)?; - if rows.is_empty() { - for statement in recall_schema(store) { - (transaction, _) = query(transaction, &statement, vec![]).await?; - } - for statement in statements(SCHEMA) { - (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; - } - let now = transaction.now; - (transaction, _) = query(transaction, - "INSERT INTO sqb_installation (singleton, installation_id, created_at) VALUES (1, $1, $2)", - vec![Parameter::Id(Uuid::now_v7()), Parameter::Time(now)]).await?; - (transaction, _) = query( - transaction, - "INSERT INTO sqb_schema_version (singleton, checksum, applied_at) VALUES (1, $1, $2)", - vec![Parameter::Text(digest), Parameter::Time(now)], - ) - .await?; - } - for statement in statements(INDEXES) { - (transaction, _) = query(transaction, statement, vec![]).await?; - } - commit(transaction).await + apply_store_migrations(store).await } diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs index fc9a7371b2..7b39d52ef4 100644 --- a/crates/sqlbuild-orchestration/src/store/models.rs +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -70,6 +70,8 @@ pub(crate) enum TargetRow { pub(crate) enum TransactionMode { Read, Write, + /// A write with SQLite foreign keys off, so a migration can rebuild a referenced table; it checks them before commit. + Migrate, } /// An owned transaction; abandoning it rolls back, including when an async task is cancelled. diff --git a/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs new file mode 100644 index 0000000000..087e8ff283 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs @@ -0,0 +1,202 @@ +//! One schema template, rendered only at the database type boundary, and the numbered migrations to it. + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, ControlTransaction}; +use crate::store_migrations::_helpers::recall_schema::recall_schema; +use crate::store_migrations::constants::{ + BASELINE_VERSION, POSTGRES_BASELINE_CHECKSUMS, SQLITE_BASELINE_CHECKSUMS, +}; +use crate::store_migrations::models::{StoreMigration, StoreMigrationPlan}; + +use crate::store::constants::{INDEXES, SCHEMA}; + +/// The migrations after the baseline, in order: entry `i` includes `store_migrations/sql/{NNNN}_{name}.{sqlite,postgres}.sql` and brings a store to version `NNNN = BASELINE_VERSION + i + 1`. +pub(crate) const MIGRATIONS: &[StoreMigration] = &[]; + +/// The schema version a store created by this build has. +pub(crate) fn current_version(migrations: &[StoreMigration]) -> i64 { + BASELINE_VERSION + i64::try_from(migrations.len()).unwrap_or(i64::MAX - BASELINE_VERSION) +} + +pub(crate) fn statements(sql: &str) -> impl Iterator { + sql.split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) +} + +/// The checksum of the schema this build creates, recorded with its version. +pub(crate) fn schema_digest(store: &ControlStore) -> String { + schema_text_digest(&recall_schema(store).join(";\n"), SCHEMA, INDEXES) +} + +/// The checksum of a schema's Recall DDL, `schema.sql` and `indexes.sql`, whatever line endings Git used. +pub(crate) fn schema_text_digest(recall: &str, schema: &str, indexes: &str) -> String { + sha256_hex( + format!("{recall}\n{schema}\n{indexes}") + .replace("\r\n", "\n") + .as_bytes(), + ) +} + +/// The statements of `migration` for `store`'s database type. +pub(crate) fn migration_sql(store: &ControlStore, migration: &StoreMigration) -> String { + render( + store, + match store { + ControlStore::Sqlite(_) => migration.sqlite, + ControlStore::Postgres(_) => migration.postgres, + }, + ) +} + +/// The store's recorded `(checksum, version)`; the version is `None` for stores from before versioning. +pub(crate) async fn recorded_schema( + store: &ControlStore, + transaction: ControlTransaction, +) -> Result<(ControlTransaction, Option<(String, Option)>), ControlError> { + let columns = match store { + ControlStore::Sqlite(_) => "SELECT name FROM pragma_table_info('sqb_schema_version')", + ControlStore::Postgres(_) => { + "SELECT column_name AS name FROM information_schema.columns \ + WHERE table_schema = current_schema() AND table_name = 'sqb_schema_version'" + } + }; + let (transaction, columns) = query(transaction, columns, vec![]).await?; + let has = |name: &str| columns.iter().any(|row| row["name"].as_str() == Some(name)); + if !has("checksum") { + return Ok((transaction, None)); + } + let select = if has("version") { + "SELECT checksum, version FROM sqb_schema_version" + } else { + "SELECT checksum, NULL AS version FROM sqb_schema_version" + }; + let (transaction, rows) = query(transaction, select, vec![]).await?; + let recorded = rows.first().map(|row| { + ( + row["checksum"].as_str().unwrap_or_default().to_owned(), + row["version"].as_i64(), + ) + }); + Ok((transaction, recorded)) +} + +/// What opening a store with `recorded` schema does, or why this build refuses it. +pub(crate) fn migration_plan( + store: &ControlStore, + recorded: Option<(&str, Option)>, + migrations: &[StoreMigration], +) -> Result { + let current = current_version(migrations); + let Some((checksum, version)) = recorded else { + return Ok(StoreMigrationPlan::Create { version: current }); + }; + let baseline_checksums = match store { + ControlStore::Sqlite(_) => SQLITE_BASELINE_CHECKSUMS, + ControlStore::Postgres(_) => POSTGRES_BASELINE_CHECKSUMS, + }; + let version = match version { + Some(version) => version, + None if baseline_checksums.contains(&checksum) => BASELINE_VERSION, + None => return Err(refusal(store, Refusal::OlderThanBaseline)), + }; + if version > current { + return Err(refusal(store, Refusal::Newer { version, current })); + } + if version < BASELINE_VERSION { + return Err(refusal(store, Refusal::OlderThanBaseline)); + } + if version == current && !matches!(recorded, Some((_, None))) { + return if checksum == schema_digest(store) { + Ok(StoreMigrationPlan::Current { version }) + } else { + Err(refusal(store, Refusal::Mismatch { version })) + }; + } + let pending = &migrations[usize::try_from(version - BASELINE_VERSION).unwrap_or(0)..]; + Ok(StoreMigrationPlan::Upgrade { + from: version, + to: current, + names: pending + .iter() + .zip(version + 1..) + .map(|(migration, number)| format!("{number:04}_{}", migration.name)) + .collect(), + }) +} + +enum Refusal { + Newer { version: i64, current: i64 }, + OlderThanBaseline, + Mismatch { version: i64 }, +} + +/// Why this build refuses a store, and how to start a new one. +fn refusal(store: &ControlStore, refusal: Refusal) -> ControlError { + let (scope, command) = recreate_command(store); + let recreate = format!( + "To start a new, empty store, which discards its run history{scope}, stop every \ + server using it and run: {command}" + ); + ControlError::Schema(match refusal { + Refusal::Newer { version, current } => format!( + "this control store's schema is version {version}, newer than this build's \ + version {current}; upgrade this server to the release that last opened it or a later one" + ), + Refusal::OlderThanBaseline => format!( + "this control store's schema is older than version {BASELINE_VERSION}, the oldest \ + this build upgrades; open it once with an earlier release that still has the \ + migrations it needs, then with this one. {recreate}" + ), + Refusal::Mismatch { version } => format!( + "this control store's schema doesn't match this build's, although both are version \ + {version}; the build's schema changed without a migration. {recreate}" + ), + }) +} + +/// The scope of what recreating `store` discards, and the command that does it. +fn recreate_command(store: &ControlStore) -> (&'static str, String) { + let quote = |text: &str| format!("'{}'", text.replace('\'', r"'\''")); + match store { + ControlStore::Sqlite(path) => { + let path = path.to_string_lossy(); + let command = format!( + "rm -f {} {} {}", + quote(&path), + quote(&format!("{path}-wal")), + quote(&format!("{path}-shm")) + ); + ("", command) + } + ControlStore::Postgres(pool) => { + let options = pool.connect_options(); + let host = options.get_socket().map_or_else( + || options.get_host().to_owned(), + |socket| socket.to_string_lossy().into_owned(), + ); + let target = format!( + "-h {} -p {} -U {} {}", + quote(&host), + options.get_port(), + quote(options.get_username()), + quote(options.get_database().unwrap_or(options.get_username())) + ); + ( + " and everything else in that database", + format!("dropdb {target} && createdb {target}"), + ) + } + } +} + +pub(crate) fn render(store: &ControlStore, sql: &str) -> String { + let postgres = matches!(store, ControlStore::Postgres(_)); + sql.replace("{uuid}", if postgres { "UUID" } else { "TEXT" }) + .replace("{time}", if postgres { "TIMESTAMPTZ" } else { "TEXT" }) + .replace("{binary}", if postgres { "BYTEA" } else { "BLOB" }) + .replace("{json}", if postgres { "JSONB" } else { "JSON" }) +} diff --git a/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration_run.rs b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration_run.rs new file mode 100644 index 0000000000..117177f693 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration_run.rs @@ -0,0 +1,108 @@ +//! Creating a control store or applying its pending migrations, with the migration list as a parameter. + +use uuid::Uuid; + +use crate::store::constants::{INDEXES, SCHEMA}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store_migrations::_helpers::migration::{ + migration_plan, migration_sql, recorded_schema, render, schema_digest, statements, +}; +use crate::store_migrations::_helpers::recall_schema::recall_schema; +use crate::store_migrations::constants::BASELINE_VERSION; +use crate::store_migrations::models::{StoreMigration, StoreMigrationPlan}; + +/// `migrate_store` with the migrations `migrations` in place of this build's. +pub(crate) async fn migrate_with( + store: &ControlStore, + migrations: &[StoreMigration], +) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Migrate).await?; + let (mut transaction, recorded) = recorded_schema(store, transaction).await?; + let plan = migration_plan( + store, + recorded + .as_ref() + .map(|(checksum, version)| (checksum.as_str(), *version)), + migrations, + )?; + let digest = schema_digest(store); + let now = transaction.now; + match plan { + StoreMigrationPlan::Current { .. } => {} + StoreMigrationPlan::Create { version } => { + (transaction, _) = query(transaction, &render(store, + "CREATE TABLE sqb_schema_version (singleton BIGINT PRIMARY KEY CHECK (singleton = 1), \ + checksum TEXT NOT NULL, applied_at {time} NOT NULL, version BIGINT)"), vec![]).await?; + for statement in recall_schema(store) { + (transaction, _) = query(transaction, &statement, vec![]).await?; + } + for statement in statements(SCHEMA) { + (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; + } + (transaction, _) = query(transaction, + "INSERT INTO sqb_installation (singleton, installation_id, created_at) VALUES (1, $1, $2)", + vec![Parameter::Id(Uuid::now_v7()), Parameter::Time(now)]).await?; + (transaction, _) = query( + transaction, + "INSERT INTO sqb_schema_version (singleton, checksum, applied_at, version) \ + VALUES (1, $1, $2, $3)", + vec![ + Parameter::Text(digest), + Parameter::Time(now), + Parameter::Integer(version), + ], + ) + .await?; + } + StoreMigrationPlan::Upgrade { from, to, .. } => { + if recorded + .as_ref() + .is_some_and(|(_, version)| version.is_none()) + { + (transaction, _) = query( + transaction, + "ALTER TABLE sqb_schema_version ADD COLUMN version BIGINT", + vec![], + ) + .await?; + } + let pending = &migrations[usize::try_from(from - BASELINE_VERSION).unwrap_or(0)..]; + for migration in pending { + let sql = migration_sql(store, migration); + for statement in statements(&sql) { + (transaction, _) = query(transaction, statement, vec![]).await?; + } + } + if !pending.is_empty() && matches!(store, ControlStore::Sqlite(_)) { + let violations; + (transaction, violations) = + query(transaction, "PRAGMA foreign_key_check", vec![]).await?; + if !violations.is_empty() { + return Err(ControlError::Schema(format!( + "migrating the control store to version {to} would break {} foreign key \ + references; the store is unchanged", + violations.len() + ))); + } + } + (transaction, _) = query( + transaction, + "UPDATE sqb_schema_version SET checksum = $1, applied_at = $2, version = $3", + vec![ + Parameter::Text(digest), + Parameter::Time(now), + Parameter::Integer(to), + ], + ) + .await?; + } + } + for statement in statements(INDEXES) { + (transaction, _) = query(transaction, statement, vec![]).await?; + } + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/store_migrations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/mod.rs new file mode 100644 index 0000000000..83115c1e5a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod migration; +pub(crate) mod migration_run; +pub(crate) mod recall_schema; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/recall_schema.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs rename to crates/sqlbuild-orchestration/src/store_migrations/_helpers/recall_schema.rs diff --git a/crates/sqlbuild-orchestration/src/store_migrations/constants.rs b/crates/sqlbuild-orchestration/src/store_migrations/constants.rs new file mode 100644 index 0000000000..56a601ec2b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/constants.rs @@ -0,0 +1,16 @@ +//! The control-store schema baseline. + +/// The oldest schema version this build upgrades; migrations at or below it are squashed away. +pub(crate) const BASELINE_VERSION: i64 = 1; + +/// What unversioned SQLite stores record for the baseline: releases built from LF, then CRLF (Windows), SQL. +pub(crate) const SQLITE_BASELINE_CHECKSUMS: &[&str] = &[ + "228a951fe04d75eedcefdb44095afe652b178d9b2f32f3a5287c143877d39ce2", + "4cb5c0c4f392a2b3f4f2229e2ab8401870cd37b65346ada92cc70aa3ca2e0542", +]; + +/// What unversioned PostgreSQL stores record for the baseline: releases built from LF, then CRLF, SQL. +pub(crate) const POSTGRES_BASELINE_CHECKSUMS: &[&str] = &[ + "163045e32da43a8b0ddf17ea22f6dd41d1ff7bffe81836f3cbb35a8754915c3e", + "90758add355967a65ddfcf233b5e66893f92449e17c84e8f9bab42fb703917ab", +]; diff --git a/crates/sqlbuild-orchestration/src/store_migrations/main/apply_store_migrations.rs b/crates/sqlbuild-orchestration/src/store_migrations/main/apply_store_migrations.rs new file mode 100644 index 0000000000..56b96f3dd2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/main/apply_store_migrations.rs @@ -0,0 +1,11 @@ +//! Bring a control store to this build's schema version. + +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::store_migrations::_helpers::migration::MIGRATIONS; +use crate::store_migrations::_helpers::migration_run::migrate_with; + +/// Create the schema or apply this build's pending migrations in one transaction, then add any missing additive indexes. +pub(crate) async fn apply_store_migrations(store: &ControlStore) -> Result<(), ControlError> { + migrate_with(store, MIGRATIONS).await +} diff --git a/crates/sqlbuild-orchestration/src/store_migrations/main/mod.rs b/crates/sqlbuild-orchestration/src/store_migrations/main/mod.rs new file mode 100644 index 0000000000..492ce6198b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/main/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod apply_store_migrations; +pub mod plan_store_migration; diff --git a/crates/sqlbuild-orchestration/src/store_migrations/main/plan_store_migration.rs b/crates/sqlbuild-orchestration/src/store_migrations/main/plan_store_migration.rs new file mode 100644 index 0000000000..67d47985e3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/main/plan_store_migration.rs @@ -0,0 +1,24 @@ +//! What opening a control store would do to its schema, without changing it. + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::store_migrations::_helpers::migration::{MIGRATIONS, migration_plan, recorded_schema}; +use crate::store_migrations::models::StoreMigrationPlan; + +/// The schema change `migrate_store` would make to `store`, or why it would refuse the store. +pub async fn plan_store_migration( + store: &ControlStore, +) -> Result { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, recorded) = recorded_schema(store, transaction).await?; + commit(transaction).await?; + migration_plan( + store, + recorded + .as_ref() + .map(|(checksum, version)| (checksum.as_str(), *version)), + MIGRATIONS, + ) +} diff --git a/crates/sqlbuild-orchestration/src/store_migrations/mod.rs b/crates/sqlbuild-orchestration/src/store_migrations/mod.rs new file mode 100644 index 0000000000..5229b9811f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/mod.rs @@ -0,0 +1,6 @@ +//! Control-store schema versions: creating a store, numbered migrations, and the dry-run plan. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/store_migrations/models.rs b/crates/sqlbuild-orchestration/src/store_migrations/models.rs new file mode 100644 index 0000000000..9fdfc2cb3e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/models.rs @@ -0,0 +1,27 @@ +//! Numbered control-store migrations and what opening a store would do with them. + +/// One numbered control-store migration, with its SQL for each store type. +#[derive(Clone, Copy, Debug)] +pub struct StoreMigration { + /// The file name without its number, such as `add_skipped_runs`. + pub name: &'static str, + /// Statements for SQLite, separated by `;`. + pub sqlite: &'static str, + /// Statements for PostgreSQL, separated by `;`. + pub postgres: &'static str, +} + +/// What opening a control store would do to its schema. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StoreMigrationPlan { + /// No store yet: create one from the current schema at `version`. + Create { version: i64 }, + /// The store is at the current `version` already. + Current { version: i64 }, + /// Apply the numbered migrations `names`, in order, from `from` to `to`. + Upgrade { + from: i64, + to: i64, + names: Vec, + }, +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/baseline/postgres.sql b/crates/sqlbuild-orchestration/tests/store/schema/baseline/postgres.sql new file mode 100644 index 0000000000..77bde76f7b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/baseline/postgres.sql @@ -0,0 +1,1431 @@ +CREATE TABLE IF NOT EXISTS "recall_meta" ( + "component" TEXT NOT NULL, + "key" TEXT NOT NULL, + "value" TEXT NOT NULL, + "recorded_at" TIMESTAMPTZ NOT NULL, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("component", "key", "recorded_at") +); + +CREATE TABLE IF NOT EXISTS "recall_invocations" ( + "command" TEXT, + "subcommand" TEXT, + "selection" JSONB, + "exclusion" JSONB, + "project_name" TEXT, + "project_dir_hash" TEXT, + "target" TEXT, + "adapter" TEXT, + "destination_database" TEXT, + "destination_schema" TEXT, + "destination_fingerprint" TEXT, + "sqlbuild_version" TEXT, + "git_revision" TEXT, + "git_dirty" BOOLEAN, + "host_token" TEXT, + "pid" BIGINT, + "external_context" JSONB, + "recall_filter_active" BOOLEAN, + "argv_digest" TEXT, + "origin" TEXT, + "command_mode" TEXT, + "selection_count" BIGINT, + "selection_truncated" BOOLEAN, + "selector_files" JSONB, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id") +); + +CREATE INDEX IF NOT EXISTS "recall_invocations_started" ON "recall_invocations" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_output" ( + "record_id" TEXT NOT NULL, + "sequence" BIGINT NOT NULL, + "stream" TEXT NOT NULL, + "occurred_at" TIMESTAMPTZ NOT NULL, + "data" BYTEA NOT NULL, + "byte_offset" BIGINT NOT NULL, + "byte_count" BIGINT NOT NULL, + "priority" TEXT, + "dropped_before" BIGINT NOT NULL, + "omitted_bytes" BIGINT, + "omitted_sequences" BIGINT, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "sequence") +); + +CREATE INDEX IF NOT EXISTS "recall_output_expiry" ON "recall_output" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_events" ( + "event_id" TEXT NOT NULL, + "invocation_sequence" BIGINT, + "occurred_at" TIMESTAMPTZ NOT NULL, + "event_type" TEXT NOT NULL, + "event_kind" TEXT NOT NULL, + "severity" TEXT NOT NULL, + "producer" TEXT NOT NULL, + "event_schema_version" BIGINT NOT NULL, + "run_id" TEXT, + "resource_id" TEXT, + "resource_attempt_id" TEXT, + "operation_id" TEXT, + "statement_id" TEXT, + "command" TEXT, + "resource_kind" TEXT, + "resource_name" TEXT, + "error_code" TEXT, + "error_type" TEXT, + "duration_ms" DOUBLE PRECISION, + "exit_code" BIGINT, + "query_id" TEXT, + "sql_digest" TEXT, + "intent" TEXT, + "statement_kind" TEXT, + "purpose" TEXT, + "outcome" TEXT, + "row_count" BIGINT, + "affected_rows" BIGINT, + "payload" JSONB NOT NULL, + "started_at" TIMESTAMPTZ, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + storage_id BIGINT GENERATED ALWAYS AS IDENTITY, + UNIQUE ("event_id") +); + +CREATE INDEX IF NOT EXISTS "recall_events_invocation" ON "recall_events" ("invocation_id", "invocation_sequence"); + +CREATE INDEX IF NOT EXISTS "recall_events_type" ON "recall_events" ("event_type", "occurred_at"); + +CREATE INDEX IF NOT EXISTS "recall_events_statement" ON "recall_events" ("statement_id"); + +CREATE INDEX IF NOT EXISTS "recall_events_expiry" ON "recall_events" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_diagnostics" ( + "record_id" TEXT NOT NULL, + "sequence" BIGINT NOT NULL, + "occurred_at" TIMESTAMPTZ NOT NULL, + "severity" TEXT NOT NULL, + "logger" TEXT NOT NULL, + "source" TEXT NOT NULL, + "message" TEXT NOT NULL, + "fields" JSONB NOT NULL, + "log_stream_id" TEXT, + "run_id" TEXT, + "resource_id" TEXT, + "resource_attempt_id" TEXT, + "operation_id" TEXT, + "statement_id" TEXT, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "sequence") +); + +CREATE INDEX IF NOT EXISTS "recall_diagnostics_severity" ON "recall_diagnostics" ("severity"); + +CREATE INDEX IF NOT EXISTS "recall_diagnostics_expiry" ON "recall_diagnostics" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_artifacts" ( + "digest" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "media_type" TEXT NOT NULL, + "size" BIGINT NOT NULL, + "content" BYTEA NOT NULL, + "seen_at" TIMESTAMPTZ NOT NULL, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("digest") +); + +CREATE INDEX IF NOT EXISTS "recall_artifacts_seen" ON "recall_artifacts" ("seen_at"); + +CREATE TABLE IF NOT EXISTS "recall_artifact_refs" ( + "digest" TEXT NOT NULL, + "ref_id" TEXT NOT NULL, + "statement_id" TEXT, + "resource_id" TEXT, + "role" TEXT NOT NULL, + "referenced_at" TIMESTAMPTZ NOT NULL, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "digest", "ref_id") +); + +CREATE INDEX IF NOT EXISTS "recall_artifact_refs_digest" ON "recall_artifact_refs" ("digest"); + +CREATE TABLE IF NOT EXISTS "recall_heartbeats" ( + "beat_at" TIMESTAMPTZ NOT NULL, + "stale_at" TIMESTAMPTZ NOT NULL, + "host_token" TEXT, + "pid" BIGINT, + "command" TEXT, + "subcommand" TEXT, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "beat_at") +); + +CREATE TABLE IF NOT EXISTS "recall_delivery" ( + "stream" TEXT NOT NULL, + "accepted" BIGINT NOT NULL, + "delivered" BIGINT NOT NULL, + "filtered" BIGINT NOT NULL, + "dropped" BIGINT NOT NULL, + "failed" BIGINT NOT NULL, + "conflicts" BIGINT NOT NULL, + "flush_complete" BOOLEAN NOT NULL, + "written_at" TIMESTAMPTZ NOT NULL, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "stream") +); + +CREATE TABLE IF NOT EXISTS "recall_conflicts" ( + "dataset" TEXT NOT NULL, + "record_key" TEXT NOT NULL, + "existing_sha256" TEXT NOT NULL, + "rejected_sha256" TEXT NOT NULL, + "detected_at" TIMESTAMPTZ NOT NULL, + "invocation_id" TEXT COLLATE "C" NOT NULL, + "invocation_started_at" TIMESTAMPTZ NOT NULL, + "schema_version" BIGINT NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TIMESTAMPTZ NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("dataset", "record_key", "rejected_sha256") +); + +CREATE INDEX IF NOT EXISTS "recall_conflicts_invocation" ON "recall_conflicts" ("invocation_id"); + +CREATE OR REPLACE VIEW "invocations" AS +WITH ids AS ( + SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at + FROM ( + SELECT invocation_id, invocation_started_at FROM recall_invocations + UNION ALL SELECT invocation_id, invocation_started_at FROM recall_heartbeats + UNION ALL SELECT invocation_id, invocation_started_at FROM recall_events + WHERE event_type IN ('invocation_started', 'invocation_completed', 'invocation_failed') + ) AS source_rows + GROUP BY invocation_id +), +started AS ( + SELECT invocation_id, MIN(occurred_at) AS started_at, MIN(command) AS command + FROM recall_events + WHERE event_type = 'invocation_started' + GROUP BY invocation_id +), +terminals AS ( + SELECT invocation_id, event_type, occurred_at, exit_code, duration_ms, error_type, error_code, + (payload #>> '{error_message}') AS error_message, + (payload #>> '{signal_number}')::BIGINT AS signal_number, + (payload #>> '{cost_status}') AS cost_status, + ROW_NUMBER() OVER ( + PARTITION BY invocation_id + ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC + ) AS position + FROM recall_events + WHERE event_type IN ('invocation_completed', 'invocation_failed') +), +beats AS ( + SELECT invocation_id, MAX(beat_at) AS last_heartbeat_at, MAX(stale_at) AS stale_at, + MIN(command) AS command, MIN(subcommand) AS subcommand + FROM recall_heartbeats + GROUP BY invocation_id +), +projected AS ( + SELECT + ids.invocation_id, + ids.invocation_started_at, + COALESCE(started.started_at, ids.invocation_started_at) AS started_at, + CASE + WHEN terminals.event_type = 'invocation_completed' THEN 'completed' + WHEN terminals.event_type = 'invocation_failed' + AND terminals.signal_number IS NOT NULL THEN 'cancelled' + WHEN terminals.event_type = 'invocation_failed' THEN 'failed' + WHEN beats.stale_at IS NULL THEN 'unknown' + WHEN beats.stale_at >= CURRENT_TIMESTAMP THEN 'running' + ELSE 'abandoned' + END AS status, + terminals.event_type AS terminal_event, + terminals.occurred_at AS terminal_at, + terminals.duration_ms AS terminal_duration_ms, + terminals.exit_code, + terminals.error_type, + terminals.error_code, + terminals.error_message, + terminals.signal_number, + terminals.cost_status, + COALESCE(context.command, started.command, beats.command) AS command, + COALESCE(context.subcommand, beats.subcommand) AS subcommand, + context.origin, + context.command_mode, + context.target, + context.adapter, + context.project_name, + context.selection, + COALESCE(context.selection_count, jsonb_array_length(context.selection)) + AS selection_count, + context.selection_truncated, + context.selector_files, + context.exclusion, + context.sqlbuild_version, + context.git_revision, + context.git_dirty, + context.host_token, + context.pid, + context.external_context, + CASE WHEN context.external_context IS NOT NULL + THEN (context.external_context #>> '{integration,name}') END + AS integration, + CASE WHEN context.external_context IS NOT NULL + THEN (context.external_context #>> '{integration,job_name}') END + AS integration_job, + CASE WHEN context.external_context IS NOT NULL + THEN (context.external_context #>> '{integration,run_id}') END + AS integration_run_id, + CASE WHEN context.external_context IS NOT NULL + THEN (context.external_context #>> '{integration,retry_number}')::BIGINT END + AS integration_retry, + context.recall_filter_active, + beats.last_heartbeat_at + FROM ids + LEFT JOIN recall_invocations AS context ON context.invocation_id = ids.invocation_id + LEFT JOIN started ON started.invocation_id = ids.invocation_id + LEFT JOIN terminals + ON terminals.invocation_id = ids.invocation_id AND terminals.position = 1 + LEFT JOIN beats ON beats.invocation_id = ids.invocation_id +) +SELECT + invocation_id, + invocation_started_at, + started_at, + status, + CASE + WHEN terminal_event IS NOT NULL THEN terminal_at + WHEN status = 'abandoned' THEN last_heartbeat_at + END AS ended_at, + CASE + WHEN terminal_event IS NOT NULL THEN terminal_duration_ms + WHEN status = 'abandoned' THEN CAST(ROUND(CAST(EXTRACT(EPOCH FROM (last_heartbeat_at - started_at)) * 1000 AS NUMERIC), 3) AS DOUBLE PRECISION) + END AS duration_ms, + CASE + WHEN terminal_event IS NOT NULL THEN 0 + WHEN status = 'abandoned' THEN 1 + END AS ended_at_estimated, + exit_code, + error_type, + error_code, + error_message, + signal_number, + cost_status, + command, + subcommand, + origin, + command_mode, + target, + adapter, + project_name, + selection, + selection_count, + selection_truncated, + selector_files, + exclusion, + sqlbuild_version, + git_revision, + git_dirty, + host_token, + pid, + external_context, + integration, + integration_job, + integration_run_id, + integration_retry, + recall_filter_active, + last_heartbeat_at +FROM projected; + +CREATE OR REPLACE VIEW "statements" AS +WITH facts AS ( + SELECT * + FROM recall_events + WHERE statement_id IS NOT NULL AND event_type LIKE 'statement!_%' ESCAPE '!' +), +firsts AS ( + SELECT invocation_id, invocation_started_at, statement_id, run_id, resource_id, resource_attempt_id, operation_id, + intent, statement_kind, purpose, sql_digest, + ROW_NUMBER() OVER ( + PARTITION BY statement_id ORDER BY invocation_sequence, occurred_at, event_id + ) AS position + FROM facts +), +started AS ( + SELECT statement_id, MIN(started_at) AS started_at + FROM ( + SELECT statement_id, occurred_at AS started_at FROM facts WHERE event_type = 'statement_started' + UNION ALL + SELECT statement_id, started_at FROM facts WHERE started_at IS NOT NULL + ) AS starts + GROUP BY statement_id +), +submitted AS ( + SELECT statement_id, MIN(occurred_at) AS submitted_at + FROM facts WHERE event_type = 'statement_submitted' GROUP BY statement_id +), +query_ids AS ( + SELECT statement_id, MAX(query_id) AS query_id FROM facts GROUP BY statement_id +), +terminals AS ( + SELECT statement_id, event_type, occurred_at, query_id, error_type, error_code, duration_ms, + row_count, affected_rows, outcome, + ROW_NUMBER() OVER ( + PARTITION BY statement_id + ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC + ) AS position + FROM facts + WHERE event_type IN ('statement_completed', 'statement_failed') +), +captures AS ( + SELECT firsts.statement_id, ( + SELECT MAX(CAST(artifact.digest IS NOT NULL AS INTEGER)) + FROM recall_artifact_refs AS ref + LEFT JOIN recall_artifacts AS artifact ON artifact.digest = ref.digest + WHERE ref.invocation_id = firsts.invocation_id + AND ref.digest = 'sha256:' || firsts.sql_digest + ) AS stored + FROM firsts + WHERE firsts.position = 1 +), +resources AS ( + SELECT invocation_id, resource_id, MIN(resource_name) AS resource_name, + MIN(resource_kind) AS resource_kind + FROM recall_events + WHERE resource_id IS NOT NULL AND resource_name IS NOT NULL + GROUP BY invocation_id, resource_id +) +SELECT + firsts.invocation_id, + firsts.invocation_started_at, + firsts.statement_id, + firsts.run_id, + firsts.resource_id, + resources.resource_name, + resources.resource_kind, + firsts.resource_attempt_id, + firsts.operation_id, + firsts.intent, + firsts.statement_kind, + firsts.purpose, + COALESCE(terminals.query_id, query_ids.query_id) AS query_id, + started.started_at, + submitted.submitted_at, + terminals.occurred_at AS ended_at, + CASE terminals.event_type + WHEN 'statement_completed' THEN 'completed' + WHEN 'statement_failed' THEN + CASE WHEN terminals.outcome = 'handled' THEN 'handled' ELSE 'failed' END + ELSE 'no_terminal' + END AS outcome, + terminals.error_type, + terminals.error_code, + terminals.duration_ms, + terminals.row_count, + terminals.affected_rows, + firsts.sql_digest, + COALESCE(captures.stored, 0) AS sql_available, + CASE captures.stored + WHEN 1 THEN NULL + WHEN 0 THEN 'expired_or_not_captured' + ELSE 'not_captured' + END AS sql_unavailable_reason +FROM firsts +LEFT JOIN started ON started.statement_id = firsts.statement_id +LEFT JOIN submitted ON submitted.statement_id = firsts.statement_id +LEFT JOIN query_ids ON query_ids.statement_id = firsts.statement_id +LEFT JOIN terminals + ON terminals.statement_id = firsts.statement_id AND terminals.position = 1 +LEFT JOIN captures ON captures.statement_id = firsts.statement_id +LEFT JOIN resources + ON resources.invocation_id = firsts.invocation_id + AND resources.resource_id = firsts.resource_id +WHERE firsts.position = 1; + +CREATE OR REPLACE VIEW "audit_results" AS +SELECT + invocation_id, + invocation_started_at, + event_id, + occurred_at, + run_id, + resource_id, + resource_attempt_id, + (payload #>> '{audit_name}') AS audit_name, + (payload #>> '{audit_definition_name}') AS audit_definition_name, + CASE WHEN (payload #>> '{target_name}') IS NULL THEN NULL ELSE + COALESCE((payload #>> '{target_database}') || '.', '') + || COALESCE((payload #>> '{target_schema}') || '.', '') + || (payload #>> '{target_name}') + END AS target_relation, + (payload #>> '{attached_target_kind}') AS attached_target_kind, + (payload #>> '{attached_target_name}') AS attached_target_name, + (payload #>> '{attached_column_name}') AS "column", + outcome, + (payload #>> '{severity}') AS severity, + (payload #>> '{violation_count}')::BIGINT AS violation_count, + (payload #>> '{measured_value}')::DOUBLE PRECISION AS measured_value, + (payload #>> '{evaluation_mode}') AS evaluation_mode, + (payload #>> '{execution_error}') AS execution_error, + sql_digest, + (payload #>> '{evidence_count}')::BIGINT AS evidence_count, + (payload #>> '{evidence_truncated}')::BOOLEAN::INTEGER AS evidence_truncated, + (payload #>> '{sample_digest}') AS sample_digest +FROM recall_events +WHERE event_type = 'audit_completed'; + +CREATE OR REPLACE VIEW "output_invocations" AS +WITH received AS ( + SELECT invocation_id, + MIN(invocation_started_at) AS invocation_started_at, + COUNT(*) AS chunks, + CAST(SUM(byte_count) AS BIGINT) AS bytes, + CAST(SUM(CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END) AS BIGINT) AS stdout_bytes, + CAST(SUM(CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END) AS BIGINT) AS stderr_bytes, + MAX(sequence) AS max_sequence, + CAST(SUM(dropped_before) AS BIGINT) AS queue_drops, + CAST(SUM(COALESCE(omitted_sequences, 1)) AS BIGINT) AS spans, + CAST(SUM(COALESCE(omitted_bytes, 0)) AS BIGINT) AS truncated_bytes + FROM recall_output + GROUP BY invocation_id +), +delivery AS ( + SELECT * FROM recall_delivery WHERE stream = 'output' +), +ids AS ( + SELECT invocation_id, invocation_started_at FROM received + UNION SELECT invocation_id, invocation_started_at FROM delivery +), +counted AS ( + SELECT + ids.invocation_id, + ids.invocation_started_at, + COALESCE(received.chunks, 0) AS chunks, + COALESCE(received.bytes, 0) AS bytes, + COALESCE(received.stdout_bytes, 0) AS stdout_bytes, + COALESCE(received.stderr_bytes, 0) AS stderr_bytes, + COALESCE(received.truncated_bytes, 0) AS truncated_bytes, + received.max_sequence, + GREATEST( + COALESCE(received.max_sequence + 1 - received.spans, 0), + COALESCE(delivery.accepted, 0) - COALESCE(received.chunks, 0) + ) AS missing_sequences, + GREATEST(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, + COALESCE(delivery.filtered, 0) AS filtered, + COALESCE(delivery.failed, 0) AS failed, + COALESCE(CAST(context.recall_filter_active AS INTEGER), 0) AS filter_active, + COALESCE(CAST(delivery.flush_complete AS INTEGER), 0) AS flush_complete, + delivery.invocation_id IS NOT NULL AS delivery_reported, + COALESCE(delivery.delivered, 0) AS delivered + FROM ids + LEFT JOIN received ON received.invocation_id = ids.invocation_id + LEFT JOIN delivery ON delivery.invocation_id = ids.invocation_id + LEFT JOIN recall_invocations AS context ON context.invocation_id = ids.invocation_id +) +SELECT + invocation_id, + invocation_started_at, + chunks, + bytes, + stdout_bytes, + stderr_bytes, + truncated_bytes, + max_sequence, + missing_sequences, + reported_drops, + filtered, + failed, + filter_active, + flush_complete, + CASE + WHEN missing_sequences = 0 AND truncated_bytes > 0 THEN 'truncated' + WHEN missing_sequences = 0 THEN NULL + WHEN filtered >= missing_sequences THEN 'filtered' + WHEN filtered + reported_drops + failed >= missing_sequences THEN 'dropped' + ELSE 'unexplained' + END AS gap_reason, + CASE + WHEN delivery_reported AND flush_complete = 1 AND missing_sequences = 0 + AND reported_drops = 0 AND filtered = 0 AND failed = 0 AND truncated_bytes = 0 + AND chunks >= delivered + THEN 1 ELSE 0 + END AS complete +FROM counted; + +CREATE OR REPLACE VIEW "output_tail" AS +SELECT invocation_id, invocation_started_at, sequence, stream, occurred_at, byte_count, + data AS text +FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY invocation_id ORDER BY sequence DESC) AS position + FROM recall_output +) AS source_rows +WHERE position <= 200; + +CREATE OR REPLACE VIEW "output_transcripts" AS +SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at, + COALESCE(string_agg(CASE WHEN stream = 'stdout' THEN data END, ''::BYTEA ORDER BY sequence), ''::BYTEA) AS stdout, + COALESCE(string_agg(CASE WHEN stream = 'stderr' THEN data END, ''::BYTEA ORDER BY sequence), ''::BYTEA) AS stderr, + COALESCE(string_agg(data, ''::BYTEA ORDER BY sequence), ''::BYTEA) AS combined +FROM recall_output GROUP BY invocation_id; + +CREATE OR REPLACE VIEW "events" AS +SELECT + event_id, + invocation_sequence, + occurred_at, + event_type, + event_kind, + severity, + producer, + event_schema_version, + run_id, + resource_id, + resource_attempt_id, + operation_id, + statement_id, + command, + resource_kind, + resource_name, + error_code, + error_type, + duration_ms, + exit_code, + query_id, + sql_digest, + intent, + statement_kind, + purpose, + outcome, + row_count, + affected_rows, + CASE WHEN producer = 'sqlbuild' THEN payload || jsonb_strip_nulls(jsonb_build_object('command', command, 'resource_kind', resource_kind, 'resource_name', resource_name, 'error_code', error_code, 'error_type', error_type, 'query_id', query_id, 'sql_digest', sql_digest, 'intent', intent, 'statement_kind', statement_kind, 'purpose', purpose, 'outcome', outcome, 'exit_code', exit_code, 'row_count', row_count, 'affected_rows', affected_rows)) ELSE payload END AS payload, + started_at, + invocation_id, + invocation_started_at, + schema_version, + producer_version, + ingested_at, + content_hash, + storage_id +FROM recall_events; + +CREATE OR REPLACE VIEW "diagnostics" AS +SELECT * FROM recall_diagnostics; + +CREATE OR REPLACE VIEW "artifacts" AS +SELECT + refs.invocation_id, + refs.invocation_started_at, + refs.digest, + refs.ref_id, + refs.role, + refs.statement_id, + refs.resource_id, + artifact.kind, + artifact.media_type, + artifact.size, + CAST(artifact.digest IS NOT NULL AS INTEGER) AS available, + CASE WHEN artifact.digest IS NULL THEN 'expired_or_not_captured' END AS unavailable_reason, + CASE WHEN artifact.media_type LIKE 'text/%' OR artifact.media_type = 'application/json' + THEN artifact.content END AS text, + artifact.content +FROM recall_artifact_refs AS refs +LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest; + +CREATE OR REPLACE VIEW "conflicts" AS +SELECT * FROM recall_conflicts; + +CREATE TABLE sqb_installation ( + singleton BIGINT PRIMARY KEY CHECK (singleton = 1), + installation_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE sqb_locations ( + location_id UUID PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + archived_at TIMESTAMPTZ +); + +CREATE TABLE sqb_source_texts ( + text_hash TEXT PRIMARY KEY, + body BYTEA NOT NULL +); + +CREATE TABLE sqb_node_definitions ( + node_hash TEXT PRIMARY KEY, + body BYTEA NOT NULL +); + +CREATE TABLE sqb_documents ( + document_hash TEXT PRIMARY KEY, + body BYTEA NOT NULL +); + +CREATE TABLE sqb_snapshots ( + snapshot_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + generation BIGINT NOT NULL CHECK (generation > 0), + git_revision TEXT, + git_dirty BOOLEAN NOT NULL, + git_author TEXT, + image_id TEXT, + environment_hash TEXT NOT NULL, + sqb_version TEXT NOT NULL, + manifest_digest TEXT NOT NULL, + published_at TIMESTAMPTZ NOT NULL, + UNIQUE (location_id, target, generation), + UNIQUE (location_id, target, snapshot_id) +); + +CREATE INDEX sqb_snapshots_location_time ON sqb_snapshots(location_id, target, published_at); + +CREATE TABLE sqb_snapshot_nodes ( + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + node_key TEXT NOT NULL, + node_kind TEXT NOT NULL, + node_hash TEXT NOT NULL REFERENCES sqb_node_definitions(node_hash), + PRIMARY KEY (snapshot_id, node_key) +); + +CREATE TABLE sqb_snapshot_sources ( + snapshot_id UUID NOT NULL, + node_key TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('raw_sql', 'compiled_sql', 'python_source')), + text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + PRIMARY KEY (snapshot_id, node_key, kind), + FOREIGN KEY (snapshot_id, node_key) REFERENCES sqb_snapshot_nodes(snapshot_id, node_key) +); + +CREATE TABLE sqb_snapshot_documents ( + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('jobs', 'groups', 'triggers', 'roles', 'lineage', 'columns', 'providers')), + document_hash TEXT NOT NULL REFERENCES sqb_documents(document_hash), + PRIMARY KEY (snapshot_id, kind) +); + +CREATE TABLE sqb_current_snapshots ( + location_id UUID NOT NULL, + target TEXT NOT NULL, + snapshot_id UUID NOT NULL, + PRIMARY KEY (location_id, target), + FOREIGN KEY (location_id, target, snapshot_id) + REFERENCES sqb_snapshots(location_id, target, snapshot_id) +); + +CREATE TABLE sqb_snapshot_changes ( + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + change_seq BIGINT NOT NULL, + change_kind TEXT NOT NULL CHECK (change_kind IN ('added', 'changed', 'removed')), + subject_key TEXT NOT NULL, + before_hash TEXT, + after_hash TEXT, + PRIMARY KEY (snapshot_id, change_seq) +); + +CREATE TABLE sqb_scheduler_lease ( + name TEXT PRIMARY KEY CHECK (name = 'scheduler'), + holder UUID NOT NULL, + epoch BIGINT NOT NULL CHECK (epoch > 0), + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE sqb_jobs ( + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + job_key TEXT NOT NULL, + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('job', 'group')), + definition_json JSONB NOT NULL, + PRIMARY KEY (location_id, target, job_key) +); + +CREATE TABLE sqb_triggers ( + trigger_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('schedule', 'sensor')), + name TEXT NOT NULL, + job_key TEXT NOT NULL, + definition_json JSONB NOT NULL, + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + removed_in_snapshot_id UUID REFERENCES sqb_snapshots(snapshot_id), + UNIQUE (location_id, target, kind, name) +); + +CREATE TABLE sqb_trigger_state ( + trigger_id UUID PRIMARY KEY REFERENCES sqb_triggers(trigger_id), + status TEXT NOT NULL CHECK (status IN ('running', 'paused')), + status_override TEXT CHECK (status_override IN ('running', 'paused')), + started_at TIMESTAMPTZ NOT NULL, + cursor_json JSONB, + last_tick_for TIMESTAMPTZ, + next_due_at TIMESTAMPTZ +); + +CREATE INDEX sqb_trigger_state_due ON sqb_trigger_state(next_due_at) WHERE status = 'running'; + +CREATE TABLE sqb_ticks ( + tick_id UUID PRIMARY KEY, + trigger_id UUID NOT NULL REFERENCES sqb_triggers(trigger_id), + scheduled_for TIMESTAMPTZ NOT NULL, + evaluated_at TIMESTAMPTZ NOT NULL, + epoch BIGINT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('requested', 'covered', 'skipped')), + run_id UUID, + covered_by_run_id UUID, + active_run_id UUID, + UNIQUE (trigger_id, scheduled_for) +); + +CREATE INDEX sqb_ticks_trigger_time ON sqb_ticks(trigger_id, scheduled_for); + +CREATE TABLE sqb_runs ( + run_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + image_id TEXT, + kind TEXT NOT NULL CHECK (kind IN ('job', 'group', 'adhoc')), + origin TEXT NOT NULL CHECK (origin IN ('manual', 'schedule', 'sensor', 'rerun', 'backfill')), + job_key TEXT, + parent_run_id UUID REFERENCES sqb_runs(run_id), + root_run_id UUID NOT NULL, + trigger_id UUID REFERENCES sqb_triggers(trigger_id), + tick_id UUID REFERENCES sqb_ticks(tick_id), + rerun_of_run_id UUID REFERENCES sqb_runs(run_id), + requested_by TEXT NOT NULL, + priority BIGINT NOT NULL DEFAULT 0, + selection_json JSONB NOT NULL, + request_json JSONB NOT NULL, + authored_json JSONB, + write_set_digest TEXT NOT NULL, + rewrite_history BOOLEAN NOT NULL DEFAULT FALSE, + status TEXT NOT NULL CHECK (status IN ('queued', 'launched', 'running', 'unresponsive', 'success', 'warn', 'partial', 'failed', 'cancelled')), + status_reason TEXT, + queued_at TIMESTAMPTZ NOT NULL, + launched_at TIMESTAMPTZ, + started_at TIMESTAMPTZ, + ended_at TIMESTAMPTZ, + cancel_requested_at TIMESTAMPTZ, + node_counts_json JSONB NOT NULL, + last_event_storage_id BIGINT NOT NULL, + worker_instance UUID, + container_id TEXT, + claimed_at TIMESTAMPTZ, + last_heartbeat_at TIMESTAMPTZ +); + +CREATE INDEX sqb_runs_location_time ON sqb_runs(location_id, target, queued_at); + +CREATE INDEX sqb_runs_location_ended ON sqb_runs(location_id, target, ended_at); + +CREATE INDEX sqb_runs_queue ON sqb_runs(priority DESC, queued_at) WHERE status = 'queued'; + +CREATE INDEX sqb_runs_active ON sqb_runs(location_id, target, job_key) WHERE status IN ('queued', 'launched', 'running', 'unresponsive'); + +CREATE TABLE sqb_run_nodes ( + run_id UUID NOT NULL REFERENCES sqb_runs(run_id), + node_key TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'success', 'warn', 'failed', 'skipped', 'cancelled')), + writes BOOLEAN NOT NULL, + started_at TIMESTAMPTZ, + ended_at TIMESTAMPTZ, + rows_affected BIGINT, + error_code TEXT, + skipped_because_of TEXT, + kind TEXT NOT NULL DEFAULT 'node' CHECK (kind IN ('node', 'check')), + finished_storage_id BIGINT, + PRIMARY KEY (run_id, node_key) +); + +CREATE INDEX sqb_run_nodes_node ON sqb_run_nodes(node_key, run_id); + +CREATE INDEX sqb_run_nodes_finished ON sqb_run_nodes(finished_storage_id) +WHERE finished_storage_id IS NOT NULL; + +CREATE INDEX sqb_run_nodes_decisive ON sqb_run_nodes(node_key, finished_storage_id) +WHERE finished_storage_id IS NOT NULL AND status IN ('success', 'failed'); + +CREATE TABLE sqb_write_claims ( + target TEXT NOT NULL, + node_key TEXT NOT NULL, + run_id UUID NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (target, node_key) +); + +CREATE INDEX sqb_write_claims_run ON sqb_write_claims(run_id); + +CREATE TABLE sqb_concurrency_slots ( + group_key TEXT NOT NULL, + slot BIGINT NOT NULL CHECK (slot > 0), + run_id UUID NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (group_key, slot), + UNIQUE (group_key, run_id) +); + +CREATE INDEX sqb_concurrency_slots_run ON sqb_concurrency_slots(run_id); + +CREATE TABLE sqb_principals ( + principal_id UUID PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL, + disabled_at TIMESTAMPTZ +); + +CREATE TABLE sqb_access_tokens ( + token_id UUID PRIMARY KEY, + secret_hash TEXT NOT NULL, + location_id UUID REFERENCES sqb_locations(location_id), + run_id UUID REFERENCES sqb_runs(run_id), + scope_json JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ, + CHECK ( + (location_id IS NOT NULL AND run_id IS NULL) + OR (location_id IS NULL AND run_id IS NOT NULL) + ) +); + +CREATE INDEX sqb_access_tokens_location ON sqb_access_tokens(location_id); + +CREATE INDEX sqb_access_tokens_run ON sqb_access_tokens(run_id); + +CREATE TABLE sqb_security_audit ( + event_id UUID PRIMARY KEY, + occurred_at TIMESTAMPTZ NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + granting_role TEXT NOT NULL, + request_id UUID NOT NULL, + ip_address TEXT, + scope_json JSONB NOT NULL, + before_json JSONB NOT NULL, + after_json JSONB NOT NULL, + storage_id BIGINT +); + +CREATE INDEX sqb_security_audit_time ON sqb_security_audit(occurred_at, event_id); + +CREATE INDEX sqb_security_audit_trigger ON sqb_security_audit((scope_json ->> 'trigger_id'), occurred_at, event_id) +WHERE action = 'trigger.changed'; + +CREATE TABLE sqb_location_instances ( + instance_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + last_seen_at TIMESTAMPTZ NOT NULL, + health_json JSONB NOT NULL +); + +CREATE INDEX sqb_location_instances_location_seen ON sqb_location_instances(location_id, last_seen_at); + +CREATE TABLE sqb_location_work ( + request_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id UUID NOT NULL REFERENCES sqb_snapshots(snapshot_id), + generation BIGINT NOT NULL, + request_json JSONB NOT NULL, + requested_by TEXT NOT NULL, + requested_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + claim_id UUID, + session_id UUID REFERENCES sqb_location_instances(instance_id), + claimed_until TIMESTAMPTZ, + result_json JSONB, + completed_at TIMESTAMPTZ +); + +CREATE INDEX sqb_location_work_pending ON sqb_location_work(location_id, target, requested_at); + +CREATE TABLE sqb_platform_sink_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); + +CREATE TABLE sqb_sensor_evaluation_events ( + event_id UUID PRIMARY KEY, + evaluation_id UUID NOT NULL, + trigger_id UUID NOT NULL REFERENCES sqb_triggers(trigger_id), + phase TEXT NOT NULL CHECK (phase IN ('requested', 'completed', 'failed', 'timed_out', 'abandoned')), + occurred_at TIMESTAMPTZ NOT NULL, + fact_json JSONB NOT NULL, + UNIQUE (evaluation_id, phase) +); + +CREATE INDEX sqb_sensor_evaluation_history ON sqb_sensor_evaluation_events(trigger_id, phase, occurred_at, evaluation_id); + +CREATE INDEX sqb_sensor_evaluation_expiry ON sqb_sensor_evaluation_events(occurred_at); + +CREATE TABLE sqb_run_requests ( + trigger_id UUID NOT NULL REFERENCES sqb_triggers(trigger_id), + run_key TEXT NOT NULL, + evaluation_id UUID NOT NULL, + run_id UUID NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (trigger_id, run_key) +); + +CREATE INDEX sqb_runs_terminal_cursor ON sqb_runs(location_id, target, job_key, last_event_storage_id) +WHERE status IN ('success', 'warn', 'partial', 'failed', 'cancelled'); + +CREATE INDEX sqb_runs_parent ON sqb_runs(parent_run_id) WHERE parent_run_id IS NOT NULL; + +CREATE TABLE sqb_warehouse_freshness ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT, + scan_id UUID +); + +CREATE INDEX sqb_warehouse_freshness_scope ON sqb_warehouse_freshness (location_id, target, observed_at, event_id); + +CREATE TABLE sqb_warehouse_fingerprints ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + definition_text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT, + node_type TEXT, + node_name TEXT, + target_database TEXT, + target_schema TEXT, + target_name TEXT, + scan_id UUID +); + +CREATE INDEX sqb_warehouse_fingerprints_scope ON sqb_warehouse_fingerprints (location_id, target, observed_at, event_id); + +CREATE TABLE sqb_development_locations ( + project_path TEXT NOT NULL, + target TEXT NOT NULL, + location_id UUID NOT NULL UNIQUE REFERENCES sqb_locations(location_id), + PRIMARY KEY (project_path, target) +); + +CREATE TABLE sqb_password_credentials ( + principal_id UUID PRIMARY KEY REFERENCES sqb_principals(principal_id), + credential_id UUID NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + changed_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE sqb_sessions ( + session_id UUID PRIMARY KEY, + principal_id UUID NOT NULL REFERENCES sqb_principals(principal_id), + credential_id UUID, + secret_hash TEXT NOT NULL, + csrf_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ +); + +CREATE INDEX sqb_sessions_person ON sqb_sessions(principal_id); + +CREATE TABLE sqb_password_login_attempts ( + attempt_id UUID PRIMARY KEY, + identity_key TEXT NOT NULL, + peer_address TEXT NOT NULL, + attempted_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX sqb_password_login_user_time ON sqb_password_login_attempts(identity_key, attempted_at); + +CREATE INDEX sqb_password_login_peer_time ON sqb_password_login_attempts(peer_address, attempted_at); + +CREATE INDEX sqb_password_login_time ON sqb_password_login_attempts(attempted_at); + +CREATE TABLE sqb_oidc_login_flows ( + flow_id UUID PRIMARY KEY, + configuration_id TEXT NOT NULL, + state_hash TEXT NOT NULL UNIQUE, + browser_hash TEXT NOT NULL, + nonce TEXT NOT NULL, + pkce_verifier TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX sqb_oidc_login_flows_expiry ON sqb_oidc_login_flows(expires_at); + +CREATE TABLE sqb_role_assignments ( + assignment_id UUID PRIMARY KEY, + principal_id UUID NOT NULL REFERENCES sqb_principals(principal_id), + role_key TEXT NOT NULL, + location_id UUID REFERENCES sqb_locations(location_id), + target TEXT, + active_key TEXT NOT NULL UNIQUE +); + +CREATE INDEX sqb_role_assignments_principal ON sqb_role_assignments(principal_id); + +CREATE TABLE sqb_person_groups ( + principal_id UUID NOT NULL REFERENCES sqb_principals(principal_id), + group_name TEXT NOT NULL, + PRIMARY KEY (principal_id, group_name) +); + +CREATE TABLE sqb_group_roles ( + group_name TEXT NOT NULL, + role_key TEXT NOT NULL, + location_id UUID, + target TEXT +); + +CREATE INDEX sqb_fingerprints_namespace ON sqb_warehouse_fingerprints (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_freshness_namespace ON sqb_warehouse_freshness (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE UNIQUE INDEX sqb_security_audit_storage_id ON sqb_security_audit(storage_id); + +CREATE TABLE sqb_warehouse_audit_results ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + audit_name TEXT NOT NULL, + outcome TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID, + binding_key TEXT +); + +CREATE INDEX sqb_audit_results_namespace ON sqb_warehouse_audit_results (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_audit_results_binding ON sqb_warehouse_audit_results (location_id,target,binding_key,observed_at,event_id); + +CREATE TABLE sqb_warehouse_janitor_events ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + event_type TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_janitor_events_namespace ON sqb_warehouse_janitor_events (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_migrations ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + target_name TEXT, + destination_model TEXT NOT NULL, + decision TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_migrations_namespace ON sqb_warehouse_migrations (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_migrations_destination ON sqb_warehouse_migrations (location_id,target,destination_model,observed_at,event_id); + +CREATE TABLE sqb_warehouse_old_name_views ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + target_name TEXT, + migration_event_id TEXT NOT NULL, + event_type TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_old_name_views_namespace ON sqb_warehouse_old_name_views (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_old_name_views_migration ON sqb_warehouse_old_name_views (location_id,target,migration_event_id,observed_at,event_id); + +CREATE TABLE sqb_warehouse_microbatches ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + model_name TEXT NOT NULL, + target_name TEXT NOT NULL, + scope_kind TEXT NOT NULL, + scope_key TEXT NOT NULL, + record_type TEXT NOT NULL, + execution_run_id TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_microbatches_namespace ON sqb_warehouse_microbatches (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_microbatches_model ON sqb_warehouse_microbatches (location_id,target,model_name); + +CREATE TABLE sqb_warehouse_column_migrations ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_column_migrations_namespace ON sqb_warehouse_column_migrations (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_node_results ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_node_results_namespace ON sqb_warehouse_node_results (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_consumption ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_consumption_namespace ON sqb_warehouse_consumption (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_observations ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + kind TEXT NOT NULL, + node TEXT NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_observations_namespace ON sqb_warehouse_observations (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_observations_latest ON sqb_warehouse_observations (location_id,target,kind,node,observed_at,event_id); + +CREATE TABLE sqb_warehouse_resource_costs ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + run_id TEXT NOT NULL, + orchestration_run_id TEXT, + attempt BIGINT NOT NULL, + resource_type TEXT NOT NULL, + resource_name TEXT NOT NULL, + run_started_at TIMESTAMPTZ NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_resource_costs_namespace ON sqb_warehouse_resource_costs (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_resource_costs_run ON sqb_warehouse_resource_costs (location_id,target,run_id,attempt); + +CREATE INDEX sqb_resource_costs_orchestration_run ON sqb_warehouse_resource_costs (orchestration_run_id); + +CREATE INDEX sqb_resource_costs_started ON sqb_warehouse_resource_costs (location_id,target,run_started_at); + +CREATE INDEX sqb_freshness_source_latest ON sqb_warehouse_freshness ( + location_id, target, (record_json ->> 'source_name'), + (record_json ->> 'target_database'), (record_json ->> 'target_schema'), + (record_json ->> 'target_name'), observed_at DESC, event_id DESC +); + +CREATE INDEX sqb_freshness_source_received ON sqb_warehouse_freshness ( + location_id, target, (record_json ->> 'source_name'), + (record_json ->> 'target_database'), (record_json ->> 'target_schema'), + (record_json ->> 'target_name'), received_at DESC, observed_at DESC, event_id DESC +); + +CREATE TABLE sqb_security_audit_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); + +CREATE TABLE sqb_warehouse_catalog ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + scan_id UUID +); + +CREATE INDEX sqb_warehouse_catalog_namespace ON sqb_warehouse_catalog + (location_id, target, state_schema, state_database, observed_at, event_id); + +CREATE INDEX sqb_fingerprint_node_latest ON sqb_warehouse_fingerprints + (location_id,target,node_type,node_name,target_database,target_schema,target_name,observed_at DESC,event_id DESC); + +CREATE TABLE sqb_warehouse_statistics ( + event_id UUID PRIMARY KEY, + location_id UUID NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + relation_name TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + record_json JSONB NOT NULL, + received_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX sqb_statistics_relation_time ON sqb_warehouse_statistics + (location_id,target,state_schema,state_database,relation_name,observed_at,event_id); + +CREATE INDEX sqb_statistics_expiration ON sqb_warehouse_statistics (observed_at,event_id); + +CREATE INDEX sqb_catalog_statistics_scan ON sqb_warehouse_catalog + (location_id,target,state_schema,state_database,event_id); + +CREATE TABLE sqb_cli_login_flows ( + flow_id UUID PRIMARY KEY, + client_hash TEXT NOT NULL, + browser_hash TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + principal_id UUID, + session_id UUID +); + +CREATE INDEX sqb_cli_login_expiry ON sqb_cli_login_flows(expires_at); + +CREATE INDEX IF NOT EXISTS sqb_runs_tick ON sqb_runs(tick_id); + +CREATE INDEX IF NOT EXISTS sqb_ticks_expiry ON sqb_ticks(scheduled_for) WHERE outcome = 'covered'; + +CREATE INDEX IF NOT EXISTS sqb_runs_snapshot ON sqb_runs(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_snapshot_nodes_hash ON sqb_snapshot_nodes(node_hash); + +CREATE INDEX IF NOT EXISTS sqb_snapshot_sources_hash ON sqb_snapshot_sources(text_hash); + +CREATE INDEX IF NOT EXISTS sqb_snapshot_documents_hash ON sqb_snapshot_documents(document_hash); + +CREATE TABLE sqb_effective_roles ( + location_id UUID NOT NULL, + target TEXT NOT NULL, + role_key TEXT NOT NULL, + resolved_grants_json TEXT NOT NULL, + PRIMARY KEY (location_id, target, role_key) +); + +CREATE INDEX IF NOT EXISTS sqb_jobs_snapshot ON sqb_jobs(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_triggers_snapshot ON sqb_triggers(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_triggers_removed ON sqb_triggers(removed_in_snapshot_id) WHERE removed_in_snapshot_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_runs_trigger ON sqb_runs(trigger_id) WHERE trigger_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_runs_rerun_of ON sqb_runs(rerun_of_run_id) WHERE rerun_of_run_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_location_work_session ON sqb_location_work(session_id) WHERE session_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_location_work_snapshot ON sqb_location_work(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_run_requests_run ON sqb_run_requests(run_id); + +CREATE INDEX IF NOT EXISTS sqb_warehouse_fingerprints_text ON sqb_warehouse_fingerprints(definition_text_hash); + +CREATE INDEX IF NOT EXISTS sqb_role_assignments_location ON sqb_role_assignments(location_id); diff --git a/crates/sqlbuild-orchestration/tests/store/schema/baseline/rows.sql b/crates/sqlbuild-orchestration/tests/store/schema/baseline/rows.sql new file mode 100644 index 0000000000..47958cc18b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/baseline/rows.sql @@ -0,0 +1,17 @@ +INSERT INTO sqb_locations (location_id, name, created_by, created_at) VALUES ('018f0000-0000-7000-8000-000000000001', 'orders', 'admin', '2026-11-01T09:00:00Z'); + +INSERT INTO sqb_snapshots (snapshot_id, location_id, target, generation, git_dirty, environment_hash, sqb_version, manifest_digest, published_at) VALUES ('018f0000-0000-7000-8000-000000000002', '018f0000-0000-7000-8000-000000000001', 'prod', 1, FALSE, 'env-1', '0.241.0', 'manifest-1', '2026-11-01T09:05:00Z'); + +INSERT INTO sqb_principals (principal_id, name, created_at) VALUES ('018f0000-0000-7000-8000-000000000003', 'ana', '2026-11-01T09:01:00Z'); + +INSERT INTO sqb_role_assignments (assignment_id, principal_id, role_key, location_id, target, active_key) VALUES ('018f0000-0000-7000-8000-000000000004', '018f0000-0000-7000-8000-000000000003', 'operator', '018f0000-0000-7000-8000-000000000001', 'prod', 'ana:operator:orders:prod'); + +INSERT INTO sqb_access_tokens (token_id, secret_hash, location_id, scope_json, created_at, expires_at) VALUES ('018f0000-0000-7000-8000-000000000005', 'hash-1', '018f0000-0000-7000-8000-000000000001', '{"kind": "location"}', '2026-11-01T09:02:00Z', '2027-11-01T09:02:00Z'); + +INSERT INTO sqb_triggers (trigger_id, location_id, target, kind, name, job_key, definition_json, snapshot_id) VALUES ('018f0000-0000-7000-8000-000000000006', '018f0000-0000-7000-8000-000000000001', 'prod', 'schedule', 'nightly', 'daily_orders', '{"cron": "0 2 * * *"}', '018f0000-0000-7000-8000-000000000002'); + +INSERT INTO sqb_trigger_state (trigger_id, status, status_override, started_at, next_due_at) VALUES ('018f0000-0000-7000-8000-000000000006', 'paused', 'paused', '2026-11-01T09:06:00Z', '2026-11-02T02:00:00Z'); + +INSERT INTO sqb_runs (run_id, location_id, target, snapshot_id, kind, origin, job_key, root_run_id, requested_by, selection_json, request_json, write_set_digest, status, queued_at, ended_at, node_counts_json, last_event_storage_id) VALUES ('018f0000-0000-7000-8000-000000000007', '018f0000-0000-7000-8000-000000000001', 'prod', '018f0000-0000-7000-8000-000000000002', 'job', 'manual', 'daily_orders', '018f0000-0000-7000-8000-000000000007', 'principal:018f0000-0000-7000-8000-000000000003', '[]', '{}', 'digest-1', 'success', '2026-11-01T10:00:00Z', '2026-11-01T10:05:00Z', '{"success": 3}', 12); + +INSERT INTO sqb_run_nodes (run_id, node_key, status, writes, started_at, ended_at, rows_affected, finished_storage_id) VALUES ('018f0000-0000-7000-8000-000000000007', 'model.orders', 'success', TRUE, '2026-11-01T10:01:00Z', '2026-11-01T10:02:00Z', 3, 11); diff --git a/crates/sqlbuild-orchestration/tests/store/schema/baseline/sqlite.sql b/crates/sqlbuild-orchestration/tests/store/schema/baseline/sqlite.sql new file mode 100644 index 0000000000..beaab06a8f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/baseline/sqlite.sql @@ -0,0 +1,1431 @@ +CREATE TABLE IF NOT EXISTS "recall_meta" ( + "component" TEXT NOT NULL, + "key" TEXT NOT NULL, + "value" TEXT NOT NULL, + "recorded_at" TEXT NOT NULL, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("component", "key", "recorded_at") +); + +CREATE TABLE IF NOT EXISTS "recall_invocations" ( + "command" TEXT, + "subcommand" TEXT, + "selection" JSON, + "exclusion" JSON, + "project_name" TEXT, + "project_dir_hash" TEXT, + "target" TEXT, + "adapter" TEXT, + "destination_database" TEXT, + "destination_schema" TEXT, + "destination_fingerprint" TEXT, + "sqlbuild_version" TEXT, + "git_revision" TEXT, + "git_dirty" INTEGER, + "host_token" TEXT, + "pid" INTEGER, + "external_context" JSON, + "recall_filter_active" INTEGER, + "argv_digest" TEXT, + "origin" TEXT, + "command_mode" TEXT, + "selection_count" INTEGER, + "selection_truncated" INTEGER, + "selector_files" JSON, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id") +); + +CREATE INDEX IF NOT EXISTS "recall_invocations_started" ON "recall_invocations" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_output" ( + "record_id" TEXT NOT NULL, + "sequence" INTEGER NOT NULL, + "stream" TEXT NOT NULL, + "occurred_at" TEXT NOT NULL, + "data" BLOB NOT NULL, + "byte_offset" INTEGER NOT NULL, + "byte_count" INTEGER NOT NULL, + "priority" TEXT, + "dropped_before" INTEGER NOT NULL, + "omitted_bytes" INTEGER, + "omitted_sequences" INTEGER, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "sequence") +); + +CREATE INDEX IF NOT EXISTS "recall_output_expiry" ON "recall_output" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_events" ( + "event_id" TEXT NOT NULL, + "invocation_sequence" INTEGER, + "occurred_at" TEXT NOT NULL, + "event_type" TEXT NOT NULL, + "event_kind" TEXT NOT NULL, + "severity" TEXT NOT NULL, + "producer" TEXT NOT NULL, + "event_schema_version" INTEGER NOT NULL, + "run_id" TEXT, + "resource_id" TEXT, + "resource_attempt_id" TEXT, + "operation_id" TEXT, + "statement_id" TEXT, + "command" TEXT, + "resource_kind" TEXT, + "resource_name" TEXT, + "error_code" TEXT, + "error_type" TEXT, + "duration_ms" REAL, + "exit_code" INTEGER, + "query_id" TEXT, + "sql_digest" TEXT, + "intent" TEXT, + "statement_kind" TEXT, + "purpose" TEXT, + "outcome" TEXT, + "row_count" INTEGER, + "affected_rows" INTEGER, + "payload" JSON NOT NULL, + "started_at" TEXT, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + storage_id INTEGER PRIMARY KEY AUTOINCREMENT, + UNIQUE ("event_id") +); + +CREATE INDEX IF NOT EXISTS "recall_events_invocation" ON "recall_events" ("invocation_id", "invocation_sequence"); + +CREATE INDEX IF NOT EXISTS "recall_events_type" ON "recall_events" ("event_type", "occurred_at"); + +CREATE INDEX IF NOT EXISTS "recall_events_statement" ON "recall_events" ("statement_id"); + +CREATE INDEX IF NOT EXISTS "recall_events_expiry" ON "recall_events" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_diagnostics" ( + "record_id" TEXT NOT NULL, + "sequence" INTEGER NOT NULL, + "occurred_at" TEXT NOT NULL, + "severity" TEXT NOT NULL, + "logger" TEXT NOT NULL, + "source" TEXT NOT NULL, + "message" TEXT NOT NULL, + "fields" JSON NOT NULL, + "log_stream_id" TEXT, + "run_id" TEXT, + "resource_id" TEXT, + "resource_attempt_id" TEXT, + "operation_id" TEXT, + "statement_id" TEXT, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "sequence") +); + +CREATE INDEX IF NOT EXISTS "recall_diagnostics_severity" ON "recall_diagnostics" ("severity"); + +CREATE INDEX IF NOT EXISTS "recall_diagnostics_expiry" ON "recall_diagnostics" ("invocation_started_at"); + +CREATE TABLE IF NOT EXISTS "recall_artifacts" ( + "digest" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "media_type" TEXT NOT NULL, + "size" INTEGER NOT NULL, + "content" BLOB NOT NULL, + "seen_at" TEXT NOT NULL, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("digest") +); + +CREATE INDEX IF NOT EXISTS "recall_artifacts_seen" ON "recall_artifacts" ("seen_at"); + +CREATE TABLE IF NOT EXISTS "recall_artifact_refs" ( + "digest" TEXT NOT NULL, + "ref_id" TEXT NOT NULL, + "statement_id" TEXT, + "resource_id" TEXT, + "role" TEXT NOT NULL, + "referenced_at" TEXT NOT NULL, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "digest", "ref_id") +); + +CREATE INDEX IF NOT EXISTS "recall_artifact_refs_digest" ON "recall_artifact_refs" ("digest"); + +CREATE TABLE IF NOT EXISTS "recall_heartbeats" ( + "beat_at" TEXT NOT NULL, + "stale_at" TEXT NOT NULL, + "host_token" TEXT, + "pid" INTEGER, + "command" TEXT, + "subcommand" TEXT, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "beat_at") +); + +CREATE TABLE IF NOT EXISTS "recall_delivery" ( + "stream" TEXT NOT NULL, + "accepted" INTEGER NOT NULL, + "delivered" INTEGER NOT NULL, + "filtered" INTEGER NOT NULL, + "dropped" INTEGER NOT NULL, + "failed" INTEGER NOT NULL, + "conflicts" INTEGER NOT NULL, + "flush_complete" INTEGER NOT NULL, + "written_at" TEXT NOT NULL, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("invocation_id", "stream") +); + +CREATE TABLE IF NOT EXISTS "recall_conflicts" ( + "dataset" TEXT NOT NULL, + "record_key" TEXT NOT NULL, + "existing_sha256" TEXT NOT NULL, + "rejected_sha256" TEXT NOT NULL, + "detected_at" TEXT NOT NULL, + "invocation_id" TEXT NOT NULL, + "invocation_started_at" TEXT NOT NULL, + "schema_version" INTEGER NOT NULL, + "producer_version" TEXT NOT NULL, + "ingested_at" TEXT NOT NULL, + "content_hash" TEXT NOT NULL, + UNIQUE ("dataset", "record_key", "rejected_sha256") +); + +CREATE INDEX IF NOT EXISTS "recall_conflicts_invocation" ON "recall_conflicts" ("invocation_id"); + +CREATE VIEW IF NOT EXISTS "invocations" AS +WITH ids AS ( + SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at + FROM ( + SELECT invocation_id, invocation_started_at FROM recall_invocations + UNION ALL SELECT invocation_id, invocation_started_at FROM recall_heartbeats + UNION ALL SELECT invocation_id, invocation_started_at FROM recall_events + WHERE event_type IN ('invocation_started', 'invocation_completed', 'invocation_failed') + ) AS source_rows + GROUP BY invocation_id +), +started AS ( + SELECT invocation_id, MIN(occurred_at) AS started_at, MIN(command) AS command + FROM recall_events + WHERE event_type = 'invocation_started' + GROUP BY invocation_id +), +terminals AS ( + SELECT invocation_id, event_type, occurred_at, exit_code, duration_ms, error_type, error_code, + json_extract(payload, '$.error_message') AS error_message, + json_extract(payload, '$.signal_number') AS signal_number, + json_extract(payload, '$.cost_status') AS cost_status, + ROW_NUMBER() OVER ( + PARTITION BY invocation_id + ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC + ) AS position + FROM recall_events + WHERE event_type IN ('invocation_completed', 'invocation_failed') +), +beats AS ( + SELECT invocation_id, MAX(beat_at) AS last_heartbeat_at, MAX(stale_at) AS stale_at, + MIN(command) AS command, MIN(subcommand) AS subcommand + FROM recall_heartbeats + GROUP BY invocation_id +), +projected AS ( + SELECT + ids.invocation_id, + ids.invocation_started_at, + COALESCE(started.started_at, ids.invocation_started_at) AS started_at, + CASE + WHEN terminals.event_type = 'invocation_completed' THEN 'completed' + WHEN terminals.event_type = 'invocation_failed' + AND terminals.signal_number IS NOT NULL THEN 'cancelled' + WHEN terminals.event_type = 'invocation_failed' THEN 'failed' + WHEN beats.stale_at IS NULL THEN 'unknown' + WHEN beats.stale_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now') THEN 'running' + ELSE 'abandoned' + END AS status, + terminals.event_type AS terminal_event, + terminals.occurred_at AS terminal_at, + terminals.duration_ms AS terminal_duration_ms, + terminals.exit_code, + terminals.error_type, + terminals.error_code, + terminals.error_message, + terminals.signal_number, + terminals.cost_status, + COALESCE(context.command, started.command, beats.command) AS command, + COALESCE(context.subcommand, beats.subcommand) AS subcommand, + context.origin, + context.command_mode, + context.target, + context.adapter, + context.project_name, + context.selection, + COALESCE(context.selection_count, json_array_length(context.selection)) + AS selection_count, + context.selection_truncated, + context.selector_files, + context.exclusion, + context.sqlbuild_version, + context.git_revision, + context.git_dirty, + context.host_token, + context.pid, + context.external_context, + CASE WHEN json_valid(context.external_context) + THEN json_extract(context.external_context, '$.integration.name') END + AS integration, + CASE WHEN json_valid(context.external_context) + THEN json_extract(context.external_context, '$.integration.job_name') END + AS integration_job, + CASE WHEN json_valid(context.external_context) + THEN json_extract(context.external_context, '$.integration.run_id') END + AS integration_run_id, + CASE WHEN json_valid(context.external_context) + THEN json_extract(context.external_context, '$.integration.retry_number') END + AS integration_retry, + context.recall_filter_active, + beats.last_heartbeat_at + FROM ids + LEFT JOIN recall_invocations AS context ON context.invocation_id = ids.invocation_id + LEFT JOIN started ON started.invocation_id = ids.invocation_id + LEFT JOIN terminals + ON terminals.invocation_id = ids.invocation_id AND terminals.position = 1 + LEFT JOIN beats ON beats.invocation_id = ids.invocation_id +) +SELECT + invocation_id, + invocation_started_at, + started_at, + status, + CASE + WHEN terminal_event IS NOT NULL THEN terminal_at + WHEN status = 'abandoned' THEN last_heartbeat_at + END AS ended_at, + CASE + WHEN terminal_event IS NOT NULL THEN terminal_duration_ms + WHEN status = 'abandoned' THEN ROUND((unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3) + END AS duration_ms, + CASE + WHEN terminal_event IS NOT NULL THEN 0 + WHEN status = 'abandoned' THEN 1 + END AS ended_at_estimated, + exit_code, + error_type, + error_code, + error_message, + signal_number, + cost_status, + command, + subcommand, + origin, + command_mode, + target, + adapter, + project_name, + selection, + selection_count, + selection_truncated, + selector_files, + exclusion, + sqlbuild_version, + git_revision, + git_dirty, + host_token, + pid, + external_context, + integration, + integration_job, + integration_run_id, + integration_retry, + recall_filter_active, + last_heartbeat_at +FROM projected; + +CREATE VIEW IF NOT EXISTS "statements" AS +WITH facts AS ( + SELECT * + FROM recall_events + WHERE statement_id IS NOT NULL AND event_type LIKE 'statement!_%' ESCAPE '!' +), +firsts AS ( + SELECT invocation_id, invocation_started_at, statement_id, run_id, resource_id, resource_attempt_id, operation_id, + intent, statement_kind, purpose, sql_digest, + ROW_NUMBER() OVER ( + PARTITION BY statement_id ORDER BY invocation_sequence, occurred_at, event_id + ) AS position + FROM facts +), +started AS ( + SELECT statement_id, MIN(started_at) AS started_at + FROM ( + SELECT statement_id, occurred_at AS started_at FROM facts WHERE event_type = 'statement_started' + UNION ALL + SELECT statement_id, started_at FROM facts WHERE started_at IS NOT NULL + ) AS starts + GROUP BY statement_id +), +submitted AS ( + SELECT statement_id, MIN(occurred_at) AS submitted_at + FROM facts WHERE event_type = 'statement_submitted' GROUP BY statement_id +), +query_ids AS ( + SELECT statement_id, MAX(query_id) AS query_id FROM facts GROUP BY statement_id +), +terminals AS ( + SELECT statement_id, event_type, occurred_at, query_id, error_type, error_code, duration_ms, + row_count, affected_rows, outcome, + ROW_NUMBER() OVER ( + PARTITION BY statement_id + ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC + ) AS position + FROM facts + WHERE event_type IN ('statement_completed', 'statement_failed') +), +captures AS ( + SELECT firsts.statement_id, ( + SELECT MAX(artifact.digest IS NOT NULL) + FROM recall_artifact_refs AS ref + LEFT JOIN recall_artifacts AS artifact ON artifact.digest = ref.digest + WHERE ref.invocation_id = firsts.invocation_id + AND ref.digest = 'sha256:' || firsts.sql_digest + ) AS stored + FROM firsts + WHERE firsts.position = 1 +), +resources AS ( + SELECT invocation_id, resource_id, MIN(resource_name) AS resource_name, + MIN(resource_kind) AS resource_kind + FROM recall_events + WHERE resource_id IS NOT NULL AND resource_name IS NOT NULL + GROUP BY invocation_id, resource_id +) +SELECT + firsts.invocation_id, + firsts.invocation_started_at, + firsts.statement_id, + firsts.run_id, + firsts.resource_id, + resources.resource_name, + resources.resource_kind, + firsts.resource_attempt_id, + firsts.operation_id, + firsts.intent, + firsts.statement_kind, + firsts.purpose, + COALESCE(terminals.query_id, query_ids.query_id) AS query_id, + started.started_at, + submitted.submitted_at, + terminals.occurred_at AS ended_at, + CASE terminals.event_type + WHEN 'statement_completed' THEN 'completed' + WHEN 'statement_failed' THEN + CASE WHEN terminals.outcome = 'handled' THEN 'handled' ELSE 'failed' END + ELSE 'no_terminal' + END AS outcome, + terminals.error_type, + terminals.error_code, + terminals.duration_ms, + terminals.row_count, + terminals.affected_rows, + firsts.sql_digest, + COALESCE(captures.stored, 0) AS sql_available, + CASE captures.stored + WHEN 1 THEN NULL + WHEN 0 THEN 'expired_or_not_captured' + ELSE 'not_captured' + END AS sql_unavailable_reason +FROM firsts +LEFT JOIN started ON started.statement_id = firsts.statement_id +LEFT JOIN submitted ON submitted.statement_id = firsts.statement_id +LEFT JOIN query_ids ON query_ids.statement_id = firsts.statement_id +LEFT JOIN terminals + ON terminals.statement_id = firsts.statement_id AND terminals.position = 1 +LEFT JOIN captures ON captures.statement_id = firsts.statement_id +LEFT JOIN resources + ON resources.invocation_id = firsts.invocation_id + AND resources.resource_id = firsts.resource_id +WHERE firsts.position = 1; + +CREATE VIEW IF NOT EXISTS "audit_results" AS +SELECT + invocation_id, + invocation_started_at, + event_id, + occurred_at, + run_id, + resource_id, + resource_attempt_id, + json_extract(payload, '$.audit_name') AS audit_name, + json_extract(payload, '$.audit_definition_name') AS audit_definition_name, + CASE WHEN json_extract(payload, '$.target_name') IS NULL THEN NULL ELSE + COALESCE(json_extract(payload, '$.target_database') || '.', '') + || COALESCE(json_extract(payload, '$.target_schema') || '.', '') + || json_extract(payload, '$.target_name') + END AS target_relation, + json_extract(payload, '$.attached_target_kind') AS attached_target_kind, + json_extract(payload, '$.attached_target_name') AS attached_target_name, + json_extract(payload, '$.attached_column_name') AS "column", + outcome, + json_extract(payload, '$.severity') AS severity, + json_extract(payload, '$.violation_count') AS violation_count, + json_extract(payload, '$.measured_value') AS measured_value, + json_extract(payload, '$.evaluation_mode') AS evaluation_mode, + json_extract(payload, '$.execution_error') AS execution_error, + sql_digest, + json_extract(payload, '$.evidence_count') AS evidence_count, + json_extract(payload, '$.evidence_truncated') AS evidence_truncated, + json_extract(payload, '$.sample_digest') AS sample_digest +FROM recall_events +WHERE event_type = 'audit_completed'; + +CREATE VIEW IF NOT EXISTS "output_invocations" AS +WITH received AS ( + SELECT invocation_id, + MIN(invocation_started_at) AS invocation_started_at, + COUNT(*) AS chunks, + SUM(byte_count) AS bytes, + SUM(CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END) AS stdout_bytes, + SUM(CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END) AS stderr_bytes, + MAX(sequence) AS max_sequence, + SUM(dropped_before) AS queue_drops, + SUM(COALESCE(omitted_sequences, 1)) AS spans, + SUM(COALESCE(omitted_bytes, 0)) AS truncated_bytes + FROM recall_output + GROUP BY invocation_id +), +delivery AS ( + SELECT * FROM recall_delivery WHERE stream = 'output' +), +ids AS ( + SELECT invocation_id, invocation_started_at FROM received + UNION SELECT invocation_id, invocation_started_at FROM delivery +), +counted AS ( + SELECT + ids.invocation_id, + ids.invocation_started_at, + COALESCE(received.chunks, 0) AS chunks, + COALESCE(received.bytes, 0) AS bytes, + COALESCE(received.stdout_bytes, 0) AS stdout_bytes, + COALESCE(received.stderr_bytes, 0) AS stderr_bytes, + COALESCE(received.truncated_bytes, 0) AS truncated_bytes, + received.max_sequence, + MAX( + COALESCE(received.max_sequence + 1 - received.spans, 0), + COALESCE(delivery.accepted, 0) - COALESCE(received.chunks, 0) + ) AS missing_sequences, + MAX(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, + COALESCE(delivery.filtered, 0) AS filtered, + COALESCE(delivery.failed, 0) AS failed, + COALESCE(context.recall_filter_active, 0) AS filter_active, + COALESCE(delivery.flush_complete, 0) AS flush_complete, + delivery.invocation_id IS NOT NULL AS delivery_reported, + COALESCE(delivery.delivered, 0) AS delivered + FROM ids + LEFT JOIN received ON received.invocation_id = ids.invocation_id + LEFT JOIN delivery ON delivery.invocation_id = ids.invocation_id + LEFT JOIN recall_invocations AS context ON context.invocation_id = ids.invocation_id +) +SELECT + invocation_id, + invocation_started_at, + chunks, + bytes, + stdout_bytes, + stderr_bytes, + truncated_bytes, + max_sequence, + missing_sequences, + reported_drops, + filtered, + failed, + filter_active, + flush_complete, + CASE + WHEN missing_sequences = 0 AND truncated_bytes > 0 THEN 'truncated' + WHEN missing_sequences = 0 THEN NULL + WHEN filtered >= missing_sequences THEN 'filtered' + WHEN filtered + reported_drops + failed >= missing_sequences THEN 'dropped' + ELSE 'unexplained' + END AS gap_reason, + CASE + WHEN delivery_reported AND flush_complete = 1 AND missing_sequences = 0 + AND reported_drops = 0 AND filtered = 0 AND failed = 0 AND truncated_bytes = 0 + AND chunks >= delivered + THEN 1 ELSE 0 + END AS complete +FROM counted; + +CREATE VIEW IF NOT EXISTS "output_tail" AS +SELECT invocation_id, invocation_started_at, sequence, stream, occurred_at, byte_count, + CAST(data AS TEXT) AS text +FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY invocation_id ORDER BY sequence DESC) AS position + FROM recall_output +) AS source_rows +WHERE position <= 200; + +CREATE VIEW IF NOT EXISTS "output_transcripts" AS +SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at, + CAST(COALESCE(group_concat(CASE WHEN stream = 'stdout' THEN data END, x'' ORDER BY sequence), x'') AS BLOB) AS stdout, + CAST(COALESCE(group_concat(CASE WHEN stream = 'stderr' THEN data END, x'' ORDER BY sequence), x'') AS BLOB) AS stderr, + CAST(COALESCE(group_concat(data, x'' ORDER BY sequence), x'') AS BLOB) AS combined +FROM recall_output GROUP BY invocation_id; + +CREATE VIEW IF NOT EXISTS "events" AS +SELECT + event_id, + invocation_sequence, + occurred_at, + event_type, + event_kind, + severity, + producer, + event_schema_version, + run_id, + resource_id, + resource_attempt_id, + operation_id, + statement_id, + command, + resource_kind, + resource_name, + error_code, + error_type, + duration_ms, + exit_code, + query_id, + sql_digest, + intent, + statement_kind, + purpose, + outcome, + row_count, + affected_rows, + CASE WHEN producer = 'sqlbuild' THEN json_patch(payload, json_remove(json_object('command', command, 'resource_kind', resource_kind, 'resource_name', resource_name, 'error_code', error_code, 'error_type', error_type, 'query_id', query_id, 'sql_digest', sql_digest, 'intent', intent, 'statement_kind', statement_kind, 'purpose', purpose, 'outcome', outcome, 'exit_code', exit_code, 'row_count', row_count, 'affected_rows', affected_rows), CASE WHEN command IS NULL THEN '$.command' ELSE '$.payload' END, CASE WHEN resource_kind IS NULL THEN '$.resource_kind' ELSE '$.payload' END, CASE WHEN resource_name IS NULL THEN '$.resource_name' ELSE '$.payload' END, CASE WHEN error_code IS NULL THEN '$.error_code' ELSE '$.payload' END, CASE WHEN error_type IS NULL THEN '$.error_type' ELSE '$.payload' END, CASE WHEN query_id IS NULL THEN '$.query_id' ELSE '$.payload' END, CASE WHEN sql_digest IS NULL THEN '$.sql_digest' ELSE '$.payload' END, CASE WHEN intent IS NULL THEN '$.intent' ELSE '$.payload' END, CASE WHEN statement_kind IS NULL THEN '$.statement_kind' ELSE '$.payload' END, CASE WHEN purpose IS NULL THEN '$.purpose' ELSE '$.payload' END, CASE WHEN outcome IS NULL THEN '$.outcome' ELSE '$.payload' END, CASE WHEN exit_code IS NULL THEN '$.exit_code' ELSE '$.payload' END, CASE WHEN row_count IS NULL THEN '$.row_count' ELSE '$.payload' END, CASE WHEN affected_rows IS NULL THEN '$.affected_rows' ELSE '$.payload' END)) ELSE payload END AS payload, + started_at, + invocation_id, + invocation_started_at, + schema_version, + producer_version, + ingested_at, + content_hash, + storage_id +FROM recall_events; + +CREATE VIEW IF NOT EXISTS "diagnostics" AS +SELECT * FROM recall_diagnostics; + +CREATE VIEW IF NOT EXISTS "artifacts" AS +SELECT + refs.invocation_id, + refs.invocation_started_at, + refs.digest, + refs.ref_id, + refs.role, + refs.statement_id, + refs.resource_id, + artifact.kind, + artifact.media_type, + artifact.size, + artifact.digest IS NOT NULL AS available, + CASE WHEN artifact.digest IS NULL THEN 'expired_or_not_captured' END AS unavailable_reason, + CASE WHEN artifact.media_type LIKE 'text/%' OR artifact.media_type = 'application/json' + THEN CAST(artifact.content AS TEXT) END AS text, + artifact.content +FROM recall_artifact_refs AS refs +LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest; + +CREATE VIEW IF NOT EXISTS "conflicts" AS +SELECT * FROM recall_conflicts; + +CREATE TABLE sqb_installation ( + singleton BIGINT PRIMARY KEY CHECK (singleton = 1), + installation_id TEXT NOT NULL, + created_at TEXT NOT NULL +); + +CREATE TABLE sqb_locations ( + location_id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_by TEXT NOT NULL, + created_at TEXT NOT NULL, + archived_at TEXT +); + +CREATE TABLE sqb_source_texts ( + text_hash TEXT PRIMARY KEY, + body BLOB NOT NULL +); + +CREATE TABLE sqb_node_definitions ( + node_hash TEXT PRIMARY KEY, + body BLOB NOT NULL +); + +CREATE TABLE sqb_documents ( + document_hash TEXT PRIMARY KEY, + body BLOB NOT NULL +); + +CREATE TABLE sqb_snapshots ( + snapshot_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + generation BIGINT NOT NULL CHECK (generation > 0), + git_revision TEXT, + git_dirty BOOLEAN NOT NULL, + git_author TEXT, + image_id TEXT, + environment_hash TEXT NOT NULL, + sqb_version TEXT NOT NULL, + manifest_digest TEXT NOT NULL, + published_at TEXT NOT NULL, + UNIQUE (location_id, target, generation), + UNIQUE (location_id, target, snapshot_id) +); + +CREATE INDEX sqb_snapshots_location_time ON sqb_snapshots(location_id, target, published_at); + +CREATE TABLE sqb_snapshot_nodes ( + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + node_key TEXT NOT NULL, + node_kind TEXT NOT NULL, + node_hash TEXT NOT NULL REFERENCES sqb_node_definitions(node_hash), + PRIMARY KEY (snapshot_id, node_key) +); + +CREATE TABLE sqb_snapshot_sources ( + snapshot_id TEXT NOT NULL, + node_key TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('raw_sql', 'compiled_sql', 'python_source')), + text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + PRIMARY KEY (snapshot_id, node_key, kind), + FOREIGN KEY (snapshot_id, node_key) REFERENCES sqb_snapshot_nodes(snapshot_id, node_key) +); + +CREATE TABLE sqb_snapshot_documents ( + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('jobs', 'groups', 'triggers', 'roles', 'lineage', 'columns', 'providers')), + document_hash TEXT NOT NULL REFERENCES sqb_documents(document_hash), + PRIMARY KEY (snapshot_id, kind) +); + +CREATE TABLE sqb_current_snapshots ( + location_id TEXT NOT NULL, + target TEXT NOT NULL, + snapshot_id TEXT NOT NULL, + PRIMARY KEY (location_id, target), + FOREIGN KEY (location_id, target, snapshot_id) + REFERENCES sqb_snapshots(location_id, target, snapshot_id) +); + +CREATE TABLE sqb_snapshot_changes ( + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + change_seq BIGINT NOT NULL, + change_kind TEXT NOT NULL CHECK (change_kind IN ('added', 'changed', 'removed')), + subject_key TEXT NOT NULL, + before_hash TEXT, + after_hash TEXT, + PRIMARY KEY (snapshot_id, change_seq) +); + +CREATE TABLE sqb_scheduler_lease ( + name TEXT PRIMARY KEY CHECK (name = 'scheduler'), + holder TEXT NOT NULL, + epoch BIGINT NOT NULL CHECK (epoch > 0), + expires_at TEXT NOT NULL +); + +CREATE TABLE sqb_jobs ( + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + job_key TEXT NOT NULL, + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('job', 'group')), + definition_json JSON NOT NULL, + PRIMARY KEY (location_id, target, job_key) +); + +CREATE TABLE sqb_triggers ( + trigger_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('schedule', 'sensor')), + name TEXT NOT NULL, + job_key TEXT NOT NULL, + definition_json JSON NOT NULL, + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + removed_in_snapshot_id TEXT REFERENCES sqb_snapshots(snapshot_id), + UNIQUE (location_id, target, kind, name) +); + +CREATE TABLE sqb_trigger_state ( + trigger_id TEXT PRIMARY KEY REFERENCES sqb_triggers(trigger_id), + status TEXT NOT NULL CHECK (status IN ('running', 'paused')), + status_override TEXT CHECK (status_override IN ('running', 'paused')), + started_at TEXT NOT NULL, + cursor_json JSON, + last_tick_for TEXT, + next_due_at TEXT +); + +CREATE INDEX sqb_trigger_state_due ON sqb_trigger_state(next_due_at) WHERE status = 'running'; + +CREATE TABLE sqb_ticks ( + tick_id TEXT PRIMARY KEY, + trigger_id TEXT NOT NULL REFERENCES sqb_triggers(trigger_id), + scheduled_for TEXT NOT NULL, + evaluated_at TEXT NOT NULL, + epoch BIGINT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('requested', 'covered', 'skipped')), + run_id TEXT, + covered_by_run_id TEXT, + active_run_id TEXT, + UNIQUE (trigger_id, scheduled_for) +); + +CREATE INDEX sqb_ticks_trigger_time ON sqb_ticks(trigger_id, scheduled_for); + +CREATE TABLE sqb_runs ( + run_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + image_id TEXT, + kind TEXT NOT NULL CHECK (kind IN ('job', 'group', 'adhoc')), + origin TEXT NOT NULL CHECK (origin IN ('manual', 'schedule', 'sensor', 'rerun', 'backfill')), + job_key TEXT, + parent_run_id TEXT REFERENCES sqb_runs(run_id), + root_run_id TEXT NOT NULL, + trigger_id TEXT REFERENCES sqb_triggers(trigger_id), + tick_id TEXT REFERENCES sqb_ticks(tick_id), + rerun_of_run_id TEXT REFERENCES sqb_runs(run_id), + requested_by TEXT NOT NULL, + priority BIGINT NOT NULL DEFAULT 0, + selection_json JSON NOT NULL, + request_json JSON NOT NULL, + authored_json JSON, + write_set_digest TEXT NOT NULL, + rewrite_history BOOLEAN NOT NULL DEFAULT FALSE, + status TEXT NOT NULL CHECK (status IN ('queued', 'launched', 'running', 'unresponsive', 'success', 'warn', 'partial', 'failed', 'cancelled')), + status_reason TEXT, + queued_at TEXT NOT NULL, + launched_at TEXT, + started_at TEXT, + ended_at TEXT, + cancel_requested_at TEXT, + node_counts_json JSON NOT NULL, + last_event_storage_id BIGINT NOT NULL, + worker_instance TEXT, + container_id TEXT, + claimed_at TEXT, + last_heartbeat_at TEXT +); + +CREATE INDEX sqb_runs_location_time ON sqb_runs(location_id, target, queued_at); + +CREATE INDEX sqb_runs_location_ended ON sqb_runs(location_id, target, ended_at); + +CREATE INDEX sqb_runs_queue ON sqb_runs(priority DESC, queued_at) WHERE status = 'queued'; + +CREATE INDEX sqb_runs_active ON sqb_runs(location_id, target, job_key) WHERE status IN ('queued', 'launched', 'running', 'unresponsive'); + +CREATE TABLE sqb_run_nodes ( + run_id TEXT NOT NULL REFERENCES sqb_runs(run_id), + node_key TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'success', 'warn', 'failed', 'skipped', 'cancelled')), + writes BOOLEAN NOT NULL, + started_at TEXT, + ended_at TEXT, + rows_affected BIGINT, + error_code TEXT, + skipped_because_of TEXT, + kind TEXT NOT NULL DEFAULT 'node' CHECK (kind IN ('node', 'check')), + finished_storage_id BIGINT, + PRIMARY KEY (run_id, node_key) +); + +CREATE INDEX sqb_run_nodes_node ON sqb_run_nodes(node_key, run_id); + +CREATE INDEX sqb_run_nodes_finished ON sqb_run_nodes(finished_storage_id) +WHERE finished_storage_id IS NOT NULL; + +CREATE INDEX sqb_run_nodes_decisive ON sqb_run_nodes(node_key, finished_storage_id) +WHERE finished_storage_id IS NOT NULL AND status IN ('success', 'failed'); + +CREATE TABLE sqb_write_claims ( + target TEXT NOT NULL, + node_key TEXT NOT NULL, + run_id TEXT NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (target, node_key) +); + +CREATE INDEX sqb_write_claims_run ON sqb_write_claims(run_id); + +CREATE TABLE sqb_concurrency_slots ( + group_key TEXT NOT NULL, + slot BIGINT NOT NULL CHECK (slot > 0), + run_id TEXT NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (group_key, slot), + UNIQUE (group_key, run_id) +); + +CREATE INDEX sqb_concurrency_slots_run ON sqb_concurrency_slots(run_id); + +CREATE TABLE sqb_principals ( + principal_id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL, + disabled_at TEXT +); + +CREATE TABLE sqb_access_tokens ( + token_id TEXT PRIMARY KEY, + secret_hash TEXT NOT NULL, + location_id TEXT REFERENCES sqb_locations(location_id), + run_id TEXT REFERENCES sqb_runs(run_id), + scope_json JSON NOT NULL, + created_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + revoked_at TEXT, + CHECK ( + (location_id IS NOT NULL AND run_id IS NULL) + OR (location_id IS NULL AND run_id IS NOT NULL) + ) +); + +CREATE INDEX sqb_access_tokens_location ON sqb_access_tokens(location_id); + +CREATE INDEX sqb_access_tokens_run ON sqb_access_tokens(run_id); + +CREATE TABLE sqb_security_audit ( + event_id TEXT PRIMARY KEY, + occurred_at TEXT NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + granting_role TEXT NOT NULL, + request_id TEXT NOT NULL, + ip_address TEXT, + scope_json JSON NOT NULL, + before_json JSON NOT NULL, + after_json JSON NOT NULL, + storage_id BIGINT +); + +CREATE INDEX sqb_security_audit_time ON sqb_security_audit(occurred_at, event_id); + +CREATE INDEX sqb_security_audit_trigger ON sqb_security_audit((scope_json ->> 'trigger_id'), occurred_at, event_id) +WHERE action = 'trigger.changed'; + +CREATE TABLE sqb_location_instances ( + instance_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + health_json JSON NOT NULL +); + +CREATE INDEX sqb_location_instances_location_seen ON sqb_location_instances(location_id, last_seen_at); + +CREATE TABLE sqb_location_work ( + request_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id TEXT NOT NULL REFERENCES sqb_snapshots(snapshot_id), + generation BIGINT NOT NULL, + request_json JSON NOT NULL, + requested_by TEXT NOT NULL, + requested_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + claim_id TEXT, + session_id TEXT REFERENCES sqb_location_instances(instance_id), + claimed_until TEXT, + result_json JSON, + completed_at TEXT +); + +CREATE INDEX sqb_location_work_pending ON sqb_location_work(location_id, target, requested_at); + +CREATE TABLE sqb_platform_sink_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at TEXT NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); + +CREATE TABLE sqb_sensor_evaluation_events ( + event_id TEXT PRIMARY KEY, + evaluation_id TEXT NOT NULL, + trigger_id TEXT NOT NULL REFERENCES sqb_triggers(trigger_id), + phase TEXT NOT NULL CHECK (phase IN ('requested', 'completed', 'failed', 'timed_out', 'abandoned')), + occurred_at TEXT NOT NULL, + fact_json JSON NOT NULL, + UNIQUE (evaluation_id, phase) +); + +CREATE INDEX sqb_sensor_evaluation_history ON sqb_sensor_evaluation_events(trigger_id, phase, occurred_at, evaluation_id); + +CREATE INDEX sqb_sensor_evaluation_expiry ON sqb_sensor_evaluation_events(occurred_at); + +CREATE TABLE sqb_run_requests ( + trigger_id TEXT NOT NULL REFERENCES sqb_triggers(trigger_id), + run_key TEXT NOT NULL, + evaluation_id TEXT NOT NULL, + run_id TEXT NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (trigger_id, run_key) +); + +CREATE INDEX sqb_runs_terminal_cursor ON sqb_runs(location_id, target, job_key, last_event_storage_id) +WHERE status IN ('success', 'warn', 'partial', 'failed', 'cancelled'); + +CREATE INDEX sqb_runs_parent ON sqb_runs(parent_run_id) WHERE parent_run_id IS NOT NULL; + +CREATE TABLE sqb_warehouse_freshness ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT, + scan_id TEXT +); + +CREATE INDEX sqb_warehouse_freshness_scope ON sqb_warehouse_freshness (location_id, target, observed_at, event_id); + +CREATE TABLE sqb_warehouse_fingerprints ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at TEXT NOT NULL, + definition_text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT, + node_type TEXT, + node_name TEXT, + target_database TEXT, + target_schema TEXT, + target_name TEXT, + scan_id TEXT +); + +CREATE INDEX sqb_warehouse_fingerprints_scope ON sqb_warehouse_fingerprints (location_id, target, observed_at, event_id); + +CREATE TABLE sqb_development_locations ( + project_path TEXT NOT NULL, + target TEXT NOT NULL, + location_id TEXT NOT NULL UNIQUE REFERENCES sqb_locations(location_id), + PRIMARY KEY (project_path, target) +); + +CREATE TABLE sqb_password_credentials ( + principal_id TEXT PRIMARY KEY REFERENCES sqb_principals(principal_id), + credential_id TEXT NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + changed_at TEXT NOT NULL +); + +CREATE TABLE sqb_sessions ( + session_id TEXT PRIMARY KEY, + principal_id TEXT NOT NULL REFERENCES sqb_principals(principal_id), + credential_id TEXT, + secret_hash TEXT NOT NULL, + csrf_hash TEXT NOT NULL, + created_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + revoked_at TEXT +); + +CREATE INDEX sqb_sessions_person ON sqb_sessions(principal_id); + +CREATE TABLE sqb_password_login_attempts ( + attempt_id TEXT PRIMARY KEY, + identity_key TEXT NOT NULL, + peer_address TEXT NOT NULL, + attempted_at TEXT NOT NULL +); + +CREATE INDEX sqb_password_login_user_time ON sqb_password_login_attempts(identity_key, attempted_at); + +CREATE INDEX sqb_password_login_peer_time ON sqb_password_login_attempts(peer_address, attempted_at); + +CREATE INDEX sqb_password_login_time ON sqb_password_login_attempts(attempted_at); + +CREATE TABLE sqb_oidc_login_flows ( + flow_id TEXT PRIMARY KEY, + configuration_id TEXT NOT NULL, + state_hash TEXT NOT NULL UNIQUE, + browser_hash TEXT NOT NULL, + nonce TEXT NOT NULL, + pkce_verifier TEXT NOT NULL, + expires_at TEXT NOT NULL +); + +CREATE INDEX sqb_oidc_login_flows_expiry ON sqb_oidc_login_flows(expires_at); + +CREATE TABLE sqb_role_assignments ( + assignment_id TEXT PRIMARY KEY, + principal_id TEXT NOT NULL REFERENCES sqb_principals(principal_id), + role_key TEXT NOT NULL, + location_id TEXT REFERENCES sqb_locations(location_id), + target TEXT, + active_key TEXT NOT NULL UNIQUE +); + +CREATE INDEX sqb_role_assignments_principal ON sqb_role_assignments(principal_id); + +CREATE TABLE sqb_person_groups ( + principal_id TEXT NOT NULL REFERENCES sqb_principals(principal_id), + group_name TEXT NOT NULL, + PRIMARY KEY (principal_id, group_name) +); + +CREATE TABLE sqb_group_roles ( + group_name TEXT NOT NULL, + role_key TEXT NOT NULL, + location_id TEXT, + target TEXT +); + +CREATE INDEX sqb_fingerprints_namespace ON sqb_warehouse_fingerprints (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_freshness_namespace ON sqb_warehouse_freshness (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE UNIQUE INDEX sqb_security_audit_storage_id ON sqb_security_audit(storage_id); + +CREATE TABLE sqb_warehouse_audit_results ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + audit_name TEXT NOT NULL, + outcome TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT, + binding_key TEXT +); + +CREATE INDEX sqb_audit_results_namespace ON sqb_warehouse_audit_results (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_audit_results_binding ON sqb_warehouse_audit_results (location_id,target,binding_key,observed_at,event_id); + +CREATE TABLE sqb_warehouse_janitor_events ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + event_type TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_janitor_events_namespace ON sqb_warehouse_janitor_events (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_migrations ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + target_name TEXT, + destination_model TEXT NOT NULL, + decision TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_migrations_namespace ON sqb_warehouse_migrations (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_migrations_destination ON sqb_warehouse_migrations (location_id,target,destination_model,observed_at,event_id); + +CREATE TABLE sqb_warehouse_old_name_views ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + target_name TEXT, + migration_event_id TEXT NOT NULL, + event_type TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_old_name_views_namespace ON sqb_warehouse_old_name_views (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_old_name_views_migration ON sqb_warehouse_old_name_views (location_id,target,migration_event_id,observed_at,event_id); + +CREATE TABLE sqb_warehouse_microbatches ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + model_name TEXT NOT NULL, + target_name TEXT NOT NULL, + scope_kind TEXT NOT NULL, + scope_key TEXT NOT NULL, + record_type TEXT NOT NULL, + execution_run_id TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_microbatches_namespace ON sqb_warehouse_microbatches (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_microbatches_model ON sqb_warehouse_microbatches (location_id,target,model_name); + +CREATE TABLE sqb_warehouse_column_migrations ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_column_migrations_namespace ON sqb_warehouse_column_migrations (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_node_results ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_node_results_namespace ON sqb_warehouse_node_results (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_consumption ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_consumption_namespace ON sqb_warehouse_consumption (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE TABLE sqb_warehouse_observations ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + kind TEXT NOT NULL, + node TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_observations_namespace ON sqb_warehouse_observations (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_observations_latest ON sqb_warehouse_observations (location_id,target,kind,node,observed_at,event_id); + +CREATE TABLE sqb_warehouse_resource_costs ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + run_id TEXT NOT NULL, + orchestration_run_id TEXT, + attempt BIGINT NOT NULL, + resource_type TEXT NOT NULL, + resource_name TEXT NOT NULL, + run_started_at TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_resource_costs_namespace ON sqb_warehouse_resource_costs (location_id,target,state_database,state_schema,observed_at,event_id); + +CREATE INDEX sqb_resource_costs_run ON sqb_warehouse_resource_costs (location_id,target,run_id,attempt); + +CREATE INDEX sqb_resource_costs_orchestration_run ON sqb_warehouse_resource_costs (orchestration_run_id); + +CREATE INDEX sqb_resource_costs_started ON sqb_warehouse_resource_costs (location_id,target,run_started_at); + +CREATE INDEX sqb_freshness_source_latest ON sqb_warehouse_freshness ( + location_id, target, (record_json ->> 'source_name'), + (record_json ->> 'target_database'), (record_json ->> 'target_schema'), + (record_json ->> 'target_name'), observed_at DESC, event_id DESC +); + +CREATE INDEX sqb_freshness_source_received ON sqb_warehouse_freshness ( + location_id, target, (record_json ->> 'source_name'), + (record_json ->> 'target_database'), (record_json ->> 'target_schema'), + (record_json ->> 'target_name'), received_at DESC, observed_at DESC, event_id DESC +); + +CREATE TABLE sqb_security_audit_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at TEXT NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); + +CREATE TABLE sqb_warehouse_catalog ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + observed_at TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL, + scan_id TEXT +); + +CREATE INDEX sqb_warehouse_catalog_namespace ON sqb_warehouse_catalog + (location_id, target, state_schema, state_database, observed_at, event_id); + +CREATE INDEX sqb_fingerprint_node_latest ON sqb_warehouse_fingerprints + (location_id,target,node_type,node_name,target_database,target_schema,target_name,observed_at DESC,event_id DESC); + +CREATE TABLE sqb_warehouse_statistics ( + event_id TEXT PRIMARY KEY, + location_id TEXT NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + relation_name TEXT NOT NULL, + observed_at TEXT NOT NULL, + record_json JSON NOT NULL, + received_at TEXT NOT NULL +); + +CREATE INDEX sqb_statistics_relation_time ON sqb_warehouse_statistics + (location_id,target,state_schema,state_database,relation_name,observed_at,event_id); + +CREATE INDEX sqb_statistics_expiration ON sqb_warehouse_statistics (observed_at,event_id); + +CREATE INDEX sqb_catalog_statistics_scan ON sqb_warehouse_catalog + (location_id,target,state_schema,state_database,event_id); + +CREATE TABLE sqb_cli_login_flows ( + flow_id TEXT PRIMARY KEY, + client_hash TEXT NOT NULL, + browser_hash TEXT NOT NULL, + expires_at TEXT NOT NULL, + principal_id TEXT, + session_id TEXT +); + +CREATE INDEX sqb_cli_login_expiry ON sqb_cli_login_flows(expires_at); + +CREATE INDEX IF NOT EXISTS sqb_runs_tick ON sqb_runs(tick_id); + +CREATE INDEX IF NOT EXISTS sqb_ticks_expiry ON sqb_ticks(scheduled_for) WHERE outcome = 'covered'; + +CREATE INDEX IF NOT EXISTS sqb_runs_snapshot ON sqb_runs(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_snapshot_nodes_hash ON sqb_snapshot_nodes(node_hash); + +CREATE INDEX IF NOT EXISTS sqb_snapshot_sources_hash ON sqb_snapshot_sources(text_hash); + +CREATE INDEX IF NOT EXISTS sqb_snapshot_documents_hash ON sqb_snapshot_documents(document_hash); + +CREATE TABLE sqb_effective_roles ( + location_id TEXT NOT NULL, + target TEXT NOT NULL, + role_key TEXT NOT NULL, + resolved_grants_json TEXT NOT NULL, + PRIMARY KEY (location_id, target, role_key) +); + +CREATE INDEX IF NOT EXISTS sqb_jobs_snapshot ON sqb_jobs(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_triggers_snapshot ON sqb_triggers(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_triggers_removed ON sqb_triggers(removed_in_snapshot_id) WHERE removed_in_snapshot_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_runs_trigger ON sqb_runs(trigger_id) WHERE trigger_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_runs_rerun_of ON sqb_runs(rerun_of_run_id) WHERE rerun_of_run_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_location_work_session ON sqb_location_work(session_id) WHERE session_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS sqb_location_work_snapshot ON sqb_location_work(snapshot_id); + +CREATE INDEX IF NOT EXISTS sqb_run_requests_run ON sqb_run_requests(run_id); + +CREATE INDEX IF NOT EXISTS sqb_warehouse_fingerprints_text ON sqb_warehouse_fingerprints(definition_text_hash); + +CREATE INDEX IF NOT EXISTS sqb_role_assignments_location ON sqb_role_assignments(location_id); diff --git a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs index fa10659ee3..b38a5fef9c 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/helpers.rs @@ -10,9 +10,13 @@ use crate::store::main::commit::commit; use crate::store::main::migrate_store::migrate_store; use crate::store::main::open_store::open_store; use crate::store::main::query::query; -use crate::store::models::{Parameter, TransactionMode}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::tests::schema::test_types::StoreDialect; use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; use crate::store::types::Row; +use crate::store_migrations::constants::{POSTGRES_BASELINE_CHECKSUMS, SQLITE_BASELINE_CHECKSUMS}; +use crate::store_migrations::models::StoreMigration; const CHILD_STORE_VARIABLE: &str = "SQLBUILD_TEST_CRASH_STORE"; const ACKNOWLEDGED: &str = "committed "; @@ -119,3 +123,213 @@ pub(crate) fn wal_bytes_after_sequential_writes(path: &Path, commits: usize, byt std::fs::metadata(wal).map_or(0, |metadata| metadata.len()) }) } + +const BASELINE_SQLITE: &str = include_str!("baseline/sqlite.sql"); +const BASELINE_POSTGRES: &str = include_str!("baseline/postgres.sql"); +pub(crate) const BASELINE_ROWS: &str = include_str!("baseline/rows.sql"); +const SHAPE_SQLITE: &str = "SELECT type || ' ' || name || ': ' || COALESCE(sql, '') AS item \ + FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' AND tbl_name <> 'sqb_schema_version'"; +const SHAPE_POSTGRES: &str = "SELECT 'column ' || table_name || '.' || column_name || ' ' || data_type \ + || ' ' || is_nullable || ' ' || COALESCE(column_default, '') AS item \ + FROM information_schema.columns WHERE table_schema = current_schema() \ + AND table_name <> 'sqb_schema_version' \ + UNION ALL SELECT 'constraint ' || conrelid::regclass::text || ' ' || conname || ' ' \ + || pg_get_constraintdef(oid) FROM pg_constraint \ + WHERE connamespace = current_schema()::regnamespace \ + AND conrelid::regclass::text <> 'sqb_schema_version' \ + UNION ALL SELECT 'index ' || indexname || ' ' || replace(indexdef, current_schema() || '.', '') \ + FROM pg_indexes \ + WHERE schemaname = current_schema() AND tablename <> 'sqb_schema_version' \ + UNION ALL SELECT 'view ' || viewname || ' ' || definition FROM pg_views \ + WHERE schemaname = current_schema()"; +/// The SQLite facts the migration contracts use. +pub(crate) const SQLITE_STORE: StoreDialect = StoreDialect { + backend: Backend::Sqlite, + baseline_schema: BASELINE_SQLITE, + baseline_checksum: SQLITE_BASELINE_CHECKSUMS[0], + time_type: "TEXT", + shape_sql: SHAPE_SQLITE, + recreate_command: "rm -f '", +}; +/// The PostgreSQL facts the migration contracts use. +pub(crate) const POSTGRES_STORE: StoreDialect = StoreDialect { + backend: Backend::Postgres, + baseline_schema: BASELINE_POSTGRES, + baseline_checksum: POSTGRES_BASELINE_CHECKSUMS[0], + time_type: "TIMESTAMPTZ", + shape_sql: SHAPE_POSTGRES, + recreate_command: "dropdb -h '", +}; +/// The tables `rows.sql` fills: installation, locations, people, roles, tokens, triggers, run nodes and runs. +pub(crate) const POPULATED_TABLES: [&str; 10] = [ + "sqb_installation", + "sqb_locations", + "sqb_snapshots", + "sqb_principals", + "sqb_role_assignments", + "sqb_access_tokens", + "sqb_triggers", + "sqb_trigger_state", + "sqb_run_nodes", + "sqb_runs", +]; + +/// Run `statements` (separated by blank lines) in one write transaction. +async fn execute_all(store: &ControlStore, statements: &[&str]) { + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("begin write"); + for statement in statements { + (transaction, _) = query(transaction, statement, vec![]) + .await + .unwrap_or_else(|error| panic!("{statement}: {error}")); + } + commit(transaction).await.expect("commit"); +} + +fn blank_line_statements(sql: &str) -> Vec<&str> { + sql.split(";\n\n") + .map(|statement| statement.trim().trim_end_matches(';')) + .filter(|statement| !statement.is_empty()) + .collect() +} + +/// A store as a release before schema versions left it: `dialect`'s baseline schema and checksum, and `rows.sql`. +pub(crate) async fn create_unversioned_baseline( + store: &ControlStore, + dialect: &StoreDialect, + checksum: &str, +) { + let version_table = format!( + "CREATE TABLE sqb_schema_version (singleton BIGINT PRIMARY KEY CHECK (singleton = 1), \ + checksum TEXT NOT NULL, applied_at {} NOT NULL)", + dialect.time_type + ); + let stamp = format!( + "INSERT INTO sqb_schema_version (singleton, checksum, applied_at) \ + VALUES (1, '{checksum}', '2026-11-01T08:00:00Z')" + ); + let installation = "INSERT INTO sqb_installation (singleton, installation_id, created_at) \ + VALUES (1, '018f0000-0000-7000-8000-000000000000', '2026-11-01T08:00:00Z')"; + let (schema, rows) = ( + dialect.baseline_schema.replace("\r\n", "\n"), + BASELINE_ROWS.replace("\r\n", "\n"), + ); + let mut statements: Vec<&str> = vec![version_table.as_str()]; + statements.extend(blank_line_statements(&schema)); + statements.extend([installation, stamp.as_str()]); + statements.extend(blank_line_statements(&rows)); + execute_all(store, &statements).await; +} + +/// Run `sql`'s `;`-separated statements in one write transaction. +pub(crate) async fn execute(store: &ControlStore, sql: &str) { + let statements: Vec<&str> = sql + .split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) + .collect(); + execute_all(store, &statements).await; +} + +/// The store's tables, columns, constraints, indexes and views, normalised so equal schemas compare equal. +pub(crate) async fn schema_shape(store: &ControlStore, dialect: &StoreDialect) -> Vec { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("begin read"); + let (transaction, rows) = query(transaction, dialect.shape_sql, vec![]) + .await + .expect("read schema"); + commit(transaction).await.expect("finish read"); + let mut shape: Vec = rows + .iter() + .map(|row| { + let item: String = text(row, "item").replace('"', ""); + let words: Vec<&str> = item.split_whitespace().collect(); + words + .join(" ") + .replace("( ", "(") + .replace(" )", ")") + .replace(" ,", ",") + }) + .collect(); + shape.sort(); + shape +} + +/// Every row of `tables`, in primary-key order. +pub(crate) async fn table_rows(store: &ControlStore, tables: &[&str]) -> Vec> { + let mut transaction = begin(store, TransactionMode::Read) + .await + .expect("begin read"); + let mut all: Vec> = Vec::new(); + for table in tables { + let rows: Vec; + (transaction, rows) = query( + transaction, + &format!("SELECT * FROM {table} ORDER BY 1"), + vec![], + ) + .await + .expect("read table"); + all.push(rows); + } + commit(transaction).await.expect("finish read"); + all +} + +/// The store's recorded schema version. +pub(crate) async fn recorded_version(store: &ControlStore) -> Option { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("begin read"); + let (transaction, rows) = query(transaction, "SELECT * FROM sqb_schema_version", vec![]) + .await + .expect("read version"); + commit(transaction).await.expect("finish read"); + rows.first() + .and_then(|row| row.get("version")) + .and_then(serde_json::Value::as_i64) +} + +/// SQLite statements rebuilding `sqb_runs` with `skipped` added to its status check, and its indexes; empty on PostgreSQL. +pub(crate) async fn rebuilt_runs_table(store: &ControlStore) -> &'static str { + let ControlStore::Sqlite(_) = store else { + return ""; + }; + let transaction = begin(store, TransactionMode::Read) + .await + .expect("begin read"); + let (transaction, rows) = query( + transaction, + "SELECT type, sql FROM sqlite_master WHERE tbl_name = 'sqb_runs' AND sql IS NOT NULL \ + ORDER BY type DESC", + vec![], + ) + .await + .expect("read sqb_runs definition"); + commit(transaction).await.expect("finish read"); + let mut definitions = rows.iter().map(|row| text(row, "sql")); + let table = definitions + .next() + .expect("sqb_runs table") + .replacen("CREATE TABLE sqb_runs", "CREATE TABLE sqb_runs_rebuilt", 1) + .replace("'cancelled'))", "'cancelled', 'skipped'))"); + let indexes: Vec = definitions.collect(); + let sql = format!( + "{table};\nINSERT INTO sqb_runs_rebuilt SELECT * FROM sqb_runs;\nDROP TABLE sqb_runs;\n\ + ALTER TABLE sqb_runs_rebuilt RENAME TO sqb_runs;\n{}", + indexes.join(";\n") + ); + Box::leak(sql.into_boxed_str()) +} + +/// The numbered names a migration plan lists for `migrations` after version `from`. +pub(crate) fn migration_names(migrations: &[StoreMigration], from: i64) -> Vec { + migrations + .iter() + .skip(usize::try_from(from - 1).expect("version at or after the baseline")) + .zip(from + 1..) + .map(|(migration, number)| format!("{number:04}_{}", migration.name)) + .collect() +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs index 6a1a10b378..325eaa9c1e 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_contracts.rs @@ -6,6 +6,8 @@ pub(crate) mod helpers; mod test_crash_recovery; #[path = "test_foreign_key_indexes.rs"] mod test_foreign_key_indexes; +#[path = "test_store_migrations.rs"] +mod test_store_migrations; #[path = "test_store_upgrade.rs"] mod test_store_upgrade; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs new file mode 100644 index 0000000000..fdcdeeccd5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs @@ -0,0 +1,460 @@ +//! Numbered control-store migrations: baseline upgrades, table rebuilds, refusals and the dry run. + +use crate::store::constants::{INDEXES, SCHEMA}; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::schema::helpers::{ + BASELINE_ROWS, POPULATED_TABLES, POSTGRES_STORE, SQLITE_STORE, create_unversioned_baseline, + execute, migration_names, rebuilt_runs_table, recorded_version, schema_shape, table_rows, +}; +use crate::store::tests::schema::test_types::{ + BaselineUpgradeTestCase, BrokenMigrationTestCase, MigrationRefusalTestCase, + PinnedChecksumTestCase, StoreMigrationPlanTestCase, TableRebuildTestCase, +}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store_migrations::_helpers::migration::{ + MIGRATIONS, current_version, migration_plan, schema_digest, schema_text_digest, +}; +use crate::store_migrations::_helpers::migration_run::migrate_with; +use crate::store_migrations::_helpers::recall_schema::recall_schema; +use crate::store_migrations::main::plan_store_migration::plan_store_migration; +use crate::store_migrations::models::{StoreMigration, StoreMigrationPlan}; + +const SKIPPED_RUN: &str = "INSERT INTO sqb_runs (run_id, location_id, target, snapshot_id, kind, \ + origin, root_run_id, parent_run_id, requested_by, selection_json, request_json, \ + write_set_digest, status, queued_at, node_counts_json, last_event_storage_id) VALUES \ + ('018f0000-0000-7000-8000-000000000008', '018f0000-0000-7000-8000-000000000001', 'prod', \ + '018f0000-0000-7000-8000-000000000002', 'job', 'manual', \ + '018f0000-0000-7000-8000-000000000007', '018f0000-0000-7000-8000-000000000007', 'admin', \ + '[]', '{}', 'digest-2', 'skipped', '2026-11-02T10:00:00Z', '{}', 13)"; +const SQLITE_CHECKSUMS: &[(i64, &str)] = &[( + 1, + "6d0e581f453370ec5156053d644dea3b22b19332cee588eb69ac3141ada5a76b", +)]; +const POSTGRES_CHECKSUMS: &[(i64, &str)] = &[( + 1, + "2a81e526eb1181eeab0b897e8b3f48ae2dd9425d85cb788874670d942bbc76c4", +)]; +const DANGLING_RUN_NODE: &str = "INSERT INTO sqb_run_nodes (run_id, node_key, status, writes) \ + VALUES ('018f0000-0000-7000-8000-0000000000ff', 'model.ghost', 'success', TRUE)"; +const WIDEN_RUN_STATUS_POSTGRES: &str = "ALTER TABLE sqb_runs DROP CONSTRAINT sqb_runs_status_check; \ + ALTER TABLE sqb_runs ADD CONSTRAINT sqb_runs_status_check CHECK (status IN ('queued', \ + 'launched', 'running', 'unresponsive', 'success', 'warn', 'partial', 'failed', 'cancelled', \ + 'skipped'))"; + +#[test] +fn given_store_at_the_baseline_when_migrating_then_it_matches_a_new_store_and_keeps_its_rows() { + let test_cases = [ + BaselineUpgradeTestCase { + description: "SQLite store from a Linux or macOS release", + dialect: &SQLITE_STORE, + recorded_checksum: "228a951fe04d75eedcefdb44095afe652b178d9b2f32f3a5287c143877d39ce2", + expected_version: current_version(MIGRATIONS), + }, + BaselineUpgradeTestCase { + description: "SQLite store from a Windows release, built from CRLF SQL", + dialect: &SQLITE_STORE, + recorded_checksum: "4cb5c0c4f392a2b3f4f2229e2ab8401870cd37b65346ada92cc70aa3ca2e0542", + expected_version: current_version(MIGRATIONS), + }, + BaselineUpgradeTestCase { + description: "PostgreSQL store from a Linux or macOS release", + dialect: &POSTGRES_STORE, + recorded_checksum: "163045e32da43a8b0ddf17ea22f6dd41d1ff7bffe81836f3cbb35a8754915c3e", + expected_version: current_version(MIGRATIONS), + }, + BaselineUpgradeTestCase { + description: "PostgreSQL store from a Windows release, built from CRLF SQL", + dialect: &POSTGRES_STORE, + recorded_checksum: "90758add355967a65ddfcf233b5e66893f92449e17c84e8f9bab42fb703917ab", + expected_version: current_version(MIGRATIONS), + }, + ]; + for test_case in test_cases { + run(async { + let (Some(upgraded), Some(created)) = ( + open_test_store(test_case.dialect.backend).await, + open_test_store(test_case.dialect.backend).await, + ) else { + return; + }; + create_unversioned_baseline( + &upgraded.store, + test_case.dialect, + test_case.recorded_checksum, + ) + .await; + let before = table_rows(&upgraded.store, &POPULATED_TABLES).await; + + migrate_store(&upgraded.store) + .await + .expect("upgrade the baseline store"); + migrate_store(&created.store).await.expect("create a store"); + + assert_eq!( + ( + schema_shape(&upgraded.store, test_case.dialect).await, + table_rows(&upgraded.store, &POPULATED_TABLES).await, + recorded_version(&upgraded.store).await, + ), + ( + schema_shape(&created.store, test_case.dialect).await, + before, + Some(test_case.expected_version), + ), + "{}: a schema change needs a numbered migration that brings the baseline to it, \ + and upgrading keeps every row", + test_case.description + ); + close_test_store(upgraded).await; + close_test_store(created).await; + }); + } +} + +#[test] +fn given_a_migration_rebuilding_a_referenced_table_when_migrating_then_rows_and_references_survive() +{ + let test_cases = [ + TableRebuildTestCase { + description: "SQLite store", + dialect: &SQLITE_STORE, + expected_runs: 2, + expected_version: 2, + }, + TableRebuildTestCase { + description: "PostgreSQL store", + dialect: &POSTGRES_STORE, + expected_runs: 2, + expected_version: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.dialect.backend).await else { + return; + }; + let store = &fixture.store; + migrate_with(store, &[]).await.expect("create the store"); + execute(store, BASELINE_ROWS).await; + let mut before = table_rows(store, &POPULATED_TABLES).await; + before.pop(); + let shape_before: Vec = schema_shape(store, test_case.dialect) + .await + .into_iter() + .filter(|item| !item.contains("'cancelled'")) + .collect(); + let migration = StoreMigration { + name: "widen_run_status", + sqlite: rebuilt_runs_table(store).await, + postgres: WIDEN_RUN_STATUS_POSTGRES, + }; + + migrate_with(store, &[migration]) + .await + .expect("rebuild sqb_runs"); + execute(store, SKIPPED_RUN).await; + + let shape_after: Vec = schema_shape(store, test_case.dialect) + .await + .into_iter() + .filter(|item| !item.contains("'cancelled'")) + .collect(); + let mut rows = table_rows(store, &POPULATED_TABLES).await; + let runs = rows.pop().expect("sqb_runs rows"); + assert_eq!( + (rows, runs.len(), shape_after, recorded_version(store).await), + ( + before, + test_case.expected_runs, + shape_before, + Some(test_case.expected_version) + ), + "{}: rows, other schema objects and version after the rebuild", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_store_this_build_cannot_upgrade_when_migrating_then_it_is_refused_unchanged() { + let test_cases = [ + MigrationRefusalTestCase { + description: "SQLite store from a newer build", + dialect: &SQLITE_STORE, + tamper: "UPDATE sqb_schema_version SET version = 99", + expected_fragments: vec!["schema is version 99, newer than this build's version"], + }, + MigrationRefusalTestCase { + description: "SQLite store older than the baseline", + dialect: &SQLITE_STORE, + tamper: "UPDATE sqb_schema_version SET checksum = 'unknown', version = NULL", + expected_fragments: vec!["older than version 1", SQLITE_STORE.recreate_command], + }, + MigrationRefusalTestCase { + description: "SQLite store whose schema changed without a migration", + dialect: &SQLITE_STORE, + tamper: "UPDATE sqb_schema_version SET checksum = 'unknown'", + expected_fragments: vec![ + "doesn't match this build's, although both are version", + SQLITE_STORE.recreate_command, + ], + }, + MigrationRefusalTestCase { + description: "PostgreSQL store from a newer build", + dialect: &POSTGRES_STORE, + tamper: "UPDATE sqb_schema_version SET version = 99", + expected_fragments: vec!["schema is version 99, newer than this build's version"], + }, + MigrationRefusalTestCase { + description: "PostgreSQL store older than the baseline", + dialect: &POSTGRES_STORE, + tamper: "UPDATE sqb_schema_version SET checksum = 'unknown', version = NULL", + expected_fragments: vec!["older than version 1", POSTGRES_STORE.recreate_command], + }, + MigrationRefusalTestCase { + description: "PostgreSQL store whose schema changed without a migration", + dialect: &POSTGRES_STORE, + tamper: "UPDATE sqb_schema_version SET checksum = 'unknown'", + expected_fragments: vec![ + "doesn't match this build's, although both are version", + POSTGRES_STORE.recreate_command, + ], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.dialect.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("create the store"); + execute(store, test_case.tamper).await; + let version_before = recorded_version(store).await; + + let error = migrate_store(store) + .await + .expect_err("refuse the store") + .to_string(); + + let found: Vec<&str> = test_case + .expected_fragments + .iter() + .copied() + .filter(|fragment| error.contains(fragment)) + .collect(); + assert_eq!( + (found, recorded_version(store).await), + (test_case.expected_fragments.clone(), version_before), + "{}: {error}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_store_when_planning_its_migration_then_the_plan_is_reported_without_changing_it() { + let current = current_version(MIGRATIONS); + let expected_plans = || { + ( + StoreMigrationPlan::Create { version: current }, + StoreMigrationPlan::Upgrade { + from: 1, + to: current, + names: migration_names(MIGRATIONS, 1), + }, + None, + ) + }; + let expected_after = || { + ( + StoreMigrationPlan::Current { version: current }, + StoreMigrationPlan::Upgrade { + from: current, + to: current + 1, + names: vec![format!("{:04}_widen_run_status", current + 1)], + }, + ) + }; + let test_cases = [ + StoreMigrationPlanTestCase { + description: "SQLite store", + dialect: &SQLITE_STORE, + expected_plans: expected_plans(), + expected_after: expected_after(), + }, + StoreMigrationPlanTestCase { + description: "PostgreSQL store", + dialect: &POSTGRES_STORE, + expected_plans: expected_plans(), + expected_after: expected_after(), + }, + ]; + for test_case in test_cases { + run(async { + let (Some(empty), Some(baseline)) = ( + open_test_store(test_case.dialect.backend).await, + open_test_store(test_case.dialect.backend).await, + ) else { + return; + }; + create_unversioned_baseline( + &baseline.store, + test_case.dialect, + test_case.dialect.baseline_checksum, + ) + .await; + + let plans = ( + plan_store_migration(&empty.store) + .await + .expect("plan an empty store"), + plan_store_migration(&baseline.store) + .await + .expect("plan the baseline store"), + recorded_version(&baseline.store).await, + ); + migrate_store(&baseline.store) + .await + .expect("upgrade the baseline store"); + let mut with_pending: Vec = MIGRATIONS.to_vec(); + with_pending.push(StoreMigration { + name: "widen_run_status", + sqlite: "", + postgres: "", + }); + let after = ( + plan_store_migration(&baseline.store) + .await + .expect("plan the upgraded store"), + migration_plan( + &baseline.store, + Some((schema_digest(&baseline.store).as_str(), Some(current))), + &with_pending, + ) + .expect("plan a pending migration"), + ); + + assert_eq!( + (plans, after), + (test_case.expected_plans, test_case.expected_after), + "{}", + test_case.description + ); + close_test_store(empty).await; + close_test_store(baseline).await; + }); + } +} + +#[test] +fn given_this_builds_schema_when_checksummed_then_it_matches_the_checksum_pinned_for_its_version() { + let test_cases = [ + PinnedChecksumTestCase { + description: "SQLite store", + dialect: &SQLITE_STORE, + line_ending: "\n", + expected_checksums: SQLITE_CHECKSUMS, + }, + PinnedChecksumTestCase { + description: "SQLite store built from a CRLF checkout", + dialect: &SQLITE_STORE, + line_ending: "\r\n", + expected_checksums: SQLITE_CHECKSUMS, + }, + PinnedChecksumTestCase { + description: "PostgreSQL store", + dialect: &POSTGRES_STORE, + line_ending: "\n", + expected_checksums: POSTGRES_CHECKSUMS, + }, + PinnedChecksumTestCase { + description: "PostgreSQL store built from a CRLF checkout", + dialect: &POSTGRES_STORE, + line_ending: "\r\n", + expected_checksums: POSTGRES_CHECKSUMS, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.dialect.backend).await else { + return; + }; + let current = current_version(MIGRATIONS); + let checked_out = |sql: &str| { + sql.replace("\r\n", "\n") + .replace('\n', test_case.line_ending) + }; + let checksum: String = schema_text_digest( + &recall_schema(&fixture.store).join(";\n"), + &checked_out(SCHEMA), + &checked_out(INDEXES), + ); + + assert_eq!( + Some(checksum.as_str()), + test_case + .expected_checksums + .iter() + .find(|(version, _)| *version == current) + .map(|(_, pinned)| *pinned), + "{}: schema.sql, indexes.sql or the Recall tables changed without a numbered \ + migration; add one and pin the new version's checksum, never edit a pinned one", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_migration_that_breaks_a_reference_when_migrating_then_it_is_refused_unchanged() { + let test_cases = [ + BrokenMigrationTestCase { + description: "SQLite store", + dialect: &SQLITE_STORE, + expected_fragment: "foreign key references; the store is unchanged", + expected_version: Some(1), + }, + BrokenMigrationTestCase { + description: "PostgreSQL store", + dialect: &POSTGRES_STORE, + expected_fragment: "violates foreign key constraint", + expected_version: Some(1), + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.dialect.backend).await else { + return; + }; + let store = &fixture.store; + migrate_with(store, &[]).await.expect("create the store"); + let migration = StoreMigration { + name: "dangling_run_node", + sqlite: DANGLING_RUN_NODE, + postgres: DANGLING_RUN_NODE, + }; + + let error = migrate_with(store, &[migration]) + .await + .expect_err("refuse the migration") + .to_string(); + + assert_eq!( + ( + error.contains(test_case.expected_fragment), + recorded_version(store).await, + table_rows(store, &["sqb_run_nodes"]).await, + ), + (true, test_case.expected_version, vec![Vec::new()]), + "{}: {error}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs index 65f75293d7..a22fbbc5ab 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_types.rs @@ -3,6 +3,7 @@ use std::collections::BTreeSet; use crate::store::tests::store_test_types::Backend; +use crate::store_migrations::models::StoreMigrationPlan; pub(crate) struct CrashRecoveryTestCase { pub(crate) description: &'static str, @@ -34,3 +35,58 @@ pub(crate) struct StoreUpgradeTestCase { pub(crate) list_indexes: &'static str, pub(crate) expected_indexes: Vec<&'static str>, } + +/// What the migration contracts need to know about one store type. +pub(crate) struct StoreDialect { + pub(crate) backend: Backend, + pub(crate) baseline_schema: &'static str, + pub(crate) baseline_checksum: &'static str, + pub(crate) time_type: &'static str, + pub(crate) shape_sql: &'static str, + pub(crate) recreate_command: &'static str, +} + +pub(crate) struct BaselineUpgradeTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static StoreDialect, + /// The checksum a release before versioning recorded for the baseline schema. + pub(crate) recorded_checksum: &'static str, + pub(crate) expected_version: i64, +} + +pub(crate) struct TableRebuildTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static StoreDialect, + pub(crate) expected_runs: usize, + pub(crate) expected_version: i64, +} + +pub(crate) struct MigrationRefusalTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static StoreDialect, + pub(crate) tamper: &'static str, + pub(crate) expected_fragments: Vec<&'static str>, +} + +pub(crate) struct StoreMigrationPlanTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static StoreDialect, + pub(crate) expected_plans: (StoreMigrationPlan, StoreMigrationPlan, Option), + pub(crate) expected_after: (StoreMigrationPlan, StoreMigrationPlan), +} + +pub(crate) struct PinnedChecksumTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static StoreDialect, + /// The line endings Git checked the SQL files out with. + pub(crate) line_ending: &'static str, + /// One `(version, checksum)` per schema version: append one with each migration and never edit one. + pub(crate) expected_checksums: &'static [(i64, &'static str)], +} + +pub(crate) struct BrokenMigrationTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static StoreDialect, + pub(crate) expected_fragment: &'static str, + pub(crate) expected_version: Option, +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index f04ae8e5bb..2246f55bc7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -177,9 +177,37 @@ sqlite3 control.sqlite3 ".backup '/backups/control.sqlite3'" ## Upgrade -Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the -server after a location or worker has connected from a newer release; the server can't read what -they wrote. +Upgrade the server first, then the locations and workers that connect to it. On start, the server +migrates its control database to the schema of its release in one transaction, keeping people, +roles, tokens, triggers and run history. Releases with an older schema refuse the database +afterwards, so back it up before you upgrade (see +[Back up the control database](#back-up-the-control-database)). To roll back, restore that backup; +runs recorded since the upgrade are lost. + +To see what an upgrade changes before you start the server, run the new release with `--dry-run`: + +```bash +sqb server migrate --dry-run +``` + +```text +The control store would migrate from schema version 1 to 2: + 0002_add_skipped_runs +``` + +`sqb server migrate` without `--dry-run` applies the migrations and exits. It takes the same +`--database-env` and `--config` options as `sqb server`, given before `migrate`. + +The server refuses a control database it can't migrate and leaves it unchanged: + +| Control database | Fix | +| --- | --- | +| Written by a newer release | Run the release that last opened it, or a later one. | +| Older than the oldest schema this release migrates | Open it once with an earlier release that still migrates it, then with this one. Or start a new, empty store with the command in the message, which discards its history. | +| Same schema version but a different schema, as from a development build | Start a new, empty store with the command in the message. | + +Don't downgrade the server after a location or worker has connected from a newer release; the +server can't read what they wrote. | Newer than the server | What happens | Fix | | --- | --- | --- | diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 780a271d22..959a12b84b 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -159,9 +159,37 @@ sqlite3 control.sqlite3 ".backup '/backups/control.sqlite3'" ## Upgrade -Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the -server after a location or worker has connected from a newer release; the server can't read what -they wrote. +Upgrade the server first, then the locations and workers that connect to it. On start, the server +migrates its control database to the schema of its release in one transaction, keeping people, +roles, tokens, triggers and run history. Releases with an older schema refuse the database +afterwards, so back it up before you upgrade (see +[Back up the control database](#back-up-the-control-database)). To roll back, restore that backup; +runs recorded since the upgrade are lost. + +To see what an upgrade changes before you start the server, run the new release with `--dry-run`: + +```bash +sqb server migrate --dry-run +``` + +```text +The control store would migrate from schema version 1 to 2: + 0002_add_skipped_runs +``` + +`sqb server migrate` without `--dry-run` applies the migrations and exits. It takes the same +`--database-env` and `--config` options as `sqb server`, given before `migrate`. + +The server refuses a control database it can't migrate and leaves it unchanged: + +| Control database | Fix | +| --- | --- | +| Written by a newer release | Run the release that last opened it, or a later one. | +| Older than the oldest schema this release migrates | Open it once with an earlier release that still migrates it, then with this one. Or start a new, empty store with the command in the message, which discards its history. | +| Same schema version but a different schema, as from a development build | Start a new, empty store with the command in the message. | + +Don't downgrade the server after a location or worker has connected from a newer release; the +server can't read what they wrote. | Newer than the server | What happens | Fix | | --- | --- | --- |