diff --git a/.gitignore b/.gitignore index 249dfe1fa1..d554ddd578 100644 --- a/.gitignore +++ b/.gitignore @@ -56,7 +56,6 @@ target/ *.so *.pyd *.dylib -.dlt/ .amp/portals/* /.env diff --git a/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs index 0f33cbb1ab..94380de640 100644 --- a/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs +++ b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs @@ -6,7 +6,7 @@ use crate::python_plan::main::upstream_closure::upstream_closure; use crate::python_plan::models::{LifecycleNode, RunLifecycle}; use crate::python_selection::constants::{ASSET_KIND, LOADER_KIND, TASK_KIND}; -/// Ingress is every selected warehouse loader and its selected upstream tasks, assets and loaders. +/// Ingress is every selected loader and its selected upstream tasks, assets and loaders. #[must_use] pub fn run_lifecycle( selected: &BTreeSet, @@ -19,7 +19,7 @@ pub fn run_lifecycle( .filter(|name| { nodes .get(name.as_str()) - .is_some_and(|node| node.kind == LOADER_KIND && !node.external_loader) + .is_some_and(|node| node.kind == LOADER_KIND) }) .cloned() .collect(); diff --git a/crates/sqlbuild-analysis/src/python_plan/models.rs b/crates/sqlbuild-analysis/src/python_plan/models.rs index 21b5825300..9d095763a8 100644 --- a/crates/sqlbuild-analysis/src/python_plan/models.rs +++ b/crates/sqlbuild-analysis/src/python_plan/models.rs @@ -5,8 +5,6 @@ pub struct LifecycleNode { /// `task`, `asset`, `loader` or `check`. pub kind: String, - /// Whether the node is a loader that owns its own warehouse connection. - pub external_loader: bool, } /// Selected Python nodes split into ingress and read-side regions, each sorted. diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs index 4489f0f16c..f26e0d968c 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs @@ -13,14 +13,13 @@ pub(crate) fn edges(rows: &[(&str, &[&str])]) -> HashMap> { .collect() } -pub(crate) fn lifecycle_nodes(rows: &[(&str, &str, bool)]) -> HashMap { +pub(crate) fn lifecycle_nodes(rows: &[(&str, &str)]) -> HashMap { rows.iter() - .map(|(name, kind, external_loader)| { + .map(|(name, kind)| { ( (*name).to_owned(), LifecycleNode { kind: (*kind).to_owned(), - external_loader: *external_loader, }, ) }) diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs index c79fd71b91..f37f49ba39 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs @@ -8,13 +8,13 @@ use crate::python_plan::tests::test_types::LifecycleTestCase; fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { let test_cases = [ LifecycleTestCase { - description: "a warehouse loader pulls its selected upstream tasks into ingress", + description: "a loader pulls its selected upstream tasks into ingress", selected: &["fetch_rates", "load_orders", "report", "check_orders"], nodes: &[ - ("fetch_rates", "task", false), - ("load_orders", "loader", false), - ("report", "asset", false), - ("check_orders", "check", false), + ("fetch_rates", "task"), + ("load_orders", "loader"), + ("report", "asset"), + ("check_orders", "check"), ], upstream: &[("load_orders", &["fetch_rates", "unselected"])], expected_ingress: &["fetch_rates", "load_orders"], @@ -22,24 +22,12 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { expected_read_side: &["report"], }, LifecycleTestCase { - description: "external loaders are neither ingress loaders nor read-side nodes", - selected: &["sync_orders", "export"], - nodes: &[("sync_orders", "loader", true), ("export", "task", false)], - upstream: &[("export", &["sync_orders"])], - expected_ingress: &[], - expected_loaders: &[], - expected_read_side: &["export"], - }, - LifecycleTestCase { - description: "a selected external loader upstream of a warehouse loader is ingress", + description: "a selected loader upstream of another loader is ingress", selected: &["sync_orders", "load_orders"], - nodes: &[ - ("sync_orders", "loader", true), - ("load_orders", "loader", false), - ], + nodes: &[("sync_orders", "loader"), ("load_orders", "loader")], upstream: &[("load_orders", &["sync_orders"])], expected_ingress: &["load_orders", "sync_orders"], - expected_loaders: &["load_orders"], + expected_loaders: &["load_orders", "sync_orders"], expected_read_side: &[], }, ]; diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs index 7a4dd8f19a..91e2c9879c 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs @@ -15,8 +15,8 @@ pub(crate) struct ClosureTestCase { pub(crate) struct LifecycleTestCase { pub(crate) description: &'static str, pub(crate) selected: &'static [&'static str], - /// `(name, kind, external loader)`. - pub(crate) nodes: &'static [(&'static str, &'static str, bool)], + /// `(name, kind)`. + pub(crate) nodes: &'static [(&'static str, &'static str)], pub(crate) upstream: &'static [(&'static str, &'static [&'static str])], pub(crate) expected_ingress: &'static [&'static str], pub(crate) expected_loaders: &'static [&'static str], diff --git a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs index 8eaa9be8ec..481848e709 100644 --- a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs +++ b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs @@ -1,7 +1,7 @@ //! The Python shape names of the natively hosted compiler shapes. /// The Python model type a natively hosted compiler shape is captured as. -pub const HOSTED_SHAPE_TYPES: [(&str, &str); 490] = [ +pub const HOSTED_SHAPE_TYPES: [(&str, &str); 488] = [ ( "sqlbuild._native:AlignedInterval", "sqlbuild.cursor_algebra.models:AlignedInterval", @@ -822,10 +822,6 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 490] = [ "sqlbuild._native:IntegerValue", "sqlbuild.cursor_algebra.models:IntegerValue", ), - ( - "sqlbuild._native:IntegrationLoaderConfig", - "sqlbuild.spec.contracts.models:IntegrationLoaderConfig", - ), ( "sqlbuild._native:InterpolationSite", "sqlbuild.lint.models:InterpolationSite", @@ -900,10 +896,6 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 490] = [ "sqlbuild._native:LoadedMacro", "sqlbuild.compiler.compile.models:LoadedMacro", ), - ( - "sqlbuild._native:LoaderConnectionMode", - "sqlbuild.compiler.discovery.types:LoaderConnectionMode", - ), ( "sqlbuild._native:LoaderDestinationParts", "sqlbuild.spec.contracts.models:LoaderDestinationParts", diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs index 19b556e8a7..5af09aa592 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs @@ -258,11 +258,10 @@ pub(crate) fn python_entry_lines( } fn source_load_label(entry: &SourceLoadPlanText) -> String { - let strategy: String = match (&entry.integration_kind, &entry.write_strategy) { - (Some(kind), _) => format!("external ({kind})"), - (None, Some(strategy)) => strategy.clone(), - (None, None) => "self-managed".to_owned(), - }; + let strategy: String = entry + .write_strategy + .clone() + .unwrap_or_else(|| "self-managed".to_owned()); let mut details: Vec = Vec::new(); if let Some(column) = &entry.cursor_column { details.push(format!("cursor: {column}")); diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 2dbb54cc38..a066c58fb9 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -349,7 +349,6 @@ pub struct SourceLoadPlanText { /// `ExecutionResourceKind` value; only `loader` and `source` loads are listed. pub resource_kind: String, pub is_reload: bool, - pub integration_kind: Option, pub write_strategy: Option, pub cursor_column: Option, pub unique_key: Vec, diff --git a/crates/sqlbuild-core/src/dlt_destination/_helpers/credentials.rs b/crates/sqlbuild-core/src/dlt_destination/_helpers/credentials.rs deleted file mode 100644 index 08bdb0c93a..0000000000 --- a/crates/sqlbuild-core/src/dlt_destination/_helpers/credentials.rs +++ /dev/null @@ -1,173 +0,0 @@ -//! Destination credentials per warehouse adapter. - -use crate::dlt_destination::constants::{ - BIGQUERY_ADAPTER, BIGQUERY_ARGUMENTS, CREDENTIALS_ARGUMENT, DATABRICKS_ADAPTER, DLT_LOADER, - MOTHERDUCK_ADAPTER, POSTGRES_ADAPTER, SNOWFLAKE_ACCOUNT_SUFFIX, SNOWFLAKE_ADAPTER, - SNOWFLAKE_OPTIONAL_SETTINGS, SQLSERVER_ADAPTER, SQLSERVER_DEFAULT_PORT, SQLSERVER_FACTORY, -}; -use crate::dlt_destination::errors::DltDestinationError; -use crate::dlt_destination::models::DltValue; -use crate::loader_connection::constants::{DEFAULT_HOST, MOTHERDUCK_PREFIX, POSTGRES_DEFAULT_PORT}; -use crate::loader_connection::main::connection_setting::connection_setting; -use crate::loader_connection::main::required_connection_setting::required_connection_setting; -use crate::loader_connection::models::LoaderConnection; - -/// Factory arguments, in call order. -type Arguments = Vec<(&'static str, DltValue)>; - -/// A required stripped connection setting of `adapter`. -pub(crate) fn required<'a>( - connection: &'a LoaderConnection, - key: &str, - adapter: &str, -) -> Result<&'a str, DltDestinationError> { - required_connection_setting(connection, key, adapter, DLT_LOADER) - .map_err(|message| DltDestinationError { message }) -} - -fn text(value: &str) -> DltValue { - DltValue::Text(value.to_owned()) -} - -/// The single `credentials` argument. -pub(crate) fn credentials(value: DltValue) -> Arguments { - vec![(CREDENTIALS_ARGUMENT, value)] -} - -fn snowflake_account(account: &str) -> &str { - account - .strip_suffix(SNOWFLAKE_ACCOUNT_SUFFIX) - .unwrap_or(account) -} - -fn motherduck(connection: &LoaderConnection) -> DltValue { - let database: &str = connection_setting(connection, "database").unwrap_or_default(); - let mut credentials: String = if database.starts_with(MOTHERDUCK_PREFIX) { - database.to_owned() - } else { - format!("{MOTHERDUCK_PREFIX}{database}") - }; - if let Some(token) = connection_setting(connection, "token") { - credentials = format!("{credentials}?motherduck_token={token}"); - } - DltValue::Text(credentials) -} - -fn postgres(connection: &LoaderConnection) -> Result { - let host: &str = connection_setting(connection, "host").unwrap_or(DEFAULT_HOST); - let port: &str = connection.port.as_deref().unwrap_or(POSTGRES_DEFAULT_PORT); - let user: &str = required(connection, "user", POSTGRES_ADAPTER)?; - let password: &str = connection_setting(connection, "password").unwrap_or_default(); - let database: &str = match connection_setting(connection, "dbname") { - Some(database) => database, - None => required(connection, "database", POSTGRES_ADAPTER)?, - }; - Ok(DltValue::Text(format!( - "postgresql://{user}:{password}@{host}:{port}/{database}" - ))) -} - -fn snowflake(connection: &LoaderConnection) -> Result { - let account: &str = required(connection, "account", SNOWFLAKE_ADAPTER)?; - let host: &str = connection_setting(connection, "host").unwrap_or(account); - let mut fields: Arguments = vec![ - ("account", text(snowflake_account(account))), - ("host", text(snowflake_account(host))), - ( - "user", - text(required(connection, "user", SNOWFLAKE_ADAPTER)?), - ), - ( - "database", - text(required(connection, "database", SNOWFLAKE_ADAPTER)?), - ), - ( - "warehouse", - text(required(connection, "warehouse", SNOWFLAKE_ADAPTER)?), - ), - ]; - for key in SNOWFLAKE_OPTIONAL_SETTINGS { - if let Some(value) = connection_setting(connection, key) { - fields.push((key, text(value))); - } - } - Ok(DltValue::Object(fields)) -} - -fn bigquery(connection: &LoaderConnection) -> Arguments { - let mut arguments: Arguments = Vec::new(); - for (argument, key) in BIGQUERY_ARGUMENTS { - if let Some(value) = connection_setting(connection, key) { - arguments.push((argument, text(value))); - } - } - arguments -} - -fn databricks(connection: &LoaderConnection) -> Result { - let mut fields: Arguments = vec![ - ( - "server_hostname", - text(required(connection, "server_hostname", DATABRICKS_ADAPTER)?), - ), - ( - "http_path", - text(required(connection, "http_path", DATABRICKS_ADAPTER)?), - ), - ( - "access_token", - text(required(connection, "token", DATABRICKS_ADAPTER)?), - ), - ]; - if let Some(catalog) = connection_setting(connection, "catalog") { - fields.push(("catalog", text(catalog))); - } - Ok(DltValue::Object(fields)) -} - -fn sqlserver(connection: &LoaderConnection) -> Result { - Ok(DltValue::Object(vec![ - ( - "host", - text(connection_setting(connection, "host").unwrap_or(DEFAULT_HOST)), - ), - ( - "port", - DltValue::Port { - default: SQLSERVER_DEFAULT_PORT, - }, - ), - ( - "database", - text(required(connection, "database", SQLSERVER_ADAPTER)?), - ), - ( - "username", - text(required(connection, "user", SQLSERVER_ADAPTER)?), - ), - ( - "password", - text(connection_setting(connection, "password").unwrap_or_default()), - ), - ( - "query", - DltValue::Object(vec![("TrustServerCertificate", text("yes"))]), - ), - ])) -} - -/// The factory and arguments of a non-DuckDB adapter, or `None` when dlt cannot load into it. -pub(crate) fn adapter_call( - adapter: &str, - connection: &LoaderConnection, -) -> Result, DltDestinationError> { - Ok(Some(match adapter { - MOTHERDUCK_ADAPTER => (MOTHERDUCK_ADAPTER, credentials(motherduck(connection))), - POSTGRES_ADAPTER => (POSTGRES_ADAPTER, credentials(postgres(connection)?)), - SNOWFLAKE_ADAPTER => (SNOWFLAKE_ADAPTER, credentials(snowflake(connection)?)), - BIGQUERY_ADAPTER => (BIGQUERY_ADAPTER, bigquery(connection)), - DATABRICKS_ADAPTER => (DATABRICKS_ADAPTER, credentials(databricks(connection)?)), - SQLSERVER_ADAPTER => (SQLSERVER_FACTORY, credentials(sqlserver(connection)?)), - _ => return Ok(None), - })) -} diff --git a/crates/sqlbuild-core/src/dlt_destination/_helpers/mod.rs b/crates/sqlbuild-core/src/dlt_destination/_helpers/mod.rs deleted file mode 100644 index 53f1c45a32..0000000000 --- a/crates/sqlbuild-core/src/dlt_destination/_helpers/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub(crate) mod credentials; -pub(crate) mod request; diff --git a/crates/sqlbuild-core/src/dlt_destination/_helpers/request.rs b/crates/sqlbuild-core/src/dlt_destination/_helpers/request.rs deleted file mode 100644 index 91f9b45ff2..0000000000 --- a/crates/sqlbuild-core/src/dlt_destination/_helpers/request.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! Destination config and dataset rules checked before any adapter credentials. - -use crate::dlt_destination::constants::SQLBUILD_OWNED_DESTINATION_KEYS; -use crate::dlt_destination::errors::DltDestinationError; -use crate::dlt_destination::models::DltDatasetName; -use crate::text::main::python_strip::python_strip; - -/// Reject a destination config that sets keys SQLBuild owns. -pub(crate) fn check_destination_keys( - adapter: &str, - destination_keys: &[Option], -) -> Result<(), DltDestinationError> { - let mut blocked: Vec<&str> = destination_keys - .iter() - .flatten() - .map(String::as_str) - .filter(|key| SQLBUILD_OWNED_DESTINATION_KEYS.contains(key)) - .collect(); - if blocked.is_empty() { - return Ok(()); - } - blocked.sort_unstable(); - Err(DltDestinationError { - message: format!( - "Adapter '{adapter}' dlt destination config cannot define SQLBuild-owned key(s): {}", - blocked.join(", ") - ), - }) -} - -/// Reject a missing or blank dataset name, which raw landing requires. -pub(crate) fn check_dataset_name( - adapter: &str, - dataset_name: &DltDatasetName, -) -> Result<(), DltDestinationError> { - let named: bool = dataset_name - .text - .as_deref() - .is_some_and(|name| !python_strip(name).is_empty()); - if named { - return Ok(()); - } - Err(DltDestinationError { - message: format!( - "Adapter '{adapter}' requires an explicit dlt source schema for raw landing" - ), - }) -} diff --git a/crates/sqlbuild-core/src/dlt_destination/constants.rs b/crates/sqlbuild-core/src/dlt_destination/constants.rs deleted file mode 100644 index a924286ff8..0000000000 --- a/crates/sqlbuild-core/src/dlt_destination/constants.rs +++ /dev/null @@ -1,41 +0,0 @@ -//! dlt destination defaults, adapters and the keys SQLBuild owns. - -/// The integration loader name dlt connection errors cite. -pub(crate) const DLT_LOADER: &str = "dlt"; -/// Destination options SQLBuild passes unless the destination config overrides them. -pub const DEFAULT_DESTINATION_OPTIONS: [(&str, &str); 1] = [("naming_convention", "sql_ci_v1")]; -/// Destination config keys SQLBuild sets itself. -pub(crate) const SQLBUILD_OWNED_DESTINATION_KEYS: [&str; 4] = [ - "credentials", - "dataset_name", - "default_schema_name", - "project_id", -]; -/// The dataset DuckDB destinations land in without an explicit one. -pub(crate) const DUCKDB_DEFAULT_DATASET: &str = "main"; -/// The dataset of SQLite's own database file; dlt attaches a file per other dataset. -pub(crate) const SQLITE_MAIN_DATASET: &str = "main"; -/// The suffix Snowflake account and host settings drop. -pub(crate) const SNOWFLAKE_ACCOUNT_SUFFIX: &str = ".snowflakecomputing.com"; -/// SQL Server's default port number. -pub(crate) const SQLSERVER_DEFAULT_PORT: i64 = 1433; -/// Snowflake settings copied into credentials when set, in order. -pub(crate) const SNOWFLAKE_OPTIONAL_SETTINGS: [&str; 4] = - ["password", "role", "authenticator", "token"]; -/// BigQuery factory arguments and the connection settings they come from, in order. -pub(crate) const BIGQUERY_ARGUMENTS: [(&str, &str); 2] = - [("project_id", "project"), ("location", "location")]; -pub(crate) const DUCKDB_ADAPTER: &str = "duckdb"; -pub(crate) const SQLITE_ADAPTER: &str = "sqlite"; -/// The dlt destination factory that writes SQLite through SQLAlchemy. -pub(crate) const SQLALCHEMY_FACTORY: &str = "sqlalchemy"; -pub(crate) const MOTHERDUCK_ADAPTER: &str = "motherduck"; -pub(crate) const POSTGRES_ADAPTER: &str = "postgres"; -pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; -pub(crate) const BIGQUERY_ADAPTER: &str = "bigquery"; -pub(crate) const DATABRICKS_ADAPTER: &str = "databricks"; -pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; -/// The dlt destination factory of SQL Server. -pub(crate) const SQLSERVER_FACTORY: &str = "mssql"; -/// The factory argument holding destination credentials. -pub(crate) const CREDENTIALS_ARGUMENT: &str = "credentials"; diff --git a/crates/sqlbuild-core/src/dlt_destination/errors.rs b/crates/sqlbuild-core/src/dlt_destination/errors.rs deleted file mode 100644 index d0ca821392..0000000000 --- a/crates/sqlbuild-core/src/dlt_destination/errors.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! dlt destination construction errors. - -/// A destination the dlt integration cannot address, with its user-facing message. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct DltDestinationError { - pub message: String, -} diff --git a/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs b/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs deleted file mode 100644 index 07f5ea30cf..0000000000 --- a/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! `dlt_destination_plan`: the dlt destination call for one adapter's connection. - -use crate::dlt_destination::_helpers::credentials::{adapter_call, credentials, required}; -use crate::dlt_destination::_helpers::request::{check_dataset_name, check_destination_keys}; -use crate::dlt_destination::constants::{ - DUCKDB_ADAPTER, DUCKDB_DEFAULT_DATASET, SQLALCHEMY_FACTORY, SQLITE_ADAPTER, SQLITE_MAIN_DATASET, -}; -use crate::dlt_destination::errors::DltDestinationError; -use crate::dlt_destination::models::{ - DltDataset, DltDestinationPlan, DltDestinationRequest, DltValue, -}; - -/// The dlt destination call for the request's adapter, connection and dataset. -pub fn dlt_destination_plan( - request: &DltDestinationRequest<'_>, -) -> Result { - let adapter: &str = request.adapter; - check_destination_keys(adapter, request.destination_keys)?; - if adapter == DUCKDB_ADAPTER { - let database: &str = required(request.connection, "database", DUCKDB_ADAPTER)?; - return Ok(DltDestinationPlan { - factory: DUCKDB_ADAPTER, - arguments: credentials(DltValue::Text(database.to_owned())), - dataset: if request.dataset_name.truthy { - DltDataset::Given - } else { - DltDataset::Default(DUCKDB_DEFAULT_DATASET) - }, - }); - } - if adapter == SQLITE_ADAPTER { - let path: &str = required(request.connection, "path", SQLITE_ADAPTER)?; - return Ok(DltDestinationPlan { - factory: SQLALCHEMY_FACTORY, - arguments: credentials(DltValue::Text(format!("sqlite:///{path}"))), - dataset: if request.dataset_name.truthy { - DltDataset::Given - } else { - DltDataset::Default(SQLITE_MAIN_DATASET) - }, - }); - } - check_dataset_name(adapter, request.dataset_name)?; - let Some((factory, arguments)) = adapter_call(adapter, request.connection)? else { - return Err(DltDestinationError { - message: format!("Adapter '{adapter}' does not support dlt integration loaders"), - }); - }; - Ok(DltDestinationPlan { - factory, - arguments, - dataset: DltDataset::Given, - }) -} diff --git a/crates/sqlbuild-core/src/dlt_destination/main/mod.rs b/crates/sqlbuild-core/src/dlt_destination/main/mod.rs deleted file mode 100644 index b8507d2941..0000000000 --- a/crates/sqlbuild-core/src/dlt_destination/main/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod dlt_destination_plan; diff --git a/crates/sqlbuild-core/src/dlt_destination/mod.rs b/crates/sqlbuild-core/src/dlt_destination/mod.rs deleted file mode 100644 index 4be5b95029..0000000000 --- a/crates/sqlbuild-core/src/dlt_destination/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The dlt destination factory and arguments for one SQLBuild warehouse adapter. - -pub(crate) mod _helpers; -pub mod constants; -pub mod errors; -pub mod main; -pub mod models; - -#[cfg(test)] -mod tests; diff --git a/crates/sqlbuild-core/src/dlt_destination/models.rs b/crates/sqlbuild-core/src/dlt_destination/models.rs deleted file mode 100644 index 3e1d881496..0000000000 --- a/crates/sqlbuild-core/src/dlt_destination/models.rs +++ /dev/null @@ -1,49 +0,0 @@ -//! A dlt destination factory call and the facts it is built from. - -use crate::loader_connection::models::LoaderConnection; - -/// One argument value of a dlt destination factory call. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum DltValue { - Text(String), - /// The connection's raw `port` setting, or `default` when unset. - Port { - default: i64, - }, - Object(Vec<(&'static str, DltValue)>), -} - -/// The dataset a dlt destination lands in. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum DltDataset { - /// The dataset name the caller gave. - Given, - Default(&'static str), -} - -/// The caller's dataset name. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct DltDatasetName { - /// The name when it is a string. - pub text: Option, - /// Python truthiness of the name. - pub truthy: bool, -} - -/// One `dlt.destinations.(**arguments, **options)` call and its dataset. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct DltDestinationPlan { - pub factory: &'static str, - pub arguments: Vec<(&'static str, DltValue)>, - pub dataset: DltDataset, -} - -/// One adapter's connection, destination config keys and dataset name. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct DltDestinationRequest<'a> { - pub adapter: &'a str, - pub connection: &'a LoaderConnection, - /// The destination config's keys, each `None` when not a string. - pub destination_keys: &'a [Option], - pub dataset_name: &'a DltDatasetName, -} diff --git a/crates/sqlbuild-core/src/dlt_destination/tests.rs b/crates/sqlbuild-core/src/dlt_destination/tests.rs deleted file mode 100644 index 48fa2dd946..0000000000 --- a/crates/sqlbuild-core/src/dlt_destination/tests.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[path = "tests/test_dlt_destination_plan.rs"] -mod dlt_destination_plan; -#[path = "tests/test_types.rs"] -mod test_types; diff --git a/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs b/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs deleted file mode 100644 index 5174d1a6d8..0000000000 --- a/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! dlt destination calls match the Python integration's. - -use crate::dlt_destination::main::dlt_destination_plan::dlt_destination_plan; -use crate::dlt_destination::models::{ - DltDataset, DltDatasetName, DltDestinationPlan, DltDestinationRequest, DltValue, -}; -use crate::dlt_destination::tests::test_types::DestinationPlanTestCase; -use crate::loader_connection::models::LoaderConnection; - -#[test] -fn given_adapter_connection_when_planning_dlt_destination_then_matches_python_mapping() { - let test_cases = [ - DestinationPlanTestCase { - description: "duckdb without dataset lands in main", - adapter: "duckdb", - strings: &[("database", " warehouse.duckdb ")], - port: None, - destination_keys: &[], - dataset_name: None, - expected_plan: Ok(DltDestinationPlan { - factory: "duckdb", - arguments: vec![("credentials", DltValue::Text("warehouse.duckdb".to_owned()))], - dataset: DltDataset::Default("main"), - }), - }, - DestinationPlanTestCase { - description: "sqlite writes its database file through sqlalchemy into main", - adapter: "sqlite", - strings: &[("path", "/data/warehouse.sqlite")], - port: None, - destination_keys: &[], - dataset_name: None, - expected_plan: Ok(DltDestinationPlan { - factory: "sqlalchemy", - arguments: vec![( - "credentials", - DltValue::Text("sqlite:////data/warehouse.sqlite".to_owned()), - )], - dataset: DltDataset::Default("main"), - }), - }, - DestinationPlanTestCase { - description: "postgres connection string uses host default and port", - adapter: "postgres", - strings: &[("user", "u"), ("database", "d")], - port: Some("6543"), - destination_keys: &["staging"], - dataset_name: Some("raw"), - expected_plan: Ok(DltDestinationPlan { - factory: "postgres", - arguments: vec![( - "credentials", - DltValue::Text("postgresql://u:@localhost:6543/d".to_owned()), - )], - dataset: DltDataset::Given, - }), - }, - DestinationPlanTestCase { - description: "owned destination keys are sorted in the error", - adapter: "postgres", - strings: &[], - port: None, - destination_keys: &["dataset_name", "credentials"], - dataset_name: Some("raw"), - expected_plan: Err( - "Adapter 'postgres' dlt destination config cannot define SQLBuild-owned key(s): credentials, dataset_name", - ), - }, - DestinationPlanTestCase { - description: "blank dataset is rejected before credentials", - adapter: "snowflake", - strings: &[], - port: None, - destination_keys: &[], - dataset_name: Some(" "), - expected_plan: Err( - "Adapter 'snowflake' requires an explicit dlt source schema for raw landing", - ), - }, - DestinationPlanTestCase { - description: "missing required setting names the dlt loader", - adapter: "snowflake", - strings: &[], - port: None, - destination_keys: &[], - dataset_name: Some("raw"), - expected_plan: Err( - "Adapter 'snowflake' requires connection.account for dlt integration loaders", - ), - }, - ]; - - for test_case in test_cases { - let connection = LoaderConnection { - strings: test_case - .strings - .iter() - .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) - .collect(), - port: test_case.port.map(str::to_owned), - }; - let destination_keys: Vec> = test_case - .destination_keys - .iter() - .map(|key| Some((*key).to_owned())) - .collect(); - let dataset_name = DltDatasetName { - text: test_case.dataset_name.map(str::to_owned), - truthy: test_case.dataset_name.is_some_and(|name| !name.is_empty()), - }; - let request = DltDestinationRequest { - adapter: test_case.adapter, - connection: &connection, - destination_keys: &destination_keys, - dataset_name: &dataset_name, - }; - assert_eq!( - dlt_destination_plan(&request).map_err(|error| error.message), - test_case.expected_plan.map_err(str::to_owned), - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-core/src/dlt_destination/tests/test_types.rs b/crates/sqlbuild-core/src/dlt_destination/tests/test_types.rs deleted file mode 100644 index ea416660fd..0000000000 --- a/crates/sqlbuild-core/src/dlt_destination/tests/test_types.rs +++ /dev/null @@ -1,11 +0,0 @@ -use crate::dlt_destination::models::DltDestinationPlan; - -pub(super) struct DestinationPlanTestCase { - pub(super) description: &'static str, - pub(super) adapter: &'static str, - pub(super) strings: &'static [(&'static str, &'static str)], - pub(super) port: Option<&'static str>, - pub(super) destination_keys: &'static [&'static str], - pub(super) dataset_name: Option<&'static str>, - pub(super) expected_plan: Result, -} diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs deleted file mode 100644 index 6a1951b26e..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs +++ /dev/null @@ -1,37 +0,0 @@ -//! Connection settings read the way the ingestr destination expects them. - -use crate::ingestr_command::constants::INGESTR_LOADER; -use crate::ingestr_command::errors::IngestrCommandError; -use crate::loader_connection::main::connection_setting::connection_setting; -use crate::loader_connection::main::required_connection_setting::required_connection_setting; -use crate::loader_connection::models::LoaderConnection; - -/// The stripped string setting `key`, or `None` when unset, not a string or blank. -pub(crate) fn optional<'a>(connection: &'a LoaderConnection, key: &str) -> Option<&'a str> { - connection_setting(connection, key) -} - -/// The stripped string setting `key`, which the adapter's ingestr destination requires. -pub(crate) fn required<'a>( - connection: &'a LoaderConnection, - key: &str, - adapter_name: &str, -) -> Result<&'a str, IngestrCommandError> { - required_connection_setting(connection, key, adapter_name, INGESTR_LOADER) - .map_err(|message| IngestrCommandError { message }) -} - -/// `(key, value)` when the setting `key` is set. -fn setting<'a>(connection: &'a LoaderConnection, key: &'a str) -> Option<(&'a str, &'a str)> { - optional(connection, key).map(|value| (key, value)) -} - -/// The settings among `keys` that are set, in `keys` order. -pub(crate) fn present<'a>( - connection: &'a LoaderConnection, - keys: &[&'a str], -) -> Vec<(&'a str, &'a str)> { - keys.iter() - .filter_map(|key| setting(connection, key)) - .collect() -} diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs deleted file mode 100644 index f485354ef2..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs +++ /dev/null @@ -1,139 +0,0 @@ -//! Destination URIs per warehouse adapter. - -use crate::ingestr_command::_helpers::connection::{optional, present, required}; -use crate::ingestr_command::_helpers::url_quote::{query_suffix, quote, urlencode}; -use crate::ingestr_command::constants::{ - BIGQUERY_PARAMETERS, DATABRICKS_PARAMETERS, SNOWFLAKE_PARAMETERS, SQLSERVER_DEFAULT_DRIVER, - SQLSERVER_DEFAULT_PORT, SQLSERVER_DRIVER_PARAMETER, SQLSERVER_PARAMETERS, - SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER, -}; -use crate::ingestr_command::errors::IngestrCommandError; -use crate::loader_connection::constants::{DEFAULT_HOST, MOTHERDUCK_PREFIX, POSTGRES_DEFAULT_PORT}; -use crate::loader_connection::models::LoaderConnection; - -/// DuckDB's database file URI. -pub(crate) fn duckdb(connection: &LoaderConnection) -> Result { - Ok(format!( - "duckdb:///{}", - required(connection, "database", "duckdb")? - )) -} - -/// SQLite's database file URI. -pub(crate) fn sqlite(connection: &LoaderConnection) -> Result { - Ok(format!( - "sqlite:///{}", - required(connection, "path", "sqlite")? - )) -} - -/// MotherDuck's database URI with its optional token. -pub(crate) fn motherduck(connection: &LoaderConnection) -> String { - let database: &str = optional(connection, "database").unwrap_or_default(); - let database: &str = database.strip_prefix(MOTHERDUCK_PREFIX).unwrap_or(database); - let token: Vec<(&str, &str)> = optional(connection, "token") - .map(|token| ("token", token)) - .into_iter() - .collect(); - format!("motherduck://{}{}", quote(database), query_suffix(&token)) -} - -/// Postgres's URI with its optional SSL mode. -pub(crate) fn postgres(connection: &LoaderConnection) -> Result { - let host: &str = optional(connection, "host").unwrap_or(DEFAULT_HOST); - let port: &str = connection.port.as_deref().unwrap_or(POSTGRES_DEFAULT_PORT); - let user: &str = required(connection, "user", "postgres")?; - let password: &str = optional(connection, "password").unwrap_or_default(); - let database: &str = match optional(connection, "dbname") { - Some(database) => database, - None => required(connection, "database", "postgres")?, - }; - let sslmode: Vec<(&str, &str)> = present(connection, &["sslmode"]); - Ok(format!( - "postgresql://{}:{}@{host}:{port}/{}{}", - quote(user), - quote(password), - quote(database), - query_suffix(&sslmode) - )) -} - -/// BigQuery's project URI with its optional credentials. -pub(crate) fn bigquery(connection: &LoaderConnection) -> Result { - let project: &str = match optional(connection, "project") { - Some(project) => project, - None => required(connection, "database", "bigquery")?, - }; - Ok(format!( - "bigquery://{project}{}", - query_suffix(&present(connection, &BIGQUERY_PARAMETERS)) - )) -} - -/// Snowflake's account URI with its optional session settings. -pub(crate) fn snowflake(connection: &LoaderConnection) -> Result { - let account: &str = required(connection, "account", "snowflake")?; - let user: &str = required(connection, "user", "snowflake")?; - let password: &str = optional(connection, "password").unwrap_or_default(); - let database: &str = required(connection, "database", "snowflake")?; - let schema: &str = required(connection, "schema", "snowflake")?; - Ok(format!( - "snowflake://{}:{}@{account}/{}/{}{}", - quote(user), - quote(password), - quote(database), - quote(schema), - query_suffix(&present(connection, &SNOWFLAKE_PARAMETERS)) - )) -} - -/// Databricks's token URI with its warehouse path. -pub(crate) fn databricks(connection: &LoaderConnection) -> Result { - let host: &str = required(connection, "server_hostname", "databricks")?; - let token: &str = required(connection, "token", "databricks")?; - let mut parameters: Vec<(&str, &str)> = vec![( - "http_path", - required(connection, "http_path", "databricks")?, - )]; - parameters.extend(present(connection, &DATABRICKS_PARAMETERS)); - Ok(format!( - "databricks://token:{}@{host}?{}", - quote(token), - urlencode(¶meters) - )) -} - -/// SQL Server's URI with its ODBC defaults. -pub(crate) fn sqlserver(connection: &LoaderConnection) -> String { - let first = |keys: [&str; 2], default: &'static str| -> String { - keys.iter() - .find_map(|key| optional(connection, key)) - .unwrap_or(default) - .to_owned() - }; - let host: String = first(["host", "server"], DEFAULT_HOST); - let port: &str = connection.port.as_deref().unwrap_or(SQLSERVER_DEFAULT_PORT); - let user: String = first(["user", "username"], "sa"); - let password: &str = optional(connection, "password").unwrap_or_default(); - let database: String = first(["database", "dbname"], "master"); - let mut parameters: Vec<(&str, &str)> = present(connection, &SQLSERVER_PARAMETERS); - let defaults: [(&str, &str); 2] = [ - (SQLSERVER_DRIVER_PARAMETER, SQLSERVER_DEFAULT_DRIVER), - (SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER, "yes"), - ]; - for (key, value) in defaults { - if !parameters - .iter() - .any(|(present_key, _)| *present_key == key) - { - parameters.push((key, value)); - } - } - format!( - "mssql://{}:{}@{host}:{port}/{}{}", - quote(&user), - quote(password), - quote(&database), - query_suffix(¶meters) - ) -} diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/mod.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/mod.rs deleted file mode 100644 index 0b39ed4d18..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/_helpers/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) mod connection; -pub(crate) mod destinations; -pub(crate) mod url_quote; diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/url_quote.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/url_quote.rs deleted file mode 100644 index d2566d4c35..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/_helpers/url_quote.rs +++ /dev/null @@ -1,42 +0,0 @@ -//! Python's `urllib.parse.quote` and `urlencode` over string pairs. - -use crate::ingestr_command::constants::URL_ALWAYS_SAFE; - -fn quote_with(text: &str, safe: &[u8]) -> String { - let mut quoted: String = String::with_capacity(text.len()); - for byte in text.bytes() { - if byte.is_ascii_alphanumeric() || URL_ALWAYS_SAFE.contains(&byte) || safe.contains(&byte) { - quoted.push(char::from(byte)); - } else { - quoted.push_str(&format!("%{byte:02X}")); - } - } - quoted -} - -/// Python's `urllib.parse.quote(text)`, which keeps `/`. -pub(crate) fn quote(text: &str) -> String { - quote_with(text, b"/") -} - -fn quote_plus(text: &str) -> String { - quote_with(text, b" ").replace(' ', "+") -} - -/// Python's `urlencode` of ordered string pairs. -pub(crate) fn urlencode(pairs: &[(&str, &str)]) -> String { - pairs - .iter() - .map(|(key, value)| format!("{}={}", quote_plus(key), quote_plus(value))) - .collect::>() - .join("&") -} - -/// `?` and the encoded pairs, or nothing when there are none. -pub(crate) fn query_suffix(pairs: &[(&str, &str)]) -> String { - if pairs.is_empty() { - String::new() - } else { - format!("?{}", urlencode(pairs)) - } -} diff --git a/crates/sqlbuild-core/src/ingestr_command/constants.rs b/crates/sqlbuild-core/src/ingestr_command/constants.rs deleted file mode 100644 index 12475e6949..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/constants.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! Ingestr command vocabulary and destination defaults. - -/// The fixed leading arguments of every ingestr run. -pub(crate) const COMMAND_PREFIX: [&str; 2] = ["ingestr", "ingest"]; -/// Progress reporting arguments ingestr runs with. -pub(crate) const COMMAND_PROGRESS: [&str; 3] = ["--yes", "--progress", "log"]; -/// Characters Python's `urllib.parse.quote` never escapes. -pub(crate) const URL_ALWAYS_SAFE: &[u8] = b"_.-~"; -/// SQL Server's default port. -pub(crate) const SQLSERVER_DEFAULT_PORT: &str = "1433"; -/// The SQL Server ODBC driver ingestr uses unless the connection names one. -pub(crate) const SQLSERVER_DEFAULT_DRIVER: &str = "ODBC Driver 18 for SQL Server"; -/// Optional BigQuery query parameters, in URI order. -pub(crate) const BIGQUERY_PARAMETERS: [&str; 3] = - ["location", "credentials_path", "credentials_base64"]; -/// Optional Snowflake query parameters, in URI order. -pub(crate) const SNOWFLAKE_PARAMETERS: [&str; 6] = [ - "warehouse", - "role", - "token", - "private_key", - "private_key_passphrase", - "authenticator", -]; -/// Optional Databricks query parameters after `http_path`, in URI order. -pub(crate) const DATABRICKS_PARAMETERS: [&str; 2] = ["catalog", "schema"]; -/// Optional SQL Server query parameters, in URI order. -pub(crate) const SQLSERVER_PARAMETERS: [&str; 3] = - ["driver", "TrustServerCertificate", "Authentication"]; -/// The SQL Server driver query parameter. -pub(crate) const SQLSERVER_DRIVER_PARAMETER: &str = "driver"; -/// The SQL Server certificate trust query parameter. -pub(crate) const SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER: &str = "TrustServerCertificate"; -/// The integration loader name ingestr connection errors cite. -pub(crate) const INGESTR_LOADER: &str = "ingestr"; diff --git a/crates/sqlbuild-core/src/ingestr_command/errors.rs b/crates/sqlbuild-core/src/ingestr_command/errors.rs deleted file mode 100644 index bb10c5e272..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/errors.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Ingestr command construction errors. - -/// A destination the ingestr integration cannot address, with its user-facing message. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct IngestrCommandError { - pub message: String, -} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/format_ingestr_command.rs b/crates/sqlbuild-core/src/ingestr_command/main/format_ingestr_command.rs deleted file mode 100644 index dc8c3ed27d..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/main/format_ingestr_command.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! The display form of an ingestr command line. - -use crate::shell_text::main::shell_quote::shell_quote; - -/// The arguments shell-quoted and joined with spaces, like Python's `shlex.join`. -#[must_use] -pub fn format_ingestr_command(arguments: &[String]) -> String { - arguments - .iter() - .map(|argument| shell_quote(argument)) - .collect::>() - .join(" ") -} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_command_arguments.rs b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_command_arguments.rs deleted file mode 100644 index 34eabc624e..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_command_arguments.rs +++ /dev/null @@ -1,43 +0,0 @@ -//! The `ingestr ingest` arguments for one source run. - -use crate::ingestr_command::constants::{COMMAND_PREFIX, COMMAND_PROGRESS}; -use crate::ingestr_command::models::{IngestrDestination, IngestrSource}; - -/// The ingestr command line that loads `source` into `destination`. -#[must_use] -pub fn ingestr_command_arguments( - source: &IngestrSource, - destination: &IngestrDestination, -) -> Vec { - let mut arguments: Vec = COMMAND_PREFIX.map(str::to_owned).to_vec(); - let pairs: [(&str, &str); 4] = [ - ("--source-uri", &source.source_uri), - ("--source-table", &source.source_table), - ("--dest-uri", &destination.uri), - ("--dest-table", &destination.table), - ]; - let optional: [(&str, Option<&String>); 2] = [ - ("--incremental-strategy", source.strategy.as_ref()), - ("--incremental-key", source.incremental_key.as_ref()), - ]; - for (flag, value) in pairs { - arguments.extend([flag.to_owned(), value.to_owned()]); - } - arguments.extend(COMMAND_PROGRESS.map(str::to_owned)); - for (flag, value) in optional { - if let Some(value) = value { - arguments.extend([flag.to_owned(), value.clone()]); - } - } - for primary_key in &source.primary_key { - arguments.extend(["--primary-key".to_owned(), primary_key.clone()]); - } - if let Some(columns) = &source.columns { - arguments.extend(["--columns".to_owned(), columns.clone()]); - } - if destination.is_reload { - arguments.push("--full-refresh".to_owned()); - } - arguments.extend(source.extra_args.iter().cloned()); - arguments -} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_table.rs b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_table.rs deleted file mode 100644 index 67fd30a1f4..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_table.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The relation an ingestr source run writes. - -/// `schema.target_name` when the source declares a schema, otherwise `target_name`. -#[must_use] -pub fn ingestr_destination_table(schema: Option<&str>, target_name: &str) -> String { - match schema { - Some(schema) => format!("{schema}.{target_name}"), - None => target_name.to_owned(), - } -} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs deleted file mode 100644 index 3a472007b0..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! The ingestr destination URI for a SQLBuild warehouse connection. - -use crate::ingestr_command::_helpers::destinations::{ - bigquery, databricks, duckdb, motherduck, postgres, snowflake, sqlite, sqlserver, -}; -use crate::ingestr_command::errors::IngestrCommandError; -use crate::loader_connection::models::LoaderConnection; - -/// The ingestr destination URI for the connection of the adapter named `adapter_name`. -pub fn ingestr_destination_uri( - adapter_name: &str, - connection: &LoaderConnection, -) -> Result { - match adapter_name { - "duckdb" => duckdb(connection), - "sqlite" => sqlite(connection), - "motherduck" => Ok(motherduck(connection)), - "postgres" => postgres(connection), - "bigquery" => bigquery(connection), - "snowflake" => snowflake(connection), - "databricks" => databricks(connection), - "sqlserver" => Ok(sqlserver(connection)), - _ => Err(IngestrCommandError { - message: format!( - "Adapter '{adapter_name}' does not support ingestr integration loaders" - ), - }), - } -} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/mod.rs b/crates/sqlbuild-core/src/ingestr_command/main/mod.rs deleted file mode 100644 index 68329553b8..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/main/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod format_ingestr_command; -pub mod ingestr_command_arguments; -pub mod ingestr_destination_table; -pub mod ingestr_destination_uri; diff --git a/crates/sqlbuild-core/src/ingestr_command/mod.rs b/crates/sqlbuild-core/src/ingestr_command/mod.rs deleted file mode 100644 index db623d7553..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The `ingestr ingest` command line for one declarative ingestr source and its warehouse. - -pub(crate) mod _helpers; -pub(crate) mod constants; -pub mod errors; -pub mod main; -pub mod models; - -#[cfg(test)] -mod tests; diff --git a/crates/sqlbuild-core/src/ingestr_command/models.rs b/crates/sqlbuild-core/src/ingestr_command/models.rs deleted file mode 100644 index bbfc6e6af1..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/models.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Inputs of one ingestr command line. - -/// A resolved ingestr source configuration. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct IngestrSource { - pub source_uri: String, - pub source_table: String, - pub strategy: Option, - pub incremental_key: Option, - pub primary_key: Vec, - pub columns: Option, - pub extra_args: Vec, -} - -/// Where one ingestr run writes. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct IngestrDestination { - pub uri: String, - pub table: String, - /// The run rebuilds the destination from scratch. - pub is_reload: bool, -} diff --git a/crates/sqlbuild-core/src/ingestr_command/tests.rs b/crates/sqlbuild-core/src/ingestr_command/tests.rs deleted file mode 100644 index ed3facc731..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/tests.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[path = "tests/test_ingestr_command.rs"] -mod ingestr_command; -#[path = "tests/test_types.rs"] -mod test_types; diff --git a/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs b/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs deleted file mode 100644 index 16bf812ffe..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs +++ /dev/null @@ -1,132 +0,0 @@ -//! Ingestr destination URIs and command lines match the Python integration's. - -use crate::ingestr_command::main::format_ingestr_command::format_ingestr_command; -use crate::ingestr_command::main::ingestr_command_arguments::ingestr_command_arguments; -use crate::ingestr_command::main::ingestr_destination_table::ingestr_destination_table; -use crate::ingestr_command::main::ingestr_destination_uri::ingestr_destination_uri; -use crate::ingestr_command::models::{IngestrDestination, IngestrSource}; -use crate::ingestr_command::tests::test_types::{CommandTestCase, DestinationUriTestCase}; -use crate::loader_connection::models::LoaderConnection; - -#[test] -fn given_connection_when_building_destination_uri_then_matches_python_quoting() { - let test_cases = [ - DestinationUriTestCase { - description: "duckdb path", - adapter_name: "duckdb", - strings: &[("database", " warehouse.duckdb ")], - port: None, - expected_uri: Ok("duckdb:///warehouse.duckdb"), - }, - DestinationUriTestCase { - description: "sqlite path", - adapter_name: "sqlite", - strings: &[("path", "/data/warehouse.sqlite")], - port: None, - expected_uri: Ok("sqlite:////data/warehouse.sqlite"), - }, - DestinationUriTestCase { - description: "motherduck prefix and token", - adapter_name: "motherduck", - strings: &[("database", "md:my db/é"), ("token", "t o/k=n&")], - port: None, - expected_uri: Ok("motherduck://my%20db/%C3%A9?token=t+o%2Fk%3Dn%26"), - }, - DestinationUriTestCase { - description: "postgres with sslmode and port", - adapter_name: "postgres", - strings: &[ - ("user", "ana lyst"), - ("password", "p@ss:/w"), - ("dbname", "orders"), - ], - port: Some("6543"), - expected_uri: Ok("postgresql://ana%20lyst:p%40ss%3A/w@localhost:6543/orders"), - }, - DestinationUriTestCase { - description: "sqlserver defaults", - adapter_name: "sqlserver", - strings: &[("password", "Sqlbuild!2026"), ("database", "analytics")], - port: None, - expected_uri: Ok("mssql://sa:Sqlbuild%212026@localhost:1433/analytics\ - ?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes"), - }, - DestinationUriTestCase { - description: "snowflake requires an account", - adapter_name: "snowflake", - strings: &[("account", " ")], - port: None, - expected_uri: Err( - "Adapter 'snowflake' requires connection.account for ingestr integration loaders", - ), - }, - DestinationUriTestCase { - description: "unsupported adapter", - adapter_name: "fabric", - strings: &[], - port: None, - expected_uri: Err("Adapter 'fabric' does not support ingestr integration loaders"), - }, - ]; - for test_case in test_cases { - let connection: LoaderConnection = LoaderConnection { - strings: test_case - .strings - .iter() - .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) - .collect(), - port: test_case.port.map(str::to_owned), - }; - let uri: Result = - ingestr_destination_uri(test_case.adapter_name, &connection) - .map_err(|error| error.message); - assert_eq!( - uri.as_deref().map_err(String::as_str), - test_case.expected_uri, - "{}", - test_case.description - ); - } -} - -#[test] -fn given_source_when_building_command_then_orders_arguments_like_python() { - let test_cases = [ - CommandTestCase { - description: "reload with schema", - schema: Some("raw"), - is_reload: true, - expected_display: "ingestr ingest --source-uri 'csv://a b' --source-table t \ - --dest-uri duckdb:///w.duckdb --dest-table raw.orders --yes --progress log \ - --primary-key id --full-refresh '--note=it'\"'\"'s'", - }, - CommandTestCase { - description: "incremental without schema", - schema: None, - is_reload: false, - expected_display: "ingestr ingest --source-uri 'csv://a b' --source-table t \ - --dest-uri duckdb:///w.duckdb --dest-table orders --yes --progress log \ - --primary-key id '--note=it'\"'\"'s'", - }, - ]; - let source: IngestrSource = IngestrSource { - source_uri: "csv://a b".to_owned(), - source_table: "t".to_owned(), - primary_key: vec!["id".to_owned()], - extra_args: vec!["--note=it's".to_owned()], - ..IngestrSource::default() - }; - for test_case in test_cases { - let destination: IngestrDestination = IngestrDestination { - uri: "duckdb:///w.duckdb".to_owned(), - table: ingestr_destination_table(test_case.schema, "orders"), - is_reload: test_case.is_reload, - }; - assert_eq!( - format_ingestr_command(&ingestr_command_arguments(&source, &destination)), - test_case.expected_display, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-core/src/ingestr_command/tests/test_types.rs b/crates/sqlbuild-core/src/ingestr_command/tests/test_types.rs deleted file mode 100644 index e406920491..0000000000 --- a/crates/sqlbuild-core/src/ingestr_command/tests/test_types.rs +++ /dev/null @@ -1,14 +0,0 @@ -pub(super) struct DestinationUriTestCase { - pub(super) description: &'static str, - pub(super) adapter_name: &'static str, - pub(super) strings: &'static [(&'static str, &'static str)], - pub(super) port: Option<&'static str>, - pub(super) expected_uri: Result<&'static str, &'static str>, -} - -pub(super) struct CommandTestCase { - pub(super) description: &'static str, - pub(super) schema: Option<&'static str>, - pub(super) is_reload: bool, - pub(super) expected_display: &'static str, -} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 318a55720a..fafe8ea212 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -26,7 +26,6 @@ pub mod dbt_process; pub mod debug_report; pub mod diagnostic_routing; pub mod diagnostic_text; -pub mod dlt_destination; pub mod event_export; pub mod fingerprint_identity; pub mod format_command; @@ -34,14 +33,12 @@ pub mod freshness_text; pub mod git_revisions; pub mod graph_walk; pub mod identifier_text; -pub mod ingestr_command; pub mod init_command; pub mod integration_results; pub mod janitor_text; pub mod json; pub mod lifecycle_events; pub mod load_command; -pub mod loader_connection; pub mod migration_values; pub mod models; pub mod nested_progress; diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index a56ae32466..98fdd9e419 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -184,6 +184,7 @@ pub const OPERATION_KINDS: [&str; 10] = [ "subprocess", "warehouse", ]; +/// Includes retired names, such as the removed dlt and ingestr loads, so stored history still decodes. pub const OPERATION_NAMES: [&str; 57] = [ "adbc_driver_install", "audit_evaluation", diff --git a/crates/sqlbuild-core/src/loader_connection/constants.rs b/crates/sqlbuild-core/src/loader_connection/constants.rs deleted file mode 100644 index 115683dca6..0000000000 --- a/crates/sqlbuild-core/src/loader_connection/constants.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Loader destination connection defaults. - -/// The host destinations connect to without one. -pub const DEFAULT_HOST: &str = "localhost"; -/// Postgres's default port. -pub const POSTGRES_DEFAULT_PORT: &str = "5432"; -/// The MotherDuck database prefix. -pub const MOTHERDUCK_PREFIX: &str = "md:"; diff --git a/crates/sqlbuild-core/src/loader_connection/main/connection_setting.rs b/crates/sqlbuild-core/src/loader_connection/main/connection_setting.rs deleted file mode 100644 index 141c17c681..0000000000 --- a/crates/sqlbuild-core/src/loader_connection/main/connection_setting.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! `connection_setting`: one stripped string connection setting. - -use crate::loader_connection::models::LoaderConnection; -use crate::text::main::python_strip::python_strip; - -/// The stripped string setting `key`, or `None` when unset, not a string or blank. -#[must_use] -pub fn connection_setting<'a>(connection: &'a LoaderConnection, key: &str) -> Option<&'a str> { - connection - .strings - .get(key) - .map(|value| python_strip(value)) - .filter(|value| !value.is_empty()) -} diff --git a/crates/sqlbuild-core/src/loader_connection/main/mod.rs b/crates/sqlbuild-core/src/loader_connection/main/mod.rs deleted file mode 100644 index 4ea9c16cda..0000000000 --- a/crates/sqlbuild-core/src/loader_connection/main/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod connection_setting; -pub mod required_connection_setting; diff --git a/crates/sqlbuild-core/src/loader_connection/main/required_connection_setting.rs b/crates/sqlbuild-core/src/loader_connection/main/required_connection_setting.rs deleted file mode 100644 index cb6ac0aecb..0000000000 --- a/crates/sqlbuild-core/src/loader_connection/main/required_connection_setting.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! `required_connection_setting`: one connection setting a loader destination requires. - -use crate::loader_connection::main::connection_setting::connection_setting; -use crate::loader_connection::models::LoaderConnection; - -/// The stripped string setting `key`, which the adapter's `loader` destination requires. -pub fn required_connection_setting<'a>( - connection: &'a LoaderConnection, - key: &str, - adapter_name: &str, - loader: &str, -) -> Result<&'a str, String> { - connection_setting(connection, key).ok_or_else(|| { - format!( - "Adapter '{adapter_name}' requires connection.{key} for {loader} integration loaders" - ) - }) -} diff --git a/crates/sqlbuild-core/src/loader_connection/mod.rs b/crates/sqlbuild-core/src/loader_connection/mod.rs deleted file mode 100644 index 28a7bc49f9..0000000000 --- a/crates/sqlbuild-core/src/loader_connection/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! Warehouse connection settings that integration loader destinations read. - -pub mod constants; -pub mod main; -pub mod models; diff --git a/crates/sqlbuild-core/src/loader_connection/models.rs b/crates/sqlbuild-core/src/loader_connection/models.rs deleted file mode 100644 index ed4f4d15bd..0000000000 --- a/crates/sqlbuild-core/src/loader_connection/models.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! The adapter connection settings a loader destination reads. - -use std::collections::BTreeMap; - -/// The adapter connection settings an ingestr or dlt destination reads. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct LoaderConnection { - /// Connection settings whose values are Python strings, unstripped. - pub strings: BTreeMap, - /// `str(connection["port"])` when the connection sets a port. - pub port: Option, -} diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index e8289a5288..17b9d155ed 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -1,7 +1,7 @@ //! Lifecycle event types, operation labels and layout of CLI progress lines. /// Operations with visible progress lines, and the label each renders with. -pub(crate) const VISIBLE_OPERATION_LABELS: [(&str, &str); 32] = [ +pub(crate) const VISIBLE_OPERATION_LABELS: [(&str, &str); 30] = [ ("clone_finalization", "Clone finalization"), ("discovery_declaration_parse", "Declaration parsing"), ("discovery_filesystem_walk", "Filesystem discovery"), @@ -12,8 +12,6 @@ pub(crate) const VISIBLE_OPERATION_LABELS: [(&str, &str); 32] = [ ("project_compile", "Project compile"), ("sql_test_planning", "SQL test planning"), ("dbt_command", "dbt command"), - ("external_source_load", "loader callable"), - ("ingestr_command", "ingestr subprocess"), ("adbc_driver_install", "DuckDB driver install"), ("managed_source_load", "loader callable"), ("python_asset", "Python asset attempt"), @@ -39,7 +37,7 @@ pub(crate) const VISIBLE_OPERATION_LABELS: [(&str, &str); 32] = [ ("relation_promotion", "Promote relation"), ]; /// Operations a live transient owner shows on a terminal when no resource attempt owns them. -pub(crate) const TTY_TRANSIENT_OWNER_OPERATIONS: [&str; 19] = [ +pub(crate) const TTY_TRANSIENT_OWNER_OPERATIONS: [&str; 17] = [ "adbc_driver_install", "dbt_command", "discovery_declaration_parse", @@ -47,8 +45,6 @@ pub(crate) const TTY_TRANSIENT_OWNER_OPERATIONS: [&str; 19] = [ "discovery_project_assembly", "discovery_python_import", "external_manifest_discovery", - "external_source_load", - "ingestr_command", "managed_source_load", "project_compile", "project_discovery", diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs index 91ea0a3256..4af09a2dea 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs @@ -9,7 +9,6 @@ use crate::project_validation::_helpers::python_nodes::{ use crate::project_validation::models::{ NamedPath, NodeFact, NodeKind, ProjectFacts, SourceFileFact, ValidationFailure, }; -use crate::yaml_entries::main::integration_loader_name::integration_loader_name; /// Validate loader and node names, node dependencies, then the loaders sources name. pub(crate) fn validate_loaders(facts: &ProjectFacts) -> Result<(), ValidationFailure> { @@ -71,10 +70,7 @@ fn source_loader_references( let loader_names: HashSet<&str> = loaders.iter().map(|loader| loader.name.as_str()).collect(); for file in sources { for entry in &file.entries { - let expected: String = match &entry.integration_loader { - Some(integration) => integration_loader_name(&integration.kind, &entry.name), - None => entry.name.clone(), - }; + let expected: String = entry.name.clone(); if entry.managed && entry.loader.as_deref() != Some(expected.as_str()) { return Err(conflict(format!( "Managed source '{}' in {} must use terminal loader '{expected}'", diff --git a/crates/sqlbuild-discovery/src/project_validation/models.rs b/crates/sqlbuild-discovery/src/project_validation/models.rs index 813c60d704..35e45ea73f 100644 --- a/crates/sqlbuild-discovery/src/project_validation/models.rs +++ b/crates/sqlbuild-discovery/src/project_validation/models.rs @@ -18,19 +18,12 @@ pub struct ModelFact { pub local_declarations: Vec, } -/// One declarative integration loader a source names. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct IntegrationLoaderFact { - pub kind: String, -} - /// One source entry of a source YAML file. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct SourceFact { pub name: String, pub managed: bool, pub loader: Option, - pub integration_loader: Option, } /// A source YAML file and its entries. diff --git a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/integrations.rs b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/integrations.rs deleted file mode 100644 index 419ee50b64..0000000000 --- a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/integrations.rs +++ /dev/null @@ -1,380 +0,0 @@ -//! Declarative integration loaders: a source's `ingestr` block and top-level `dlt_sources`. - -use sqlbuild_core::text::main::python_strip::python_strip; -use sqlbuild_model_config::errors::ConfigError; -use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; - -use crate::yaml_entries::_helpers::fields::{ - Site, get, optional_mapping, optional_text, reject_unknown_keys, required_text, - stripped_strings, -}; -use crate::yaml_entries::constants::DLT_LOADER_KIND; -use crate::yaml_entries::main::integration_loader_name::integration_loader_name; -use crate::yaml_entries::models::{ - ParsedDltResource, ParsedDltSource, ParsedIngestrConfig, ParsedIntegrationLoader, - ParsedSourceEntry, -}; - -const INGESTR_KEYS: [&str; 8] = [ - "columns", - "extra_args", - "incremental_key", - "primary_key", - "source_table", - "source_uri", - "strategy", - "unique_key", -]; -const INGESTR_STRATEGIES: [&str; 5] = [ - "append", - "delete+insert", - "merge", - "replace", - "truncate+insert", -]; -const DLT_GROUP_KEYS: [&str; 5] = ["config", "destination", "resources", "schema", "type"]; -const DLT_RESOURCE_KEYS: [&str; 7] = [ - "description", - "incremental", - "merge_key", - "name", - "primary_key", - "schema", - "write_disposition", -]; -const FILESYSTEM: &str = "filesystem"; -const REST_API: &str = "rest_api"; -const SQL_DATABASE: &str = "sql_database"; -const BLOCKED_DESTINATION_KEYS: [&str; 3] = ["credentials", "dataset_name", "default_schema_name"]; -const RAW_CONFIG_EXCLUDED: [&str; 8] = [ - "name", - "table", - "write_disposition", - "primary_key", - "merge_key", - "incremental", - "schema", - "description", -]; - -/// `parse_source_integration_loader`: the source's `ingestr` block, if any. -pub(crate) fn source_integration_loader( - entries: &[(N, N)], - path: &str, -) -> Result>, ConfigError> { - let Some(raw) = get(entries, "ingestr").filter(|value| value.kind() != NodeKind::Null) else { - return Ok(None); - }; - if raw.kind() != NodeKind::Map { - return Err(ConfigError::compile(format!( - "{path} source 'ingestr' must be a mapping" - ))); - } - let config: Vec<(N, N)> = raw.entries(); - let site: Site<'_> = Site { - path, - label: "source ingestr", - }; - reject_unknown_keys(&config, &INGESTR_KEYS, site)?; - let source_uri: N = required_text(&config, "source_uri", site)?; - let source_table: N = required_text(&config, "source_table", site)?; - let strategy: Option = optional_text(&config, "strategy", site)?; - let incremental_key: Option = optional_text(&config, "incremental_key", site)?; - let primary_key: Vec = ingestr_strings(&config, "primary_key", path)?; - let unique_key: Vec = ingestr_strings(&config, "unique_key", path)?; - if !primary_key.is_empty() && !unique_key.is_empty() { - return Err(site.error("cannot define both primary_key and unique_key")); - } - let columns: Option = optional_text(&config, "columns", site)?; - let extra_args: Vec = ingestr_strings(&config, "extra_args", path)?; - if let Some(text) = strategy.as_ref().and_then(AuthoredNode::text) - && !INGESTR_STRATEGIES.contains(&text.as_str()) - { - return Err(site.error(&format!( - "strategy must be one of: {}", - INGESTR_STRATEGIES.join(", ") - ))); - } - Ok(Some(ParsedIntegrationLoader::Ingestr( - ParsedIngestrConfig { - source_uri, - source_table, - strategy, - incremental_key, - primary_key: if primary_key.is_empty() { - unique_key - } else { - primary_key - }, - columns, - extra_args, - }, - ))) -} - -/// `_optional_ingestr_string_tuple`: one string or a list of strings, each stripped. -fn ingestr_strings( - config: &[(N, N)], - key: &str, - path: &str, -) -> Result, ConfigError> { - let error = |text: &str| ConfigError::compile(format!("{path} source ingestr '{key}' {text}")); - let Some(value) = get(config, key).filter(|value| value.kind() != NodeKind::Null) else { - return Ok(Vec::new()); - }; - stripped_strings(value, error) -} - -/// `parse_dlt_sources`: one managed source per resource of each `dlt_sources` group. -pub(crate) fn dlt_sources( - payload: &[(N, N)], - path: &str, -) -> Result>, ConfigError> { - let Some(raw_groups) = get(payload, "dlt_sources") else { - return Ok(Vec::new()); - }; - if raw_groups.kind() != NodeKind::List { - return Err(ConfigError::compile(format!( - "{path} dlt_sources must be a list" - ))); - } - let mut sources: Vec> = Vec::new(); - for (group_index, group) in raw_groups.items().iter().enumerate() { - if group.kind() != NodeKind::Map { - return Err(ConfigError::compile(format!( - "{path} dlt_sources must contain only mappings" - ))); - } - sources.extend(dlt_group(&group.entries(), path, group_index)?); - } - Ok(sources) -} - -fn dlt_group( - group: &[(N, N)], - path: &str, - group_index: usize, -) -> Result>, ConfigError> { - let site: Site<'_> = Site { - path, - label: "dlt source", - }; - reject_unknown_keys(group, &DLT_GROUP_KEYS, site)?; - let source_type: N = required_text(group, "type", site)?; - let type_text: String = source_type.text().unwrap_or_default(); - if ![FILESYSTEM, REST_API, SQL_DATABASE].contains(&type_text.as_str()) { - return Err(site.error("type must be one of: filesystem, rest_api, sql_database")); - } - let config: N = match get(group, "config") { - Some(config) if config.kind() == NodeKind::Map => config.clone(), - _ => return Err(site.error("config must be a mapping")), - }; - let destination: Option = optional_mapping(group, "destination", site)?; - if let Some(destination) = &destination { - let entries: Vec<(N, N)> = destination.entries(); - for blocked in BLOCKED_DESTINATION_KEYS { - if get(&entries, blocked).is_some() { - return Err(site.error(&format!("destination cannot define '{blocked}'"))); - } - } - } - let schema: Option = optional_text(group, "schema", site)?; - validate_group_config(&type_text, &config.entries(), path)?; - let resources: Vec = match get(group, "resources") { - Some(resources) if resources.kind() == NodeKind::List && !resources.items().is_empty() => { - resources.items() - } - _ => return Err(site.error("resources must be a non-empty list")), - }; - let mut sources: Vec> = Vec::with_capacity(resources.len()); - for resource in &resources { - if resource.kind() != NodeKind::Map { - return Err(site.error("resources must contain only mappings")); - } - let group: DltGroup<'_, N> = DltGroup { - source_type: &source_type, - type_text: &type_text, - schema: schema.as_ref(), - config: &config, - destination: destination.as_ref(), - index: group_index, - }; - sources.push(dlt_resource(&group, &resource.entries(), path)?); - } - Ok(sources) -} - -/// What every resource of one dlt source group shares. -struct DltGroup<'a, N> { - source_type: &'a N, - type_text: &'a str, - schema: Option<&'a N>, - config: &'a N, - destination: Option<&'a N>, - index: usize, -} - -fn validate_group_config( - source_type: &str, - config: &[(N, N)], - path: &str, -) -> Result<(), ConfigError> { - let error = |text: &str| ConfigError::compile(format!("{path} dlt {text}")); - let is_text = |value: Option<&N>| value.is_some_and(|value| value.kind() == NodeKind::Str); - match source_type { - REST_API => { - let base_url: bool = get(config, "client") - .filter(|client| client.kind() == NodeKind::Map) - .is_some_and(|client| is_text(get(&client.entries(), "base_url"))); - if !base_url { - return Err(error("rest_api config requires client.base_url")); - } - } - SQL_DATABASE if !is_text(get(config, "credentials")) => { - return Err(error("sql_database config requires credentials")); - } - FILESYSTEM if !is_text(get(config, "bucket_url")) => { - return Err(error("filesystem config requires bucket_url")); - } - _ => {} - } - Ok(()) -} - -fn dlt_resource( - group: &DltGroup<'_, N>, - resource: &[(N, N)], - path: &str, -) -> Result, ConfigError> { - let name: N = required_text( - resource, - "name", - Site { - path, - label: "dlt resource", - }, - )?; - let name_text: String = name.text().unwrap_or_default(); - let error = - |text: &str| ConfigError::compile(format!("{path} dlt resource '{name_text}' {text}")); - if get(resource, "write_strategy").is_some() { - return Err(error("must use dlt write_disposition, not write_strategy")); - } - let type_key: &str = match group.type_text { - FILESYSTEM => "reader", - REST_API => "endpoint", - _ => "table", - }; - let mut allowed: Vec<&str> = DLT_RESOURCE_KEYS.to_vec(); - allowed.push(type_key); - let typed_label: String = format!("dlt {} resource '{name_text}'", group.type_text); - reject_unknown_keys( - resource, - &allowed, - Site { - path, - label: &typed_label, - }, - )?; - let dlt_name: N = dlt_resource_name(group.type_text, resource, path)?; - let write_disposition: Option = get(resource, "write_disposition").cloned(); - let disposition: Option = write_disposition.as_ref().and_then(AuthoredNode::text); - if disposition.as_deref() == Some("delete_insert") { - return Err(error("does not support delete_insert; use a Python loader")); - } - let primary_key: Option = get(resource, "primary_key") - .filter(|value| value.kind() != NodeKind::Null) - .cloned(); - if disposition.as_deref() == Some("merge") && primary_key.is_none() { - return Err(error("merge requires primary_key")); - } - let resource_label: String = format!("dlt resource '{name_text}'"); - let resource_site: Site<'_> = Site { - path, - label: &resource_label, - }; - let schema: Option = optional_text(resource, "schema", resource_site)? - .filter(|value| python_truthy_text(value)) - .or_else(|| group.schema.cloned()); - let raw_config: Vec<(N, N)> = if group.type_text == REST_API { - resource - .iter() - .filter(|(key, _)| key.is_text("endpoint")) - .cloned() - .collect() - } else { - resource - .iter() - .filter(|(key, _)| !excluded_from_raw_config(key)) - .cloned() - .collect() - }; - let incremental: Option = optional_mapping(resource, "incremental", resource_site)?; - let description: Option = optional_text(resource, "description", resource_site)?; - let mut entry: ParsedSourceEntry = ParsedSourceEntry::named(name.clone()); - entry.description = description; - entry.schema = schema.clone(); - entry.table = Some(name.clone()); - entry.managed = true; - entry.loader = Some(integration_loader_name(DLT_LOADER_KIND, &name_text)); - entry.integration_loader = Some(ParsedIntegrationLoader::Dlt(Box::new(ParsedDltSource { - source_type: group.source_type.clone(), - schema: schema.clone(), - config: group.config.clone(), - destination: group.destination.cloned(), - resource: ParsedDltResource { - name, - dlt_name, - schema, - raw_config, - write_disposition, - primary_key: get(resource, "primary_key").cloned(), - merge_key: get(resource, "merge_key").cloned(), - incremental, - }, - group_index: group.index, - }))); - Ok(entry) -} - -/// Whether `value or other` keeps `value`: a non-empty string is always truthy. -fn python_truthy_text(value: &N) -> bool { - value.with_text(|text| !text.is_empty()).unwrap_or(true) -} - -fn dlt_resource_name( - source_type: &str, - resource: &[(N, N)], - path: &str, -) -> Result { - let site = |label| Site { path, label }; - match source_type { - SQL_DATABASE => required_text(resource, "table", site("dlt sql_database resource")), - REST_API => { - let endpoint: Vec<(N, N)> = match get(resource, "endpoint") { - Some(endpoint) if endpoint.kind() == NodeKind::Map => endpoint.entries(), - _ => { - return Err(ConfigError::compile(format!( - "{path} dlt rest_api resource must define endpoint" - ))); - } - }; - if let Some(name) = get(&endpoint, "name") - && name - .with_text(|text| !python_strip(text).is_empty()) - .unwrap_or(false) - { - return Ok(name.clone()); - } - required_text(resource, "name", site("dlt rest_api resource")) - } - _ => required_text(resource, "name", site("dlt filesystem resource")), - } -} - -/// Whether `key` is a resource setting SQLBuild reads itself rather than passing to dlt. -fn excluded_from_raw_config(key: &N) -> bool { - RAW_CONFIG_EXCLUDED - .iter() - .any(|excluded| key.is_text(excluded)) -} diff --git a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/mod.rs b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/mod.rs index 4bccc8d43a..90b120281d 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/mod.rs @@ -1,6 +1,6 @@ pub(crate) mod columns; pub(crate) mod fields; pub(crate) mod freshness; -pub(crate) mod integrations; +pub(crate) mod removed_integrations; pub(crate) mod seeds; pub(crate) mod sources; diff --git a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/removed_integrations.rs b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/removed_integrations.rs new file mode 100644 index 0000000000..9445b75079 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/removed_integrations.rs @@ -0,0 +1,40 @@ +//! The removed declarative integrations: a source's `ingestr` block and top-level `dlt_sources`. + +use sqlbuild_model_config::errors::ConfigError; +use sqlbuild_model_config::types::AuthoredNode; + +use crate::yaml_entries::_helpers::fields::{Site, get}; + +const LOADER_HELP: &str = "with `managed: true` and load it with a Python `@loader` named after \ + the source instead"; + +/// A source that still declares an `ingestr` block, which SQLBuild no longer runs. +pub(crate) fn reject_ingestr_source( + entry: &[(N, N)], + site: Site<'_>, +) -> Result<(), ConfigError> { + if get(entry, "ingestr").is_none() { + return Ok(()); + } + let name: String = get(entry, "name") + .and_then(AuthoredNode::text) + .map_or_else(String::new, |name| format!("'{name}' ")); + Err(site.error(&format!( + "{name}declares an ingestr integration, which SQLBuild no longer supports; declare the \ + source {LOADER_HELP}" + ))) +} + +/// A sources file that still declares `dlt_sources`, which SQLBuild no longer runs. +pub(crate) fn reject_dlt_sources( + payload: &[(N, N)], + path: &str, +) -> Result<(), ConfigError> { + if get(payload, "dlt_sources").is_none() { + return Ok(()); + } + Err(ConfigError::compile(format!( + "{path} declares dlt_sources, a dlt integration, which SQLBuild no longer supports; \ + declare each source under `sources:` {LOADER_HELP}" + ))) +} diff --git a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs index b917e30008..d36343f0cc 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs @@ -10,11 +10,10 @@ use crate::yaml_entries::_helpers::fields::{ stripped_strings, }; use crate::yaml_entries::_helpers::freshness::freshness; -use crate::yaml_entries::_helpers::integrations::source_integration_loader; -use crate::yaml_entries::main::integration_loader_name::integration_loader_name; -use crate::yaml_entries::models::{ParsedIntegrationLoader, ParsedSourceEntry, ParsedYamlColumn}; +use crate::yaml_entries::_helpers::removed_integrations::reject_ingestr_source; +use crate::yaml_entries::models::{ParsedSourceEntry, ParsedYamlColumn}; -const ENTRY_KEYS: [&str; 19] = [ +const ENTRY_KEYS: [&str; 18] = [ "audits", "columns", "contract", @@ -24,7 +23,6 @@ const ENTRY_KEYS: [&str; 19] = [ "expression", "final", "freshness", - "ingestr", "load_batch_size", "managed", "meta", @@ -60,6 +58,7 @@ pub(crate) fn source_entry( path, label: "source", }; + reject_ingestr_source(entry, site)?; let shape: SourceShape = source_shape(entry, site)?; let name: N = required_text(entry, "name", site)?; let name_text: String = name.text().unwrap_or_default(); @@ -102,28 +101,18 @@ fn source_shape( }) } -/// The entry named `name`, managed by its own loader or by its integration loader. +/// The entry named `name`, managed by the loader named after it. fn managed_source( entry: &[(N, N)], name: N, name_text: &str, site: Site<'_>, ) -> Result, ConfigError> { - let integration_loader: Option> = - source_integration_loader(entry, site.path)?; - let mut managed: bool = optional_bool(entry, "managed", site)? + let managed: bool = optional_bool(entry, "managed", site)? .is_some_and(|node| node.kind() == NodeKind::Bool(true)); - let mut loader: Option = managed.then(|| name_text.to_owned()); - if loader.is_none() - && let Some(integration) = &integration_loader - { - loader = Some(integration_loader_name(integration.kind(), name_text)); - managed = true; - } let mut parsed: ParsedSourceEntry = ParsedSourceEntry::named(name); parsed.managed = managed; - parsed.loader = loader; - parsed.integration_loader = integration_loader; + parsed.loader = managed.then(|| name_text.to_owned()); Ok(parsed) } @@ -243,15 +232,6 @@ fn validate_source( "uses expression with type_enforcement but has no typed columns", )); } - if let Some(integration) = &entry.integration_loader { - let kind: &str = integration.kind(); - if entry.loader.as_deref() != Some(integration_loader_name(kind, name).as_str()) { - return Err(error(&format!("cannot override loader for {kind}"))); - } - if entry.write_strategy.is_some() { - return Err(error(&format!("cannot define write_strategy with {kind}"))); - } - } let strategy: Option<&str> = entry.write_strategy; let has_unique_key: bool = !entry.unique_key.is_empty(); let has_cursor: bool = entry.cursor_column.is_some(); diff --git a/crates/sqlbuild-discovery/src/yaml_entries/constants.rs b/crates/sqlbuild-discovery/src/yaml_entries/constants.rs index aecfb9e032..387b31a7ed 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/constants.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/constants.rs @@ -1,9 +1,6 @@ -//! The top-level keys of source and schema YAML documents and integration loader kinds. +//! The top-level keys of source and schema YAML documents. /// The keys a schema YAML file under `seeds/` may declare. pub(crate) const SEED_FILE_KEYS: [&str; 1] = ["seeds"]; /// The keys a sources YAML file may declare. -pub(crate) const SOURCE_FILE_KEYS: [&str; 2] = ["sources", "dlt_sources"]; - -/// The integration kind whose loaders run under the bare source name. -pub(crate) const DLT_LOADER_KIND: &str = "dlt"; +pub(crate) const SOURCE_FILE_KEYS: [&str; 1] = ["sources"]; diff --git a/crates/sqlbuild-discovery/src/yaml_entries/main/integration_loader_name.rs b/crates/sqlbuild-discovery/src/yaml_entries/main/integration_loader_name.rs deleted file mode 100644 index 6a496fb4ac..0000000000 --- a/crates/sqlbuild-discovery/src/yaml_entries/main/integration_loader_name.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! The loader name a declarative integration source runs under. - -use crate::yaml_entries::constants::DLT_LOADER_KIND; - -/// The source name for dlt sources, `__` for every other integration. -pub fn integration_loader_name(kind: &str, source_name: &str) -> String { - if kind == DLT_LOADER_KIND { - source_name.to_owned() - } else { - format!("{kind}__{source_name}") - } -} diff --git a/crates/sqlbuild-discovery/src/yaml_entries/main/mod.rs b/crates/sqlbuild-discovery/src/yaml_entries/main/mod.rs index 2ffa29d821..cb3f72f7bd 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/main/mod.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/main/mod.rs @@ -1,3 +1,2 @@ -pub mod integration_loader_name; pub mod parse_schema_document; pub mod parse_sources_document; diff --git a/crates/sqlbuild-discovery/src/yaml_entries/main/parse_sources_document.rs b/crates/sqlbuild-discovery/src/yaml_entries/main/parse_sources_document.rs index d30212c6fb..9e51b9dc28 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/main/parse_sources_document.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/main/parse_sources_document.rs @@ -4,12 +4,12 @@ use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; use crate::yaml_entries::_helpers::fields::{Site, get, reject_unknown_keys}; -use crate::yaml_entries::_helpers::integrations::dlt_sources; +use crate::yaml_entries::_helpers::removed_integrations::reject_dlt_sources; use crate::yaml_entries::_helpers::sources::source_entry; use crate::yaml_entries::constants::SOURCE_FILE_KEYS; use crate::yaml_entries::models::ParsedSourceEntry; -/// Parse a loaded sources document: its `sources`, then the managed sources of `dlt_sources`. +/// Parse a loaded sources document's `sources`. pub fn parse_sources_document( loaded: &N, path: &str, @@ -23,6 +23,7 @@ pub fn parse_sources_document( ))); } }; + reject_dlt_sources(&payload, path)?; reject_unknown_keys( &payload, &SOURCE_FILE_KEYS, @@ -47,6 +48,5 @@ pub fn parse_sources_document( sources.push(source_entry(&item.entries(), path)?); } } - sources.extend(dlt_sources(&payload, path)?); Ok(sources) } diff --git a/crates/sqlbuild-discovery/src/yaml_entries/models.rs b/crates/sqlbuild-discovery/src/yaml_entries/models.rs index 23d3105ef4..5824b05d4e 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/models.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/models.rs @@ -71,61 +71,6 @@ pub struct ParsedFreshness { pub age_policy: Option>, } -/// A source-level `ingestr` loader; key lists hold Python's stripped strings. -#[derive(Clone, Debug, PartialEq)] -pub struct ParsedIngestrConfig { - pub source_uri: N, - pub source_table: N, - pub strategy: Option, - pub incremental_key: Option, - pub primary_key: Vec, - pub columns: Option, - pub extra_args: Vec, -} - -/// One dlt resource as `DltResourceConfig` stores it. -#[derive(Clone, Debug, PartialEq)] -pub struct ParsedDltResource { - pub name: N, - pub dlt_name: N, - pub schema: Option, - /// The resource entries the loader passes to dlt, in authored order. - pub raw_config: Vec<(N, N)>, - pub write_disposition: Option, - pub primary_key: Option, - pub merge_key: Option, - pub incremental: Option, -} - -/// A dlt source group's settings shared by the source entry of each resource. -#[derive(Clone, Debug, PartialEq)] -pub struct ParsedDltSource { - pub source_type: N, - pub schema: Option, - pub config: N, - /// The destination mapping, or `None` for Python's empty default. - pub destination: Option, - pub resource: ParsedDltResource, - pub group_index: usize, -} - -/// The declarative integration loader a source is managed by. -#[derive(Clone, Debug, PartialEq)] -pub enum ParsedIntegrationLoader { - Ingestr(ParsedIngestrConfig), - Dlt(Box>), -} - -impl ParsedIntegrationLoader { - /// The loader kind `IntegrationLoaderConfig` records. - pub fn kind(&self) -> &'static str { - match self { - Self::Ingestr(_) => "ingestr", - Self::Dlt(_) => "dlt", - } - } -} - /// One source declaration of a `sources/*.yml` file. #[derive(Clone, Debug, PartialEq)] pub struct ParsedSourceEntry { @@ -137,7 +82,6 @@ pub struct ParsedSourceEntry { pub read_final: bool, pub loader: Option, pub managed: bool, - pub integration_loader: Option>, pub freshness: Option>, /// `append`, `delete_insert`, `merge` or `table`. pub write_strategy: Option<&'static str>, @@ -165,7 +109,6 @@ impl ParsedSourceEntry { read_final: false, loader: None, managed: false, - integration_loader: None, freshness: None, write_strategy: None, load_batch_size: None, diff --git a/crates/sqlbuild-discovery/src/yaml_entries/tests/helpers.rs b/crates/sqlbuild-discovery/src/yaml_entries/tests/helpers.rs index 62e5b95790..668a5f8eff 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/tests/helpers.rs @@ -41,7 +41,7 @@ pub(super) fn sources(entry: Vec<(&str, Node)>) -> Node { map(vec![("sources", Node::List(vec![map(entry)]))]) } -/// A `dlt_sources` document with one group. +/// A `dlt_sources` document with one group, which SQLBuild rejects. pub(super) fn dlt_group(group: Vec<(&str, Node)>) -> Node { map(vec![("dlt_sources", Node::List(vec![map(group)]))]) } diff --git a/crates/sqlbuild-discovery/src/yaml_entries/tests/test_parse_sources_document.rs b/crates/sqlbuild-discovery/src/yaml_entries/tests/test_parse_sources_document.rs index 382737cc62..96133bcc97 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/tests/test_parse_sources_document.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/tests/test_parse_sources_document.rs @@ -8,23 +8,35 @@ use crate::yaml_entries::tests::test_types::{ fn given_sources_documents_when_parsing_then_matches_python_sources_and_errors() { let test_cases = [ SourcesDocumentTestCase { - description: "a dlt resource becomes a managed source named after it", + description: "a dlt_sources group is a removed integration", document: dlt_group(vec![ ("type", text("sql_database")), ( - "config", - map(vec![("credentials", text("duckdb:///shop.db"))]), + "resources", + Node::List(vec![map(vec![("name", text("orders"))])]), ), - ("schema", text("raw")), + ]), + expected_sources: parse_error( + "/project/sources/shop.yml declares dlt_sources, a dlt integration, which \ + SQLBuild no longer supports; declare each source under `sources:` with \ + `managed: true` and load it with a Python `@loader` named after the source \ + instead", + ), + }, + SourcesDocumentTestCase { + description: "an ingestr block on a source is a removed integration", + document: sources(vec![ + ("name", text("orders")), ( - "resources", - Node::List(vec![map(vec![ - ("name", text("orders")), - ("table", text("orders_v1")), - ])]), + "ingestr", + map(vec![("source_table", text("public.orders"))]), ), ]), - expected_sources: lines(&["orders loader=orders managed=true schema=raw table=orders"]), + expected_sources: parse_error( + "/project/sources/shop.yml source 'orders' declares an ingestr integration, \ + which SQLBuild no longer supports; declare the source with `managed: true` and \ + load it with a Python `@loader` named after the source instead", + ), }, SourcesDocumentTestCase { description: "final reads a table through its deduplicating form", @@ -96,24 +108,6 @@ fn given_sources_documents_when_parsing_then_matches_python_sources_and_errors() "/project/sources/shop.yml source freshness type and filter require column or query", ), }, - SourcesDocumentTestCase { - description: "a dlt resource with write_strategy", - document: dlt_group(vec![ - ("type", text("filesystem")), - ("config", map(vec![("bucket_url", text("s3://orders"))])), - ( - "resources", - Node::List(vec![map(vec![ - ("name", text("orders")), - ("write_strategy", text("append")), - ])]), - ), - ]), - expected_sources: parse_error( - "/project/sources/shop.yml dlt resource 'orders' must use dlt write_disposition, \ - not write_strategy", - ), - }, ]; for test_case in test_cases { assert_eq!( diff --git a/crates/sqlbuild-host/src/entries/_helpers/third_party.rs b/crates/sqlbuild-host/src/entries/_helpers/third_party.rs index 2319edd97b..ff5de7773d 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/third_party.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/third_party.rs @@ -139,51 +139,6 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Exception, edge: HostEdge::ThirdParty, }, - HostEntry { - name: "DLT_INTEGRATION_KIND", - kind: HostKind::Value, - edge: HostEdge::ThirdParty, - }, - HostEntry { - name: "DltIntegrationError", - kind: HostKind::Exception, - edge: HostEdge::ThirdParty, - }, - HostEntry { - name: "dlt_loader_function", - kind: HostKind::Function, - edge: HostEdge::ThirdParty, - }, - HostEntry { - name: "DltResourceConfig", - kind: HostKind::Class, - edge: HostEdge::ThirdParty, - }, - HostEntry { - name: "DltSourceConfig", - kind: HostKind::Class, - edge: HostEdge::ThirdParty, - }, - HostEntry { - name: "INGESTR_INTEGRATION_KIND", - kind: HostKind::Value, - edge: HostEdge::ThirdParty, - }, - HostEntry { - name: "IngestrIntegrationError", - kind: HostKind::Exception, - edge: HostEdge::ThirdParty, - }, - HostEntry { - name: "ingestr_loader_function", - kind: HostKind::Function, - edge: HostEdge::ThirdParty, - }, - HostEntry { - name: "IngestrSourceConfig", - kind: HostKind::Class, - edge: HostEdge::ThirdParty, - }, HostEntry { name: "RIVERS_ASSET_NODE_KINDS", kind: HostKind::Value, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 70f6e3a9ab..1922abc77d 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -424,15 +424,6 @@ pub enum ThirdPartyHost { LoadDbtYaml, RenderSelectedDbtProfileOutput, DbtProfileError, - DltIntegrationKind, - DltIntegrationError, - DltLoaderFunction, - DltResourceConfig, - DltSourceConfig, - IngestrIntegrationKind, - IngestrIntegrationError, - IngestrLoaderFunction, - IngestrSourceConfig, RiversAssetNodeKinds, } diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compile_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compile_records/constants.rs index 4d3ce8b6a2..1eda7ce0f7 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compile_records/constants.rs @@ -940,10 +940,6 @@ const R_DISCOVERED_LOADER_FUNCTION: RecordSpec = RecordSpec { FieldSpec::new("columns", FieldDefault::EmptyTuple), FieldSpec::new("contract", FieldDefault::None), FieldSpec::new("description", FieldDefault::None), - FieldSpec::new( - "connection_mode", - FieldDefault::NativeMember(RecordId::LoaderConnectionMode, "SQLBUILD"), - ), FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), FieldSpec { name: "identity", diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs index 6c3b9ce8ae..b9d86c1362 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs @@ -713,11 +713,6 @@ const E_HARD_CODED_RELATION_OWNER_KIND: EnumSpec = EnumSpec { doc: "The kind of Python code that names a project relation in SQL.", members: &[("NODE", "node"), ("HOOK", "hook"), ("LOADER", "loader")], }; -const E_LOADER_CONNECTION_MODE: EnumSpec = EnumSpec { - name: "LoaderConnectionMode", - doc: "How a source loader owns the destination warehouse connection.", - members: &[("SQLBUILD", "sqlbuild"), ("EXTERNAL", "external")], -}; const E_RULE_SUBJECT: EnumSpec = EnumSpec { name: "RuleSubject", doc: "Compiler-owned unit used to invoke and cache one custom rule.", @@ -796,6 +791,5 @@ pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ E_SCENARIO_ARTIFACT_KIND, E_SQL_REFERENCE_KIND, E_HARD_CODED_RELATION_OWNER_KIND, - E_LOADER_CONNECTION_MODE, E_RULE_SUBJECT, ]; diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs index 8f2aff3869..fa83395d14 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs @@ -497,10 +497,6 @@ const R_DISCOVERED_PYTHON_LOADER_METADATA: RecordSpec = RecordSpec { FieldSpec::new("unique_key", FieldDefault::EmptyTuple), FieldSpec::new("columns", FieldDefault::EmptyTuple), FieldSpec::new("contract", FieldDefault::None), - FieldSpec::new( - "connection_mode", - FieldDefault::NativeMember(RecordId::LoaderConnectionMode, "SQLBUILD"), - ), ], }; const R_DISCOVERED_PYTHON_TASK_METADATA: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index 3ac29f7c87..33fbe90f14 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -659,11 +659,6 @@ const R_SCHEMA_SEED_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("audits", FieldDefault::EmptyTuple), ], }; -const R_INTEGRATION_LOADER_CONFIG: RecordSpec = RecordSpec { - name: "IntegrationLoaderConfig", - doc: "Typed configuration for one declarative integration loader.", - fields: &[FieldSpec::required("kind"), FieldSpec::required("config")], -}; const R_SOURCE_FRESHNESS_AGE_POLICY: RecordSpec = RecordSpec { name: "SourceFreshnessAgePolicy", doc: "Age-based source freshness warning/error thresholds.", @@ -708,7 +703,6 @@ const R_SOURCE_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("final", FieldDefault::False), FieldSpec::new("loader", FieldDefault::None), FieldSpec::new("managed", FieldDefault::False), - FieldSpec::new("integration_loader", FieldDefault::None), FieldSpec::new("freshness", FieldDefault::None), FieldSpec::new("write_strategy", FieldDefault::None), FieldSpec::new("load_batch_size", FieldDefault::None), @@ -780,7 +774,6 @@ pub(crate) const CONTRACT_RECORDS: &[RecordSpec] = &[ R_SCHEMA_DYNAMIC_COLUMN_FAMILY, R_SCHEMA_MODEL_ENTRY, R_SCHEMA_SEED_ENTRY, - R_INTEGRATION_LOADER_CONFIG, R_SOURCE_FRESHNESS_AGE_POLICY, R_SOURCE_FRESHNESS_CONFIG, R_SOURCE_COLUMN_ENTRY, diff --git a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs index 3a3a2c281d..ac4ad9478e 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs @@ -280,7 +280,6 @@ native_id_names! { RecordId::InferredNullability => "InferredNullability", RecordId::InitialValidFrom => "InitialValidFrom", RecordId::IntegerValue => "IntegerValue", - RecordId::IntegrationLoaderConfig => "IntegrationLoaderConfig", RecordId::InterpolationSite => "InterpolationSite", RecordId::InvalidSqlReferenceCall => "InvalidSqlReferenceCall", RecordId::JanitorCommandRequest => "JanitorCommandRequest", @@ -302,7 +301,6 @@ native_id_names! { RecordId::LiteralSqlRelation => "LiteralSqlRelation", RecordId::LoadCommandRequest => "LoadCommandRequest", RecordId::LoadedMacro => "LoadedMacro", - RecordId::LoaderConnectionMode => "LoaderConnectionMode", RecordId::LoaderDestinationParts => "LoaderDestinationParts", RecordId::LocalClonePolicy => "LocalClonePolicy", RecordId::LocalConfig => "LocalConfig", diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index c849c68bd0..9d56005e6d 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -714,7 +714,6 @@ const R_SOURCE_LOAD_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("cursor_column", FieldDefault::None), FieldSpec::new("unique_key", FieldDefault::EmptyTuple), FieldSpec::new("is_reload", FieldDefault::False), - FieldSpec::new("integration_kind", FieldDefault::None), ], }; const R_FUNCTION_PLAN_ENTRY: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/types.rs b/crates/sqlbuild-python-base/src/records/types.rs index 8bbe5811e5..ea187f9426 100644 --- a/crates/sqlbuild-python-base/src/records/types.rs +++ b/crates/sqlbuild-python-base/src/records/types.rs @@ -273,7 +273,6 @@ pub enum RecordId { InferredNullability, InitialValidFrom, IntegerValue, - IntegrationLoaderConfig, InterpolationSite, InvalidSqlReferenceCall, JanitorCommandRequest, @@ -295,7 +294,6 @@ pub enum RecordId { LiteralSqlRelation, LoadCommandRequest, LoadedMacro, - LoaderConnectionMode, LoaderDestinationParts, LocalClonePolicy, LocalConfig, diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/integration_loaders.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/integration_loaders.rs deleted file mode 100644 index c4d836a00d..0000000000 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/integration_loaders.rs +++ /dev/null @@ -1,86 +0,0 @@ -//! Synthetic loader records for sources declaring an ingestr or dlt integration loader. - -use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyTuple; -use sqlbuild_discovery::yaml_entries::main::integration_loader_name::integration_loader_name; -use sqlbuild_host::entries::models::ThirdPartyHost; - -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; - -/// One integration: its kind constant and the runtime module building its loader functions. -struct Integration { - kind_constant: ThirdPartyHost, - loader_factory: ThirdPartyHost, -} - -/// Integrations in loader order: every ingestr loader precedes every dlt loader. -const INTEGRATIONS: [Integration; 2] = [ - Integration { - kind_constant: ThirdPartyHost::IngestrIntegrationKind, - loader_factory: ThirdPartyHost::IngestrLoaderFunction, - }, - Integration { - kind_constant: ThirdPartyHost::DltIntegrationKind, - loader_factory: ThirdPartyHost::DltLoaderFunction, - }, -]; - -/// The external-connection `DiscoveredLoaderFunction` of every integration source, in file order. -pub(crate) fn integration_loader_functions<'py>( - py: Python<'py>, - source_files: &Bound<'py, PyAny>, -) -> PyResult> { - let mut declared: Vec<(String, Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); - for source_file in source_files.try_iter()? { - let source_file: Bound<'py, PyAny> = source_file?; - for source_entry in native_field(&source_file, "source_entries")?.try_iter()? { - let source_entry: Bound<'py, PyAny> = source_entry?; - let loader: Bound<'py, PyAny> = native_field(&source_entry, "integration_loader")?; - if loader.is_none() { - continue; - } - let kind: String = native_field(&loader, "kind")?.extract()?; - declared.push((kind, source_file.clone(), source_entry)); - } - } - if declared.is_empty() { - return Ok(PyTuple::empty(py).into_any()); - } - let external: Bound<'py, PyAny> = enum_member(py, RecordId::LoaderConnectionMode, "EXTERNAL")?; - let mut loaders: Vec> = Vec::new(); - for integration in &INTEGRATIONS { - let kind: String = host_entry(py, integration.kind_constant)?.extract()?; - let factory: Bound<'py, PyAny> = host_entry(py, integration.loader_factory)?; - for (_, source_file, source_entry) in declared.iter().filter(|loader| loader.0 == kind) { - let name: String = integration_loader_name( - &kind, - &native_field(source_entry, "name")?.extract::()?, - ); - let function: Bound<'py, PyAny> = call_with( - &factory, - &[ - ("source_entry", source_entry.clone()), - ("name", name.clone().into_pyobject(py)?.into_any()), - ], - )?; - loaders.push(new_record_with( - py, - RecordId::DiscoveredLoaderFunction, - &[ - ("file_path", native_field(source_file, "file_path")?), - ("relative_path", native_field(source_file, "relative_path")?), - ("name", name.into_pyobject(py)?.into_any()), - ("function", function), - ("connection_mode", external.clone()), - ], - )?); - } - } - Ok(PyTuple::new(py, loaders)?.into_any()) -} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs index ba9e864c07..34b8da1b5a 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod inputs; -pub(crate) mod integration_loaders; pub(crate) mod pipeline; pub(crate) mod pipeline_entries; pub(crate) mod project; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/discovery.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/discovery.rs index a55fbc50db..ada549062c 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/discovery.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/discovery.rs @@ -6,7 +6,6 @@ use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::compile_driver::_helpers::setup::integration_loaders::integration_loader_functions; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -457,9 +456,7 @@ fn project_inputs<'py>( )? .into_any()) }; - let source_files: Bound<'py, PyAny> = declarations.get_item("source_files")?; - let integration_loaders: Bound<'py, PyAny> = integration_loader_functions(py, &source_files)?; - let loader_functions: Bound<'py, PyAny> = tuple_of("loaders")?.add(integration_loaders)?; + let loader_functions: Bound<'py, PyAny> = tuple_of("loaders")?; let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ ("project_config", project_config.clone()), ("local_config", local_config.clone()), diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs index 1fdf94746a..19d4f79fdf 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs @@ -10,9 +10,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTypeMethods}; use sqlbuild_discovery::project_validation::errors::CsvDialectError; use sqlbuild_discovery::project_validation::models::{ - AuditFileFact, CsvSettings, DependencyFact, IntegrationLoaderFact, ModelFact, NamedPath, - NodeFact, NodeKind, SchemaFileFact, ScopedDeclarationsFact, SeedEntryFact, SeedFileFact, - SourceFact, SourceFileFact, TestBlockFact, TestFileFact, + AuditFileFact, CsvSettings, DependencyFact, ModelFact, NamedPath, NodeFact, NodeKind, + SchemaFileFact, ScopedDeclarationsFact, SeedEntryFact, SeedFileFact, SourceFact, + SourceFileFact, TestBlockFact, TestFileFact, }; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -123,19 +123,10 @@ pub(crate) fn schema_fact(py: Python<'_>, schema: &Bound<'_, PyAny>) -> PyResult } fn source_fact(entry: &Bound<'_, PyAny>) -> PyResult { - let integration: Bound<'_, PyAny> = native_field(entry, "integration_loader")?; - let integration_loader: Option = if integration.is_none() { - None - } else { - Some(IntegrationLoaderFact { - kind: native_field(&integration, "kind")?.extract()?, - }) - }; Ok(SourceFact { name: column_name(entry)?, managed: native_field(entry, "managed")?.extract()?, loader: optional_text(entry, "loader")?, - integration_loader, }) } diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs index cdfa999a41..29d3b4d2c2 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs @@ -6,8 +6,8 @@ use pyo3::{FromPyObject, pyfunction}; use sqlbuild_discovery::yaml_entries::main::parse_schema_document::parse_schema_document as parse_schema; use sqlbuild_discovery::yaml_entries::main::parse_sources_document::parse_sources_document as parse_sources; use sqlbuild_discovery::yaml_entries::models::{ - ParsedCsvSettings, ParsedDltSource, ParsedFreshness, ParsedIngestrConfig, - ParsedIntegrationLoader, ParsedNaValues, ParsedSeedEntry, ParsedSourceEntry, ParsedYamlColumn, + ParsedCsvSettings, ParsedFreshness, ParsedNaValues, ParsedSeedEntry, ParsedSourceEntry, + ParsedYamlColumn, }; use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; @@ -34,10 +34,6 @@ pub(crate) struct YamlClasses<'py> { source_column_entry: Bound<'py, PyAny>, source_freshness_config: Bound<'py, PyAny>, source_freshness_age_policy: Bound<'py, PyAny>, - integration_loader_config: Bound<'py, PyAny>, - ingestr_source_config: Bound<'py, PyAny>, - dlt_source_config: Bound<'py, PyAny>, - dlt_resource_config: Bound<'py, PyAny>, source_write_strategy: Bound<'py, PyAny>, source_freshness_strategy: Bound<'py, PyAny>, source_freshness_value_kind: Bound<'py, PyAny>, @@ -217,79 +213,11 @@ fn freshness<'py>( classes.source_freshness_config.call((), Some(&fields)) } -fn ingestr_config<'py>( - py: Python<'py>, - classes: &YamlClasses<'py>, - config: &ParsedIngestrConfig>, -) -> PyResult> { - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("source_uri", &config.source_uri.0)?; - fields.set_item("source_table", &config.source_table.0)?; - fields.set_item("strategy", optional(py, config.strategy.as_ref()))?; - fields.set_item( - "incremental_key", - optional(py, config.incremental_key.as_ref()), - )?; - fields.set_item("primary_key", strings(py, &config.primary_key)?)?; - fields.set_item("columns", optional(py, config.columns.as_ref()))?; - fields.set_item("extra_args", strings(py, &config.extra_args)?)?; - classes.ingestr_source_config.call((), Some(&fields)) -} - -fn dlt_config<'py>( - py: Python<'py>, - classes: &YamlClasses<'py>, - source: &ParsedDltSource>, -) -> PyResult> { - let resource = &source.resource; - let raw_config: Bound<'py, PyDict> = PyDict::new(py); - for (key, value) in &resource.raw_config { - raw_config.set_item(&key.0, &value.0)?; - } - let resource_fields: Bound<'py, PyDict> = PyDict::new(py); - resource_fields.set_item("name", &resource.name.0)?; - resource_fields.set_item("dlt_name", &resource.dlt_name.0)?; - resource_fields.set_item("schema", optional(py, resource.schema.as_ref()))?; - resource_fields.set_item("raw_config", raw_config)?; - resource_fields.set_item( - "write_disposition", - optional(py, resource.write_disposition.as_ref()), - )?; - resource_fields.set_item("primary_key", optional(py, resource.primary_key.as_ref()))?; - resource_fields.set_item("merge_key", optional(py, resource.merge_key.as_ref()))?; - resource_fields.set_item("incremental", meta(py, resource.incremental.as_ref()))?; - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("source_type", &source.source_type.0)?; - fields.set_item("schema", optional(py, source.schema.as_ref()))?; - fields.set_item("config", &source.config.0)?; - fields.set_item("destination", meta(py, source.destination.as_ref()))?; - fields.set_item( - "resource", - classes - .dlt_resource_config - .call((), Some(&resource_fields))?, - )?; - fields.set_item("group_index", source.group_index)?; - classes.dlt_source_config.call((), Some(&fields)) -} - fn source_entry<'py>( py: Python<'py>, classes: &YamlClasses<'py>, source: &ParsedSourceEntry>, ) -> PyResult> { - let integration: Bound<'py, PyAny> = match &source.integration_loader { - None => py.None().into_bound(py), - Some(loader) => { - let config: Bound<'py, PyAny> = match loader { - ParsedIntegrationLoader::Ingestr(config) => ingestr_config(py, classes, config)?, - ParsedIntegrationLoader::Dlt(source) => dlt_config(py, classes, source)?, - }; - classes - .integration_loader_config - .call1((loader.kind(), config))? - } - }; let fields: Bound<'py, PyDict> = PyDict::new(py); fields.set_item("name", &source.name.0)?; fields.set_item("database", optional(py, source.database.as_ref()))?; @@ -298,7 +226,6 @@ fn source_entry<'py>( fields.set_item("final", source.read_final)?; fields.set_item("loader", source.loader.as_deref())?; fields.set_item("managed", source.managed)?; - fields.set_item("integration_loader", integration)?; let freshness_config: Bound<'py, PyAny> = match &source.freshness { Some(config) => freshness(py, classes, config)?, None => py.None().into_bound(py), @@ -370,9 +297,7 @@ pub(crate) fn parse_sources_document<'py>( fn loaded_sources(sources: &[ParsedSourceEntry>]) -> Vec { sources .iter() - .filter(|source| { - source.managed || source.loader.is_some() || source.integration_loader.is_some() - }) + .filter(|source| source.managed || source.loader.is_some()) .filter_map(|source| { let Ok(name) = source.name.0.extract::() else { return None; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs index 6a4c93a030..0329261a25 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs @@ -4,7 +4,7 @@ use crate::model_config::main::model_nodes::contract_classes::contract_classes; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods}; -use sqlbuild_host::entries::models::{FacadeHost, ThirdPartyHost}; +use sqlbuild_host::entries::models::FacadeHost; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -25,7 +25,7 @@ const CONTRACT_CLASSES: [(&str, RecordId); 6] = [ RecordId::MeasurementThresholdBound, ), ]; -const YAML_ENTRY_CLASSES: [(&str, RecordId); 10] = [ +const YAML_ENTRY_CLASSES: [(&str, RecordId); 9] = [ ("schema_seed_entry", RecordId::SchemaSeedEntry), ("seed_csv_settings", RecordId::SeedCsvSettings), ("source_entry", RecordId::SourceEntry), @@ -35,10 +35,6 @@ const YAML_ENTRY_CLASSES: [(&str, RecordId); 10] = [ "source_freshness_age_policy", RecordId::SourceFreshnessAgePolicy, ), - ( - "integration_loader_config", - RecordId::IntegrationLoaderConfig, - ), ("source_write_strategy", RecordId::SourceWriteStrategy), ( "source_freshness_strategy", @@ -49,12 +45,6 @@ const YAML_ENTRY_CLASSES: [(&str, RecordId); 10] = [ RecordId::SourceFreshnessValueKind, ), ]; -const INTEGRATION_ENTRY_CLASSES: [(&str, ThirdPartyHost); 3] = [ - ("ingestr_source_config", ThirdPartyHost::IngestrSourceConfig), - ("dlt_source_config", ThirdPartyHost::DltSourceConfig), - ("dlt_resource_config", ThirdPartyHost::DltResourceConfig), -]; - pub(crate) static CONTRACT_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); pub(crate) static YAML_ENTRY_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); @@ -80,9 +70,6 @@ pub(crate) fn built_yaml_entry_classes(py: Python<'_>) -> PyResult) -> PyResult { "value", )?)?, is_reload: native_field(entry, "is_reload")?.is_truthy()?, - integration_kind: optional_text(&native_field(entry, "integration_kind")?)?, write_strategy: if strategy.is_none() { None } else { diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs index c808eadbc4..1e743fe847 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs @@ -67,7 +67,6 @@ fn table_name(entry: &Bound<'_, PyAny>) -> PyResult { fn external_table(entry: &Bound<'_, PyAny>) -> PyResult { Ok(!native_field(entry, "managed")?.is_truthy()? && native_field(entry, "loader")?.is_none() - && native_field(entry, "integration_loader")?.is_none() && native_field(entry, "expression")?.is_none() && !native_field(entry, "schema")?.is_none() && !table_name(entry)?.contains(WILDCARD_TABLE_CHARACTER)) diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs index b9c88351a2..217782ff6b 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs @@ -4,15 +4,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::python_plan::models::LifecycleNode; use sqlbuild_python_base::records::main::fields::field::field as native_field; -const EXTERNAL_CONNECTION_MODE: &str = "external"; - pub(crate) fn lifecycle_node(node: &Bound<'_, PyAny>) -> PyResult { - let loader: Bound<'_, PyAny> = native_field(node, "loader")?; - let external_loader: bool = !loader.is_none() - && native_field(&loader, "connection_mode")?.extract::()? - == EXTERNAL_CONNECTION_MODE; Ok(LifecycleNode { kind: native_field(node, "kind")?.extract()?, - external_loader, }) } diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs index 3a20a71b6d..ccd8d7dd2e 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs @@ -64,7 +64,6 @@ fn kind_metadata<'py>( "unique_key", "columns", "contract", - "connection_mode", ], )?, ), diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs index 36585d9e3c..d6ca143475 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs @@ -83,15 +83,6 @@ pub(crate) fn build_source_load_entries<'py>( kinds.getattr(if intermediate { "LOADER" } else { "SOURCE" })?, )?; kwargs.set_item("is_reload", is_reload)?; - let integration = native_field(&entry, "integration_loader")?; - kwargs.set_item( - "integration_kind", - if integration.is_none() { - py.None().into_bound(py) - } else { - native_field(&integration, "kind")? - }, - )?; entries.push(new_record( py, RecordId::SourceLoadPlanEntry, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs deleted file mode 100644 index 37571cebf7..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs +++ /dev/null @@ -1,125 +0,0 @@ -//! The dlt destination factory, arguments and dataset for one SQLBuild warehouse adapter. - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::dlt_destination::constants::DEFAULT_DESTINATION_OPTIONS; -use sqlbuild_core::dlt_destination::main::dlt_destination_plan::dlt_destination_plan; -use sqlbuild_core::dlt_destination::models::{ - DltDataset, DltDatasetName, DltDestinationPlan, DltDestinationRequest, DltValue, -}; -use sqlbuild_core::loader_connection::models::LoaderConnection; -use sqlbuild_host::entries::models::ThirdPartyHost; -use sqlbuild_python_base::native_host::main::host_error::host_error; -use sqlbuild_python_base::positional_record; - -use crate::bindings::_helpers::load::ingestr_command::connection; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; - -const DLT_ERROR: ThirdPartyHost = ThirdPartyHost::DltIntegrationError; - -positional_record! { - into; - /// `(factory, arguments, options, dataset_name)` of one dlt destination. - struct DestinationCall<'py> { - factory: &'static str, - arguments: Bound<'py, PyDict>, - options: Bound<'py, PyDict>, - dataset: Bound<'py, PyAny>, - } -} - -/// A `DltIntegrationError`, or the failure to construct one. -fn dlt_error(py: Python<'_>, message: String) -> PyErr { - host_error(py, DLT_ERROR, &message) -} - -/// The text of a string value, or `None` for any other value. -fn string(value: &Bound<'_, PyAny>) -> PyResult> { - if value.is_instance_of::() { - return Ok(Some(value.extract()?)); - } - Ok(None) -} - -/// One factory argument as a Python value. -fn argument<'py>( - value: &DltValue, - connection_config: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = connection_config.py(); - match value { - DltValue::Text(text) => Ok(PyString::new(py, text).into_any()), - DltValue::Port { default } => mapping_get_or(connection_config, "port", *default), - DltValue::Object(fields) => Ok(arguments(fields, connection_config)?.into_any()), - } -} - -fn arguments<'py>( - fields: &[(&str, DltValue)], - connection_config: &Bound<'py, PyAny>, -) -> PyResult> { - let values: Bound<'py, PyDict> = PyDict::new(connection_config.py()); - for (key, value) in fields { - values.set_item(*key, argument(value, connection_config)?)?; - } - Ok(values) -} - -/// `(factory, arguments, options, dataset_name)`; callers unpack `arguments` and `options` apart. -#[pyfunction] -#[pyo3(signature = (*, adapter_name, connection_config, destination_config, dataset_name))] -fn dlt_destination_arguments<'py>( - adapter_name: &str, - connection_config: &Bound<'py, PyAny>, - destination_config: &Bound<'py, PyAny>, - dataset_name: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = connection_config.py(); - let dict: Bound<'py, PyAny> = imported(py, "builtins", "dict")?; - let destination: Bound<'py, PyDict> = if destination_config.is_truthy()? { - dict.call1((destination_config,))?.downcast_into()? - } else { - PyDict::new(py) - }; - let mut destination_keys: Vec> = Vec::new(); - for key in destination.keys() { - destination_keys.push(string(&key)?); - } - let settings: Bound<'py, PyDict> = dict.call1((connection_config,))?.downcast_into()?; - let name = DltDatasetName { - text: string(dataset_name)?, - truthy: dataset_name.is_truthy()?, - }; - let connection: LoaderConnection = connection(&settings)?; - let request = DltDestinationRequest { - adapter: adapter_name, - connection: &connection, - destination_keys: &destination_keys, - dataset_name: &name, - }; - let plan: DltDestinationPlan = - dlt_destination_plan(&request).map_err(|error| dlt_error(py, error.message))?; - let options: Bound<'py, PyDict> = PyDict::new(py); - for (key, value) in DEFAULT_DESTINATION_OPTIONS { - options.set_item(key, value)?; - } - options.update(destination.as_mapping())?; - let dataset: Bound<'py, PyAny> = match plan.dataset { - DltDataset::Given => dataset_name.clone(), - DltDataset::Default(name) => PyString::new(py, name).into_any(), - }; - Ok(DestinationCall { - factory: plan.factory, - arguments: arguments(&plan.arguments, connection_config)?, - options, - dataset, - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(dlt_destination_arguments, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs deleted file mode 100644 index 6e9b4b0015..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs +++ /dev/null @@ -1,201 +0,0 @@ -//! The ingestr command line for one declarative ingestr source, built from its runtime values. - -use sqlbuild_python_base::native_host::main::host_error::host_error; -use std::collections::BTreeMap; - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::ingestr_command::main::format_ingestr_command::format_ingestr_command as native_format_ingestr_command; -use sqlbuild_core::ingestr_command::main::ingestr_command_arguments::ingestr_command_arguments; -use sqlbuild_core::ingestr_command::main::ingestr_destination_table::ingestr_destination_table; -use sqlbuild_core::ingestr_command::main::ingestr_destination_uri::ingestr_destination_uri; -use sqlbuild_core::ingestr_command::models::{IngestrDestination, IngestrSource}; -use sqlbuild_core::loader_connection::models::LoaderConnection; -use sqlbuild_host::entries::models::ThirdPartyHost; -use sqlbuild_model_config::templates::models::TemplateOptions; - -use sqlbuild_python_base::boundary::models::Request; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::model_config::main::config::expanded_template_data::expanded_template_data; - -const INGESTR_ERROR: ThirdPartyHost = ThirdPartyHost::IngestrIntegrationError; -const INGESTR_SOURCE_CONFIG: ThirdPartyHost = ThirdPartyHost::IngestrSourceConfig; -const INGESTR_KIND: ThirdPartyHost = ThirdPartyHost::IngestrIntegrationKind; -const TEMPLATE_LABEL: &str = "ingestr source config"; -const TEMPLATE_FIELDS: [&str; 7] = [ - "source_uri", - "source_table", - "strategy", - "incremental_key", - "primary_key", - "columns", - "extra_args", -]; -const LIST_FIELDS: [&str; 2] = ["primary_key", "extra_args"]; - -/// An `IngestrIntegrationError`, or the failure to construct one. -fn ingestr_error(py: Python<'_>, message: String) -> PyErr { - host_error(py, INGESTR_ERROR, &message) -} - -/// The source's ingestr config, raising when the source declares no valid one. -fn ingestr_config<'py>(source_entry: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = source_entry.py(); - let name: String = native_field(source_entry, "name")?.str()?.extract()?; - let loader: Bound<'py, PyAny> = native_field(source_entry, "integration_loader")?; - if loader.is_none() || !native_field(&loader, "kind")?.eq(host_entry(py, INGESTR_KIND)?)? { - return Err(ingestr_error( - py, - format!("Source '{name}' does not define ingestr config"), - )); - } - let config: Bound<'py, PyAny> = native_field(&loader, "config")?; - if !config.is_instance(&host_entry(py, INGESTR_SOURCE_CONFIG)?)? { - return Err(ingestr_error( - py, - format!("Source '{name}' defines invalid ingestr config"), - )); - } - Ok(config) -} - -/// `str(value)`, or `None` when the value is `None` or renders empty. -fn optional_text(value: Option>) -> PyResult> { - let Some(value) = value.filter(|value| !value.is_none()) else { - return Ok(None); - }; - let text: String = value.str()?.extract()?; - Ok((!text.is_empty()).then_some(text)) -} - -/// One string per item of a list value, or the value itself as one string. -fn text_list(value: Option>) -> PyResult> { - let Some(value) = value.filter(|value| !value.is_none()) else { - return Ok(Vec::new()); - }; - if value.is_instance_of::() { - return Ok(vec![value.extract()?]); - } - match value.try_iter() { - Ok(items) => items - .map(|item| item?.str()?.extract()) - .collect::>>(), - Err(_) => Ok(vec![value.str()?.extract()?]), - } -} - -/// The source config with `${...}` templates expanded against vars and run context. -fn resolved_source( - config: &Bound<'_, PyAny>, - vars: &Bound<'_, PyDict>, - (environment, run_id): (Option<&str>, &str), -) -> PyResult { - let py: Python<'_> = config.py(); - let value: Bound<'_, PyDict> = PyDict::new(py); - for field in TEMPLATE_FIELDS { - let raw: Bound<'_, PyAny> = native_field(config, field)?; - if LIST_FIELDS.contains(&field) { - value.set_item( - field, - PyList::new(py, raw.try_iter()?.collect::>>()?)?, - )?; - } else { - value.set_item(field, raw)?; - } - } - let context: Bound<'_, PyDict> = PyDict::new(py); - context.set_item("environment", environment)?; - context.set_item("run_id", run_id)?; - let expanded: Bound<'_, PyAny> = expanded_template_data( - value.as_any(), - (vars, &context), - ( - TemplateOptions { - allow_context: true, - preserve_context_tokens: false, - preserve_unknown_context: true, - }, - TEMPLATE_LABEL, - ), - )?; - let expanded: &Bound<'_, PyDict> = expanded.downcast()?; - let get = |key: &str| expanded.get_item(key); - Ok(IngestrSource { - source_uri: expanded.as_any().get_item("source_uri")?.str()?.extract()?, - source_table: expanded - .as_any() - .get_item("source_table")? - .str()? - .extract()?, - strategy: optional_text(get("strategy")?)?, - incremental_key: optional_text(get("incremental_key")?)?, - primary_key: text_list(get("primary_key")?)?, - columns: optional_text(get("columns")?)?, - extra_args: text_list(get("extra_args")?)?, - }) -} - -/// The string settings of a connection and its rendered port. -pub(crate) fn connection(connection_config: &Bound<'_, PyDict>) -> PyResult { - let mut strings: BTreeMap = BTreeMap::new(); - for (key, value) in connection_config.iter() { - if let (Ok(key), Ok(value)) = (key.extract::(), value.downcast::()) { - strings.insert(key, value.extract()?); - } - } - let port: Option = connection_config - .get_item("port")? - .map(|port| port.str()?.extract()) - .transpose()?; - Ok(LoaderConnection { strings, port }) -} - -/// The ingestr CLI command for one source entry. -#[pyfunction] -#[pyo3(signature = (**request))] -fn build_ingestr_command<'py>( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request: Request<'_, 'py> = Request::new("build_ingestr_command", request)?; - let py: Python<'py> = request.py(); - let source_entry: Bound<'py, PyAny> = request.get("source_entry")?; - let config: Bound<'py, PyAny> = ingestr_config(&source_entry)?; - let environment: Option = request.get("environment")?.extract()?; - let run_id: String = request.get("run_id")?.extract()?; - let source: IngestrSource = resolved_source( - &config, - request.get("vars")?.downcast()?, - (environment.as_deref(), &run_id), - )?; - let adapter_name: String = request.get("adapter_name")?.extract()?; - let uri: String = ingestr_destination_uri( - &adapter_name, - &connection(request.get("connection_config")?.downcast()?)?, - ) - .map_err(|error| ingestr_error(py, error.message))?; - let schema: Option = native_field(&source_entry, "schema")?.extract()?; - let destination_name: String = request.get("destination_name")?.extract()?; - let destination: IngestrDestination = IngestrDestination { - uri, - table: ingestr_destination_table(schema.as_deref(), &destination_name), - is_reload: request.get("is_reload")?.is_truthy()?, - }; - PyTuple::new(py, ingestr_command_arguments(&source, &destination)) -} - -/// The display-safe, shell-quoted form of an ingestr command. -#[pyfunction] -#[pyo3(signature = (command, /))] -fn format_ingestr_command(command: Vec) -> String { - native_format_ingestr_command(&command) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_ingestr_command, module)?)?; - module.add_function(wrap_pyfunction!(format_ingestr_command, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs index 59c61242c6..68a14122c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs @@ -1,5 +1,3 @@ -pub(crate) mod dlt_destination; -pub(crate) mod ingestr_command; pub(crate) mod load_command; pub(crate) mod load_progress; pub(crate) mod load_selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs index a6a6c41ef0..198a9ed693 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs @@ -2,13 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::load::{ - dlt_destination, ingestr_command, load_command, load_progress, load_selection, -}; +use crate::bindings::_helpers::load::{load_command, load_progress, load_selection}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - dlt_destination::register(module)?; - ingestr_command::register(module)?; load_command::register(module)?; load_progress::register(module)?; load_selection::register(module) diff --git a/fensu.toml b/fensu.toml index bb7e60942e..98ab180ea3 100644 --- a/fensu.toml +++ b/fensu.toml @@ -587,46 +587,6 @@ modules = ["sqlbuild.integrations.dbt.exceptions"] symbols = ["DbtProfileError"] reason = "Native code resolves these through the sqlbuild._native_host name table." -[[targets.python.dead_code.roots]] -modules = ["sqlbuild.integrations.dlt.constants"] -symbols = ["DLT_INTEGRATION_KIND"] -reason = "Native code resolves these through the sqlbuild._native_host name table." - -[[targets.python.dead_code.roots]] -modules = ["sqlbuild.integrations.dlt.exceptions"] -symbols = ["DltIntegrationError"] -reason = "Native code resolves these through the sqlbuild._native_host name table." - -[[targets.python.dead_code.roots]] -modules = ["sqlbuild.integrations.dlt.main._loaders"] -symbols = ["dlt_loader_function"] -reason = "Native code resolves these through the sqlbuild._native_host name table." - -[[targets.python.dead_code.roots]] -modules = ["sqlbuild.integrations.dlt.models"] -symbols = ["DltResourceConfig", "DltSourceConfig"] -reason = "Native code resolves these through the sqlbuild._native_host name table." - -[[targets.python.dead_code.roots]] -modules = ["sqlbuild.integrations.ingestr.constants"] -symbols = ["INGESTR_INTEGRATION_KIND"] -reason = "Native code resolves these through the sqlbuild._native_host name table." - -[[targets.python.dead_code.roots]] -modules = ["sqlbuild.integrations.ingestr.exceptions"] -symbols = ["IngestrIntegrationError"] -reason = "Native code resolves these through the sqlbuild._native_host name table." - -[[targets.python.dead_code.roots]] -modules = ["sqlbuild.integrations.ingestr.main._loaders"] -symbols = ["ingestr_loader_function"] -reason = "Native code resolves these through the sqlbuild._native_host name table." - -[[targets.python.dead_code.roots]] -modules = ["sqlbuild.integrations.ingestr.models"] -symbols = ["IngestrSourceConfig"] -reason = "Native code resolves these through the sqlbuild._native_host name table." - [[targets.python.dead_code.roots]] modules = ["sqlbuild.integrations.rivers.constants"] symbols = ["RIVERS_ASSET_NODE_KINDS"] @@ -1516,18 +1476,6 @@ path = "scripts/cli_preview/_helpers/workflow.py" symbols = ["_run_process._forward_signal"] reason = "This callback preserves signal.signal's required positional handler protocol." -[[targets.python.rule_exceptions]] -rule = "FFS120" -path = "src/sqlbuild/integrations/dlt/classes/sqlbuild_dlt_progress_collector.py" -symbols = [ - "SqlbuildDltProgressCollector.on_start_trace", - "SqlbuildDltProgressCollector.on_start_trace_step", - "SqlbuildDltProgressCollector.on_end_trace_step", - "SqlbuildDltProgressCollector.on_end_trace", - "SqlbuildDltProgressCollector.update", -] -reason = "These collector methods preserve dlt's external positional progress protocol." - [[targets.python.rule_exceptions]] rule = "FFS120" path = "src/sqlbuild/runtime/user_code/classes/user_code_finder.py" @@ -2381,7 +2329,7 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs=1", "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs=3", "crates/sqlbuild-python-front/src/discovery/_helpers/sources/sql_test_blocks.rs=2", - "crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs=14", + "crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs=10", "crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs=1", "crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs=1", "crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs=6", @@ -2787,8 +2735,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=2", @@ -2934,7 +2880,7 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs=12", "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/native_classes.rs=2", "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs=1", - "crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs=13", + "crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs=9", "crates/sqlbuild-python-front/src/discovery/models.rs=7", "crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs=9", "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs=1", @@ -3063,7 +3009,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=4", diff --git a/pyproject.toml b/pyproject.toml index 1ba942caea..897bda1a46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,13 +47,6 @@ sqlserver = [ clickhouse = [ "clickhouse-connect>=0.15", ] -ingestr = [ - "dbc>=0.3.1", - "ingestr>=1.1.73", -] -dlt = [ - "dlt[duckdb,filesystem,sql_database]>=1.28.0,<2", -] rivers = [ "rivers>=0.1.4", ] @@ -71,9 +64,6 @@ dev = [ "dbt-postgres>=1.9.0,<1.11", "dbt-snowflake==1.12.0b1", "dbt-sqlserver>=1.9.0,<1.11", - "dlt[duckdb,filesystem,sql_database]>=1.28.0,<2", - "dbc>=0.3.1", - "ingestr>=1.1.73", "jsonschema>=4.23.0", "maturin>=1.9,<2", "pre-commit>=4.3.0", diff --git a/python-boundary.toml b/python-boundary.toml index 93d911cd83..896154dc6b 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -397,16 +397,6 @@ path = "integrations/rivers" rule = 4 note = "Documented orchestrator integration and rule 4 Rivers glue: translator invocation and AssetDef construction; the asset projection of the DAG artifact is native." -[[stays]] -path = "integrations/dlt" -rule = 4 -note = "Loader runtime that runs inside user loaders and calls the dlt destination factories; synthetic loader record building, naming and adapter-config to destination mapping are native (dlt_destination_arguments)." - -[[stays]] -path = "integrations/ingestr" -rule = 1 -note = "Loader runtime that runs the ingestr subprocess inside the executor load phase; synthetic loader record building, naming, destination URIs and command lines are native." - [[stays]] path = "integrations/dbt/_helpers/profile/load.py" rule = 4 @@ -720,7 +710,7 @@ note = "Shape rule: data enums and aliases only (ExecutionResourceKind, Connecti [[stays]] path = "errors/contracts/exceptions.py" rule = 1 -note = "Shape rule: data exceptions only (SharedInputError, ExecutorInputError), no logic; consumed by executor/audit_results/_helpers/sql.py, executor/build/_helpers/external_source_loads.py, executor/build/_helpers/retention.py and 53 more." +note = "Shape rule: data exceptions only (SharedInputError, ExecutorInputError), no logic; consumed by executor/audit_results/_helpers/sql.py, executor/build/_helpers/retention.py, executor/build/classes/build_scheduler.py and 52 more." [[stays]] path = "compiler/fingerprints/constants.py" diff --git a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py index a72a2b462a..5d62b97bb1 100644 --- a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py @@ -270,6 +270,29 @@ def _yaml_file_cases() -> tuple[FailureCase, ...]: ) }, ), + failure_case( + name="source-removed-ingestr-block", + expected_code="D006", + expected_message="declares an ingestr integration, which SQLBuild no longer supports", + files={ + FAILURE_SOURCES_PATH: FAILURE_BASE_FILES[FAILURE_SOURCES_PATH].replace( + " description: Orders feed.\n", + " description: Orders feed.\n ingestr:\n" + " source_table: public.orders\n", + ) + }, + ), + failure_case( + name="sources-removed-dlt-sources", + expected_code="D006", + expected_message="declares dlt_sources, a dlt integration", + files={ + "sources/loaded.yml": ( + "dlt_sources:\n - type: sql_database\n resources:\n" + " - name: loaded_orders\n table: orders\n" + ) + }, + ), failure_case( name="seed-declaration-without-csv", expected_code="D008", diff --git a/scripts/setting_help/constants.py b/scripts/setting_help/constants.py index 6041387d9e..1b33022ae7 100644 --- a/scripts/setting_help/constants.py +++ b/scripts/setting_help/constants.py @@ -17,6 +17,7 @@ ) YAML_FILE_TEXT_MODULES: frozenset[str] = frozenset( { + "crates/sqlbuild-discovery/src/yaml_entries/_helpers/removed_integrations.rs", "crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs", } ) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 12ccec2ff4..c4db851627 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -123,8 +123,6 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [Overview](integrations/dagster.md) (`integrations/dagster`) - Orchestrate SQLBuild pipelines with Dagster scheduling, retries, and asset UI. - [API Reference](integrations/dagster-reference.md) (`integrations/dagster-reference`) - Dagster integration classes, decorators, and translator hooks. - [Rivers](integrations/rivers.md) (`integrations/rivers`) - Orchestrate SQLBuild pipelines with Rivers scheduling, jobs, and asset tracking. -- [dlt](integrations/dlt.md) (`integrations/dlt`) - Declarative dlt sources in YAML, or full dlt pipelines inside Python source loaders. -- [ingestr](integrations/ingestr.md) (`integrations/ingestr`) - Declarative data ingestion from 50+ sources using ingestr. ## Reference diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md index e2315e19fc..3191029e63 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md @@ -88,6 +88,3 @@ model SQL as written: rejects later non-null values. Cast it, for example `CAST(NULL AS DECIMAL(18, 2)) AS discount`. - Integer division `//` and modulo `%` by zero raise an error instead of returning `NULL`. Wrap the expression in `TRY(...)` to get `NULL`. - -dlt's dataset read API calls a DuckDB method that DuckDB 2.0 removed when it reads a loaded table -back as one whole Arrow table, so that read fails. Dataframe and chunked Arrow reads still work. Loading data with dlt is not affected. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md index 4a2a535e3a..4747f831bb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md @@ -110,7 +110,7 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly | `sqb diff`, `sqb clone` | Supported; clones are physical copies | | Scenarios, `sqb scenario capture`, `--local` replay | Supported; see below | | Statement time limits | Connection interrupt | -| `sqb load`: Python, dlt and ingestr loaders | Supported; dlt writes through its `sqlalchemy` destination | +| `sqb load`: Python loaders | Supported | ## Scenarios diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index b0a06f4675..94cda6a657 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -79,7 +79,7 @@ error[P010]: model 'orders_summary' has no description | Model | `MODEL (description "...")`, or the `description` of its `model_schema` | | Scenario | `SCENARIO (description "...")` | | Seed | `description:` on its entry in `seeds/*.yml` | -| Source | `description:` on its entry in `sources/*.yml` (or on the dlt resource) | +| Source | `description:` on its entry in `sources/*.yml` | | SQL function or table function | `FUNCTION (description "...")` | | Python UDF | `@udf(description="...")` or the function docstring | | Named SQL hook | `HOOK (description "...")` | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index 94e291ab16..74df44465d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -237,7 +237,7 @@ sources: type: VARCHAR ``` -Self-managed loaders must not declare a `write_strategy`. They are useful when you want to use adapter-specific SQL (e.g. `COPY INTO`, external tables), call an external ingestion tool like [dlt](../../integrations/dlt.md), or handle writes in a way that doesn't fit the dict-return pattern. +Self-managed loaders must not declare a `write_strategy`. They are useful when you want to use adapter-specific SQL (e.g. `COPY INTO`, external tables), call an external ingestion tool, or handle writes in a way that doesn't fit the dict-return pattern. ## Loader context diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md index a3b3bba6e0..36d598b7e2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md @@ -128,13 +128,6 @@ Managed sources support incremental write strategies (`table`, `append`, `delete See [Loaders](python-nodes/loaders.md) for the full guide on writing loader functions, write strategies, the loader context API, and auto-load behavior during builds. -### Declarative integrations (no Python) - -For common ingestion you can declare the source entirely in YAML, with no `@loader` function. SQLBuild generates the loader for you and runs it during the build: - -- [dlt](../integrations/dlt.md) - declare `dlt_sources` for `rest_api`, `sql_database`, and `filesystem` sources. -- [ingestr](../integrations/ingestr.md) - add an `ingestr` block to a source to pull from 50+ sources. - ## Source freshness Source freshness lets SQLBuild observe whether a source's data changed between runs. This feeds into @@ -294,11 +287,10 @@ without building. | `audits` | Source-level audits | | `meta` | Free-form metadata mapping | | `load_batch_size` | Positive row batch size for managed loaders | -| `ingestr` | Declarative ingestr loader config (see [ingestr](../integrations/ingestr.md)) | -A sources file contains only `sources` and `dlt_sources`. Unknown keys in a sources file, a source, -a source column, `freshness`, `freshness.age_policy`, or `ingestr` fail discovery with `D006`, -naming the key and suggesting the nearest supported key. +A sources file contains only `sources`. Unknown keys in a sources file, a source, a source column, +`freshness`, or `freshness.age_policy` fail discovery with `D006`, naming the key and suggesting the +nearest supported key. ## YAML that SQLBuild reads diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md index c1564311b6..734bcce0d1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/feature-comparison.md @@ -112,7 +112,6 @@ SQLBuild, dbt, and SQLMesh are all SQL pipeline frameworks. They share common gr | Feature | SQLBuild | dbt | SQLMesh | |---------|----------|-----|---------| | Source loaders | Python `@loader` functions with table/append/delete_insert/merge strategies | No (external to dbt) | No (external to SQLMesh) | -| Declarative ingestion | dlt and ingestr integrations - YAML-only source config, no Python | No | No | | Auto-load during builds | Managed sources loaded before dependent models | No | No | | Source deferral | `--defer-sources-to` reads source data from another target | No | No | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md deleted file mode 100644 index 25edbba368..0000000000 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md +++ /dev/null @@ -1,236 +0,0 @@ - - -# dlt - -> Declarative dlt sources in YAML, or full dlt pipelines inside Python source loaders. - -Online: https://sqlbuild.com/docs/integrations/dlt/ - -## Contents - -- Install -- Declarative dlt sources (YAML) -- dlt inside a Python loader -- Build integration - -[dlt](https://dlthub.com) is an open-source Python library for loading data from APIs, databases, cloud storage, and more. SQLBuild integrates with dlt two ways: - -- **Declarative dlt sources** - configure dlt directly in your source YAML, no Python code. SQLBuild runs the dlt pipeline as part of the build lifecycle. Best for the common REST API, SQL database, and filesystem cases. -- **dlt inside a Python loader** - wrap any `dlt.pipeline(...)` in a SQLBuild [source loader](../concepts/python-nodes/loaders.md) for full control when you need dlt features beyond the declarative surface. - -## Install - -```bash -pip install 'sqlbuild[dlt]' -# or -uv pip install 'sqlbuild[dlt]' -``` - -This installs dlt with the `duckdb`, `filesystem`, and `sql_database` extras alongside SQLBuild. - -## Declarative dlt sources (YAML) - -Declare a `dlt_sources` list in a `sources/*.yml` file. Each entry has a `type`, a `config` block passed to the dlt source, and a list of `resources` that become SQLBuild managed sources. SQLBuild generates a synthetic loader per resource and writes into the database its adapter manages, no Python and no destination setup required. - -Supported source types: `rest_api`, `sql_database`, `filesystem`. - -### REST API - -```yaml -# sources/github.yml -dlt_sources: - - type: rest_api - config: - client: - base_url: https://api.github.com/ - headers: - Authorization: "Bearer ${github_token}" - paginator: - type: header_link - resources: - - name: raw_github_issues - description: Issues from the GitHub API - write_disposition: append - endpoint: - path: "repos/${github_owner}/${github_repo}/issues" - params: - state: all - per_page: 100 -``` - -`rest_api` requires `client.base_url` in `config`. Each resource needs an `endpoint` mapping; the dlt resource name comes from `endpoint.name` if present, otherwise the resource `name`. - -### SQL database - -```yaml -# sources/postgres.yml -dlt_sources: - - type: sql_database - config: - credentials: "${postgres_connection_string}" - resources: - - name: raw_customers - description: Customers from the CRM database - table: customers - write_disposition: merge - primary_key: id -``` - -`sql_database` requires `credentials` in `config`, and each resource requires a `table`. - -### Filesystem - -```yaml -# sources/files.yml -dlt_sources: - - type: filesystem - config: - bucket_url: "s3://my-bucket/events/" - resources: - - name: raw_events - description: Events from the event export files - reader: csv - write_disposition: append -``` - -`filesystem` requires `bucket_url` in `config`. - -### Resource options - -| Key | Description | -|-----|-------------| -| `name` | Source name SQLBuild exposes (referenced via `__source("name")`). Required. | -| `description` | What the loaded rows are. Required ([Descriptions](../concepts/models/configuration.md#descriptions)). | -| `table` | Source table to replicate (`sql_database` only). Required for that type. | -| `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | -| `reader` | File reader: `csv`, `jsonl`, or `parquet` (`filesystem` only). Required for that type. | -| `write_disposition` | dlt write disposition: `replace`, `append`, or `merge`. `merge` requires `primary_key`. | -| `primary_key` / `merge_key` | dlt keys for `merge`. A `merge_key` replaces every row whose key value arrives, like a SQL model's `delete_insert` with [`replace_key`](../concepts/incremental.md#delete_insert). | -| `incremental` | dlt incremental config mapping (e.g. cursor column and start value). | -| `schema` | Per-resource schema override (falls back to the group `schema`). | - -A `dlt_sources` entry accepts only `type`, `config`, `destination`, `schema`, and `resources`, and each resource accepts only the keys above for its source type. Any other key fails compilation with the nearest supported key as a hint. - -Use SQLBuild's `write_disposition` (dlt's term), not `write_strategy`. `delete_insert` is not available declaratively; use a Python loader for that. Config values support SQLBuild's [interpolation](../concepts/interpolation.md): `${name}` for a project variable, `${ENV:NAME}` for an environment variable, and `${CTX:...}` for context, resolved at load time, so credentials stay out of the YAML. - -### Destination - -By default each resource loads into the database SQLBuild's adapter manages, with the dataset/schema derived from your target. On SQLite, dlt writes through its `sqlalchemy` destination into the project's database file (dataset `main`); the `sqlbuild[dlt]` extra installs the SQLAlchemy it needs. An optional `destination` mapping passes extra settings to the dlt destination, but it cannot set `credentials`, `dataset_name`, or `default_schema_name` (SQLBuild manages those): - -```yaml -dlt_sources: - - type: rest_api - destination: - loader_file_format: parquet - config: - client: - base_url: https://api.example.com/ - resources: - - name: raw_widgets - endpoint: - path: widgets -``` - -### Reference in models - -Declared resources are managed sources, reference them like any other source: - -```sql -SELECT id, title, state FROM __source("raw_github_issues") -``` - -## dlt inside a Python loader - -When you need dlt capabilities beyond the declarative surface (custom transforms, `delete_insert`, sources not covered above), wrap a dlt pipeline in a [source loader](../concepts/python-nodes/loaders.md). The loader calls `dlt.pipeline(...).run(...)` and returns `None`; dlt handles the writes and SQLBuild treats the source as loaded. - -```python -# python/loaders/github_sources.py -import dlt -from dlt.sources.rest_api import rest_api_source -from sqlbuild.loaders import loader -from sqlbuild.executor.load.models import LoaderContext - -@loader -def raw_github_issues(ctx: LoaderContext): - """Github issues.""" - source = rest_api_source({ - "client": { - "base_url": "https://api.github.com/", - "headers": {"Authorization": f"Bearer {ctx.vars['github_token']}"}, - "paginator": {"type": "header_link"}, - }, - "resources": [{ - "name": "issues", - "endpoint": { - "path": "repos/{owner}/{repo}/issues", - "params": { - "owner": ctx.vars["github_owner"], - "repo": ctx.vars["github_repo"], - "state": "all", - "per_page": 100, - }, - }, - }], - }) - - pipeline = dlt.pipeline( - pipeline_name="github_issues", - destination=dlt.destinations.duckdb(ctx.connection), - dataset_name=ctx.destination_schema or "main", - ) - pipeline.run(source) -``` - -Bind it to a source with `managed: true` in `sources/*.yml`: - -```yaml -# sources/github.yml -sources: - - name: raw_github_issues - description: Github issues - managed: true - table: issues - columns: - - name: id - type: INTEGER - - name: title - type: VARCHAR -``` - -### DuckDB connection sharing - -With the DuckDB adapter, pass `ctx.connection` to dlt's DuckDB destination to reuse SQLBuild's open connection, so dlt writes into the same database without a separate connection string: - -```python -pipeline = dlt.pipeline( - pipeline_name="my_pipeline", - destination=dlt.destinations.duckdb(ctx.connection), - dataset_name=ctx.destination_schema or "main", -) -``` - -### Warehouse destinations - -For Snowflake, BigQuery, or Databricks, configure dlt with its own credentials. dlt writes directly to the warehouse, and SQLBuild reads the resulting tables as sources: - -```python -pipeline = dlt.pipeline( - pipeline_name="api_data", - destination="snowflake", - dataset_name=ctx.destination_schema or "public", -) -``` - -Configure dlt credentials via its own `secrets.toml` or environment variables, as in the [dlt documentation](https://dlthub.com/docs/general-usage/credentials/setup). - -## Build integration - -Whether declarative or Python, loaders run automatically during `sqb build` (when `auto_load_sources` is enabled), so dlt pipelines execute as part of the normal build lifecycle: - -```bash -sqb build # loaders run, then models build -sqb build --no-load # skip loading, use existing source data -sqb load # run loaders standalone -``` - -See [Loaders](../concepts/python-nodes/loaders.md) for write strategies, the loader context API, auto-load behavior, and source deferral. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md deleted file mode 100644 index 8fd6812a78..0000000000 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md +++ /dev/null @@ -1,177 +0,0 @@ - - -# ingestr - -> Declarative data ingestion from 50+ sources using ingestr. - -Online: https://sqlbuild.com/docs/integrations/ingestr/ - -## Contents - -- Install -- How it works -- Example: PostgreSQL to DuckDB -- Example: Stripe with incremental merge -- Configuration reference -- Template variables -- Destination URI -- Reload -- Build integration -- Supported sources - -[ingestr](https://github.com/bruin-data/ingestr) is an open-source CLI tool by [Bruin](https://github.com/bruin-data) that copies data from any source to any destination using a single command. SQLBuild integrates with ingestr as a declarative source loader - you configure the ingestion directly in your source YAML, and SQLBuild handles execution as part of the build lifecycle. - -## Install - -```bash -pip install 'sqlbuild[ingestr]' -# or -uv pip install 'sqlbuild[ingestr]' -``` - -This installs `ingestr` alongside SQLBuild. The `ingestr` CLI must be available on `PATH`. - -ingestr reads and writes DuckDB through an ADBC driver. Before the first ingestr run, SQLBuild -installs the DuckDB driver that matches its own `duckdb` package (with `dbc`, included in the extra) -into `/etc/adbc/drivers`, so the files ingestr writes stay readable by SQLBuild. Set -`ADBC_DRIVER_PATH` to manage the driver yourself. - -## How it works - -1. Declare an `ingestr` block on a source in `sources/*.yml` -2. SQLBuild generates a synthetic loader that calls `ingestr ingest` as a subprocess -3. ingestr reads from the configured source and writes directly to the SQLBuild adapter's database -4. The destination URI is built automatically from your SQLBuild connection config - no manual destination setup - -No Python loader code is needed. The YAML declaration is the entire configuration. - -## Example: PostgreSQL to DuckDB - -Replicate a table from PostgreSQL into your local DuckDB project: - -```yaml -# sources/raw.yml -sources: - - name: raw_orders - description: Orders - table: orders - ingestr: - source_uri: "postgresql://user:pass@host:5432/mydb" - source_table: "public.orders" -``` - -That's it. Run `sqb load` or `sqb build` and ingestr copies the `orders` table into your project. - -## Example: Stripe with incremental merge - -Load Stripe charges with incremental merge on a primary key: - -```yaml -sources: - - name: raw_stripe_charges - description: Stripe charges - table: charges - ingestr: - source_uri: "stripe://${stripe_api_key}" - source_table: "charges" - strategy: merge - primary_key: id - incremental_key: created -``` - -On subsequent runs, ingestr merges new and updated records based on the `id` column, using `created` to detect changes. - -## Configuration reference - -The `ingestr` block on a source supports the following fields: - -| Field | Required | Description | -|-------|----------|-------------| -| `source_uri` | Yes | ingestr source connection URI (e.g. `postgresql://...`, `stripe://...`, `shopify://...`) | -| `source_table` | Yes | Source table or resource name to ingest | -| `strategy` | No | Ingestion strategy: `replace`, `append`, `merge`, `delete+insert`, or `truncate+insert` | -| `incremental_key` | No | Column used for incremental change detection | -| `primary_key` | No | Primary key column(s) for merge strategy (string or list) | -| `columns` | No | Comma-separated column list to select from the source | -| `extra_args` | No | Additional CLI arguments passed to `ingestr ingest` (list of strings) | - -### Strategy mapping - -| Strategy | Behavior | -|----------|----------| -| `replace` | Drop and recreate the destination table (default when no strategy is set) | -| `append` | Insert all rows without deduplication | -| `merge` | Upsert based on `primary_key`, using `incremental_key` for change detection | -| `delete+insert` | Delete matching rows by `incremental_key` range, then insert replacements. A SQL model does the same with `delete_insert` and a `cursor`; use [`replace_key`](../concepts/incremental.md#delete_insert) to replace groups of rows by key value instead | -| `truncate+insert` | Truncate destination table, then insert all rows | - -## Template variables - -All string fields in the `ingestr` block support SQLBuild project variable substitution and context templates: - -```yaml -sources: - - name: raw_orders - description: Orders - ingestr: - source_uri: "postgresql://${pg_user}:${pg_password}@${pg_host}:5432/${pg_database}" - source_table: "public.orders" -``` - -Variables are resolved from the project's merged variable config (project + target + local). - -Set sensitive values in `sqlbuild_local.toml` (gitignored): - -```toml -[vars] -pg_user = "readonly" -pg_password = "secret" -pg_host = "prod-db.example.com" -pg_database = "analytics" -stripe_api_key = "sk_live_..." -``` - -## Destination URI - -SQLBuild automatically builds the ingestr destination URI from your adapter connection config. All supported adapters work without manual destination configuration: - -| Adapter | Destination URI format | -|---------|----------------------| -| DuckDB | `duckdb:///path/to/db.duckdb` | -| SQLite | `sqlite:///path/to/db.sqlite` | -| MotherDuck | `motherduck://database` | -| PostgreSQL | `postgresql://user:pass@host:port/db` | -| Snowflake | `snowflake://user:pass@account/db/schema` | -| BigQuery | `bigquery://project` | -| Databricks | `databricks://token:...@host` | -| SQL Server | `mssql://user:pass@host:port/db` | - -## Reload - -When `--reload` is passed, ingestr runs with `--full-refresh`, forcing a complete reload regardless of the configured strategy: - -```bash -sqb load --reload -sqb build --reload -``` - -## Build integration - -ingestr sources are managed sources - they participate in the same lifecycle as Python loaders: - -```bash -# ingestr runs automatically before dependent models -sqb build - -# run ingestr sources standalone -sqb load - -# skip loading -sqb build --no-load -``` - -See [Loaders](../concepts/python-nodes/loaders.md) for details on auto-load behavior, source deferral, and the `--load` / `--no-load` / `--reload` flags. - -## Supported sources - -ingestr supports 50+ sources including databases, SaaS APIs, and file systems. See the [ingestr documentation](https://bruin-data.github.io/ingestr/) for the full list of supported sources and their URI formats. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 798647548d..5a1da1984f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3360,25 +3360,6 @@ def format_seed_execution_json( *, results: tuple[SeedExecutionResult, ...], plan: PlanOutput ) -> str: ... def format_load_execution_json(*, results: tuple[LoadExecutionResult, ...]) -> str: ... -def build_ingestr_command( - *, - source_entry: SourceEntry, - adapter_name: str, - connection_config: dict[str, object], - destination_name: str, - vars: dict[str, object], - environment: str | None, - run_id: str, - is_reload: bool, -) -> tuple[str, ...]: ... -def format_ingestr_command(command: Sequence[str], /) -> str: ... -def dlt_destination_arguments( - *, - adapter_name: str, - connection_config: Mapping[str, object], - destination_config: Mapping[str, object] | None, - dataset_name: str | None, -) -> tuple[str, dict[str, Any], dict[str, Any], str | None]: ... def format_clone_execution_json( *, result: CloneExecutionResult, @@ -10518,7 +10499,6 @@ class DiscoveredPythonLoaderMetadata(NativeRecord): unique_key: tuple[str, ...] columns: tuple[SourceColumnEntry, ...] contract: str | None - connection_mode: LoaderConnectionMode def __init__( self, destination: str | None = ..., @@ -10527,7 +10507,6 @@ class DiscoveredPythonLoaderMetadata(NativeRecord): unique_key: tuple[str, ...] = ..., columns: tuple[SourceColumnEntry, ...] = ..., contract: str | None = ..., - connection_mode: LoaderConnectionMode = ..., ) -> None: ... class DiscoveredPythonTaskMetadata(NativeRecord): @@ -12655,17 +12634,6 @@ class SchemaSeedEntry(NativeRecord): audits: tuple[SchemaAuditInstance, ...] = ..., ) -> None: ... -class IntegrationLoaderConfig(NativeRecord): - """Typed configuration for one declarative integration loader.""" - - kind: str - config: object - def __init__( - self, - kind: str, - config: object, - ) -> None: ... - class SourceFreshnessAgePolicy(NativeRecord): """Age-based source freshness warning/error thresholds.""" @@ -12727,7 +12695,6 @@ class SourceEntry(NativeRecord): final: bool loader: str | None managed: bool - integration_loader: IntegrationLoaderConfig | None freshness: SourceFreshnessConfig | None write_strategy: SourceWriteStrategy | None load_batch_size: int | None @@ -12749,7 +12716,6 @@ class SourceEntry(NativeRecord): final: bool = ..., loader: str | None = ..., managed: bool = ..., - integration_loader: IntegrationLoaderConfig | None = ..., freshness: SourceFreshnessConfig | None = ..., write_strategy: SourceWriteStrategy | None = ..., load_batch_size: int | None = ..., @@ -14204,7 +14170,6 @@ class SourceLoadPlanEntry(NativeRecord): cursor_column: str | None unique_key: tuple[str, ...] is_reload: bool - integration_kind: str | None def __init__( self, key: CompiledObjectKey, @@ -14216,7 +14181,6 @@ class SourceLoadPlanEntry(NativeRecord): cursor_column: str | None = ..., unique_key: tuple[str, ...] = ..., is_reload: bool = ..., - integration_kind: str | None = ..., ) -> None: ... class FunctionPlanEntry(NativeRecord): @@ -14644,12 +14608,6 @@ class HardCodedRelationOwnerKind(StrEnum): HOOK = "hook" LOADER = "loader" -class LoaderConnectionMode(StrEnum): - """How a source loader owns the destination warehouse connection.""" - - SQLBUILD = "sqlbuild" - EXTERNAL = "external" - class SqlHookEntry(NativeRecord): """A model lifecycle hook that executes SQL.""" @@ -14854,7 +14812,6 @@ class DiscoveredLoaderFunction(NativeRecord): columns: tuple[SourceColumnEntry, ...] contract: str | None description: str | None - connection_mode: LoaderConnectionMode provider_usages: tuple[DiscoveredProviderUsage, ...] identity: PythonNodeIdentity | None def __init__( @@ -14871,7 +14828,6 @@ class DiscoveredLoaderFunction(NativeRecord): columns: tuple[SourceColumnEntry, ...] = ..., contract: str | None = ..., description: str | None = ..., - connection_mode: LoaderConnectionMode = ..., provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., identity: PythonNodeIdentity | None = ..., ) -> None: ... diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 46a0ef8207..c6f7428a5a 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -748,24 +748,6 @@ "render_selected_dbt_profile_output", ), "DbtProfileError": ("sqlbuild.integrations.dbt.exceptions", "DbtProfileError"), - "DLT_INTEGRATION_KIND": ("sqlbuild.integrations.dlt.constants", "DLT_INTEGRATION_KIND"), - "DltIntegrationError": ("sqlbuild.integrations.dlt.exceptions", "DltIntegrationError"), - "dlt_loader_function": ("sqlbuild.integrations.dlt.main._loaders", "dlt_loader_function"), - "DltResourceConfig": ("sqlbuild.integrations.dlt.models", "DltResourceConfig"), - "DltSourceConfig": ("sqlbuild.integrations.dlt.models", "DltSourceConfig"), - "INGESTR_INTEGRATION_KIND": ( - "sqlbuild.integrations.ingestr.constants", - "INGESTR_INTEGRATION_KIND", - ), - "IngestrIntegrationError": ( - "sqlbuild.integrations.ingestr.exceptions", - "IngestrIntegrationError", - ), - "ingestr_loader_function": ( - "sqlbuild.integrations.ingestr.main._loaders", - "ingestr_loader_function", - ), - "IngestrSourceConfig": ("sqlbuild.integrations.ingestr.models", "IngestrSourceConfig"), "RIVERS_ASSET_NODE_KINDS": ( "sqlbuild.integrations.rivers.constants", "RIVERS_ASSET_NODE_KINDS", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index c3d24a82bd..39099b17ac 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -425,21 +425,6 @@ from sqlbuild.integrations.dbt._helpers.profile.render import ( render_selected_dbt_profile_output as render_selected_dbt_profile_output, ) from sqlbuild.integrations.dbt.exceptions import DbtProfileError as DbtProfileError -from sqlbuild.integrations.dlt.constants import DLT_INTEGRATION_KIND as DLT_INTEGRATION_KIND -from sqlbuild.integrations.dlt.exceptions import DltIntegrationError as DltIntegrationError -from sqlbuild.integrations.dlt.main._loaders import dlt_loader_function as dlt_loader_function -from sqlbuild.integrations.dlt.models import DltResourceConfig as DltResourceConfig -from sqlbuild.integrations.dlt.models import DltSourceConfig as DltSourceConfig -from sqlbuild.integrations.ingestr.constants import ( - INGESTR_INTEGRATION_KIND as INGESTR_INTEGRATION_KIND, -) -from sqlbuild.integrations.ingestr.exceptions import ( - IngestrIntegrationError as IngestrIntegrationError, -) -from sqlbuild.integrations.ingestr.main._loaders import ( - ingestr_loader_function as ingestr_loader_function, -) -from sqlbuild.integrations.ingestr.models import IngestrSourceConfig as IngestrSourceConfig from sqlbuild.integrations.rivers.constants import ( RIVERS_ASSET_NODE_KINDS as RIVERS_ASSET_NODE_KINDS, ) diff --git a/src/sqlbuild/executor/build/_helpers/external_source_loads.py b/src/sqlbuild/executor/build/_helpers/external_source_loads.py deleted file mode 100644 index 25840ab945..0000000000 --- a/src/sqlbuild/executor/build/_helpers/external_source_loads.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Pre-connection execution for external source-load nodes.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import ( - CompiledObjectKey, - DiscoveredLoaderFunction, - LoaderConnectionMode, - PlanOutput, - SourceEntry, - SourceLoadPlanEntry, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.build._helpers.source_node import execute_build_source_node -from sqlbuild.executor.build.models import ( - BuildCallbacks, - BuildRuntimeParams, - ExternalSourceLoadResults, -) -from sqlbuild.executor.load.main._build_execution_indexes import build_load_execution_indexes -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( - ResourceAttemptLifecycle, -) - - -def run_external_source_loads_before_connections( - *, - plan: PlanOutput, - loader_functions: tuple[DiscoveredLoaderFunction, ...], - adapter: BaseAdapter, - connection_config: dict[str, object], - runtime: BuildRuntimeParams, - callbacks: BuildCallbacks, - precompleted_keys: frozenset[CompiledObjectKey] = frozenset(), -) -> ExternalSourceLoadResults: - """Run external source-load nodes before SQLBuild opens warehouse connections.""" - - loader_functions_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in loader_functions - } - external_entries_by_key: dict[CompiledObjectKey, SourceLoadPlanEntry] = { - entry.key: entry - for entry in plan.source_load_entries - if loader_functions_by_name[entry.loader].connection_mode == LoaderConnectionMode.EXTERNAL - and entry.key not in precompleted_keys - } - if not external_entries_by_key: - return ExternalSourceLoadResults( - results=(), completed_keys=frozenset(), failed_keys=frozenset() - ) - _validate_external_entries_are_preconnect_runnable( - plan=plan, - external_entries_by_key=external_entries_by_key, - loader_functions_by_name=loader_functions_by_name, - ) - loader_ref_entries: dict[Callable[..., object], SourceEntry] = build_load_execution_indexes( - sources=tuple(plan.source_map.values()), - loader_functions=loader_functions, - ).loader_ref_entries - results: list[LoadExecutionResult] = [] - completed_keys: set[CompiledObjectKey] = set() - failed_keys: set[CompiledObjectKey] = set() - key: CompiledObjectKey - for key in plan.execution_order: - source_load_entry: SourceLoadPlanEntry | None = external_entries_by_key.get(key) - if source_load_entry is None: - continue - source_entry: SourceEntry = plan.source_map[key.name] - if any(upstream in failed_keys for upstream in plan.upstream_deps.get(key, ())): - with ResourceAttemptLifecycle( - resource_id=f"source:{source_entry.name}", - resource_kind=source_load_entry.resource_kind.value, - resource_name=source_entry.name, - run_id=runtime.run_id, - ) as lifecycle: - lifecycle.skipped(skip_code="dependency") - failed_keys.add(key) - completed_keys.add(key) - continue - with ResourceAttemptLifecycle( - resource_id=f"source:{source_entry.name}", - resource_kind=source_load_entry.resource_kind.value, - resource_name=source_entry.name, - run_id=runtime.run_id, - ) as lifecycle: - result: LoadExecutionResult = execute_build_source_node( - key=key, - plan=plan, - loader_functions_by_name=loader_functions_by_name, - loader_ref_entries=loader_ref_entries, - adapter=adapter, - connection_config=connection_config, - connection=None, - runtime=runtime, - callbacks=callbacks, - ) - if result.status == ExecutionStatus.FAILED: - lifecycle.failed() - elif result.status == ExecutionStatus.SKIPPED: - lifecycle.skipped( - skip_code="explicit", - skip_mode=result.skip_mode.value if result.skip_mode is not None else None, - ) - results.append(result) - completed_keys.add(key) - if result.status != ExecutionStatus.SUCCESS: - failed_keys.add(key) - if callbacks.on_node_complete is not None: - callbacks.on_node_complete(result) - return ExternalSourceLoadResults( - results=tuple(results), - completed_keys=frozenset(completed_keys), - failed_keys=frozenset(failed_keys), - ) - - -def _validate_external_entries_are_preconnect_runnable( - *, - plan: PlanOutput, - external_entries_by_key: dict[CompiledObjectKey, SourceLoadPlanEntry], - loader_functions_by_name: dict[str, DiscoveredLoaderFunction], -) -> None: - source_entries_by_key: dict[CompiledObjectKey, SourceLoadPlanEntry] = { - entry.key: entry for entry in plan.source_load_entries - } - key: CompiledObjectKey - for key in external_entries_by_key: - upstream: CompiledObjectKey - for upstream in plan.upstream_deps.get(key, ()): - if upstream not in plan.execution_order: - continue - upstream_source_entry: SourceLoadPlanEntry | None = source_entries_by_key.get(upstream) - if upstream_source_entry is not None: - upstream_loader: DiscoveredLoaderFunction = loader_functions_by_name[ - upstream_source_entry.loader - ] - if upstream_loader.connection_mode == LoaderConnectionMode.EXTERNAL: - continue - raise ExecutorInputError( - f"External loader '{external_entries_by_key[key].loader}' cannot depend on " - f"SQLBuild-connection loader '{upstream_loader.name}'. External loaders must " - "be runnable before SQLBuild opens warehouse connections." - ) - raise ExecutorInputError( - f"External loader '{external_entries_by_key[key].loader}' cannot depend on " - f"in-plan node '{upstream.name}'. External loaders must be runnable before " - "SQLBuild opens warehouse connections." - ) diff --git a/src/sqlbuild/executor/build/main/_external_source_loads.py b/src/sqlbuild/executor/build/main/_external_source_loads.py deleted file mode 100644 index 4c8ce832c3..0000000000 --- a/src/sqlbuild/executor/build/main/_external_source_loads.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Public entrypoint for pre-connection external source-load execution.""" - -from __future__ import annotations - -from sqlbuild._native import CompiledObjectKey, DiscoveredLoaderFunction, PlanOutput -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.executor.build._helpers.external_source_loads import ( - run_external_source_loads_before_connections as _run_external_source_loads_before_connections, -) -from sqlbuild.executor.build.models import ( - BuildCallbacks, - BuildRuntimeParams, - ExternalSourceLoadResults, -) - - -def run_external_source_loads_before_connections( - *, - plan: PlanOutput, - loader_functions: tuple[DiscoveredLoaderFunction, ...], - adapter: BaseAdapter, - connection_config: dict[str, object], - runtime: BuildRuntimeParams, - callbacks: BuildCallbacks, - precompleted_keys: frozenset[CompiledObjectKey] = frozenset(), -) -> ExternalSourceLoadResults: - """Run external source-load nodes before SQLBuild opens warehouse connections.""" - - return _run_external_source_loads_before_connections( - plan=plan, - loader_functions=loader_functions, - adapter=adapter, - connection_config=connection_config, - runtime=runtime, - callbacks=callbacks, - precompleted_keys=precompleted_keys, - ) diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 5b9521edf0..b7d4ecc4f3 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -212,15 +212,6 @@ class NodeCompletion: error: BaseException | None = None -@dataclass(frozen=True) -class ExternalSourceLoadResults: - """Results and scheduler state from pre-connection external source loads.""" - - results: tuple[LoadExecutionResult, ...] - completed_keys: frozenset[CompiledObjectKey] - failed_keys: frozenset[CompiledObjectKey] - - @dataclass(frozen=True) class BuildExecutionResult: """Aggregate outcome of a full build execution.""" diff --git a/src/sqlbuild/executor/load/_helpers/external.py b/src/sqlbuild/executor/load/_helpers/external.py deleted file mode 100644 index c85d333541..0000000000 --- a/src/sqlbuild/executor/load/_helpers/external.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Helpers for external source loader execution.""" - -from __future__ import annotations - -import logging -import time -from collections.abc import Callable - -from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.load.models import ( - LoaderContext, - LoaderDestination, - LoaderResult, - LoaderSkipResult, - LoadExecutionResult, - LoadRuntimeParams, -) -from sqlbuild.executor.node_results.models import NodeResultEnvelope -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.provider.main.runtime import ( - _empty_provider_container, - invoke_with_providers, -) -from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def execute_external_source_load( - *, - source_entry: SourceEntry, - loader_function: DiscoveredLoaderFunction, - adapter: BaseAdapter, - connection_config: dict[str, object], - destination: LoaderDestination, - runtime: LoadRuntimeParams, - statement_recorder: StatementRecorder, - resource_kind: ExecutionResourceKind, - start: float, - on_progress: Callable[[str], None] | None = None, -) -> LoadExecutionResult: - """Run one external writer while SQLBuild holds no destination connection.""" - - destination_relation: str = destination.relation - try: - context: LoaderContext = LoaderContext( - adapter=adapter, - connection_config=connection_config, - connection=None, - destination=destination_relation, - destination_database=source_entry.database, - destination_schema=source_entry.schema, - destination_name=destination.name, - run_id=runtime.run_id, - runtime_dir=runtime.runtime_dir, - target=runtime.target, - target_config=runtime.target_config, - vars=runtime.vars, - is_reload=runtime.is_reload, - use_color=runtime.use_color, - current_cursor_value=None, - logger=logging.getLogger(f"sqlbuild.loader.{loader_function.name}"), - statement_recorder=statement_recorder, - start_cursor_ts=runtime.start_cursor_ts, - end_cursor_ts=runtime.end_cursor_ts, - start_cursor_int=runtime.start_cursor_int, - end_cursor_int=runtime.end_cursor_int, - providers=( - runtime.providers if runtime.providers is not None else _empty_provider_container() - ), - result_store=runtime.result_store, - on_progress=on_progress, - ) - with OperationLifecycle(operation_kind="loader", operation_name="external_source_load"): - raw_rows: object = invoke_with_providers( - function=loader_function.function, - context=context, - providers=runtime.providers, - ) - if isinstance(raw_rows, LoaderSkipResult): - return LoadExecutionResult( - source_name=source_entry.name, - loader_name=loader_function.name, - status=ExecutionStatus.SKIPPED, - target=destination_relation, - resource_kind=resource_kind, - staging_relation=None, - rows_loaded=0, - duration_ms=int((time.monotonic() - start) * 1000), - lifecycle_events=statement_recorder.snapshot(), - skip_mode=raw_rows.mode, - skip_reason=raw_rows.reason, - ) - if isinstance(raw_rows, LoaderResult): - return LoadExecutionResult( - source_name=source_entry.name, - loader_name=loader_function.name, - status=ExecutionStatus.SUCCESS, - target=destination_relation, - resource_kind=resource_kind, - staging_relation=None, - rows_loaded=0, - duration_ms=int((time.monotonic() - start) * 1000), - lifecycle_events=statement_recorder.snapshot(), - result_payload=raw_rows.payload, - result_metadata=raw_rows.metadata, - result_materialized=raw_rows.materialized, - ) - if isinstance(raw_rows, NodeResultEnvelope): - raise ExecutorInputError( - f"Loader '{loader_function.name}' returned a node result envelope; " - "use ctx.result(...) to return this loader's own result" - ) - if raw_rows is not None: - raise ExecutorInputError( - f"External loader '{loader_function.name}' returned rows, but external " - "loaders must write their own destination and return None" - ) - return LoadExecutionResult( - source_name=source_entry.name, - loader_name=loader_function.name, - status=ExecutionStatus.SUCCESS, - target=destination_relation, - resource_kind=resource_kind, - staging_relation=None, - rows_loaded=0, - duration_ms=int((time.monotonic() - start) * 1000), - lifecycle_events=statement_recorder.snapshot(), - ) - except Exception as error: - return LoadExecutionResult( - source_name=source_entry.name, - loader_name=loader_function.name, - status=ExecutionStatus.FAILED, - target=destination_relation, - resource_kind=resource_kind, - staging_relation=None, - duration_ms=int((time.monotonic() - start) * 1000), - lifecycle_events=statement_recorder.snapshot(), - error_message=str(error), - ) diff --git a/src/sqlbuild/executor/load/_helpers/scheduling.py b/src/sqlbuild/executor/load/_helpers/scheduling.py index e7737c7d6e..9b0ae9f1ec 100644 --- a/src/sqlbuild/executor/load/_helpers/scheduling.py +++ b/src/sqlbuild/executor/load/_helpers/scheduling.py @@ -1,4 +1,4 @@ -"""Load pipeline phases: external preloads, connections, and DAG scheduling.""" +"""Load pipeline phases: connections and DAG scheduling.""" from __future__ import annotations @@ -10,10 +10,8 @@ from functools import partial from typing import Any -from sqlbuild._native import DiscoveredLoaderFunction, LoaderConnectionMode, SourceEntry +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.python_nodes.types import SkipMode -from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.dag_runtime import ( build_load_dag_state, complete_dag_source, @@ -23,10 +21,8 @@ from sqlbuild.executor.load._helpers.execution import ( build_source_downstream_names, build_source_upstream_names, - dependency_node_names, ) from sqlbuild.executor.load.models import ( - ExternalLoadPhaseResult, LoadCallbacks, LoadDagState, LoadDispatchInputs, @@ -34,107 +30,12 @@ LoadExecutionResult, LoadRuntimeParams, ) -from sqlbuild.executor.load.types import LoadProgressCallback from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( raise_warehouse_connection_failure, ) -from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback -def validate_external_loaders_are_preconnect_runnable( - *, sources: tuple[SourceEntry, ...], indexes: LoadExecutionIndexes -) -> None: - """Reject external loaders that depend on SQLBuild-connection loaders.""" - - source_by_name: dict[str, SourceEntry] = {source.name: source for source in sources} - source: SourceEntry - for source in sources: - if source.loader is None: - continue - loader: DiscoveredLoaderFunction = indexes.loader_by_name[source.loader] - if loader.connection_mode != LoaderConnectionMode.EXTERNAL: - continue - dependency_name: str - for dependency_name in dependency_node_names(source=source, indexes=indexes): - dependency_source: SourceEntry | None = source_by_name.get(dependency_name) - if dependency_source is None or dependency_source.loader is None: - continue - dependency_loader: DiscoveredLoaderFunction = indexes.loader_by_name[ - dependency_source.loader - ] - if dependency_loader.connection_mode == LoaderConnectionMode.SQLBUILD: - raise ExecutorInputError( - f"External loader '{loader.name}' cannot depend on SQLBuild-connection " - f"loader '{dependency_loader.name}'. External loaders must be runnable " - "before SQLBuild opens warehouse connections." - ) - - -def run_external_source_loads( - *, - sources: tuple[SourceEntry, ...], - indexes: LoadExecutionIndexes, - source_by_name: dict[str, SourceEntry], - adapter: BaseAdapter, - connection_config: dict[str, object], - runtime: LoadRuntimeParams, - callbacks: LoadCallbacks, -) -> ExternalLoadPhaseResult: - """Run external-connection loaders before opening warehouse connections.""" - - external_sources: tuple[SourceEntry, ...] = tuple( - source - for source in sources - if source.loader is not None - and indexes.loader_by_name[source.loader].connection_mode == LoaderConnectionMode.EXTERNAL - ) - sqlbuild_sources: tuple[SourceEntry, ...] = tuple( - source - for source in sources - if source.loader is None - or indexes.loader_by_name[source.loader].connection_mode == LoaderConnectionMode.SQLBUILD - ) - preloaded_results: list[LoadExecutionResult] = [] - failed_or_hard_skipped: set[str] = set() - on_load_progress: LoadProgressCallback | None = callbacks.on_load_progress - external_source: SourceEntry - for external_source in external_sources: - result: LoadExecutionResult = execute_ready_dag_source( - source_name=external_source.name, - dispatch=LoadDispatchInputs( - source_by_name=source_by_name, - indexes=indexes, - failed_or_hard_skipped=failed_or_hard_skipped, - results_by_name={}, - ), - adapter=adapter, - connection_config=connection_config, - connection=None, - runtime=runtime, - on_load_start=callbacks.on_load_start, - on_progress=( - None - if on_load_progress is None - else lambda message, source=external_source: on_load_progress( - source, message=message - ) - ), - ) - preloaded_results.append(result) - if result.status == ExecutionStatus.FAILED or ( - result.status == ExecutionStatus.SKIPPED and result.skip_mode == SkipMode.HARD - ): - failed_or_hard_skipped.add(external_source.name) - if callbacks.on_load_complete is not None: - callbacks.on_load_complete(result) - return ExternalLoadPhaseResult( - preloaded_results=tuple(preloaded_results), - failed_or_hard_skipped=frozenset(failed_or_hard_skipped), - sqlbuild_sources=sqlbuild_sources, - ) - - def open_load_connections( *, adapter: BaseAdapter, @@ -167,38 +68,36 @@ def open_load_connections( def run_load_dag( *, - sqlbuild_sources: tuple[SourceEntry, ...], + sources: tuple[SourceEntry, ...], source_by_name: dict[str, SourceEntry], indexes: LoadExecutionIndexes, adapter: BaseAdapter, connection_config: dict[str, object], connections: tuple[Any, ...], effective_concurrency: int, - initial_failed_or_hard_skipped: frozenset[str], runtime: LoadRuntimeParams, callbacks: LoadCallbacks, ) -> tuple[LoadExecutionResult, ...]: - """Execute the SQLBuild-connection loader DAG sequentially or concurrently.""" + """Execute the loader DAG sequentially or concurrently.""" source_index_by_name: dict[str, int] = { - source.name: index for index, source in enumerate(sqlbuild_sources) + source.name: index for index, source in enumerate(sources) } upstream_names: dict[str, tuple[str, ...]] = build_source_upstream_names( - sources=sqlbuild_sources, + sources=sources, indexes=indexes, ) downstream_names: dict[str, tuple[str, ...]] = build_source_downstream_names( upstream_names=upstream_names ) - results: list[LoadExecutionResult | None] = [None] * len(sqlbuild_sources) + results: list[LoadExecutionResult | None] = [None] * len(sources) state: LoadDagState = build_load_dag_state( - sources=sqlbuild_sources, + sources=sources, results=results, source_index_by_name=source_index_by_name, upstream_names=upstream_names, downstream_names=downstream_names, ) - state.failed_or_skipped.update(initial_failed_or_hard_skipped) if effective_concurrency == 1: _run_sequential_load_dag( state=state, diff --git a/src/sqlbuild/executor/load/main/_execute.py b/src/sqlbuild/executor/load/main/_execute.py index eef2c5c56b..d2b6c0c3a8 100644 --- a/src/sqlbuild/executor/load/main/_execute.py +++ b/src/sqlbuild/executor/load/main/_execute.py @@ -10,7 +10,6 @@ from sqlbuild._native import ( CostContext, DiscoveredLoaderFunction, - LoaderConnectionMode, SourceEntry, SourceWriteStrategy, ) @@ -27,7 +26,6 @@ load_staging_cursor_bounds, ) from sqlbuild.executor.load._helpers.execution import load_resource_kind, row_write_entry -from sqlbuild.executor.load._helpers.external import execute_external_source_load from sqlbuild.executor.load._helpers.loader_invocation import ( build_loader_context, build_loader_relation_guard, @@ -126,19 +124,6 @@ def _execute_source_load( phase="load", ): resource_kind: ExecutionResourceKind = load_resource_kind(source_entry) - if loader_function.connection_mode == LoaderConnectionMode.EXTERNAL: - return execute_external_source_load( - source_entry=source_entry, - loader_function=loader_function, - adapter=adapter, - connection_config=connection_config, - destination=destination, - runtime=runtime, - statement_recorder=statement_recorder, - resource_kind=resource_kind, - start=start, - on_progress=on_progress, - ) validate_source_write_strategy(source_entry) if not runtime.schema_prepared: adapter.ensure_schema( diff --git a/src/sqlbuild/executor/load/main/run.py b/src/sqlbuild/executor/load/main/run.py index 676e1ed458..e7553518f0 100644 --- a/src/sqlbuild/executor/load/main/run.py +++ b/src/sqlbuild/executor/load/main/run.py @@ -9,12 +9,9 @@ from sqlbuild.executor.load._helpers.execution import build_load_execution_indexes from sqlbuild.executor.load._helpers.scheduling import ( open_load_connections, - run_external_source_loads, run_load_dag, - validate_external_loaders_are_preconnect_runnable, ) from sqlbuild.executor.load.models import ( - ExternalLoadPhaseResult, LoadCallbacks, LoadExecutionIndexes, LoadExecutionResult, @@ -44,24 +41,8 @@ def run_load_pipeline( source_count: int = sum(1 for source in sources if source.loader is not None) if source_count == 0: return () - validate_external_loaders_are_preconnect_runnable(sources=sources, indexes=indexes) source_by_name: dict[str, SourceEntry] = {source.name: source for source in index_sources} - external_phase: ExternalLoadPhaseResult = run_external_source_loads( - sources=sources, - indexes=indexes, - source_by_name=source_by_name, - adapter=adapter, - connection_config=connection_config, - runtime=runtime, - callbacks=resolved_callbacks, - ) - if not external_phase.sqlbuild_sources: - return external_phase.preloaded_results - - remaining_source_count: int = sum( - 1 for source in external_phase.sqlbuild_sources if source.loader is not None - ) - effective_concurrency: int = max(1, min(max_concurrency, remaining_source_count)) + effective_concurrency: int = max(1, min(max_concurrency, source_count)) connections: tuple[Any, ...] = open_load_connections( adapter=adapter, connection_config=connection_config, @@ -72,14 +53,13 @@ def run_load_pipeline( ) try: dag_results: tuple[LoadExecutionResult, ...] = run_load_dag( - sqlbuild_sources=external_phase.sqlbuild_sources, + sources=sources, source_by_name=source_by_name, indexes=indexes, adapter=adapter, connection_config=connection_config, connections=connections, effective_concurrency=effective_concurrency, - initial_failed_or_hard_skipped=external_phase.failed_or_hard_skipped, runtime=runtime, callbacks=resolved_callbacks, ) @@ -87,4 +67,4 @@ def run_load_pipeline( connection: Any for connection in connections: adapter.close(connection) - return external_phase.preloaded_results + dag_results + return dag_results diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 38f6e970b0..1632edc59b 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -334,13 +334,4 @@ class LoadCallbacks: on_connection_error: ConnectionElapsedCallback | None = None -@dataclass(frozen=True) -class ExternalLoadPhaseResult: - """Outcome of running external-connection loaders before warehouse connect.""" - - preloaded_results: tuple[LoadExecutionResult, ...] - failed_or_hard_skipped: frozenset[str] - sqlbuild_sources: tuple[SourceEntry, ...] - - from sqlbuild.executor.load.classes.dag_state import LoadDagState # noqa: E402,F401 diff --git a/src/sqlbuild/executor/pipeline/main/run.py b/src/sqlbuild/executor/pipeline/main/run.py index eba798e2f5..c6a893e488 100644 --- a/src/sqlbuild/executor/pipeline/main/run.py +++ b/src/sqlbuild/executor/pipeline/main/run.py @@ -10,9 +10,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.build.main._execute import execute_build_plan -from sqlbuild.executor.build.main._external_source_loads import ( - run_external_source_loads_before_connections, -) from sqlbuild.executor.build.models import ( BuildCallbacks, BuildCustomizations, @@ -20,7 +17,6 @@ BuildExecutionTimings, BuildInitialState, BuildRuntimeParams, - ExternalSourceLoadResults, ) from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.pipeline._helpers.auditing import ( @@ -169,28 +165,6 @@ def _run_build_pipeline( effective_concurrency: int = min( max(1, runtime.max_concurrency), runnable_graph_width(plan=plan) ) - external_source_load_results: ExternalSourceLoadResults = ( - run_external_source_loads_before_connections( - plan=plan, - loader_functions=inputs.customizations.loader_functions, - adapter=adapter, - connection_config=connection_config, - runtime=inputs.runtime, - callbacks=inputs.callbacks, - precompleted_keys=inputs.initial_state.precompleted_keys, - ) - ) - merged_initial_state: BuildInitialState = replace( - inputs.initial_state, - precompleted_keys=inputs.initial_state.precompleted_keys - | external_source_load_results.completed_keys, - initial_load_results=( - *inputs.initial_state.initial_load_results, - *external_source_load_results.results, - ), - initial_failed_keys=inputs.initial_state.initial_failed_keys - | external_source_load_results.failed_keys, - ) preparation: BuildConnectionPreparation = prepare_build_connections( plan=plan, adapter=adapter, @@ -211,7 +185,7 @@ def _run_build_pipeline( runtime=inputs.runtime, callbacks=inputs.callbacks, customizations=inputs.customizations, - initial_state=merged_initial_state, + initial_state=inputs.initial_state, schema_prepared=True, ) execution_seconds: float = time.monotonic() - execution_start diff --git a/src/sqlbuild/integrations/dlt/__init__.py b/src/sqlbuild/integrations/dlt/__init__.py deleted file mode 100644 index 99241d8a56..0000000000 --- a/src/sqlbuild/integrations/dlt/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""dlt integration loader helpers.""" diff --git a/src/sqlbuild/integrations/dlt/_helpers/__init__.py b/src/sqlbuild/integrations/dlt/_helpers/__init__.py deleted file mode 100644 index 9c4f6f2dae..0000000000 --- a/src/sqlbuild/integrations/dlt/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""dlt helper modules.""" diff --git a/src/sqlbuild/integrations/dlt/_helpers/config.py b/src/sqlbuild/integrations/dlt/_helpers/config.py deleted file mode 100644 index 9027e67e0d..0000000000 --- a/src/sqlbuild/integrations/dlt/_helpers/config.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Configuration resolution helpers for declarative dlt sources.""" - -from __future__ import annotations - -from dataclasses import replace -from typing import cast - -from sqlbuild._native import expand_template_data -from sqlbuild.integrations.dlt.models import DltResourceConfig, DltSourceConfig - - -def resolve_dlt_config( - *, config: DltSourceConfig, vars: dict[str, object], environment: str | None, run_id: str -) -> DltSourceConfig: - """Resolve supported templates in dlt source config values.""" - - values: object = expand_template_data( - value={ - "config": config.config, - "destination": config.destination, - "schema": config.schema, - "resource": config.resource.raw_config, - "resource_schema": config.resource.schema, - "write_disposition": config.resource.write_disposition, - "primary_key": config.resource.primary_key, - "merge_key": config.resource.merge_key, - "incremental": config.resource.incremental, - }, - variables=vars, - context_values={"environment": environment, "run_id": run_id}, - context_label="dlt source config", - allow_context=True, - preserve_context_tokens=False, - preserve_unknown_context=True, - ) - if not isinstance(values, dict): - return config - resolved_values: dict[str, object] = cast(dict[str, object], values) - resolved_resource: DltResourceConfig = replace( - config.resource, - schema=_optional_string(resolved_values.get("resource_schema")), - raw_config=_mapping(resolved_values.get("resource")), - write_disposition=resolved_values.get("write_disposition"), - primary_key=resolved_values.get("primary_key"), - merge_key=resolved_values.get("merge_key"), - incremental=_mapping(resolved_values.get("incremental")), - ) - return replace( - config, - schema=_optional_string(resolved_values.get("schema")), - config=_mapping(resolved_values.get("config")), - destination=_mapping(resolved_values.get("destination")), - resource=resolved_resource, - ) - - -def _mapping(value: object | None) -> dict[str, object]: - if not isinstance(value, dict): - return {} - return dict(cast(dict[str, object], value)) - - -def _optional_string(value: object | None) -> str | None: - if not isinstance(value, str) or not value.strip(): - return None - return value.strip() diff --git a/src/sqlbuild/integrations/dlt/_helpers/destination.py b/src/sqlbuild/integrations/dlt/_helpers/destination.py deleted file mode 100644 index 3553745ec4..0000000000 --- a/src/sqlbuild/integrations/dlt/_helpers/destination.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Build dlt destinations from SQLBuild adapter connection config.""" - -from __future__ import annotations - -from sqlbuild._native import dlt_destination_arguments -from sqlbuild.integrations.dlt.exceptions import DltIntegrationError -from sqlbuild.integrations.dlt.models import DltDestinationConfig - - -def build_dlt_destination( - *, - adapter_name: str, - connection_config: dict[str, object], - destination_config: dict[str, object] | None = None, - dataset_name: str | None, -) -> DltDestinationConfig: - """Return a dlt destination object and dataset for a SQLBuild target.""" - - try: - import dlt.destinations as destinations - except ImportError as error: - raise DltIntegrationError( - "This source uses dlt, but dlt is not installed. Install it with: " - "pip install 'sqlbuild[dlt]'" - ) from error - - factory, arguments, options, resolved_dataset_name = dlt_destination_arguments( - adapter_name=adapter_name, - connection_config=connection_config, - destination_config=destination_config, - dataset_name=dataset_name, - ) - return DltDestinationConfig( - destination=getattr(destinations, factory)(**arguments, **options), - dataset_name=resolved_dataset_name, - ) diff --git a/src/sqlbuild/integrations/dlt/_helpers/runner.py b/src/sqlbuild/integrations/dlt/_helpers/runner.py deleted file mode 100644 index e316fb60f7..0000000000 --- a/src/sqlbuild/integrations/dlt/_helpers/runner.py +++ /dev/null @@ -1,77 +0,0 @@ -"""In-process runner helpers for dlt.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any, cast - -from sqlbuild.executor.load.models import LoaderContext -from sqlbuild.integrations.dlt._helpers.config import resolve_dlt_config -from sqlbuild.integrations.dlt._helpers.destination import build_dlt_destination -from sqlbuild.integrations.dlt._helpers.source import build_dlt_source -from sqlbuild.integrations.dlt.classes.sqlbuild_dlt_progress_collector import ( - SqlbuildDltProgressCollector, -) -from sqlbuild.integrations.dlt.exceptions import DltIntegrationError -from sqlbuild.integrations.dlt.models import DltDestinationConfig, DltSourceConfig - - -def run_dlt_source(*, config: DltSourceConfig, ctx: LoaderContext) -> None: - """Run one declarative dlt resource into the SQLBuild target dataset.""" - - try: - import dlt - except ImportError as error: - raise DltIntegrationError( - "This source uses dlt, but dlt is not installed. Install it with: " - "pip install 'sqlbuild[dlt]'" - ) from error - - resolved: DltSourceConfig = resolve_dlt_config( - config=config, - vars=ctx.vars, - environment=ctx.target, - run_id=ctx.run_id, - ) - destination: DltDestinationConfig = build_dlt_destination( - adapter_name=ctx.adapter.adapter_name, - connection_config=ctx.connection_config, - destination_config=resolved.destination, - dataset_name=resolved.resource.schema, - ) - dlt_source: Any = build_dlt_source(resolved) - pipelines_dir: Path = ctx.runtime_dir / "dlt" - pipelines_dir.mkdir(parents=True, exist_ok=True) - progress_collector: SqlbuildDltProgressCollector = SqlbuildDltProgressCollector( - on_progress=ctx.progress - ) - pipeline: Any = dlt.pipeline( - pipeline_name=_pipeline_name(config=resolved, target=ctx.target), - pipelines_dir=str(pipelines_dir), - destination=destination.destination, - dataset_name=cast(str, destination.dataset_name), - progress=cast(Any, progress_collector), - ) - pipeline.drop() - load_info: Any = pipeline.run( - dlt_source, - destination=destination.destination, - dataset_name=cast(str, destination.dataset_name), - table_name=resolved.resource.name, - write_disposition=cast(Any, resolved.resource.write_disposition), - primary_key=cast(Any, resolved.resource.primary_key), - refresh=cast(Any, "drop_resources" if ctx.is_reload else None), - ) - load_info.raise_on_failed_jobs() - progress_summary: str = progress_collector.format_summary() - if progress_summary: - ctx.log(progress_summary) - ctx.log(str(load_info)) - return None - - -def _pipeline_name(*, config: DltSourceConfig, target: str | None) -> str: - target_part: str = target or "default" - return ( - f"sqlbuild_{target_part}_{config.source_type}_{config.group_index}_{config.resource.name}" - ) diff --git a/src/sqlbuild/integrations/dlt/_helpers/source.py b/src/sqlbuild/integrations/dlt/_helpers/source.py deleted file mode 100644 index 9bcb826bfe..0000000000 --- a/src/sqlbuild/integrations/dlt/_helpers/source.py +++ /dev/null @@ -1,99 +0,0 @@ -"""dlt source construction helpers.""" - -from __future__ import annotations - -from copy import deepcopy -from typing import Any, cast - -from sqlbuild.integrations.dlt.constants import ( - DLT_FILESYSTEM_READER_CSV, - DLT_FILESYSTEM_READER_JSONL, - DLT_FILESYSTEM_READER_PARQUET, - DLT_SOURCE_TYPE_FILESYSTEM, - DLT_SOURCE_TYPE_REST_API, - DLT_SOURCE_TYPE_SQL_DATABASE, -) -from sqlbuild.integrations.dlt.exceptions import DltIntegrationError -from sqlbuild.integrations.dlt.models import DltSourceConfig - - -def build_dlt_source(config: DltSourceConfig) -> Any: - """Build a dlt source/resource for one declarative SQLBuild resource.""" - - if config.source_type == DLT_SOURCE_TYPE_SQL_DATABASE: - return _build_sql_database_source(config) - if config.source_type == DLT_SOURCE_TYPE_REST_API: - return _build_rest_api_source(config) - if config.source_type == DLT_SOURCE_TYPE_FILESYSTEM: - return _build_filesystem_resource(config) - raise DltIntegrationError(f"Unsupported dlt source type '{config.source_type}'") - - -def _build_sql_database_source(config: DltSourceConfig) -> Any: - try: - from dlt.sources.sql_database import sql_database - except ImportError as error: - raise DltIntegrationError( - "dlt sql_database sources require dlt SQL database dependencies. Install with: " - "pip install 'sqlbuild[dlt]'" - ) from error - - source_config: dict[str, object] = deepcopy(config.config) - source_config["table_names"] = [config.resource.dlt_name] - source_factory: Any = sql_database - source: Any = source_factory(**cast(Any, source_config)) - return source.with_resources(config.resource.dlt_name) - - -def _build_rest_api_source(config: DltSourceConfig) -> Any: - try: - from dlt.sources.rest_api import rest_api_source - except ImportError as error: - raise DltIntegrationError( - "dlt rest_api sources require dlt. Install with: pip install 'sqlbuild[dlt]'" - ) from error - - source_config: dict[str, object] = deepcopy(config.config) - source_config["resources"] = [_rest_resource_config(config)] - source: Any = rest_api_source(cast(Any, source_config)) - return source.with_resources(config.resource.dlt_name) - - -def _build_filesystem_resource(config: DltSourceConfig) -> Any: - try: - from dlt.sources.filesystem import filesystem, read_csv, read_jsonl, read_parquet - except ImportError as error: - raise DltIntegrationError( - "dlt filesystem sources require dlt filesystem dependencies. Install with: " - "pip install 'sqlbuild[dlt]'" - ) from error - - reader: object | None = config.resource.raw_config.get("reader") - if not isinstance(reader, str): - raise DltIntegrationError( - f"dlt filesystem source '{config.resource.name}' must define reader" - ) - source_config: dict[str, object] = deepcopy(config.config) - filesystem_factory: Any = filesystem - resource: Any = filesystem_factory(**cast(Any, source_config)) - if reader == DLT_FILESYSTEM_READER_CSV: - return resource | read_csv().with_name(config.resource.dlt_name) - if reader == DLT_FILESYSTEM_READER_JSONL: - return resource | read_jsonl().with_name(config.resource.dlt_name) - if reader == DLT_FILESYSTEM_READER_PARQUET: - return resource | read_parquet().with_name(config.resource.dlt_name) - raise DltIntegrationError( - f"dlt filesystem source '{config.resource.name}' reader must be one of: csv, jsonl, parquet" - ) - - -def _rest_resource_config(config: DltSourceConfig) -> dict[str, object]: - resource_config: dict[str, object] = deepcopy(config.resource.raw_config) - resource_config["name"] = config.resource.dlt_name - if config.resource.write_disposition is not None: - resource_config["write_disposition"] = config.resource.write_disposition - if config.resource.primary_key is not None: - resource_config["primary_key"] = config.resource.primary_key - if config.resource.merge_key is not None: - resource_config["merge_key"] = config.resource.merge_key - return resource_config diff --git a/src/sqlbuild/integrations/dlt/classes/__init__.py b/src/sqlbuild/integrations/dlt/classes/__init__.py deleted file mode 100644 index 7b5d754daa..0000000000 --- a/src/sqlbuild/integrations/dlt/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""dlt integration runtime classes.""" diff --git a/src/sqlbuild/integrations/dlt/classes/dlt_progress_counter.py b/src/sqlbuild/integrations/dlt/classes/dlt_progress_counter.py deleted file mode 100644 index 00d51f8bc8..0000000000 --- a/src/sqlbuild/integrations/dlt/classes/dlt_progress_counter.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Mutable dlt progress counter.""" - -from __future__ import annotations - - -class DltProgressCounter: - def __init__( - self, - *, - step: str, - name: str, - label: str | None, - count: int = 0, - total: int | None = None, - message: str | None = None, - ) -> None: - self.step = step - self.name = name - self.label = label - self.count = count - self.total = total - self.message = message diff --git a/src/sqlbuild/integrations/dlt/classes/sqlbuild_dlt_progress_collector.py b/src/sqlbuild/integrations/dlt/classes/sqlbuild_dlt_progress_collector.py deleted file mode 100644 index c946801af6..0000000000 --- a/src/sqlbuild/integrations/dlt/classes/sqlbuild_dlt_progress_collector.py +++ /dev/null @@ -1,156 +0,0 @@ -"""dlt progress collector for SQLBuild source loaders.""" - -from __future__ import annotations - -import time -from collections.abc import Callable -from typing import Any - -from sqlbuild.integrations.dlt.classes.dlt_progress_counter import DltProgressCounter -from sqlbuild.integrations.dlt.constants import DLT_FORCE_PROGRESS_COUNTER_NAMES -from sqlbuild.integrations.dlt.models import DltProgressEvent - - -class SqlbuildDltProgressCollector: - """Collector compatible with dlt's progress collector protocol.""" - - def __init__(self, *, on_progress: Callable[[str], None] | None = None) -> None: - self.step: str = "" - self.events: list[DltProgressEvent] = [] - self._counters: dict[tuple[str, str, str | None], DltProgressCounter] = {} - self._on_progress: Callable[[str], None] | None = on_progress - self._last_live_message: str = "" - self._last_live_at: float = 0.0 - - def __call__(self, step: str) -> SqlbuildDltProgressCollector: - self.step = step - return self - - def __enter__(self) -> SqlbuildDltProgressCollector: - self._start(self.step) - return self - - def __exit__( - self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: Any - ) -> None: - self._stop() - - def on_start_trace(self, trace: Any, step: Any, pipeline: Any) -> None: - return None - - def on_start_trace_step(self, trace: Any, step: Any, pipeline: Any) -> None: - return None - - def on_end_trace_step( - self, trace: Any, step: Any, pipeline: Any, step_info: Any, send_state: bool - ) -> None: - return None - - def on_end_trace(self, trace: Any, pipeline: Any, send_state: bool) -> None: - return None - - def update( - self, - name: str, - inc: int = 1, - total: int | None = None, - inc_total: int | None = None, - message: str | None = None, - label: str | None = None, - ) -> None: - event: DltProgressEvent = DltProgressEvent( - step=self.step, - name=name, - inc=inc, - total=total, - inc_total=inc_total, - message=message, - label=label, - ) - self.events.append(event) - key: tuple[str, str, str | None] = (self.step, name, label) - counter: DltProgressCounter = self._counters.setdefault( - key, - DltProgressCounter(step=self.step, name=name, label=label, total=total), - ) - counter.count += inc - if total is not None: - counter.total = total - if inc_total is not None: - counter.total = (counter.total or 0) + inc_total - if message is not None: - counter.message = message - self._emit_live_progress(counter) - - def _start(self, step: str) -> None: - self.step = step - self._emit_live_message(message=f"dlt {self._display_step(step)}", force=True) - - def _stop(self) -> None: - return None - - def format_summary(self) -> str: - if not self._counters: - return "" - lines: list[str] = ["dlt progress"] - step: str - for step in self._ordered_steps(): - rendered: str = self._format_step(step) - if rendered: - lines.append(f"{self._display_step(step)}: {rendered}") - return "\n".join(lines) - - def _emit_live_progress(self, counter: DltProgressCounter) -> None: - message: str = f"dlt {self._display_step(counter.step)}: {self._format_counter(counter)}" - self._emit_live_message( - message=message, force=counter.name in DLT_FORCE_PROGRESS_COUNTER_NAMES - ) - - def _emit_live_message(self, *, message: str, force: bool = False) -> None: - if self._on_progress is None: - return - now: float = time.monotonic() - if not force and message == self._last_live_message: - return - live_refresh_interval_seconds: float = 0.5 - if not force and now - self._last_live_at < live_refresh_interval_seconds: - return - self._last_live_message = message - self._last_live_at = now - self._on_progress(message) - - def _ordered_steps(self) -> tuple[str, ...]: - ordered: list[str] = [] - event: DltProgressEvent - for event in self.events: - if event.step not in ordered: - ordered.append(event.step) - return tuple(ordered) - - def _format_step(self, step: str) -> str: - counters: list[DltProgressCounter] = [ - counter for counter in self._counters.values() if counter.step == step - ] - counters.sort( - key=lambda counter: (counter.name.startswith("_"), counter.name, counter.label or "") - ) - return ", ".join(self._format_counter(counter) for counter in counters if counter.count) - - def _display_step(self, step: str) -> str: - normalized: str = step.strip().lower() - if normalized.startswith("extract"): - return "extract" - if normalized.startswith("normalize"): - return "normalize" - if normalized.startswith("load"): - return "load" - return normalized or "dlt" - - def _format_counter(self, counter: DltProgressCounter) -> str: - name: str = counter.name if counter.label is None else f"{counter.name} ({counter.label})" - value: str = ( - f"{counter.count}/{counter.total}" if counter.total is not None else f"{counter.count}" - ) - if counter.message is not None: - return f"{name} {value} [{counter.message}]" - return f"{name} {value}" diff --git a/src/sqlbuild/integrations/dlt/constants.py b/src/sqlbuild/integrations/dlt/constants.py deleted file mode 100644 index bd492e9b06..0000000000 --- a/src/sqlbuild/integrations/dlt/constants.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Constants for dlt integration loaders.""" - -from __future__ import annotations - -DLT_INTEGRATION_KIND: str = "dlt" -DLT_SOURCE_TYPE_SQL_DATABASE: str = "sql_database" -DLT_SOURCE_TYPE_REST_API: str = "rest_api" -DLT_SOURCE_TYPE_FILESYSTEM: str = "filesystem" -DLT_FILESYSTEM_READER_CSV: str = "csv" -DLT_FILESYSTEM_READER_JSONL: str = "jsonl" -DLT_FILESYSTEM_READER_PARQUET: str = "parquet" -DLT_FORCE_PROGRESS_COUNTER_NAMES: frozenset[str] = frozenset({"Resources", "Files", "Jobs"}) diff --git a/src/sqlbuild/integrations/dlt/exceptions.py b/src/sqlbuild/integrations/dlt/exceptions.py deleted file mode 100644 index fbeb3537ed..0000000000 --- a/src/sqlbuild/integrations/dlt/exceptions.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Expected dlt integration exception types.""" - -from __future__ import annotations - - -class DltIntegrationError(RuntimeError): - """Raised when declarative dlt integration configuration or execution fails.""" diff --git a/src/sqlbuild/integrations/dlt/main/__init__.py b/src/sqlbuild/integrations/dlt/main/__init__.py deleted file mode 100644 index ab036476b3..0000000000 --- a/src/sqlbuild/integrations/dlt/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""dlt integration loader entrypoints.""" diff --git a/src/sqlbuild/integrations/dlt/main/_loaders.py b/src/sqlbuild/integrations/dlt/main/_loaders.py deleted file mode 100644 index 8c8bc80459..0000000000 --- a/src/sqlbuild/integrations/dlt/main/_loaders.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Loader functions that run declarative dlt sources.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import SourceEntry -from sqlbuild.executor.load.models import LoaderContext -from sqlbuild.integrations.dlt._helpers.runner import run_dlt_source -from sqlbuild.integrations.dlt.models import DltSourceConfig - - -def dlt_loader_function(*, source_entry: SourceEntry, name: str) -> Callable[[LoaderContext], None]: - """The loader function that runs one declarative dlt source under `name`.""" - - def run(ctx: LoaderContext) -> None: - config: object = ( - source_entry.integration_loader.config if source_entry.integration_loader else None - ) - if not isinstance(config, DltSourceConfig): - return None - return run_dlt_source(config=config, ctx=ctx) - - run.__name__ = name - return run diff --git a/src/sqlbuild/integrations/dlt/models.py b/src/sqlbuild/integrations/dlt/models.py deleted file mode 100644 index 0c21602a7c..0000000000 --- a/src/sqlbuild/integrations/dlt/models.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Typed models for dlt integration loaders.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any - - -@dataclass(frozen=True) -class DltResourceConfig: - """One SQLBuild-managed dlt resource inside a declarative source group.""" - - name: str - dlt_name: str - raw_config: dict[str, object] - schema: str | None = None - write_disposition: object | None = None - primary_key: object | None = None - merge_key: object | None = None - incremental: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class DltSourceConfig: - """Configuration for one dlt resource loaded by a synthetic loader.""" - - source_type: str - config: dict[str, object] - destination: dict[str, object] - resource: DltResourceConfig - group_index: int - schema: str | None = None - - -@dataclass(frozen=True) -class DltDestinationConfig: - """Resolved dlt destination and dataset for a SQLBuild target.""" - - destination: Any - dataset_name: str | None - - -@dataclass(frozen=True) -class DltProgressEvent: - step: str - name: str - inc: int - total: int | None - inc_total: int | None - message: str | None - label: str | None diff --git a/src/sqlbuild/integrations/ingestr/__init__.py b/src/sqlbuild/integrations/ingestr/__init__.py deleted file mode 100644 index 07e1f8015d..0000000000 --- a/src/sqlbuild/integrations/ingestr/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""ingestr integration loader helpers.""" diff --git a/src/sqlbuild/integrations/ingestr/_helpers/__init__.py b/src/sqlbuild/integrations/ingestr/_helpers/__init__.py deleted file mode 100644 index ad3f6fa95f..0000000000 --- a/src/sqlbuild/integrations/ingestr/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""ingestr helper modules.""" diff --git a/src/sqlbuild/integrations/ingestr/_helpers/output.py b/src/sqlbuild/integrations/ingestr/_helpers/output.py deleted file mode 100644 index 4c7264204a..0000000000 --- a/src/sqlbuild/integrations/ingestr/_helpers/output.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Output formatting helpers for ingestr subprocess execution.""" - -from __future__ import annotations - -from sqlbuild.executor.load.models import LoaderContext -from sqlbuild.integrations.ingestr.models import IngestrCommandResult - - -def record_ingestr_output(*, ctx: LoaderContext, result: IngestrCommandResult) -> None: - """Attach captured ingestr output to the loader lifecycle.""" - - ctx.log("ingestr execution") - ctx.log(result.command_display) - if result.stdout: - ctx.log(_format_output_block(label="ingestr stdout", output=result.stdout)) - if result.stderr: - ctx.log(_format_output_block(label="ingestr stderr", output=result.stderr)) - - -def _format_output_block(*, label: str, output: str) -> str: - return f"{label}\n{output.rstrip()}" diff --git a/src/sqlbuild/integrations/ingestr/_helpers/runner.py b/src/sqlbuild/integrations/ingestr/_helpers/runner.py deleted file mode 100644 index 730bc6c822..0000000000 --- a/src/sqlbuild/integrations/ingestr/_helpers/runner.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Subprocess runner helpers for ingestr.""" - -from __future__ import annotations - -import shutil -import subprocess -import sys -import tomllib -from importlib import metadata -from os import environ -from pathlib import Path - -from sqlbuild._native import format_ingestr_command -from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError -from sqlbuild.integrations.ingestr.models import IngestrCommandResult -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - -_DUCKDB_DRIVER_BOOTSTRAP_ERROR: str = "ADBC driver still not available after installation" -_DUCKDB_MANIFEST: str = "duckdb.toml" - - -def run_ingestr_command( - command: tuple[str, ...], -) -> IngestrCommandResult: - """Run ingestr and raise a clear integration error when execution fails.""" - - if shutil.which(command[0]) is None: - raise IngestrIntegrationError( - "This source uses ingestr, but the ingestr CLI is not available. " - "Install it with: pip install 'sqlbuild[ingestr]'" - ) - ensure_duckdb_driver() - command_display: str = format_ingestr_command(command) - try: - completed: subprocess.CompletedProcess[str] = _run_observed_ingestr_subprocess( - command=command - ) - if completed.returncode != 0 and _DUCKDB_DRIVER_BOOTSTRAP_ERROR in completed.stdout: - completed = _run_observed_ingestr_subprocess(command=command) - except OSError as error: - raise IngestrIntegrationError(f"failed to execute ingestr: {error}") from error - result: IngestrCommandResult = IngestrCommandResult( - command_display=command_display, - stdout=completed.stdout, - stderr=completed.stderr, - ) - if completed.returncode != 0: - raise IngestrIntegrationError( - f"ingestr failed with exit code {completed.returncode}", result - ) - return result - - -def _run_ingestr_subprocess(*, command: tuple[str, ...]) -> subprocess.CompletedProcess[str]: - return subprocess.run( - command, - capture_output=True, - check=False, - env=_ingestr_subprocess_env(), - text=True, - ) - - -def _run_observed_ingestr_subprocess( - *, command: tuple[str, ...] -) -> subprocess.CompletedProcess[str]: - with OperationLifecycle( - operation_kind="subprocess", - operation_name="ingestr_command", - auto_fail_base_exceptions=False, - ) as lifecycle: - completed: subprocess.CompletedProcess[str] = _run_ingestr_subprocess(command=command) - if completed.returncode == 0: - lifecycle.completed(exit_code=0) - else: - lifecycle.failed( - error_code=f"exit_{completed.returncode}", - exit_code=completed.returncode, - signal_number=-completed.returncode if completed.returncode < 0 else None, - ) - return completed - - -def _ingestr_subprocess_env() -> dict[str, str]: - env: dict[str, str] = dict(environ) - if env.get("ADBC_DRIVER_PATH"): - return env - driver_path: Path = _managed_driver_path() - if driver_path.exists(): - env["ADBC_DRIVER_PATH"] = str(driver_path) - return env - - -def _managed_driver_path() -> Path: - return Path(sys.prefix) / "etc" / "adbc" / "drivers" - - -def ensure_duckdb_driver() -> None: - """Install the DuckDB ADBC driver matching the duckdb package unless the user set one.""" - - if environ.get("ADBC_DRIVER_PATH"): - return - try: - duckdb_version: str = metadata.version("duckdb") - except metadata.PackageNotFoundError: - return - driver_path: Path = _managed_driver_path() - if _installed_driver_version(driver_path / _DUCKDB_MANIFEST) == duckdb_version: - return - dbc: str | None = _dbc_executable() - if dbc is None: - raise IngestrIntegrationError( - "ingestr needs the dbc driver installer to pin its DuckDB driver to duckdb " - f"{duckdb_version}. Install it with: pip install 'sqlbuild[ingestr]'" - ) - driver_path.mkdir(parents=True, exist_ok=True) - with OperationLifecycle( - operation_kind="subprocess", - operation_name="adbc_driver_install", - auto_fail_base_exceptions=False, - ) as lifecycle: - completed: subprocess.CompletedProcess[str] = subprocess.run( - (dbc, "install", "--quiet", f"duckdb={duckdb_version}"), - capture_output=True, - check=False, - env={**environ, "ADBC_DRIVER_PATH": str(driver_path)}, - text=True, - ) - if completed.returncode == 0: - lifecycle.completed(exit_code=0) - else: - lifecycle.failed( - error_code=f"exit_{completed.returncode}", - exit_code=completed.returncode, - signal_number=-completed.returncode if completed.returncode < 0 else None, - ) - if completed.returncode != 0: - raise IngestrIntegrationError( - f"could not install the DuckDB ADBC driver for duckdb {duckdb_version}: " - f"{(completed.stderr or completed.stdout).strip()}" - ) - - -def _installed_driver_version(manifest: Path) -> str | None: - try: - with manifest.open("rb") as handle: - version: object = tomllib.load(handle).get("version") - except (OSError, tomllib.TOMLDecodeError): - return None - return version if isinstance(version, str) else None - - -def _dbc_executable() -> str | None: - for name in ("dbc", "dbc.exe"): - beside_python: Path = Path(sys.executable).parent / name - if beside_python.exists(): - return str(beside_python) - return shutil.which("dbc") diff --git a/src/sqlbuild/integrations/ingestr/constants.py b/src/sqlbuild/integrations/ingestr/constants.py deleted file mode 100644 index 47b3f3f671..0000000000 --- a/src/sqlbuild/integrations/ingestr/constants.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Constants for ingestr integration loaders.""" - -from __future__ import annotations - -INGESTR_INTEGRATION_KIND: str = "ingestr" diff --git a/src/sqlbuild/integrations/ingestr/exceptions.py b/src/sqlbuild/integrations/ingestr/exceptions.py deleted file mode 100644 index 780c18eab2..0000000000 --- a/src/sqlbuild/integrations/ingestr/exceptions.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Expected ingestr integration exception types.""" - -from __future__ import annotations - - -class IngestrIntegrationError(RuntimeError): - """Raised when ingestr command construction or execution fails.""" - - def __init__(self, message: str, run_result: object | None = None) -> None: - super().__init__(message) - self._run_result = run_result - - @property - def run_result(self) -> object | None: - return self._run_result diff --git a/src/sqlbuild/integrations/ingestr/main/__init__.py b/src/sqlbuild/integrations/ingestr/main/__init__.py deleted file mode 100644 index 23d6535363..0000000000 --- a/src/sqlbuild/integrations/ingestr/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""ingestr integration loader entrypoints.""" diff --git a/src/sqlbuild/integrations/ingestr/main/_loaders.py b/src/sqlbuild/integrations/ingestr/main/_loaders.py deleted file mode 100644 index d5704629c2..0000000000 --- a/src/sqlbuild/integrations/ingestr/main/_loaders.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Loader functions that run declarative ingestr sources.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import SourceEntry, build_ingestr_command -from sqlbuild.executor.load.models import LoaderContext -from sqlbuild.integrations.ingestr._helpers.output import record_ingestr_output -from sqlbuild.integrations.ingestr._helpers.runner import run_ingestr_command -from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError -from sqlbuild.integrations.ingestr.models import IngestrCommandResult - - -def ingestr_loader_function( - *, source_entry: SourceEntry, name: str -) -> Callable[[LoaderContext], None]: - """The loader function that runs one declarative ingestr source under `name`.""" - - def run(ctx: LoaderContext) -> None: - command: tuple[str, ...] = build_ingestr_command( - source_entry=source_entry, - adapter_name=ctx.adapter.adapter_name, - connection_config=ctx.connection_config, - destination_name=ctx.destination_name, - vars=ctx.vars, - environment=ctx.target, - run_id=ctx.run_id, - is_reload=ctx.is_reload, - ) - try: - result: IngestrCommandResult = run_ingestr_command(command) - except IngestrIntegrationError as error: - if isinstance(error.run_result, IngestrCommandResult): - _ = record_ingestr_output(ctx=ctx, result=error.run_result) - raise - _ = record_ingestr_output(ctx=ctx, result=result) - return None - - run.__name__ = name - return run diff --git a/src/sqlbuild/integrations/ingestr/models.py b/src/sqlbuild/integrations/ingestr/models.py deleted file mode 100644 index ae25cb839d..0000000000 --- a/src/sqlbuild/integrations/ingestr/models.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Typed models for ingestr integration loaders.""" - -from __future__ import annotations - -from dataclasses import dataclass, field - - -@dataclass(frozen=True) -class IngestrCommandResult: - """Captured output from one ingestr subprocess run.""" - - command_display: str - stdout: str - stderr: str - - -@dataclass(frozen=True) -class IngestrSourceConfig: - """Configuration for one source loaded by the ingestr integration loader.""" - - source_uri: str - source_table: str - strategy: str | None = None - incremental_key: str | None = None - primary_key: tuple[str, ...] = field(default_factory=tuple) - columns: str | None = None - extra_args: tuple[str, ...] = field(default_factory=tuple) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/_test_types.py index e6a33bccfe..336c367461 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/_test_types.py @@ -1,14 +1,6 @@ from dataclasses import dataclass, field -@dataclass(frozen=True) -class BigQueryDltE2ETestCase: - description: str - expected_loaded_rows: tuple[tuple[object, ...], ...] - expected_model_rows: tuple[tuple[object, ...], ...] - expected_return_code: int = 0 - - @dataclass(frozen=True) class BigQueryCliTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_dlt_bigquery.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_dlt_bigquery.py deleted file mode 100644 index e900cbb35c..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_dlt_bigquery.py +++ /dev/null @@ -1,118 +0,0 @@ -"""BigQuery E2E tests for declarative dlt source loading.""" - -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.bigquery._test_types import ( - BigQueryDltE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.bigquery.helpers import ( - build_bigquery_project_toml, - build_unique_dataset_name, - cleanup_bigquery_dataset, - ensure_bigquery_dataset_ready, - fetch_bigquery_rows, - relation_name, -) -from tests.e2e.src.sqlbuild.cli.commands.main.load.helpers import ( - write_sqlite_orders_source_database, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - BigQueryDltE2ETestCase( - description="loads sqlite dlt source into bigquery raw dataset", - expected_loaded_rows=((1, 10), (2, 20), (3, 30)), - expected_model_rows=((2, 20), (3, 30)), - ) - ], - ids=lambda case: case.description, -) -def test_given_dlt_source_when_loading_to_bigquery_then_table_is_materialized_and_queryable( - tmp_path: Path, - test_case: BigQueryDltE2ETestCase, -) -> None: - dataset_base: str = build_unique_dataset_name(prefix="sqb_dlt_bigquery") - model_dataset_name: str = f"{dataset_base}_models" - raw_dataset_name: str = f"{dataset_base}_raw" - project_name: str = "dlt_bigquery_project" - source_db_path: Path = tmp_path / project_name / "source.db" - try: - ensure_bigquery_dataset_ready(dataset_name=model_dataset_name) - ensure_bigquery_dataset_ready(dataset_name=raw_dataset_name) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=project_name, - repo_files={ - "sqlbuild_project.toml": build_bigquery_project_toml( - project_name=project_name, - dataset_name=model_dataset_name, - ), - "sources/raw.yml": ( - "dlt_sources:\n" - " - type: sql_database\n" - f" schema: {raw_dataset_name}\n" - " config:\n" - f' credentials: "sqlite:///{source_db_path}"\n' - " resources:\n" - " - name: raw_orders\n" - " description: Test source raw_orders.\n" - " table: orders\n" - " write_disposition: replace\n" - ), - "models/order_totals.sql": ( - "MODEL (description 'Test model order_totals.', materialized table);\n\n" - 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' - ), - }, - ) - write_sqlite_orders_source_database(source_db_path) - - load_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "load", "--select", "raw_orders"), - project_dir=project_dir, - ) - - assert load_result.returncode == test_case.expected_return_code, ( - load_result.stdout + load_result.stderr - ) - loaded_rows: tuple[tuple[object, ...], ...] = fetch_bigquery_rows( - dataset_name=raw_dataset_name, - sql=( - "SELECT order_id, amount FROM " - f"{relation_name(dataset_name=raw_dataset_name, name='raw_orders')} " - "ORDER BY order_id" - ), - ) - assert loaded_rows == test_case.expected_loaded_rows - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "order_totals"), - project_dir=project_dir, - ) - - assert build_result.returncode == test_case.expected_return_code, ( - build_result.stdout + build_result.stderr - ) - model_rows: tuple[tuple[object, ...], ...] = fetch_bigquery_rows( - dataset_name=model_dataset_name, - sql=( - "SELECT order_id, amount FROM " - f"{relation_name(dataset_name=model_dataset_name, name='order_totals')} " - "ORDER BY order_id" - ), - ) - assert model_rows == test_case.expected_model_rows - finally: - cleanup_bigquery_dataset(dataset_name=raw_dataset_name) - cleanup_bigquery_dataset(dataset_name=model_dataset_name) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py index 688cd7f480..dbfbf41d12 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py @@ -100,24 +100,6 @@ 'replay_on_change = "bounded-14d"', ), ), - ModelHeaderKeyCompileCase( - description="misspelled dlt resource key", - repo_files={ - "sqlbuild_project.toml": _PROJECT_TOML, - "sources/raw.yml": ( - "dlt_sources:\n - type: sql_database\n config:\n" - " credentials: duckdb:///inputs.duckdb\n resources:\n" - " - name: raw_orders\n table: orders\n" - " primary_keys: order_id\n" - ), - "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.');\n\nSELECT 1 AS order_id\n", - }, - expected_fragments=( - "error[D006]", - "dlt sql_database resource 'raw_orders' has unknown keys: primary_keys", - "did you mean 'primary_key'?", - ), - ), ], ids=lambda case: case.description, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/_test_types.py index 32f2480162..0efbff8e75 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/_test_types.py @@ -1,14 +1,6 @@ from dataclasses import dataclass, field -@dataclass(frozen=True) -class DatabricksDltE2ETestCase: - description: str - expected_loaded_rows: tuple[tuple[object, ...], ...] - expected_model_rows: tuple[tuple[object, ...], ...] - expected_return_code: int = 0 - - @dataclass(frozen=True) class DatabricksCliTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_dlt_databricks.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_dlt_databricks.py deleted file mode 100644 index b1c5dcfaff..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_dlt_databricks.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Databricks E2E tests for declarative dlt source loading.""" - -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.databricks._test_types import ( - DatabricksDltE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.databricks.helpers import ( - build_databricks_project_toml, - build_unique_schema_name, - cleanup_databricks_schema, - ensure_databricks_schema_ready, - fetch_databricks_rows, - relation_name, -) -from tests.e2e.src.sqlbuild.cli.commands.main.load.helpers import ( - write_sqlite_orders_source_database, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - DatabricksDltE2ETestCase( - description="loads sqlite dlt source into databricks raw schema", - expected_loaded_rows=((1, 10), (2, 20), (3, 30)), - expected_model_rows=((2, 20), (3, 30)), - ) - ], - ids=lambda case: case.description, -) -def test_given_dlt_source_when_loading_to_databricks_then_table_is_materialized_and_queryable( - tmp_path: Path, - test_case: DatabricksDltE2ETestCase, -) -> None: - schema_base: str = build_unique_schema_name(prefix="sqb_dlt_databricks") - model_schema_name: str = f"{schema_base}_models" - raw_schema_name: str = f"{schema_base}_raw" - project_name: str = "dlt_databricks_project" - source_db_path: Path = tmp_path / project_name / "source.db" - try: - ensure_databricks_schema_ready(schema_name=model_schema_name) - ensure_databricks_schema_ready(schema_name=raw_schema_name) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=project_name, - repo_files={ - "sqlbuild_project.toml": build_databricks_project_toml( - project_name=project_name, - schema_name=model_schema_name, - ), - "sources/raw.yml": ( - "dlt_sources:\n" - " - type: sql_database\n" - f" schema: {raw_schema_name}\n" - " config:\n" - f' credentials: "sqlite:///{source_db_path}"\n' - " resources:\n" - " - name: raw_orders\n" - " description: Test source raw_orders.\n" - " table: orders\n" - " write_disposition: replace\n" - ), - "models/order_totals.sql": ( - "MODEL (description 'Test model order_totals.', materialized table);\n\n" - 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' - ), - }, - ) - write_sqlite_orders_source_database(source_db_path) - - load_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "load", "--select", "raw_orders"), - project_dir=project_dir, - ) - - assert load_result.returncode == test_case.expected_return_code, ( - load_result.stdout + load_result.stderr - ) - loaded_rows: tuple[tuple[object, ...], ...] = fetch_databricks_rows( - schema_name=raw_schema_name, - sql=( - "SELECT order_id, amount FROM " - f"{relation_name(schema_name=raw_schema_name, name='raw_orders')} " - "ORDER BY order_id" - ), - ) - assert loaded_rows == test_case.expected_loaded_rows - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "order_totals"), - project_dir=project_dir, - ) - - assert build_result.returncode == test_case.expected_return_code, ( - build_result.stdout + build_result.stderr - ) - model_rows: tuple[tuple[object, ...], ...] = fetch_databricks_rows( - schema_name=model_schema_name, - sql=( - "SELECT order_id, amount FROM " - f"{relation_name(schema_name=model_schema_name, name='order_totals')} " - "ORDER BY order_id" - ), - ) - assert model_rows == test_case.expected_model_rows - finally: - cleanup_databricks_schema(schema_name=raw_schema_name) - cleanup_databricks_schema(schema_name=model_schema_name) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py index 7a71fd98f2..286b6bb96e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py @@ -107,20 +107,6 @@ class SourceLoaderErrorE2ETestCase: expected_return_code: int = 1 -@dataclass(frozen=True) -class DltSqlDatabaseE2ETestCase: - description: str - expected_loaded_rows: tuple[tuple[object, ...], ...] - expected_model_rows: tuple[tuple[object, ...], ...] - - -@dataclass(frozen=True) -class DltLocalSourceE2ETestCase: - description: str - expected_loaded_rows: tuple[tuple[object, ...], ...] - expected_model_rows: tuple[tuple[object, ...], ...] - - @dataclass(frozen=True) class LoaderRelationGuardE2ETestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py deleted file mode 100644 index 17b823732c..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py +++ /dev/null @@ -1,316 +0,0 @@ -"""E2E tests for declarative dlt source loading.""" - -from __future__ import annotations - -import sqlite3 -import subprocess -from contextlib import closing -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.load._test_types import ( - DltLocalSourceE2ETestCase, - DltSqlDatabaseE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.load.helpers import ( - serve_orders_api, - write_sqlite_orders_source_database, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - query_duckdb, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - DltSqlDatabaseE2ETestCase( - description="loads sqlite sql_database source into duckdb", - expected_loaded_rows=((1, 10), (2, 20), (3, 30)), - expected_model_rows=((2, 20), (3, 30)), - ) - ], - ids=lambda case: case.description, -) -def test_given_dlt_sql_database_source_when_loading_then_table_is_materialized_and_queryable( - tmp_path: Path, test_case: DltSqlDatabaseE2ETestCase -) -> None: - project_name: str = "dlt_sql_database_project" - warehouse_db_path: Path = tmp_path / project_name / "warehouse.duckdb" - source_db_path: Path = tmp_path / project_name / "source.db" - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=project_name, - repo_files={ - "sqlbuild_project.toml": ( - 'name = "dlt_sql_database_project"\n' - 'adapter = "duckdb"\n\n' - "[connection]\n" - f'database = "{warehouse_db_path}"\n\n' - "[defaults]\n" - 'materialized = "table"\n' - ), - "sources/raw.yml": ( - "dlt_sources:\n" - " - type: sql_database\n" - " config:\n" - f' credentials: "sqlite:///{source_db_path}"\n' - " resources:\n" - " - name: raw_orders\n" - " description: Test source raw_orders.\n" - " table: orders\n" - " write_disposition: replace\n" - ), - "models/order_totals.sql": ( - "MODEL (description 'Test model order_totals.', materialized table);\n\n" - 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' - ), - }, - ) - write_sqlite_orders_source_database(source_db_path) - - load_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "load", "--select", "raw_orders"), - project_dir=project_dir, - ) - - assert load_result.returncode == 0, load_result.stdout + load_result.stderr - loaded_rows: list[tuple[object, ...]] = query_duckdb( - db_path=warehouse_db_path, - sql="SELECT order_id, amount FROM raw_orders ORDER BY order_id", - ) - assert tuple(loaded_rows) == test_case.expected_loaded_rows - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "order_totals"), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - model_rows: list[tuple[object, ...]] = query_duckdb( - db_path=warehouse_db_path, - sql="SELECT order_id, amount FROM order_totals ORDER BY order_id", - ) - assert tuple(model_rows) == test_case.expected_model_rows - - -@pytest.mark.parametrize( - "test_case", - [ - DltLocalSourceE2ETestCase( - description="loads local jsonl filesystem source into duckdb", - expected_loaded_rows=((1, 10), (2, 20), (3, 30)), - expected_model_rows=((2, 20), (3, 30)), - ) - ], - ids=lambda case: case.description, -) -def test_given_dlt_filesystem_source_when_loading_then_table_is_materialized_and_queryable( - tmp_path: Path, test_case: DltLocalSourceE2ETestCase -) -> None: - project_name: str = "dlt_filesystem_project" - project_root: Path = tmp_path / project_name - warehouse_db_path: Path = project_root / "warehouse.duckdb" - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=project_name, - repo_files={ - "sqlbuild_project.toml": ( - 'name = "dlt_filesystem_project"\n' - 'adapter = "duckdb"\n\n' - "[connection]\n" - f'database = "{warehouse_db_path}"\n\n' - "[defaults]\n" - 'materialized = "table"\n' - ), - "sources/raw.yml": ( - "dlt_sources:\n" - " - type: filesystem\n" - " schema: main\n" - " config:\n" - f' bucket_url: "file://{project_root / "data"}"\n' - ' file_glob: "orders.jsonl"\n' - " resources:\n" - " - name: raw_orders\n" - " description: Test source raw_orders.\n" - " reader: jsonl\n" - " write_disposition: replace\n" - ), - "data/orders.jsonl": ( - '{"order_id": 1, "amount": 10}\n' - '{"order_id": 2, "amount": 20}\n' - '{"order_id": 3, "amount": 30}\n' - ), - "models/order_totals.sql": ( - "MODEL (description 'Test model order_totals.', materialized table);\n\n" - 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' - ), - }, - ) - - load_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "load", "--select", "raw_orders"), - project_dir=project_dir, - ) - - assert load_result.returncode == 0, load_result.stdout + load_result.stderr - loaded_rows: list[tuple[object, ...]] = query_duckdb( - db_path=warehouse_db_path, - sql="SELECT order_id, amount FROM raw_orders ORDER BY order_id", - ) - assert tuple(loaded_rows) == test_case.expected_loaded_rows - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "order_totals"), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - model_rows: list[tuple[object, ...]] = query_duckdb( - db_path=warehouse_db_path, - sql="SELECT order_id, amount FROM order_totals ORDER BY order_id", - ) - assert tuple(model_rows) == test_case.expected_model_rows - - -@pytest.mark.parametrize( - "test_case", - [ - DltLocalSourceE2ETestCase( - description="loads local rest api source into duckdb", - expected_loaded_rows=((1, 10), (2, 20), (3, 30)), - expected_model_rows=((2, 20), (3, 30)), - ) - ], - ids=lambda case: case.description, -) -def test_given_dlt_rest_api_source_when_loading_then_table_is_materialized_and_queryable( - tmp_path: Path, test_case: DltLocalSourceE2ETestCase -) -> None: - project_name: str = "dlt_rest_api_project" - warehouse_db_path: Path = tmp_path / project_name / "warehouse.duckdb" - with serve_orders_api() as base_url: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=project_name, - repo_files={ - "sqlbuild_project.toml": ( - 'name = "dlt_rest_api_project"\n' - 'adapter = "duckdb"\n\n' - "[connection]\n" - f'database = "{warehouse_db_path}"\n\n' - "[defaults]\n" - 'materialized = "table"\n' - ), - "sources/raw.yml": ( - "dlt_sources:\n" - " - type: rest_api\n" - " schema: main\n" - " config:\n" - " client:\n" - f' base_url: "{base_url}"\n' - " resources:\n" - " - name: raw_orders\n" - " description: Test source raw_orders.\n" - " endpoint:\n" - ' path: "orders"\n' - ' data_selector: "$"\n' - " write_disposition: replace\n" - ), - "models/order_totals.sql": ( - "MODEL (description 'Test model order_totals.', materialized table);\n\n" - 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' - ), - }, - ) - - load_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "load", "--select", "raw_orders"), - project_dir=project_dir, - ) - - assert load_result.returncode == 0, load_result.stdout + load_result.stderr - loaded_rows: list[tuple[object, ...]] = query_duckdb( - db_path=warehouse_db_path, - sql="SELECT order_id, amount FROM raw_orders ORDER BY order_id", - ) - assert tuple(loaded_rows) == test_case.expected_loaded_rows - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "order_totals"), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - model_rows: list[tuple[object, ...]] = query_duckdb( - db_path=warehouse_db_path, - sql="SELECT order_id, amount FROM order_totals ORDER BY order_id", - ) - assert tuple(model_rows) == test_case.expected_model_rows - - -@pytest.mark.parametrize( - "test_case", - [ - DltLocalSourceE2ETestCase( - description="loads local jsonl filesystem source into sqlite", - expected_loaded_rows=((1, 10), (2, 20), (3, 30)), - expected_model_rows=((2, 20), (3, 30)), - ) - ], - ids=lambda case: case.description, -) -def test_given_sqlite_project_when_loading_dlt_filesystem_source_then_models_read_it( - tmp_path: Path, test_case: DltLocalSourceE2ETestCase -) -> None: - project_root: Path = tmp_path / "dlt_sqlite_project" - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="dlt_sqlite_project", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "dlt_sqlite_project"\nadapter = "sqlite"\n\n' - '[connection]\npath = "warehouse.sqlite"\n\n' - '[defaults]\nmaterialized = "table"\n' - ), - "sources/raw.yml": ( - "dlt_sources:\n" - " - type: filesystem\n" - " config:\n" - f' bucket_url: "file://{project_root / "data"}"\n' - ' file_glob: "orders.jsonl"\n' - " resources:\n" - " - name: raw_orders\n" - " description: Test source raw_orders.\n" - " reader: jsonl\n" - " write_disposition: replace\n" - ), - "data/orders.jsonl": ( - '{"order_id": 1, "amount": 10}\n' - '{"order_id": 2, "amount": 20}\n' - '{"order_id": 3, "amount": 30}\n' - ), - "models/order_totals.sql": ( - "MODEL (description 'Test model order_totals.', materialized table);\n\n" - 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' - ), - }, - ) - - built: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - - assert built.returncode == 0, built.stdout + built.stderr - with closing(sqlite3.connect(project_dir / "warehouse.sqlite")) as warehouse: - loaded_rows: list[tuple[object, ...]] = warehouse.execute( - "SELECT order_id, amount FROM raw_orders ORDER BY order_id" - ).fetchall() - model_rows: list[tuple[object, ...]] = warehouse.execute( - "SELECT order_id, amount FROM order_totals ORDER BY order_id" - ).fetchall() - assert tuple(loaded_rows) == test_case.expected_loaded_rows - assert tuple(model_rows) == test_case.expected_model_rows diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py index 354f07f82e..6da08b7adc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py @@ -42,14 +42,6 @@ class PostgresSourceLoaderDagE2ETestCase: expected_return_code: int = 0 -@dataclass(frozen=True) -class PostgresDltE2ETestCase: - description: str - expected_loaded_rows: tuple[tuple[object, ...], ...] - expected_model_rows: tuple[tuple[object, ...], ...] - expected_return_code: int = 0 - - @dataclass(frozen=True) class PostgresIntermediateDagStrategyE2ETestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_dlt_postgres.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_dlt_postgres.py deleted file mode 100644 index f02490f61e..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_dlt_postgres.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Postgres E2E tests for declarative dlt source loading.""" - -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.load.helpers import ( - write_sqlite_orders_source_database, -) -from tests.e2e.src.sqlbuild.cli.commands.main.postgres._test_types import ( - PostgresDltE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.postgres.helpers import ( - build_postgres_project_toml, - build_unique_schema_name, - cleanup_postgres_schema, - ensure_postgres_schema_ready, - fetch_postgres_rows, - relation_name, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresDltE2ETestCase( - description="loads sqlite dlt source into postgres raw schema", - expected_loaded_rows=((1, 10), (2, 20), (3, 30)), - expected_model_rows=((2, 20), (3, 30)), - ) - ], - ids=lambda case: case.description, -) -def test_given_dlt_source_when_loading_to_postgres_then_table_is_materialized_and_queryable( - tmp_path: Path, - postgres_e2e_config: dict[str, object], - test_case: PostgresDltE2ETestCase, -) -> None: - schema_base: str = build_unique_schema_name(prefix="sqb_dlt_postgres") - model_schema_name: str = f"{schema_base}_models" - raw_schema_name: str = f"{schema_base}_raw" - project_name: str = "dlt_postgres_project" - source_db_path: Path = tmp_path / project_name / "source.db" - try: - ensure_postgres_schema_ready(schema_name=model_schema_name, config=postgres_e2e_config) - ensure_postgres_schema_ready(schema_name=raw_schema_name, config=postgres_e2e_config) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=project_name, - repo_files={ - "sqlbuild_project.toml": build_postgres_project_toml( - project_name=project_name, - schema_name=model_schema_name, - config=postgres_e2e_config, - ), - "sources/raw.yml": ( - "dlt_sources:\n" - " - type: sql_database\n" - f" schema: {raw_schema_name}\n" - " config:\n" - f' credentials: "sqlite:///{source_db_path}"\n' - " resources:\n" - " - name: raw_orders\n" - " description: Test source raw_orders.\n" - " table: orders\n" - " write_disposition: replace\n" - ), - "models/order_totals.sql": ( - "MODEL (description 'Test model order_totals.', materialized table);\n\n" - 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' - ), - }, - ) - write_sqlite_orders_source_database(source_db_path) - - load_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "load", "--select", "raw_orders"), - project_dir=project_dir, - ) - - assert load_result.returncode == test_case.expected_return_code, ( - load_result.stdout + load_result.stderr - ) - loaded_rows: tuple[tuple[object, ...], ...] = fetch_postgres_rows( - config=postgres_e2e_config, - sql=( - "SELECT order_id, amount FROM " - f"{relation_name(schema_name=raw_schema_name, name='raw_orders')} " - "ORDER BY order_id" - ), - ) - assert loaded_rows == test_case.expected_loaded_rows - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "order_totals"), - project_dir=project_dir, - ) - - assert build_result.returncode == test_case.expected_return_code, ( - build_result.stdout + build_result.stderr - ) - model_rows: tuple[tuple[object, ...], ...] = fetch_postgres_rows( - config=postgres_e2e_config, - sql=( - "SELECT order_id, amount FROM " - f"{relation_name(schema_name=model_schema_name, name='order_totals')} " - "ORDER BY order_id" - ), - ) - assert model_rows == test_case.expected_model_rows - finally: - cleanup_postgres_schema(schema_name=raw_schema_name, config=postgres_e2e_config) - cleanup_postgres_schema(schema_name=model_schema_name, config=postgres_e2e_config) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/_test_types.py index 7b52fbd292..53d64b08f5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/_test_types.py @@ -1,14 +1,6 @@ from dataclasses import dataclass, field -@dataclass(frozen=True) -class SnowflakeDltE2ETestCase: - description: str - expected_loaded_rows: tuple[tuple[object, ...], ...] - expected_model_rows: tuple[tuple[object, ...], ...] - expected_return_code: int = 0 - - @dataclass(frozen=True) class SnowflakeCliTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_dlt_snowflake.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_dlt_snowflake.py deleted file mode 100644 index 1b47a05b6a..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_dlt_snowflake.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Snowflake E2E tests for declarative dlt source loading.""" - -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.load.helpers import ( - write_sqlite_orders_source_database, -) -from tests.e2e.src.sqlbuild.cli.commands.main.snowflake._test_types import ( - SnowflakeDltE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.snowflake.helpers import ( - build_snowflake_project_toml, - build_unique_schema_name, - cleanup_snowflake_schema, - ensure_query_schema_ready, - fetch_snowflake_rows, - relation_name, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - SnowflakeDltE2ETestCase( - description="loads sqlite dlt source into snowflake raw schema", - expected_loaded_rows=((1, 10), (2, 20), (3, 30)), - expected_model_rows=((2, 20), (3, 30)), - ) - ], - ids=lambda case: case.description, -) -def test_given_dlt_source_when_loading_to_snowflake_then_table_is_materialized_and_queryable( - tmp_path: Path, - test_case: SnowflakeDltE2ETestCase, -) -> None: - schema_base: str = build_unique_schema_name(prefix="sqb_dlt_snowflake") - model_schema_name: str = f"{schema_base}_MODELS" - raw_schema_name: str = f"{schema_base}_raw".lower() - project_name: str = "dlt_snowflake_project" - source_db_path: Path = tmp_path / project_name / "source.db" - try: - ensure_query_schema_ready(schema_name=model_schema_name) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=project_name, - repo_files={ - "sqlbuild_project.toml": build_snowflake_project_toml( - project_name=project_name, - schema_name=model_schema_name, - ), - "sources/raw.yml": ( - "dlt_sources:\n" - " - type: sql_database\n" - f" schema: {raw_schema_name}\n" - " config:\n" - f' credentials: "sqlite:///{source_db_path}"\n' - " resources:\n" - " - name: raw_orders\n" - " description: Test source raw_orders.\n" - " table: orders\n" - " write_disposition: replace\n" - ), - "models/order_totals.sql": ( - "MODEL (description 'Test model order_totals.', materialized table);\n\n" - 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' - ), - }, - ) - write_sqlite_orders_source_database(source_db_path) - - load_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "load", "--select", "raw_orders"), - project_dir=project_dir, - ) - - assert load_result.returncode == test_case.expected_return_code, ( - load_result.stdout + load_result.stderr - ) - loaded_rows: tuple[tuple[object, ...], ...] = fetch_snowflake_rows( - schema_name=raw_schema_name, - sql=( - "SELECT order_id, amount FROM " - f"{relation_name(schema_name=raw_schema_name, name='raw_orders')} " - "ORDER BY order_id" - ), - ) - assert loaded_rows == test_case.expected_loaded_rows - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "order_totals"), - project_dir=project_dir, - ) - - assert build_result.returncode == test_case.expected_return_code, ( - build_result.stdout + build_result.stderr - ) - model_rows: tuple[tuple[object, ...], ...] = fetch_snowflake_rows( - schema_name=model_schema_name, - sql=( - "SELECT order_id, amount FROM " - f"{relation_name(schema_name=model_schema_name, name='order_totals')} " - "ORDER BY order_id" - ), - ) - assert model_rows == test_case.expected_model_rows - finally: - cleanup_snowflake_schema(schema_name=raw_schema_name) - cleanup_snowflake_schema(schema_name=model_schema_name) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/_test_types.py index 5faf5172ac..6945299c9a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/_test_types.py @@ -1,14 +1,6 @@ from dataclasses import dataclass, field -@dataclass(frozen=True) -class SqlServerDltE2ETestCase: - description: str - expected_loaded_rows: tuple[tuple[object, ...], ...] - expected_model_rows: tuple[tuple[object, ...], ...] - expected_return_code: int = 0 - - @dataclass(frozen=True) class SqlServerBuildE2ETestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_dlt_sqlserver.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_dlt_sqlserver.py deleted file mode 100644 index c500293d0d..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_dlt_sqlserver.py +++ /dev/null @@ -1,121 +0,0 @@ -"""SQL Server E2E tests for declarative dlt source loading.""" - -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.load.helpers import ( - write_sqlite_orders_source_database, -) -from tests.e2e.src.sqlbuild.cli.commands.main.sqlserver._test_types import ( - SqlServerDltE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.sqlserver.helpers import ( - build_sqlserver_config, - build_sqlserver_project_toml, - build_unique_schema_name, - cleanup_sqlserver_schema, - ensure_sqlserver_schema_ready, - fetch_sqlserver_rows, - relation_name, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - SqlServerDltE2ETestCase( - description="loads sqlite dlt source into sqlserver raw schema", - expected_loaded_rows=((1, 10), (2, 20), (3, 30)), - expected_model_rows=((2, 20), (3, 30)), - ) - ], - ids=lambda case: case.description, -) -def test_given_dlt_source_when_loading_to_sqlserver_then_table_is_materialized_and_queryable( - tmp_path: Path, - test_case: SqlServerDltE2ETestCase, -) -> None: - config: dict[str, object] = build_sqlserver_config() - schema_base: str = build_unique_schema_name(prefix="sqb_dlt_sqlserver") - model_schema_name: str = f"{schema_base}_models" - raw_schema_name: str = f"{schema_base}_raw" - project_name: str = "dlt_sqlserver_project" - source_db_path: Path = tmp_path / project_name / "source.db" - try: - ensure_sqlserver_schema_ready(schema_name=model_schema_name, config=config) - ensure_sqlserver_schema_ready(schema_name=raw_schema_name, config=config) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=project_name, - repo_files={ - "sqlbuild_project.toml": build_sqlserver_project_toml( - project_name=project_name, - schema_name=model_schema_name, - config=config, - ), - "sources/raw.yml": ( - "dlt_sources:\n" - " - type: sql_database\n" - f" schema: {raw_schema_name}\n" - " config:\n" - f' credentials: "sqlite:///{source_db_path}"\n' - " resources:\n" - " - name: raw_orders\n" - " description: Test source raw_orders.\n" - " table: orders\n" - " write_disposition: replace\n" - ), - "models/order_totals.sql": ( - "MODEL (description 'Test model order_totals.', materialized table);\n\n" - 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' - ), - }, - ) - write_sqlite_orders_source_database(source_db_path) - - load_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "load", "--select", "raw_orders"), - project_dir=project_dir, - ) - - assert load_result.returncode == test_case.expected_return_code, ( - load_result.stdout + load_result.stderr - ) - loaded_rows: tuple[tuple[object, ...], ...] = fetch_sqlserver_rows( - config=config, - sql=( - "SELECT order_id, amount FROM " - f"{relation_name(schema_name=raw_schema_name, name='raw_orders')} " - "ORDER BY order_id" - ), - ) - assert loaded_rows == test_case.expected_loaded_rows - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "order_totals"), - project_dir=project_dir, - ) - - assert build_result.returncode == test_case.expected_return_code, ( - build_result.stdout + build_result.stderr - ) - model_rows: tuple[tuple[object, ...], ...] = fetch_sqlserver_rows( - config=config, - sql=( - "SELECT order_id, amount FROM " - f"{relation_name(schema_name=model_schema_name, name='order_totals')} " - "ORDER BY order_id" - ), - ) - assert model_rows == test_case.expected_model_rows - finally: - cleanup_sqlserver_schema(schema_name=raw_schema_name, config=config) - cleanup_sqlserver_schema(schema_name=model_schema_name, config=config) diff --git a/tests/e2e/src/sqlbuild/integrations/ingestr/_test_types.py b/tests/e2e/src/sqlbuild/integrations/ingestr/_test_types.py deleted file mode 100644 index d9265b8598..0000000000 --- a/tests/e2e/src/sqlbuild/integrations/ingestr/_test_types.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class IngestrE2ETestCase: - description: str - expected_rows: tuple[tuple[object, ...], ...] - expected_stdout_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/integrations/ingestr/conftest.py b/tests/e2e/src/sqlbuild/integrations/ingestr/conftest.py deleted file mode 100644 index 3fe94eae29..0000000000 --- a/tests/e2e/src/sqlbuild/integrations/ingestr/conftest.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import annotations - -from collections.abc import Iterator -from typing import Any - -import pytest - - -@pytest.fixture(scope="session") -def postgres_container() -> Iterator[Any]: - from testcontainers.postgres import PostgresContainer - - try: - with PostgresContainer("postgres:17") as container: - yield container - except Exception as exc: - pytest.skip(f"Docker not available for Postgres testcontainer: {exc}") - - -@pytest.fixture(scope="session") -def postgres_config(postgres_container: Any) -> dict[str, object]: - return { - "host": postgres_container.get_container_host_ip(), - "port": int(postgres_container.get_exposed_port(5432)), - "dbname": postgres_container.dbname, - "user": postgres_container.username, - "password": postgres_container.password, - } diff --git a/tests/e2e/src/sqlbuild/integrations/ingestr/helpers.py b/tests/e2e/src/sqlbuild/integrations/ingestr/helpers.py deleted file mode 100644 index eeab4cd46a..0000000000 --- a/tests/e2e/src/sqlbuild/integrations/ingestr/helpers.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import annotations - -import os -import subprocess -from collections.abc import Mapping -from pathlib import Path -from typing import Any -from urllib.parse import quote - -from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter - -REPO_ROOT: Path = Path(__file__).resolve().parents[6] - - -def run_sqb_with_ingestr( - *, command: tuple[str, ...], project_dir: Path -) -> subprocess.CompletedProcess[str]: - process_env: dict[str, str] = dict(os.environ) - return subprocess.run( - [ - "uv", - "run", - "--with", - "ingestr", - "sqb", - "--project-dir", - str(project_dir), - *command, - ], - cwd=REPO_ROOT, - capture_output=True, - text=True, - env=process_env, - check=False, - ) - - -def postgres_uri(config: Mapping[str, object]) -> str: - user: str = quote(str(config["user"])) - password: str = quote(str(config["password"])) - host: str = str(config["host"]) - port: object = config["port"] - database: str = quote(str(config["dbname"])) - return f"postgresql://{user}:{password}@{host}:{port}/{database}" - - -def postgres_project_toml(*, project_name: str, config: Mapping[str, object]) -> str: - return ( - f""" -name = "{project_name}" -adapter = "postgres" - -[connection] -host = "{config["host"]}" -port = {config["port"]} -dbname = "{config["dbname"]}" -user = "{config["user"]}" -password = "{config["password"]}" -""".strip() - + "\n" - ) - - -def duckdb_project_toml(*, project_name: str, database_path: Path) -> str: - return ( - f""" -name = "{project_name}" -adapter = "duckdb" - -[connection] -database = "{database_path}" -""".strip() - + "\n" - ) - - -def execute_postgres_sql(*, config: Mapping[str, object], sql: str) -> None: - adapter: PostgresAdapter = PostgresAdapter() - connection: Any = adapter.connect(dict(config)) - try: - adapter.execute(connection=connection, sql=sql) - finally: - adapter.close(connection) - - -def fetch_postgres_rows( - *, config: Mapping[str, object], sql: str -) -> tuple[tuple[object, ...], ...]: - adapter: PostgresAdapter = PostgresAdapter() - connection: Any = adapter.connect(dict(config)) - try: - cursor: Any = adapter.execute(connection=connection, sql=sql) - return tuple(cursor.fetchall()) - finally: - adapter.close(connection) - - -def fetch_duckdb_rows(*, database_path: Path, sql: str) -> tuple[tuple[object, ...], ...]: - import duckdb - - connection: duckdb.DuckDBPyConnection = duckdb.connect(str(database_path), read_only=True) - try: - return tuple(connection.execute(sql).fetchall()) - finally: - connection.close() - - -def write_project_files(*, project_dir: Path, files: Mapping[str, str]) -> None: - relative_path: str - contents: str - for relative_path, contents in files.items(): - file_path: Path = project_dir / relative_path - file_path.parent.mkdir(parents=True, exist_ok=True) - file_path.write_text(contents, encoding="utf-8") diff --git a/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py b/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py deleted file mode 100644 index 4db0cacd83..0000000000 --- a/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py +++ /dev/null @@ -1,414 +0,0 @@ -from __future__ import annotations - -import sqlite3 -import subprocess -from contextlib import closing -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.integrations.ingestr._test_types import IngestrE2ETestCase -from tests.e2e.src.sqlbuild.integrations.ingestr.helpers import ( - duckdb_project_toml, - execute_postgres_sql, - fetch_duckdb_rows, - fetch_postgres_rows, - postgres_project_toml, - postgres_uri, - run_sqb_with_ingestr, - write_project_files, -) - - -@pytest.mark.parametrize( - "test_case", - [ - IngestrE2ETestCase( - description="loads postgres source into duckdb destination", - expected_rows=((1, "postgres-source"), (2, "postgres-source")), - expected_stdout_fragments=("ingestr execution", "raw_pg_orders", "OK"), - ) - ], - ids=lambda case: case.description, -) -@pytest.mark.postgres -def test_given_postgres_source_when_loading_with_ingestr_then_duckdb_target_has_rows( - tmp_path: Path, - test_case: IngestrE2ETestCase, - postgres_config: dict[str, object], -) -> None: - source_table: str = "ingestr_pg_source_orders" - execute_postgres_sql( - config=postgres_config, - sql=( - f"DROP TABLE IF EXISTS public.{source_table}; " - f"CREATE TABLE public.{source_table} (order_id INTEGER, status TEXT); " - f"INSERT INTO public.{source_table} VALUES " - "(1, 'postgres-source'), (2, 'postgres-source')" - ), - ) - project_dir: Path = tmp_path / "pg_to_duckdb" - duckdb_path: Path = project_dir / "target.duckdb" - write_project_files( - project_dir=project_dir, - files={ - "sqlbuild_project.toml": duckdb_project_toml( - project_name="pg_to_duckdb", database_path=duckdb_path - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_pg_orders\n description: Test source raw_pg_orders.\n" - " ingestr:\n" - f' source_uri: "{postgres_uri(postgres_config)}"\n' - f" source_table: public.{source_table}\n" - ), - }, - ) - - result: subprocess.CompletedProcess[str] = run_sqb_with_ingestr( - command=("--no-color", "load"), project_dir=project_dir - ) - - assert result.returncode == 0, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert ( - fetch_duckdb_rows( - database_path=duckdb_path, - sql="SELECT order_id, status FROM raw_pg_orders ORDER BY order_id", - ) - == test_case.expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - IngestrE2ETestCase( - description="loads a sqlite source into a sqlite destination", - expected_rows=((1, "new"), (2, "paid")), - expected_stdout_fragments=("ingestr execution", "raw_sqlite_orders", "OK"), - ) - ], - ids=lambda case: case.description, -) -def test_given_sqlite_project_when_loading_with_ingestr_then_target_has_rows( - tmp_path: Path, - test_case: IngestrE2ETestCase, -) -> None: - source_path: Path = tmp_path / "source.db" - with closing(sqlite3.connect(source_path)) as source, source: - source.execute("CREATE TABLE orders (order_id INTEGER, status TEXT)") - source.execute("INSERT INTO orders VALUES (1, 'new'), (2, 'paid')") - project_dir: Path = tmp_path / "sqlite_to_sqlite" - write_project_files( - project_dir=project_dir, - files={ - "sqlbuild_project.toml": ( - 'name = "sqlite_to_sqlite"\nadapter = "sqlite"\n\n' - '[connection]\npath = "target.sqlite"\n' - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_sqlite_orders\n description: Test source raw_sqlite_orders.\n" - " ingestr:\n" - f' source_uri: "sqlite:///{source_path}"\n' - " source_table: orders\n" - ), - }, - ) - - result: subprocess.CompletedProcess[str] = run_sqb_with_ingestr( - command=("--no-color", "load"), project_dir=project_dir - ) - - assert result.returncode == 0, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - with closing(sqlite3.connect(project_dir / "target.sqlite")) as target: - rows: list[tuple[object, ...]] = target.execute( - "SELECT order_id, status FROM raw_sqlite_orders ORDER BY order_id" - ).fetchall() - assert tuple(rows) == test_case.expected_rows - - -@pytest.mark.parametrize( - "test_case", - [ - IngestrE2ETestCase( - description="loads duckdb source into duckdb destination without destination lock", - expected_rows=((1, "new"), (2, "paid")), - expected_stdout_fragments=("ingestr execution", "raw_duckdb_orders", "OK"), - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_source_when_loading_with_ingestr_to_duckdb_then_target_has_rows( - tmp_path: Path, - test_case: IngestrE2ETestCase, -) -> None: - import duckdb - - source_duckdb_path: Path = tmp_path / "source.duckdb" - source_connection: duckdb.DuckDBPyConnection = duckdb.connect(str(source_duckdb_path)) - try: - source_connection.execute("CREATE TABLE orders (order_id INTEGER, status VARCHAR)") - source_connection.execute("INSERT INTO orders VALUES (1, 'new'), (2, 'paid')") - finally: - source_connection.close() - project_dir: Path = tmp_path / "duckdb_to_duckdb" - target_duckdb_path: Path = project_dir / "target.duckdb" - write_project_files( - project_dir=project_dir, - files={ - "sqlbuild_project.toml": duckdb_project_toml( - project_name="duckdb_to_duckdb", database_path=target_duckdb_path - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_duckdb_orders\n description: Test source raw_duckdb_orders.\n" - " ingestr:\n" - f' source_uri: "duckdb:///{source_duckdb_path}"\n' - " source_table: orders\n" - ), - }, - ) - - result: subprocess.CompletedProcess[str] = run_sqb_with_ingestr( - command=("--no-color", "load"), project_dir=project_dir - ) - - assert result.returncode == 0, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert ( - fetch_duckdb_rows( - database_path=target_duckdb_path, - sql="SELECT order_id, status FROM raw_duckdb_orders ORDER BY order_id", - ) - == test_case.expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - IngestrE2ETestCase( - description="build runs ingestr duckdb source before building model without lock", - expected_rows=((1, "new"), (2, "paid")), - expected_stdout_fragments=( - "ingestr execution", - "raw_duckdb_orders", - "stg_duckdb_orders", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_ingestr_duckdb_source_when_building_with_load_then_model_reads_loaded_source( - tmp_path: Path, - test_case: IngestrE2ETestCase, -) -> None: - import duckdb - - source_duckdb_path: Path = tmp_path / "source.duckdb" - source_connection: duckdb.DuckDBPyConnection = duckdb.connect(str(source_duckdb_path)) - try: - source_connection.execute("CREATE TABLE orders (order_id INTEGER, status VARCHAR)") - source_connection.execute("INSERT INTO orders VALUES (1, 'new'), (2, 'paid')") - finally: - source_connection.close() - project_dir: Path = tmp_path / "build_duckdb_to_duckdb" - target_duckdb_path: Path = project_dir / "target.duckdb" - write_project_files( - project_dir=project_dir, - files={ - "sqlbuild_project.toml": duckdb_project_toml( - project_name="build_duckdb_to_duckdb", database_path=target_duckdb_path - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_duckdb_orders\n description: Test source raw_duckdb_orders.\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: status\n" - " type: VARCHAR\n" - " ingestr:\n" - f' source_uri: "duckdb:///{source_duckdb_path}"\n' - " source_table: orders\n" - ), - "models/stg_duckdb_orders.sql": """ -MODEL (description "Test model stg_duckdb_orders.", materialized table); - -SELECT order_id, status FROM __source("raw_duckdb_orders") -""".strip() - + "\n", - }, - ) - - result: subprocess.CompletedProcess[str] = run_sqb_with_ingestr( - command=("--no-color", "build", "--load"), project_dir=project_dir - ) - - assert result.returncode == 0, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert ( - fetch_duckdb_rows( - database_path=target_duckdb_path, - sql="SELECT order_id, status FROM stg_duckdb_orders ORDER BY order_id", - ) - == test_case.expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - IngestrE2ETestCase( - description="build loads postgres source with ingestr before building model", - expected_rows=((1, "new"), (2, "paid")), - expected_stdout_fragments=("ingestr execution", "raw_pg_orders", "stg_pg_orders"), - ) - ], - ids=lambda case: case.description, -) -@pytest.mark.postgres -def test_given_ingestr_source_when_building_with_load_then_model_reads_loaded_source( - tmp_path: Path, - test_case: IngestrE2ETestCase, - postgres_config: dict[str, object], -) -> None: - source_table: str = "orders_build_source" - execute_postgres_sql( - config=postgres_config, - sql=f"DROP TABLE IF EXISTS public.{source_table}", - ) - execute_postgres_sql( - config=postgres_config, - sql=f"CREATE TABLE public.{source_table} (order_id INTEGER, status TEXT)", - ) - execute_postgres_sql( - config=postgres_config, - sql=f"INSERT INTO public.{source_table} VALUES (1, 'new'), (2, 'paid')", - ) - duckdb_path: Path = tmp_path / "build_ingestr.duckdb" - project_dir: Path = tmp_path / "build_pg_to_duckdb" - write_project_files( - project_dir=project_dir, - files={ - "sqlbuild_project.toml": duckdb_project_toml( - project_name="build_pg_to_duckdb", database_path=duckdb_path - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_pg_orders\n description: Test source raw_pg_orders.\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: status\n" - " type: VARCHAR\n" - " ingestr:\n" - f' source_uri: "{postgres_uri(postgres_config)}"\n' - f" source_table: public.{source_table}\n" - ), - "models/stg_pg_orders.sql": """ -MODEL (description "Test model stg_pg_orders.", materialized table); - -SELECT order_id, status FROM __source("raw_pg_orders") -""".strip() - + "\n", - }, - ) - - result: subprocess.CompletedProcess[str] = run_sqb_with_ingestr( - command=("--no-color", "build", "--load"), project_dir=project_dir - ) - - assert result.returncode == 0, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert ( - fetch_duckdb_rows( - database_path=duckdb_path, - sql="SELECT order_id, status FROM stg_pg_orders ORDER BY order_id", - ) - == test_case.expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - IngestrE2ETestCase( - description="loads duckdb source into postgres destination", - expected_rows=((7, "duckdb-source"), (8, "duckdb-source")), - expected_stdout_fragments=("ingestr execution", "raw_duckdb_orders", "OK"), - ) - ], - ids=lambda case: case.description, -) -@pytest.mark.postgres -def test_given_duckdb_source_when_loading_with_ingestr_then_postgres_target_has_rows( - tmp_path: Path, - test_case: IngestrE2ETestCase, - postgres_config: dict[str, object], -) -> None: - import duckdb - - source_duckdb_path: Path = tmp_path / "source.duckdb" - source_connection: duckdb.DuckDBPyConnection = duckdb.connect(str(source_duckdb_path)) - try: - source_connection.execute("CREATE TABLE source_orders (order_id INTEGER, status VARCHAR)") - source_connection.execute( - "INSERT INTO source_orders VALUES (7, 'duckdb-source'), (8, 'duckdb-source')" - ) - finally: - source_connection.close() - target_schema: str = "ingestr_duckdb_to_pg" - execute_postgres_sql( - config=postgres_config, - sql=f"DROP SCHEMA IF EXISTS {target_schema} CASCADE", - ) - project_dir: Path = tmp_path / "duckdb_to_pg" - write_project_files( - project_dir=project_dir, - files={ - "sqlbuild_project.toml": postgres_project_toml( - project_name="duckdb_to_pg", config=postgres_config - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_duckdb_orders\n description: Test source raw_duckdb_orders.\n" - f" schema: {target_schema}\n" - " ingestr:\n" - f' source_uri: "duckdb:///{source_duckdb_path}"\n' - " source_table: source_orders\n" - ), - }, - ) - - try: - result: subprocess.CompletedProcess[str] = run_sqb_with_ingestr( - command=("--no-color", "load"), project_dir=project_dir - ) - - assert result.returncode == 0, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert ( - fetch_postgres_rows( - config=postgres_config, - sql=( - f"SELECT order_id, status FROM {target_schema}.raw_duckdb_orders " - "ORDER BY order_id" - ), - ) - == test_case.expected_rows - ) - finally: - execute_postgres_sql( - config=postgres_config, - sql=f"DROP SCHEMA IF EXISTS {target_schema} CASCADE", - ) diff --git a/tests/goldens/compiler/failure/source-removed-ingestr-block.json b/tests/goldens/compiler/failure/source-removed-ingestr-block.json new file mode 100644 index 0000000000..2261883f87 --- /dev/null +++ b/tests/goldens/compiler/failure/source-removed-ingestr-block.json @@ -0,0 +1,18 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "D006", + "message": "/sources/raw.yml source 'raw_orders' declares an ingestr integration, which SQLBuild no longer supports; declare the source with `managed: true` and load it with a Python `@loader` named after the source instead" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/source-unknown-key.json b/tests/goldens/compiler/failure/source-unknown-key.json index 63ad391a74..cb5e919763 100644 --- a/tests/goldens/compiler/failure/source-unknown-key.json +++ b/tests/goldens/compiler/failure/source-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D006", - "message": "/sources/raw.yml source has unknown keys: freshnes; allowed keys: audits, columns, contract, cursor_column, database, description, expression, final, freshness, ingestr, load_batch_size, managed, meta, name, schema, table, type_enforcement, unique_key, write_strategy", + "message": "/sources/raw.yml source has unknown keys: freshnes; allowed keys: audits, columns, contract, cursor_column, database, description, expression, final, freshness, load_batch_size, managed, meta, name, schema, table, type_enforcement, unique_key, write_strategy", "help": "did you mean 'freshness'?" } ] diff --git a/tests/goldens/compiler/failure/sources-removed-dlt-sources.json b/tests/goldens/compiler/failure/sources-removed-dlt-sources.json new file mode 100644 index 0000000000..61f3d75708 --- /dev/null +++ b/tests/goldens/compiler/failure/sources-removed-dlt-sources.json @@ -0,0 +1,18 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "D006", + "message": "/sources/loaded.yml declares dlt_sources, a dlt integration, which SQLBuild no longer supports; declare each source under `sources:` with `managed: true` and load it with a Python `@loader` named after the source instead" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/python_oracle/project_assembly_error_and_python_value_edges.json b/tests/goldens/python_oracle/project_assembly_error_and_python_value_edges.json index 65dda44c3c..ef859e81ae 100644 --- a/tests/goldens/python_oracle/project_assembly_error_and_python_value_edges.json +++ b/tests/goldens/python_oracle/project_assembly_error_and_python_value_edges.json @@ -117,6 +117,6 @@ ], [ "project_assembly", -"77a21099e4cf0ed8378806e7" +"442ec6090c98a2fe91f076da" ] ] diff --git a/tests/goldens/python_oracle/project_assembly_managed_sources_templated_seeds_and_targets_hooks_audits_a_function.json b/tests/goldens/python_oracle/project_assembly_managed_sources_templated_seeds_and_targets_hooks_audits_a_function.json index c51fc0ba0c..66f103e6e3 100644 --- a/tests/goldens/python_oracle/project_assembly_managed_sources_templated_seeds_and_targets_hooks_audits_a_function.json +++ b/tests/goldens/python_oracle/project_assembly_managed_sources_templated_seeds_and_targets_hooks_audits_a_function.json @@ -1,34 +1,34 @@ [ [ "project_assembly", -"c0cc684fdc0838367c4aa5a5" +"5b069ce112672c6d87f7f806" ], [ "project_assembly", -"89e88c0fa0a676eefb1838f8" +"61e216e0ab313fb2f53ec2c0" ], [ "project_assembly", -"9bcbea02e4f8292c20f34a7e" +"efef4bd131cdadfb20ebab06" ], [ "project_assembly", -"3b563ba79ba36067df383ee6" +"aaeb7dff9aedc0f80aa72d4e" ], [ "project_assembly", -"b9e8a53734face2e3949dd37" +"755e2183b256167de99c2ce4" ], [ "project_assembly", -"f934f6542c039b80ea0ee187" +"139b1d2944dae16473432c49" ], [ "project_assembly", -"5186cd735cf200339781e260" +"98862c046dbafe5eccf5bae1" ], [ "project_assembly", -"9a072d9d5401f86627a5830b" +"56b1b1b9a92f2ef32b2ea4ec" ] ] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index aaf1acff51..5dc94e83c5 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -23,10 +23,8 @@ WAREHOUSE_AND_INTEGRATION_MODULES: tuple[str, ...] = ( "dagster", "databricks", - "dlt", "duckdb", "google.cloud.bigquery", - "ingestr", "jinja2", "psycopg", "pyarrow", diff --git a/tests/native_field_fallbacks.txt b/tests/native_field_fallbacks.txt index 6ebb260f60..c51c8afde5 100644 --- a/tests/native_field_fallbacks.txt +++ b/tests/native_field_fallbacks.txt @@ -218,7 +218,6 @@ crates/sqlbuild-python-base/src/frontier/models.rs FutureCursorAction.name crates/sqlbuild-python-base/src/frontier/models.rs GrantKind.name crates/sqlbuild-python-base/src/frontier/models.rs HardCodedRelationOwnerKind.name crates/sqlbuild-python-base/src/frontier/models.rs InferredNullability.name -crates/sqlbuild-python-base/src/frontier/models.rs LoaderConnectionMode.name crates/sqlbuild-python-base/src/frontier/models.rs MaterializationType.name crates/sqlbuild-python-base/src/frontier/models.rs MicrobatchLimitAction.name crates/sqlbuild-python-base/src/frontier/models.rs MissingMigrationOriginPolicy.name diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py index d0312a9858..d17591c683 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py @@ -8,7 +8,6 @@ DiscoveredLoaderFunction, DiscoveredProjectInputs, DiscoveredSourceFile, - IntegrationLoaderConfig, LocalConfig, ProjectConfig, SourceEntry, @@ -27,10 +26,6 @@ def load_raw_orders(_ctx: object) -> list[dict[str, object]]: return [] -def ingestr_raw_orders(_ctx: object) -> list[dict[str, object]]: - return [] - - def build_load_selection_inputs() -> DiscoveredProjectInputs: """Build reachable load-selection inputs with terminal and intermediate loaders.""" @@ -67,34 +62,3 @@ def build_load_selection_inputs() -> DiscoveredProjectInputs: ), ), ) - - -def build_integration_load_selection_inputs() -> DiscoveredProjectInputs: - """Build load-selection inputs with a terminal integration loader.""" - - return DiscoveredProjectInputs( - project_config=ProjectConfig(name="demo", adapter="duckdb"), - local_config=LocalConfig(), - source_files=( - DiscoveredSourceFile( - file_path=Path("sources/raw.yml"), - relative_path=Path("sources/raw.yml"), - contents="", - source_entries=( - SourceEntry( - name="raw_orders", - loader="ingestr__raw_orders", - integration_loader=IntegrationLoaderConfig(kind="ingestr", config=object()), - ), - ), - ), - ), - loader_functions=( - DiscoveredLoaderFunction( - file_path=Path("sources/raw.yml"), - relative_path=Path("sources/raw.yml"), - name="ingestr__raw_orders", - function=ingestr_raw_orders, - ), - ), - ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py index 8ff72a5fa5..8b625cc844 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py @@ -16,16 +16,13 @@ TargetConfig, resolve_loader_defaults, select_load_entries, - select_load_reference_entries, ) from tests.unit.src.sqlbuild.cli.commands._helpers.load_selection._test_types import ( LoaderDefaultResolutionTestCase, LoaderDestinationSelectionTestCase, - LoadReferenceSelectionTestCase, LoadSelectionTestCase, ) from tests.unit.src.sqlbuild.cli.commands._helpers.load_selection.helpers import ( - build_integration_load_selection_inputs, build_load_selection_inputs, ) @@ -65,37 +62,6 @@ def test_given_load_selector_when_selecting_then_returns_expected_source_entries ) -@pytest.mark.parametrize( - "test_case", - [ - LoadReferenceSelectionTestCase( - description="integration terminal loader is selected, not reference-only", - select=("raw_orders",), - expected_entry_names=(), - ) - ], - ids=lambda case: case.description, -) -def test_given_integration_loader_selected_when_selecting_references_then_returns_no_terminal_ref( - test_case: LoadReferenceSelectionTestCase, -) -> None: - discovered_inputs: DiscoveredProjectInputs = build_integration_load_selection_inputs() - selected_entries: tuple[SourceEntry, ...] = select_load_entries( - discovered_inputs=discovered_inputs, - select=test_case.select, - exclude=(), - target_config=None, - ) - - reference_entries: tuple[SourceEntry, ...] = select_load_reference_entries( - discovered_inputs=discovered_inputs, - selected_sources=selected_entries, - target_config=None, - ) - - assert tuple(entry.name for entry in reference_entries) == test_case.expected_entry_names - - @pytest.mark.parametrize( "test_case", ( diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 94cbc8af7a..1ab6d7b9ce 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -876,7 +876,7 @@ def test_given_model_node_when_reporting_progress_then_writes_materialization_la description="completed source load renders multiline lifecycle logs", expected_fragments=( "source raw_orders", - "log ingestr stdout", + "log loader stdout", " Starting data ingestion", " Rows: 3", ), @@ -886,7 +886,7 @@ def test_given_model_node_when_reporting_progress_then_writes_materialization_la description="completed source load colors log label and content semantically", expected_fragments=( "\033[2m log \033[0m", - "\033[2mingestr stdout\033[0m", + "\033[2mloader stdout\033[0m", "\033[2m Starting data ingestion\033[0m", ), use_color=True, @@ -908,13 +908,13 @@ def test_given_source_load_logs_when_reporting_progress_then_writes_indented_log callbacks.on_node_complete( LoadExecutionResult( source_name="raw_orders", - loader_name="ingestr__raw_orders", + loader_name="raw_orders", status=ExecutionStatus.SUCCESS, target="raw_orders", lifecycle_events=( LifeCycleEvent( kind=LifeCycleEventKind.LOG, - content="ingestr stdout\nStarting data ingestion\nRows: 3", + content="loader stdout\nStarting data ingestion\nRows: 3", ), ), ) diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py index fa879d11af..2e8c8438a8 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py @@ -213,7 +213,6 @@ def build_source_load_entry( cursor_column: str | None = None, unique_key: tuple[str, ...] = (), is_reload: bool = False, - integration_kind: str | None = None, ) -> SourceLoadPlanEntry: """Build a minimal SourceLoadPlanEntry for formatter tests.""" @@ -226,7 +225,6 @@ def build_source_load_entry( cursor_column=cursor_column, unique_key=unique_key, is_reload=is_reload, - integration_kind=integration_kind, ) diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index 01747e9e02..88b5e76b41 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -763,22 +763,15 @@ write_strategy=SourceWriteStrategy.MERGE, unique_key=("order_id", "updated_at"), ), - build_source_load_entry( - name="raw_ingestr", - write_strategy=None, - integration_kind="ingestr", - ), build_source_load_entry(name="raw_self_managed", write_strategy=None), ), ), expected_fragments=( - "Plan ready 0 selected, 4 sources to load", + "Plan ready 0 selected, 3 sources to load", "raw_events", "delete_insert (cursor: event_at)", "raw_orders", "merge (unique_key: order_id, updated_at)", - "raw_ingestr", - "external (ingestr)", "raw_self_managed", "self-managed", ), diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index e6241256b3..315cff9f79 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -615,19 +615,6 @@ class ParseSqlScenarioFileErrorTestCase: expected_error_fragment: str -@dataclass(frozen=True) -class ParseSourcesYamlDltTestCase: - description: str - contents: str - expected_source_names: tuple[str, ...] - expected_loaders: tuple[str, ...] - expected_kind: str - expected_dlt_names: tuple[str, ...] - expected_schemas: tuple[str | None, ...] - expected_destination_config: dict[str, object] = field(default_factory=dict) - expected_descriptions: tuple[str | None, ...] = () - - @dataclass(frozen=True) class ParseSqlAuditFileTestCase: description: str @@ -736,24 +723,6 @@ class ParseSourcesYamlErrorTestCase: expected_error_fragment: str -@dataclass(frozen=True) -class ParseSourcesYamlHelpTestCase: - description: str - contents: str - expected_error_fragment: str - expected_help: str - - -@dataclass(frozen=True) -class ParseSourcesYamlIngestrTestCase: - description: str - contents: str - expected_loader: str - expected_kind: str - expected_primary_key: tuple[str, ...] - expected_extra_args: tuple[str, ...] - - @dataclass(frozen=True) class DiscoverMaterializationFilesTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py index fd57f1a902..1be9ee29a8 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py @@ -1,18 +1,12 @@ from __future__ import annotations from pathlib import Path -from typing import Any, cast import pytest -from sqlbuild._native import SourceEntry, SourceParseError, SourceWriteStrategy -from sqlbuild.integrations.dlt.models import DltSourceConfig -from sqlbuild.integrations.ingestr.models import IngestrSourceConfig +from sqlbuild._native import SourceEntry, SourceWriteStrategy from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( - ParseSourcesYamlDltTestCase, ParseSourcesYamlErrorTestCase, - ParseSourcesYamlHelpTestCase, - ParseSourcesYamlIngestrTestCase, ParseSourcesYamlTestCase, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import expected_or_actual @@ -179,30 +173,6 @@ expected_source_audit_names=((), (), (), ()), expected_column_audit_names=((), (), (), ()), ), - ParseSourcesYamlTestCase( - description="parses ingestr integration loader metadata", - contents=""" - sources: - - name: raw_orders - table: orders - ingestr: - source_uri: stripe://token - source_table: charges - strategy: merge - primary_key: [id, account_id] - incremental_key: updated_at - columns: id,account_id,updated_at - extra_args: [--debug] - """, - expected_source_names=("raw_orders",), - expected_column_names=((),), - expected_type_enforcement_values=(None,), - expected_contract_values=(None,), - expected_expressions=(None,), - expected_loaders=("ingestr__raw_orders",), - expected_source_audit_names=((),), - expected_column_audit_names=((),), - ), ParseSourcesYamlTestCase( description="parses source freshness strategies", contents=""" @@ -532,120 +502,6 @@ def test_given_sources_yaml_variants_when_parsing_then_it_returns_expected_raw_m assert tuple(actual_column_audit_names) == test_case.expected_column_audit_names -@pytest.mark.parametrize( - "test_case", - [ - ParseSourcesYamlIngestrTestCase( - description="stores typed ingestr integration config", - contents=""" - sources: - - name: raw_orders - table: orders - ingestr: - source_uri: stripe://token - source_table: charges - strategy: merge - primary_key: [id, account_id] - incremental_key: updated_at - columns: id,account_id,updated_at - extra_args: [--debug] - """, - expected_loader="ingestr__raw_orders", - expected_kind="ingestr", - expected_primary_key=("id", "account_id"), - expected_extra_args=("--debug",), - ) - ], - ids=lambda case: case.description, -) -def test_given_ingestr_source_yaml_when_parsing_then_stores_typed_integration_config( - test_case: ParseSourcesYamlIngestrTestCase, -) -> None: - source_entries: tuple[SourceEntry, ...] = parse_sources_yml( - contents=test_case.contents, - file_path=Path("sources/raw.yml"), - ) - - assert source_entries[0].loader == test_case.expected_loader - assert source_entries[0].integration_loader is not None - assert source_entries[0].integration_loader.kind == test_case.expected_kind - ingestr_config: object = source_entries[0].integration_loader.config - assert isinstance(ingestr_config, IngestrSourceConfig) - assert ingestr_config.primary_key == test_case.expected_primary_key - assert ingestr_config.extra_args == test_case.expected_extra_args - - -@pytest.mark.parametrize( - "test_case", - [ - ParseSourcesYamlDltTestCase( - description="expands dlt sources into managed source entries", - contents=""" - dlt_sources: - - type: sql_database - schema: raw - destination: - naming_convention: sql_cs_v1 - create_indexes: true - config: - credentials: duckdb:///source.duckdb - schema: main - resources: - - name: raw_orders - description: Orders from the storefront database - table: orders - write_disposition: merge - primary_key: order_id - - name: raw_customers - schema: raw_customers_override - table: customers - write_disposition: append - """, - expected_source_names=("raw_orders", "raw_customers"), - expected_loaders=("raw_orders", "raw_customers"), - expected_kind="dlt", - expected_dlt_names=("orders", "customers"), - expected_schemas=("raw", "raw_customers_override"), - expected_destination_config={"naming_convention": "sql_cs_v1", "create_indexes": True}, - expected_descriptions=("Orders from the storefront database", None), - ) - ], - ids=lambda case: case.description, -) -def test_given_dlt_sources_yaml_when_parsing_then_expands_managed_sources( - test_case: ParseSourcesYamlDltTestCase, -) -> None: - source_entries: tuple[SourceEntry, ...] = parse_sources_yml( - contents=test_case.contents, - file_path=Path("sources/raw.yml"), - ) - - assert tuple(entry.name for entry in source_entries) == test_case.expected_source_names - assert tuple(entry.loader for entry in source_entries) == test_case.expected_loaders - assert all(entry.managed for entry in source_entries) - assert all(entry.integration_loader is not None for entry in source_entries) - assert tuple(cast(Any, entry.integration_loader).kind for entry in source_entries) == ( - test_case.expected_kind, - test_case.expected_kind, - ) - configs: tuple[object, ...] = tuple( - cast(Any, entry.integration_loader).config for entry in source_entries - ) - assert all(isinstance(config, DltSourceConfig) for config in configs) - typed_configs: tuple[DltSourceConfig, ...] = cast(tuple[DltSourceConfig, ...], configs) - assert ( - tuple(config.resource.dlt_name for config in typed_configs) == test_case.expected_dlt_names - ) - assert tuple(entry.schema for entry in source_entries) == test_case.expected_schemas - assert tuple(entry.description for entry in source_entries) == test_case.expected_descriptions - assert all("description" not in config.resource.raw_config for config in typed_configs) - assert tuple(config.resource.schema for config in typed_configs) == test_case.expected_schemas - assert tuple(config.destination for config in typed_configs) == ( - test_case.expected_destination_config, - test_case.expected_destination_config, - ) - - @pytest.mark.parametrize( "test_case", [ @@ -709,16 +565,33 @@ def test_given_dlt_sources_yaml_when_parsing_then_expands_managed_sources( expected_error_fragment="source freshness age_policy has unknown keys: warn_afer", ), ParseSourcesYamlErrorTestCase( - description="raises when ingestr config declares an unknown key", + description="raises when a source declares the removed ingestr integration", contents=""" sources: - name: raw_orders ingestr: source_uri: postgresql://localhost/shop source_table: public.orders - incremental_column: updated_at """, - expected_error_fragment="source ingestr has unknown keys: incremental_column", + expected_error_fragment=( + "source 'raw_orders' declares an ingestr integration, which SQLBuild no longer " + "supports" + ), + ), + ParseSourcesYamlErrorTestCase( + description="raises when a sources file declares the removed dlt integration", + contents=""" + dlt_sources: + - type: sql_database + config: + credentials: duckdb:///source.duckdb + resources: + - name: raw_orders + table: orders + """, + expected_error_fragment=( + "declares dlt_sources, a dlt integration, which SQLBuild no longer supports" + ), ), ParseSourcesYamlErrorTestCase( description="raises when source meta is not a mapping", @@ -835,117 +708,6 @@ def test_given_dlt_sources_yaml_when_parsing_then_expands_managed_sources( """, expected_error_fragment="source 'loader' is not supported", ), - ParseSourcesYamlErrorTestCase( - description="raises when source defines both loader and ingestr", - contents=""" - sources: - - name: raw_orders - managed: true - ingestr: - source_uri: stripe://token - source_table: charges - """, - expected_error_fragment="cannot override loader for ingestr", - ), - ParseSourcesYamlErrorTestCase( - description="raises when ingestr strategy is unknown", - contents=""" - sources: - - name: raw_orders - ingestr: - source_uri: stripe://token - source_table: charges - strategy: nope - """, - expected_error_fragment="ingestr strategy must be one of", - ), - ParseSourcesYamlErrorTestCase( - description="raises when dlt merge omits primary key", - contents=""" - dlt_sources: - - type: sql_database - config: - credentials: duckdb:///source.duckdb - resources: - - name: raw_orders - table: orders - write_disposition: merge - """, - expected_error_fragment="merge requires primary_key", - ), - ParseSourcesYamlErrorTestCase( - description="raises when dlt uses sqlbuild write strategy", - contents=""" - dlt_sources: - - type: sql_database - config: - credentials: duckdb:///source.duckdb - resources: - - name: raw_orders - table: orders - write_strategy: table - """, - expected_error_fragment="must use dlt write_disposition", - ), - ParseSourcesYamlErrorTestCase( - description="raises when dlt destination overrides credentials", - contents=""" - dlt_sources: - - type: sql_database - destination: - credentials: other - config: - credentials: duckdb:///source.duckdb - resources: - - name: raw_orders - table: orders - """, - expected_error_fragment="dlt source destination cannot define 'credentials'", - ), - ParseSourcesYamlErrorTestCase( - description="raises when dlt source group has an unknown key", - contents=""" - dlt_sources: - - type: sql_database - confg: - credentials: duckdb:///source.duckdb - resources: - - name: raw_orders - table: orders - """, - expected_error_fragment="dlt source has unknown keys: confg", - ), - ParseSourcesYamlErrorTestCase( - description="raises when dlt resource has an unknown key", - contents=""" - dlt_sources: - - type: sql_database - config: - credentials: duckdb:///source.duckdb - resources: - - name: raw_orders - table: orders - primary_keys: id - """, - expected_error_fragment=( - "dlt sql_database resource 'raw_orders' has unknown keys: primary_keys" - ), - ), - ParseSourcesYamlErrorTestCase( - description="raises when dlt resource uses a key of another source type", - contents=""" - dlt_sources: - - type: sql_database - config: - credentials: duckdb:///source.duckdb - resources: - - name: raw_orders - table: orders - endpoint: - path: orders - """, - expected_error_fragment="dlt sql_database resource 'raw_orders' has unknown keys: endpoint", - ), ParseSourcesYamlErrorTestCase( description="raises when write strategy is unknown", contents=""" @@ -1527,38 +1289,3 @@ def test_given_invalid_sources_yaml_when_parsing_then_it_raises_clear_errors( ) -> None: with pytest.raises(ValueError, match=test_case.expected_error_fragment): parse_sources_yml(contents=test_case.contents, file_path=Path("sources/raw.yml")) - - -@pytest.mark.parametrize( - "test_case", - [ - ParseSourcesYamlHelpTestCase( - description="misspelled rest_api resource key", - contents=""" -dlt_sources: - - type: rest_api - config: - client: - base_url: https://api.example.com/ - resources: - - name: raw_orders - write_dispositon: append - endpoint: - path: orders -""", - expected_error_fragment=( - "dlt rest_api resource 'raw_orders' has unknown keys: write_dispositon" - ), - expected_help="did you mean 'write_disposition'?", - ), - ], - ids=lambda case: case.description, -) -def test_given_misspelled_dlt_resource_key_when_parsing_then_help_names_nearest_key( - test_case: ParseSourcesYamlHelpTestCase, -) -> None: - with pytest.raises(SourceParseError) as error_info: - parse_sources_yml(contents=test_case.contents, file_path=Path("sources/raw.yml")) - - assert test_case.expected_error_fragment in str(error_info.value) - assert error_info.value.help == test_case.expected_help diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index e9e35a166a..820bc643ce 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -35,7 +35,6 @@ DiscoveredPythonNodeFunctions, DiscoveredSourceFile, DiscoveredTaskFunction, - LoaderConnectionMode, LocalConfig, ProjectConfig, ProjectGraph, @@ -112,24 +111,6 @@ def check_loaded_orders(_ctx: object) -> bool: return True -def build_external_loader_python_node_graph() -> PythonNodeGraph: - return build_python_node_graph( - discovered_inputs=DiscoveredProjectInputs( - project_config=ProjectConfig(name="demo", adapter="duckdb"), - local_config=LocalConfig(), - loader_functions=( - DiscoveredLoaderFunction( - file_path=Path("/project/loaders/events.py"), - relative_path=Path("python/loaders/events.py"), - name="load_events", - function=load_events, - connection_mode=LoaderConnectionMode.EXTERNAL, - ), - ), - ) - ) - - def build_terminal_loader_python_node_graph() -> PythonNodeGraph: """Build a graph containing one loader owned by a managed source.""" return build_python_node_graph( @@ -498,7 +479,6 @@ def build_python_node_graph_for_case(case_name: str) -> PythonNodeGraph: "intermediate_loader_check_dependency": ( build_intermediate_loader_check_dependency_python_node_graph ), - "external_loader": build_external_loader_python_node_graph, "orders": build_orders_python_node_graph, } ) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py index 4d6878bf34..aa5e96ce3b 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py @@ -18,7 +18,6 @@ DiscoveredProvider, DiscoveredPythonNode, DiscoveredTaskFunction, - LoaderConnectionMode, LocalConfig, ProjectConfig, PythonNodeGraph, @@ -284,7 +283,6 @@ def test_given_fully_configured_python_nodes_when_building_then_identity_config_ unique_key=("event_id",), columns=columns, contract="enforced", - connection_mode=LoaderConnectionMode.EXTERNAL, ), ), task_functions=( diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py index 09196671c4..75f75307ad 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py @@ -70,17 +70,6 @@ expected_ingress_loader_names=frozenset({"fetch_pages"}), expected_read_side_python_names=frozenset({"export_orders"}), ), - PythonSqlRunLifecycleTestCase( - description="excludes external loader from ingress so it runs pre-connection", - python_graph_case="external_loader", - selection=PythonSqlRunSelection( - sql_keys=frozenset(), - python_node_names=frozenset({"load_events"}), - ), - expected_ingress_python_names=frozenset(), - expected_ingress_loader_names=frozenset(), - expected_read_side_python_names=frozenset(), - ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py index e30da1baae..b9fa3b4e93 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py @@ -121,18 +121,6 @@ class BuildSourceNodeExecutionTestCase: expected_rows: tuple[tuple[object, ...], ...] -@dataclass(frozen=True) -class ExternalBuildSourceLoadTestCase: - """One pre-connection build source-load behavior case.""" - - description: str - source_name: str - loader_name: str - expected_status: ExecutionStatus - expected_completed_key_count: int - expected_lifecycle_message: str - - @dataclass(frozen=True) class MaterializationRecreatesRelationTestCase: description: str diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py deleted file mode 100644 index 7c3c246bf8..0000000000 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Tests for pre-connection external source-load execution.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild._native import ( - CompiledObjectKey, - CompiledResourceType, - DiscoveredLoaderFunction, - LoaderConnectionMode, - PlanOutput, - SourceEntry, - SourceLoadPlanEntry, -) -from sqlbuild.executor.build._helpers.external_source_loads import ( - run_external_source_loads_before_connections, -) -from sqlbuild.executor.build.models import ( - BuildCallbacks, - BuildRuntimeParams, - ExternalSourceLoadResults, -) -from sqlbuild.executor.load.models import LoaderContext, LoadExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.observability import ( - EventDispatcher, - LifecycleEvent, - dispatcher_scope, - invocation_scope, -) -from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( - ExternalBuildSourceLoadTestCase, -) -from tests.unit.src.sqlbuild.executor.load.helpers import CountingLoaderContextTestAdapter - - -@pytest.mark.parametrize( - "test_case", - [ - ExternalBuildSourceLoadTestCase( - description="runs external source before build connections", - source_name="raw_orders", - loader_name="external_orders_loader", - expected_status=ExecutionStatus.SUCCESS, - expected_completed_key_count=1, - expected_lifecycle_message="external build loader ran", - ) - ], - ids=lambda case: case.description, -) -def test_given_external_source_load_when_preloading_then_records_progress_and_no_connection( - test_case: ExternalBuildSourceLoadTestCase, -) -> None: - adapter: CountingLoaderContextTestAdapter = CountingLoaderContextTestAdapter() - key: CompiledObjectKey = CompiledObjectKey( - resource_type=CompiledResourceType.SOURCE, - name=test_case.source_name, - ) - completed_results: list[object] = [] - started_nodes: list[tuple[str, object]] = [] - observed_connections: list[object] = [] - order: list[str] = [] - dispatcher: EventDispatcher = EventDispatcher() - - def record_event(event: LifecycleEvent) -> None: - order.append(event.event_type) - - def record_start(name: str, *, resource_kind: ExecutionResourceKind) -> None: - order.append("callback_start") - started_nodes.append((name, resource_kind)) - - def record_complete(result: object) -> None: - order.append("callback_complete") - completed_results.append(result) - - dispatcher.subscribe_lifecycle(subscriber=record_event, accepts_opaque=False) - - def external_loader(ctx: LoaderContext) -> None: - observed_connections.append(ctx.connection) - ctx.log(test_case.expected_lifecycle_message) - return None - - with invocation_scope("external-build-invocation"), dispatcher_scope(dispatcher): - results: ExternalSourceLoadResults = run_external_source_loads_before_connections( - plan=PlanOutput( - execution_order=(key,), - selected_keys=frozenset({key}), - source_load_entries=( - SourceLoadPlanEntry( - key=key, - name=test_case.source_name, - loader=test_case.loader_name, - destination=test_case.source_name, - ), - ), - source_map={ - test_case.source_name: SourceEntry( - name=test_case.source_name, - loader=test_case.loader_name, - ) - }, - ), - loader_functions=( - DiscoveredLoaderFunction( - file_path=Path("python/loaders/raw.py"), - relative_path=Path("python/loaders/raw.py"), - name=test_case.loader_name, - function=external_loader, - connection_mode=LoaderConnectionMode.EXTERNAL, - ), - ), - adapter=adapter, - connection_config={}, - runtime=BuildRuntimeParams( - run_id="run-1", - target="dev", - effective_vars={}, - ), - callbacks=BuildCallbacks( - on_node_start=record_start, - on_node_complete=record_complete, - ), - ) - - assert adapter.connection_count == 0 - assert observed_connections == [None] - assert len(results.completed_keys) == test_case.expected_completed_key_count - assert tuple(result.status for result in results.results) == (test_case.expected_status,) - assert isinstance(completed_results[0], LoadExecutionResult) - assert started_nodes[0][0] == test_case.source_name - assert results.results[0].lifecycle_events[0].content == test_case.expected_lifecycle_message - operation_entries: tuple[str, ...] = tuple( - filter(lambda entry: entry.startswith("operation_"), order) - ) - assert tuple(order) == ( - "resource_attempt_started", - "callback_start", - *operation_entries, - "resource_attempt_completed", - "callback_complete", - ) diff --git a/tests/unit/src/sqlbuild/executor/load/_test_types.py b/tests/unit/src/sqlbuild/executor/load/_test_types.py index 20f6a9fc0a..9e83615b8d 100644 --- a/tests/unit/src/sqlbuild/executor/load/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/load/_test_types.py @@ -6,7 +6,7 @@ from datetime import date, datetime from typing import Any -from sqlbuild._native import LoaderConnectionMode, SourceColumnEntry +from sqlbuild._native import SourceColumnEntry from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cursor_algebra.types import CursorScalar @@ -148,19 +148,6 @@ class LoadDagStateSchedulingTestCase: expected_callback_sources: tuple[str, ...] -@dataclass(frozen=True) -class ExternalLoadPipelineTestCase: - """One external loader pipeline behavior case.""" - - description: str - source_name: str - loader_name: str - expected_connection_count: int - expected_connection_is_none: bool - expected_status: ExecutionStatus - expected_lifecycle_message: str - - @dataclass(frozen=True) class LoadPipelineSkipFanInTestCase: """One loader skip fan-in behavior case.""" @@ -183,7 +170,6 @@ class ConcurrentLoadProgressTestCase: @dataclass(frozen=True) class LoadStartOrderingTestCase: description: str - connection_mode: LoaderConnectionMode expected_connection_count: int @@ -231,9 +217,3 @@ class LoaderOperationLifecycleTestCase: description: str expected_status: ExecutionStatus expected_event_types: tuple[str, ...] - - -@dataclass(frozen=True) -class ExternalLoaderContractTestCase: - description: str - expected_error_fragment: str diff --git a/tests/unit/src/sqlbuild/executor/load/test_execute.py b/tests/unit/src/sqlbuild/executor/load/test_execute.py index 4b277f089a..0048124707 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/load/test_execute.py @@ -12,7 +12,6 @@ from sqlbuild._native import ( CostContext, DiscoveredLoaderFunction, - LoaderConnectionMode, SourceEntry, SourceWriteStrategy, ) @@ -24,7 +23,6 @@ LoadExecutionResult, LoadRuntimeParams, ) -from sqlbuild.executor.node_results.models import NodeResultEnvelope from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.observability import ( EventDispatcher, @@ -36,7 +34,6 @@ from sqlbuild.provider.classes.session import ProviderSession from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle from tests.unit.src.sqlbuild.executor.load._test_types import ( - ExternalLoaderContractTestCase, LoaderOperationLifecycleTestCase, SourceLoadExecutionContextTestCase, SourceLoadNoneReturnTestCase, @@ -497,282 +494,6 @@ def raw_orders_loader( ) in (result.error_message or "") -@pytest.mark.parametrize( - "test_case", - [ - SourceLoadExecutionContextTestCase( - description="injects provider into external source loader execution", - source_name="raw_orders", - loader_name="raw_orders_loader", - target_table="orders", - database="analytics", - schema="raw", - run_id="run-123", - target="dev", - vars={}, - is_reload=False, - start_cursor_ts=None, - end_cursor_ts=None, - start_cursor_int=None, - end_cursor_int=None, - expected_target="analytics.raw.orders", - expected_current_cursor_value=None, - expected_status=ExecutionStatus.SUCCESS, - expected_rows_loaded=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_provider_parameter_when_executing_external_source_loader_then_provider_is_injected( - test_case: SourceLoadExecutionContextTestCase, -) -> None: - observed_labels: list[str] = [] - events: list[LifecycleEvent] = [] - - def raw_orders_loader( - ctx: LoaderContext, - slack_provider: ExecutionSlackProvider, - ) -> None: - observed_labels.append(f"{ctx.connection}:{ctx.destination}:{slack_provider.label}") - return None - - providers: ProviderContainer = ProviderSession( - {"slack_provider": ExecutionSlackProvider(label="slack")} - ).providers - - dispatcher: EventDispatcher = EventDispatcher() - dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - with invocation_scope("inv-external-loader"), dispatcher_scope(dispatcher): - result: LoadExecutionResult = execute_source_load( - source_entry=SourceEntry( - name=test_case.source_name, - database=test_case.database, - schema=test_case.schema, - table=test_case.target_table, - loader=test_case.loader_name, - ), - loader_function=DiscoveredLoaderFunction( - file_path=Path("python/loaders/raw.py"), - relative_path=Path("python/loaders/raw.py"), - name=test_case.loader_name, - function=raw_orders_loader, - connection_mode=LoaderConnectionMode.EXTERNAL, - ), - adapter=LoaderContextTestAdapter(), - connection_config={}, - connection=object(), - statement_recorder=StatementRecorder(), - runtime=LoadRuntimeParams( - run_id=test_case.run_id, - target=test_case.target, - vars=test_case.vars, - is_reload=test_case.is_reload, - providers=providers, - ), - ) - - assert result.status == test_case.expected_status - assert result.rows_loaded == test_case.expected_rows_loaded - assert tuple(observed_labels) == ("None:analytics.raw.orders:slack",) - observed_operations: tuple[LifecycleEvent, ...] = operation_events(events) - assert observed_operations[0].payload["operation_name"] == "external_source_load" - assert tuple(event.event_type for event in observed_operations) == ( - "operation_started", - "operation_completed", - ) - - -@pytest.mark.parametrize( - "test_case", - ( - ExternalLoaderContractTestCase( - description="external loader returns unsupported generator", - expected_error_fragment="external loaders must write their own destination", - ), - ), - ids=lambda case: case.description, -) -def test_given_external_loader_generator_when_validating_return_then_fails_without_consuming( - test_case: ExternalLoaderContractTestCase, -) -> None: - events: list[LifecycleEvent] = [] - consumed: list[str] = [] - dispatcher: EventDispatcher = EventDispatcher() - dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - - def external_generator(ctx: LoaderContext) -> Iterator[dict[str, object]]: - del ctx - consumed.append("consumed") - yield {"order_id": 1} - - with invocation_scope("inv-external-generator"), dispatcher_scope(dispatcher): - result: LoadExecutionResult = execute_source_load( - source_entry=SourceEntry(name="raw_orders", table="orders", loader="external"), - loader_function=DiscoveredLoaderFunction( - file_path=Path("python/loaders/external.py"), - relative_path=Path("python/loaders/external.py"), - name="external", - function=external_generator, - connection_mode=LoaderConnectionMode.EXTERNAL, - ), - adapter=LoaderContextTestAdapter(), - connection_config={}, - connection=object(), - statement_recorder=StatementRecorder(), - runtime=LoadRuntimeParams( - run_id="run-external", target="dev", vars={}, is_reload=False - ), - ) - - observed_operations: tuple[LifecycleEvent, ...] = operation_events(events) - assert result.status == ExecutionStatus.FAILED - assert test_case.expected_error_fragment in (result.error_message or "").lower() - assert consumed == [] - assert tuple(event.event_type for event in observed_operations) == ( - "operation_started", - "operation_failed", - ) - - -@pytest.mark.parametrize( - "test_case", - ( - ExternalLoaderContractTestCase( - description="external loader returns node result envelope", - expected_error_fragment="returned a node result envelope", - ), - ), - ids=lambda case: case.description, -) -def test_given_external_loader_node_envelope_when_validating_return_then_operation_fails( - test_case: ExternalLoaderContractTestCase, -) -> None: - events: list[LifecycleEvent] = [] - dispatcher: EventDispatcher = EventDispatcher() - dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - - def external_envelope(ctx: LoaderContext) -> NodeResultEnvelope: - del ctx - return NodeResultEnvelope( - node_type="loader", - node_name="other", - run_id="old-run", - status="success", - payload=None, - metadata={}, - error_message=None, - materialized=None, - ts=datetime(2026, 1, 1), - ) - - with invocation_scope("inv-external-envelope"), dispatcher_scope(dispatcher): - result: LoadExecutionResult = execute_source_load( - source_entry=SourceEntry(name="raw_orders", table="orders", loader="external"), - loader_function=DiscoveredLoaderFunction( - file_path=Path("python/loaders/external.py"), - relative_path=Path("python/loaders/external.py"), - name="external", - function=external_envelope, - connection_mode=LoaderConnectionMode.EXTERNAL, - ), - adapter=LoaderContextTestAdapter(), - connection_config={}, - connection=object(), - statement_recorder=StatementRecorder(), - runtime=LoadRuntimeParams( - run_id="run-external", target="dev", vars={}, is_reload=False - ), - ) - - observed_operations: tuple[LifecycleEvent, ...] = operation_events(events) - assert result.status == ExecutionStatus.FAILED - assert test_case.expected_error_fragment in (result.error_message or "") - assert tuple(event.event_type for event in observed_operations) == ( - "operation_started", - "operation_failed", - ) - - -@pytest.mark.parametrize( - "test_case", - [ - SourceLoadExecutionContextTestCase( - description="exposes providers on external source loader context", - source_name="raw_orders", - loader_name="raw_orders_loader", - target_table="orders", - database="analytics", - schema="raw", - run_id="run-123", - target="dev", - vars={}, - is_reload=False, - start_cursor_ts=None, - end_cursor_ts=None, - start_cursor_int=None, - end_cursor_int=None, - expected_target="analytics.raw.orders", - expected_current_cursor_value=None, - expected_status=ExecutionStatus.SUCCESS, - expected_rows_loaded=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_provider_container_when_executing_external_loader_then_context_exposes_providers( - test_case: SourceLoadExecutionContextTestCase, -) -> None: - observed_labels: list[str] = [] - - def raw_orders_loader(ctx: LoaderContext) -> None: - attr_provider: ExecutionSlackProvider = cast( - ExecutionSlackProvider, ctx.providers.slack_provider - ) - item_provider: ExecutionSlackProvider = cast( - ExecutionSlackProvider, ctx.providers["slack_provider"] - ) - observed_labels.append( - f"{ctx.connection}:{ctx.destination}:{attr_provider.label}:{item_provider.label}" - ) - return None - - providers: ProviderContainer = ProviderSession( - {"slack_provider": ExecutionSlackProvider(label="slack")} - ).providers - - result: LoadExecutionResult = execute_source_load( - source_entry=SourceEntry( - name=test_case.source_name, - database=test_case.database, - schema=test_case.schema, - table=test_case.target_table, - loader=test_case.loader_name, - ), - loader_function=DiscoveredLoaderFunction( - file_path=Path("python/loaders/raw.py"), - relative_path=Path("python/loaders/raw.py"), - name=test_case.loader_name, - function=raw_orders_loader, - connection_mode=LoaderConnectionMode.EXTERNAL, - ), - adapter=LoaderContextTestAdapter(), - connection_config={}, - connection=object(), - statement_recorder=StatementRecorder(), - runtime=LoadRuntimeParams( - run_id=test_case.run_id, - target=test_case.target, - vars=test_case.vars, - is_reload=test_case.is_reload, - providers=providers, - ), - ) - - assert result.status == test_case.expected_status - assert result.rows_loaded == test_case.expected_rows_loaded - assert tuple(observed_labels) == ("None:analytics.raw.orders:slack:slack",) - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/executor/load/test_run.py b/tests/unit/src/sqlbuild/executor/load/test_run.py index 1d5bc8a8e9..c0af7947ca 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_run.py +++ b/tests/unit/src/sqlbuild/executor/load/test_run.py @@ -10,7 +10,6 @@ from sqlbuild._native import ( DiscoveredLoaderFunction, - LoaderConnectionMode, LoadProgressReporter, NativeProgressProjector, SourceEntry, @@ -32,87 +31,17 @@ ) from tests.unit.src.sqlbuild.executor.load._test_types import ( ConcurrentLoadProgressTestCase, - ExternalLoadPipelineTestCase, LoadPipelineSkipFanInTestCase, LoadStartOrderingTestCase, ) from tests.unit.src.sqlbuild.executor.load.helpers import CountingLoaderContextTestAdapter -@pytest.mark.parametrize( - "test_case", - [ - ExternalLoadPipelineTestCase( - description="runs external loader without opening SQLBuild connection", - source_name="raw_orders", - loader_name="external_orders_loader", - expected_connection_count=0, - expected_connection_is_none=True, - expected_status=ExecutionStatus.SUCCESS, - expected_lifecycle_message="external loader ran", - ) - ], - ids=lambda case: case.description, -) -def test_given_external_loader_when_running_load_pipeline_then_does_not_open_connection( - test_case: ExternalLoadPipelineTestCase, -) -> None: - adapter: CountingLoaderContextTestAdapter = CountingLoaderContextTestAdapter() - observed_connections: list[object] = [] - completed_results: list[LoadExecutionResult] = [] - - def external_loader(ctx: LoaderContext) -> None: - observed_connections.append(ctx.connection) - ctx.log(test_case.expected_lifecycle_message) - return None - - results: tuple[LoadExecutionResult, ...] = run_load_pipeline( - sources=( - SourceEntry( - name=test_case.source_name, - loader=test_case.loader_name, - ), - ), - loader_functions=( - DiscoveredLoaderFunction( - file_path=Path("python/loaders/raw.py"), - relative_path=Path("python/loaders/raw.py"), - name=test_case.loader_name, - function=external_loader, - connection_mode=LoaderConnectionMode.EXTERNAL, - ), - ), - connection_config={}, - adapter=adapter, - runtime=LoadRuntimeParams( - run_id="run-1", - target=None, - vars={}, - is_reload=False, - ), - callbacks=LoadCallbacks( - on_load_complete=completed_results.append, - ), - ) - - assert adapter.connection_count == test_case.expected_connection_count - assert (observed_connections[0] is None) is test_case.expected_connection_is_none - assert tuple(result.status for result in results) == (test_case.expected_status,) - assert completed_results == list(results) - assert results[0].lifecycle_events[0].content == test_case.expected_lifecycle_message - - @pytest.mark.parametrize( "test_case", ( - LoadStartOrderingTestCase( - description="external load start callback follows canonical start", - connection_mode=LoaderConnectionMode.EXTERNAL, - expected_connection_count=0, - ), LoadStartOrderingTestCase( description="sequential load start callback follows canonical start", - connection_mode=LoaderConnectionMode.SQLBUILD, expected_connection_count=1, ), ), @@ -142,7 +71,6 @@ def loader(_ctx: LoaderContext) -> None: relative_path=Path("python/loaders/ordered.py"), name="ordered_loader", function=loader, - connection_mode=test_case.connection_mode, ), ), connection_config={}, diff --git a/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py b/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py deleted file mode 100644 index 5df8511ae4..0000000000 --- a/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Shared test types for dlt integration tests.""" - -from __future__ import annotations - -from dataclasses import dataclass, field - - -@dataclass(frozen=True) -class DltLoaderDiscoveryTestCase: - description: str - repo_files: dict[str, str] - expected_loader_names: tuple[str, ...] - expected_relative_paths: tuple[str, ...] - - -@dataclass(frozen=True) -class DltDestinationTestCase: - description: str - adapter_name: str - connection_config: dict[str, object] - dataset_name: str | None - expected_destination_name: str - expected_dataset_name: str | None - destination_config: dict[str, object] | None = None - expected_config_params: dict[str, object] = field(default_factory=dict) - expected_caps_params: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class DltDestinationErrorTestCase: - description: str - adapter_name: str - connection_config: dict[str, object] - dataset_name: str | None - expected_error_fragment: str - destination_config: dict[str, object] | None = None - - -@dataclass(frozen=True) -class DltProgressCollectorTestCase: - description: str - updates: tuple[tuple[str, str, int, int | None], ...] - expected_fragments: tuple[str, ...] - expected_live_fragments: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/integrations/dlt/test_destination.py b/tests/unit/src/sqlbuild/integrations/dlt/test_destination.py deleted file mode 100644 index 37f92ee27f..0000000000 --- a/tests/unit/src/sqlbuild/integrations/dlt/test_destination.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Unit tests for dlt destination mapping.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.integrations.dlt._helpers.destination import build_dlt_destination -from sqlbuild.integrations.dlt.exceptions import DltIntegrationError -from sqlbuild.integrations.dlt.models import DltDestinationConfig -from tests.unit.src.sqlbuild.integrations.dlt._test_types import ( - DltDestinationErrorTestCase, - DltDestinationTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - DltDestinationTestCase( - description="maps duckdb destination", - adapter_name="duckdb", - connection_config={"database": "warehouse.duckdb"}, - dataset_name="raw", - expected_destination_name="duckdb", - expected_dataset_name="raw", - expected_caps_params={"naming_convention": "sql_ci_v1"}, - ), - DltDestinationTestCase( - description="defaults duckdb dataset to main", - adapter_name="duckdb", - connection_config={"database": "warehouse.duckdb"}, - dataset_name=None, - expected_destination_name="duckdb", - expected_dataset_name="main", - expected_caps_params={"naming_convention": "sql_ci_v1"}, - ), - DltDestinationTestCase( - description="allows destination overrides", - adapter_name="duckdb", - connection_config={"database": "warehouse.duckdb"}, - dataset_name="raw", - destination_config={"naming_convention": "sql_cs_v1", "create_indexes": True}, - expected_destination_name="duckdb", - expected_dataset_name="raw", - expected_config_params={"create_indexes": True}, - expected_caps_params={"naming_convention": "sql_cs_v1"}, - ), - DltDestinationTestCase( - description="maps motherduck destination", - adapter_name="motherduck", - connection_config={"database": "analytics", "token": "token"}, - dataset_name="raw", - expected_destination_name="motherduck", - expected_dataset_name="raw", - ), - DltDestinationTestCase( - description="maps snowflake destination", - adapter_name="snowflake", - connection_config={ - "account": "acct", - "user": "user", - "password": "pass", - "warehouse": "wh", - "database": "db", - }, - dataset_name="raw", - expected_destination_name="snowflake", - expected_dataset_name="raw", - ), - DltDestinationTestCase( - description="maps bigquery destination", - adapter_name="bigquery", - connection_config={"project": "proj", "location": "US"}, - dataset_name="raw", - expected_destination_name="bigquery", - expected_dataset_name="raw", - ), - DltDestinationTestCase( - description="maps databricks destination", - adapter_name="databricks", - connection_config={ - "server_hostname": "dbc.example.com", - "http_path": "/sql/1.0/warehouses/abc", - "token": "token", - "catalog": "main", - }, - dataset_name="raw", - expected_destination_name="databricks", - expected_dataset_name="raw", - ), - DltDestinationTestCase( - description="maps postgres destination", - adapter_name="postgres", - connection_config={ - "host": "localhost", - "dbname": "db", - "user": "user", - "password": "pass", - }, - dataset_name="raw", - expected_destination_name="postgres", - expected_dataset_name="raw", - ), - DltDestinationTestCase( - description="maps sqlserver destination", - adapter_name="sqlserver", - connection_config={ - "host": "localhost", - "database": "db", - "user": "sa", - "password": "pass", - }, - dataset_name="raw", - expected_destination_name="mssql", - expected_dataset_name="raw", - ), - ], - ids=lambda case: case.description, -) -def test_given_sqlbuild_connection_when_building_dlt_destination_then_maps_destination( - test_case: DltDestinationTestCase, -) -> None: - result: DltDestinationConfig = build_dlt_destination( - adapter_name=test_case.adapter_name, - connection_config=test_case.connection_config, - destination_config=test_case.destination_config, - dataset_name=test_case.dataset_name, - ) - - assert result.destination.destination_name == test_case.expected_destination_name - assert result.dataset_name == test_case.expected_dataset_name - expected_key: str - for expected_key, expected_value in test_case.expected_config_params.items(): - assert result.destination.config_params[expected_key] == expected_value - for expected_key, expected_value in test_case.expected_caps_params.items(): - assert result.destination.caps_params[expected_key] == expected_value - - -@pytest.mark.parametrize( - "test_case", - [ - DltDestinationErrorTestCase( - description="requires schema for real warehouse", - adapter_name="postgres", - connection_config={"host": "localhost", "dbname": "db", "user": "user"}, - dataset_name=None, - expected_error_fragment="requires an explicit dlt source schema", - ), - DltDestinationErrorTestCase( - description="rejects sqlbuild owned destination keys", - adapter_name="duckdb", - connection_config={"database": "warehouse.duckdb"}, - dataset_name="raw", - destination_config={"credentials": "other.duckdb"}, - expected_error_fragment="cannot define SQLBuild-owned key", - ), - ], - ids=lambda case: case.description, -) -def test_given_real_warehouse_without_schema_when_building_dlt_destination_then_raises( - test_case: DltDestinationErrorTestCase, -) -> None: - with pytest.raises(DltIntegrationError) as exc_info: - build_dlt_destination( - adapter_name=test_case.adapter_name, - connection_config=test_case.connection_config, - destination_config=test_case.destination_config, - dataset_name=test_case.dataset_name, - ) - - assert test_case.expected_error_fragment in str(exc_info.value) diff --git a/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py b/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py deleted file mode 100644 index 0be18f8519..0000000000 --- a/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Unit tests for synthetic dlt loader discovery.""" - -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path -from typing import Any, cast - -import pytest - -from sqlbuild._native import DiscoveredProjectInputs, LoaderConnectionMode, discover_project_inputs -from tests.unit.src.sqlbuild.integrations.dlt._test_types import DltLoaderDiscoveryTestCase - - -@pytest.mark.parametrize( - "test_case", - [ - DltLoaderDiscoveryTestCase( - description="builds deterministic synthetic loaders per resource", - repo_files={ - "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "sources/raw.yml": ( - "dlt_sources:\n" - " - type: sql_database\n" - " config:\n" - ' credentials: "duckdb:///source.duckdb"\n' - " resources:\n" - " - name: raw_orders\n" - " table: orders\n" - " - name: raw_customers\n" - " table: customers\n" - ), - }, - expected_loader_names=("raw_orders", "raw_customers"), - expected_relative_paths=("sources/raw.yml", "sources/raw.yml"), - ) - ], - ids=lambda case: case.description, -) -def test_given_dlt_source_when_discovering_inputs_then_loader_functions_are_deterministic( - test_case: DltLoaderDiscoveryTestCase, - tmp_path: Path, - write_repo_files: Callable[[Path, dict[str, str]], None], -) -> None: - write_repo_files(tmp_path, test_case.repo_files) - - inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - - assert tuple(loader.name for loader in inputs.loader_functions) == ( - test_case.expected_loader_names - ) - assert tuple(str(loader.relative_path) for loader in inputs.loader_functions) == ( - test_case.expected_relative_paths - ) - assert tuple(cast(Any, loader.function).__name__ for loader in inputs.loader_functions) == ( - test_case.expected_loader_names - ) - assert {loader.connection_mode for loader in inputs.loader_functions} == { - LoaderConnectionMode.EXTERNAL - } diff --git a/tests/unit/src/sqlbuild/integrations/dlt/test_progress.py b/tests/unit/src/sqlbuild/integrations/dlt/test_progress.py deleted file mode 100644 index 4d1265a180..0000000000 --- a/tests/unit/src/sqlbuild/integrations/dlt/test_progress.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Unit tests for dlt progress collection.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.integrations.dlt.classes.sqlbuild_dlt_progress_collector import ( - SqlbuildDltProgressCollector, -) -from tests.unit.src.sqlbuild.integrations.dlt._test_types import DltProgressCollectorTestCase - - -@pytest.mark.parametrize( - "test_case", - ( - DltProgressCollectorTestCase( - description="summarizes row, file, and job counters", - updates=( - ("extract", "raw_orders", 5000, None), - ("extract", "Resources", 1, 1), - ("normalize", "Files", 2, 2), - ("load", "Jobs", 2, 2), - ), - expected_fragments=( - "dlt progress", - "extract: Resources 1/1, raw_orders 5000", - "normalize: Files 2/2", - "load: Jobs 2/2", - ), - expected_live_fragments=("dlt extract", "dlt normalize", "dlt load"), - ), - DltProgressCollectorTestCase( - description="keeps labelled counters distinct", - updates=( - ("extract", "Resources", 1, 2), - ("extract", "Resources", 1, 2), - ("load", "Jobs", 1, 1), - ), - expected_fragments=("extract: Resources 2/2", "load: Jobs 1/1"), - expected_live_fragments=("dlt extract", "dlt load"), - ), - ), - ids=lambda case: case.description, -) -def test_given_dlt_progress_updates_when_collecting_then_summarizes_counters( - test_case: DltProgressCollectorTestCase, -) -> None: - live_messages: list[str] = [] - collector: SqlbuildDltProgressCollector = SqlbuildDltProgressCollector( - on_progress=live_messages.append - ) - - step: str - name: str - inc: int - total: int | None - for step, name, inc, total in test_case.updates: - with collector(step): - collector.update(name, inc=inc, total=total) - summary: str = collector.format_summary() - - expected_fragment: str - for expected_fragment in test_case.expected_fragments: - assert expected_fragment in summary - expected_live_fragment: str - for expected_live_fragment in test_case.expected_live_fragments: - assert any(expected_live_fragment in message for message in live_messages) diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py b/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py deleted file mode 100644 index 1747f36fe3..0000000000 --- a/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Shared test types for ingestr integration tests.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild._native import SourceEntry - - -@dataclass(frozen=True) -class IngestrCommandTestCase: - description: str - adapter_name: str - connection_config: dict[str, object] - expected_command: tuple[str, ...] - - -@dataclass(frozen=True) -class IngestrCommandErrorTestCase: - description: str - source_entry: SourceEntry - adapter_name: str - connection_config: dict[str, object] - expected_message: str - - -@dataclass(frozen=True) -class IngestrLoaderDiscoveryTestCase: - description: str - repo_files: dict[str, str] - expected_loader_names: tuple[str, ...] - expected_relative_paths: tuple[str, ...] - - -@dataclass(frozen=True) -class IngestrRunnerTestCase: - description: str - returncode: int - expected_signal_number: int - expected_event_types: tuple[str, ...] = () - - -@dataclass(frozen=True) -class IngestrDriverTestCase: - description: str - driver_files: tuple[tuple[str, str], ...] - environment: dict[str, str] - expected_installs: tuple[tuple[str, ...], ...] - - -@dataclass(frozen=True) -class IngestrDriverFailureTestCase: - description: str - stderr: str - expected_message: str diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py b/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py deleted file mode 100644 index bced98e655..0000000000 --- a/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Unit tests for ingestr integration-loader command construction.""" - -from __future__ import annotations - -import pytest - -from sqlbuild._native import IntegrationLoaderConfig, SourceEntry, build_ingestr_command -from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError -from sqlbuild.integrations.ingestr.models import IngestrSourceConfig -from tests.unit.src.sqlbuild.integrations.ingestr._test_types import ( - IngestrCommandErrorTestCase, - IngestrCommandTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - IngestrCommandTestCase( - description="uses sqlbuild destination and resolved templates", - adapter_name="duckdb", - connection_config={"database": "warehouse.duckdb"}, - expected_command=( - "ingestr", - "ingest", - "--source-uri", - "stripe://secret", - "--source-table", - "charges", - "--dest-uri", - "duckdb:///warehouse.duckdb", - "--dest-table", - "raw.orders", - "--yes", - "--progress", - "log", - "--incremental-strategy", - "merge", - "--incremental-key", - "updated_at", - "--primary-key", - "id", - "--columns", - "id,updated_at", - "--full-refresh", - "--debug", - ), - ), - IngestrCommandTestCase( - description="supports sqlserver destination uri", - adapter_name="sqlserver", - connection_config={ - "host": "localhost", - "port": 1433, - "user": "sa", - "password": "Sqlbuild!2026", - "database": "analytics", - }, - expected_command=( - "ingestr", - "ingest", - "--source-uri", - "stripe://secret", - "--source-table", - "charges", - "--dest-uri", - "mssql://sa:Sqlbuild%212026@localhost:1433/analytics" - "?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes", - "--dest-table", - "raw.orders", - "--yes", - "--progress", - "log", - "--incremental-strategy", - "merge", - "--incremental-key", - "updated_at", - "--primary-key", - "id", - "--columns", - "id,updated_at", - "--full-refresh", - "--debug", - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_ingestr_source_when_building_command_then_it_uses_sqlbuild_destination( - test_case: IngestrCommandTestCase, -) -> None: - source_entry: SourceEntry = SourceEntry( - name="raw_orders", - schema="raw", - table="orders", - integration_loader=IntegrationLoaderConfig( - kind="ingestr", - config=IngestrSourceConfig( - source_uri="stripe://${token}", - source_table="charges", - strategy="merge", - incremental_key="updated_at", - primary_key=("id",), - columns="id,updated_at", - extra_args=("--debug",), - ), - ), - ) - - command: tuple[str, ...] = build_ingestr_command( - source_entry=source_entry, - adapter_name=test_case.adapter_name, - connection_config=test_case.connection_config, - destination_name="orders", - vars={"token": "secret"}, - environment="dev", - run_id="run-1", - is_reload=True, - ) - - assert command == test_case.expected_command - - -@pytest.mark.parametrize( - "test_case", - [ - IngestrCommandErrorTestCase( - description="unsupported adapter", - source_entry=SourceEntry( - name="raw_orders", - integration_loader=IntegrationLoaderConfig( - kind="ingestr", - config=IngestrSourceConfig(source_uri="csv://x", source_table="orders"), - ), - ), - adapter_name="fabric", - connection_config={}, - expected_message="Adapter 'fabric' does not support ingestr integration loaders", - ), - IngestrCommandErrorTestCase( - description="blank required connection setting", - source_entry=SourceEntry( - name="raw_orders", - integration_loader=IntegrationLoaderConfig( - kind="ingestr", - config=IngestrSourceConfig(source_uri="csv://x", source_table="orders"), - ), - ), - adapter_name="postgres", - connection_config={"user": " "}, - expected_message=( - "Adapter 'postgres' requires connection.user for ingestr integration loaders" - ), - ), - IngestrCommandErrorTestCase( - description="source without ingestr config", - source_entry=SourceEntry(name="raw_orders"), - adapter_name="duckdb", - connection_config={"database": "warehouse.duckdb"}, - expected_message="Source 'raw_orders' does not define ingestr config", - ), - ], - ids=lambda case: case.description, -) -def test_given_unbuildable_ingestr_source_when_building_command_then_raises_integration_error( - test_case: IngestrCommandErrorTestCase, -) -> None: - with pytest.raises(IngestrIntegrationError) as raised: - _ = build_ingestr_command( - source_entry=test_case.source_entry, - adapter_name=test_case.adapter_name, - connection_config=test_case.connection_config, - destination_name="orders", - vars={}, - environment=None, - run_id="run-1", - is_reload=False, - ) - - assert str(raised.value) == test_case.expected_message diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py b/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py deleted file mode 100644 index 466917cf40..0000000000 --- a/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Unit tests for synthetic ingestr loader discovery.""" - -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path -from typing import Any, cast - -import pytest - -from sqlbuild._native import DiscoveredProjectInputs, LoaderConnectionMode, discover_project_inputs -from tests.unit.src.sqlbuild.integrations.ingestr._test_types import ( - IngestrLoaderDiscoveryTestCase, -) - -_INGESTR_SOURCES: str = ( - "sources:\n" - " - name: raw_orders\n" - " ingestr:\n" - ' source_uri: "stripe://token"\n' - " source_table: charges\n" - " - name: plain_orders\n" - " schema: public\n" - " table: orders\n" -) -_DLT_SOURCES: str = ( - "dlt_sources:\n" - " - type: sql_database\n" - " config:\n" - ' credentials: "duckdb:///source.duckdb"\n' - " resources:\n" - " - name: dlt_orders\n" - " table: orders\n" -) - - -@pytest.mark.parametrize( - "test_case", - [ - IngestrLoaderDiscoveryTestCase( - description="builds deterministic synthetic loader", - repo_files={ - "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "sources/raw.yml": _INGESTR_SOURCES, - }, - expected_loader_names=("ingestr__raw_orders",), - expected_relative_paths=("sources/raw.yml",), - ), - IngestrLoaderDiscoveryTestCase( - description="orders every ingestr loader before every dlt loader", - repo_files={ - "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "sources/a_dlt.yml": _DLT_SOURCES, - "sources/b_raw.yml": _INGESTR_SOURCES, - }, - expected_loader_names=("ingestr__raw_orders", "dlt_orders"), - expected_relative_paths=("sources/b_raw.yml", "sources/a_dlt.yml"), - ), - ], - ids=lambda case: case.description, -) -def test_given_ingestr_source_when_discovering_inputs_then_loader_functions_are_deterministic( - test_case: IngestrLoaderDiscoveryTestCase, - tmp_path: Path, - write_repo_files: Callable[[Path, dict[str, str]], None], -) -> None: - write_repo_files(tmp_path, test_case.repo_files) - - inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - - assert tuple(loader.name for loader in inputs.loader_functions) == ( - test_case.expected_loader_names - ) - assert tuple(str(loader.relative_path) for loader in inputs.loader_functions) == ( - test_case.expected_relative_paths - ) - assert tuple(cast(Any, loader.function).__name__ for loader in inputs.loader_functions) == ( - test_case.expected_loader_names - ) - assert {loader.connection_mode for loader in inputs.loader_functions} == { - LoaderConnectionMode.EXTERNAL - } diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/test_runner.py b/tests/unit/src/sqlbuild/integrations/ingestr/test_runner.py deleted file mode 100644 index 2e8fea69b1..0000000000 --- a/tests/unit/src/sqlbuild/integrations/ingestr/test_runner.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Canonical subprocess lifecycle tests for ingestr.""" - -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from sqlbuild.integrations.ingestr._helpers.runner import ensure_duckdb_driver, run_ingestr_command -from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError -from sqlbuild.observability import ( - EventDispatcher, - LifecycleEvent, - dispatcher_scope, - invocation_scope, -) -from tests.unit.src.sqlbuild.integrations.ingestr._test_types import ( - IngestrDriverFailureTestCase, - IngestrDriverTestCase, - IngestrRunnerTestCase, -) - -_RUNNER: str = "sqlbuild.integrations.ingestr._helpers.runner" -_INSTALL: tuple[str, ...] = ("dbc", "install", "--quiet", "duckdb=1.5.2") - - -@pytest.mark.parametrize( - "test_case", - ( - IngestrRunnerTestCase( - description="signalled child excludes streams and arguments", - returncode=-15, - expected_signal_number=15, - ), - ), - ids=lambda case: case.description, -) -def test_given_signalled_ingestr_when_run_then_safe_terminal_excludes_child_streams( - test_case: IngestrRunnerTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - events: list[LifecycleEvent] = [] - dispatcher: EventDispatcher = EventDispatcher() - dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - monkeypatch.setattr( - "sqlbuild.integrations.ingestr._helpers.runner.shutil.which", - lambda executable: executable, - ) - monkeypatch.setattr( - "sqlbuild.integrations.ingestr._helpers.runner.ensure_duckdb_driver", lambda: None - ) - monkeypatch.setattr( - "sqlbuild.integrations.ingestr._helpers.runner._run_ingestr_subprocess", - lambda **kwargs: subprocess.CompletedProcess( - args=kwargs["command"], - returncode=test_case.returncode, - stdout="private child stdout", - stderr="private child stderr", - ), - ) - - with ( - invocation_scope("inv-ingestr-signal"), - dispatcher_scope(dispatcher), - pytest.raises(IngestrIntegrationError, match=f"exit code {test_case.returncode}"), - ): - _ = run_ingestr_command(("ingestr", "--secret", "private-value")) - - assert tuple(event.event_type for event in events) == ( - "operation_started", - "operation_failed", - ) - assert events[0].payload == { - "operation_kind": "subprocess", - "operation_name": "ingestr_command", - } - assert events[1].payload["exit_code"] == test_case.returncode - assert events[1].payload["signal_number"] == test_case.expected_signal_number - assert "private" not in repr(events) - - -@pytest.mark.parametrize( - "test_case", - ( - IngestrRunnerTestCase( - description="interrupted child has no fabricated terminal", - returncode=0, - expected_signal_number=0, - expected_event_types=("operation_started",), - ), - ), - ids=lambda case: case.description, -) -def test_given_unobserved_ingestr_interruption_when_run_then_only_start_is_published( - test_case: IngestrRunnerTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - events: list[LifecycleEvent] = [] - dispatcher: EventDispatcher = EventDispatcher() - dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - monkeypatch.setattr( - "sqlbuild.integrations.ingestr._helpers.runner.shutil.which", - lambda executable: executable, - ) - monkeypatch.setattr( - "sqlbuild.integrations.ingestr._helpers.runner.ensure_duckdb_driver", lambda: None - ) - monkeypatch.setattr( - "sqlbuild.integrations.ingestr._helpers.runner._run_ingestr_subprocess", - lambda **kwargs: (_ for _ in ()).throw(KeyboardInterrupt), - ) - - with ( - invocation_scope(f"inv-{test_case.description}"), - dispatcher_scope(dispatcher), - pytest.raises(KeyboardInterrupt), - ): - _ = run_ingestr_command(("ingestr", "run")) - - assert tuple(event.event_type for event in events) == test_case.expected_event_types - - -@pytest.mark.parametrize( - "test_case", - ( - IngestrDriverTestCase( - description="missing driver installs the duckdb package version", - driver_files=(), - environment={}, - expected_installs=(_INSTALL,), - ), - IngestrDriverTestCase( - description="driver of another version is replaced", - driver_files=(("duckdb.toml", "version = '2.0.0'\n"),), - environment={}, - expected_installs=(_INSTALL,), - ), - IngestrDriverTestCase( - description="matching driver is kept", - driver_files=(("duckdb.toml", "version = '1.5.2'\n"),), - environment={}, - expected_installs=(), - ), - IngestrDriverTestCase( - description="user driver path is left alone", - driver_files=(), - environment={"ADBC_DRIVER_PATH": "/custom/adbc"}, - expected_installs=(), - ), - ), - ids=lambda case: case.description, -) -def test_given_duckdb_driver_state_when_ensuring_then_installs_only_a_mismatch( - test_case: IngestrDriverTestCase, - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - driver_path: Path = tmp_path / "etc" / "adbc" / "drivers" - driver_path.mkdir(parents=True) - _ = [(driver_path / name).write_text(text) for name, text in test_case.driver_files] - calls: list[tuple[tuple[str, ...], dict[str, str]]] = [] - monkeypatch.setattr(f"{_RUNNER}.sys.prefix", str(tmp_path)) - monkeypatch.setattr(f"{_RUNNER}.environ", dict(test_case.environment)) - monkeypatch.setattr(f"{_RUNNER}.metadata.version", lambda name: "1.5.2") - monkeypatch.setattr(f"{_RUNNER}._dbc_executable", lambda: "dbc") - monkeypatch.setattr( - f"{_RUNNER}.subprocess.run", - lambda args, **kwargs: ( - calls.append((args, kwargs["env"])) or subprocess.CompletedProcess(args, 0, "", "") - ), - ) - - ensure_duckdb_driver() - - assert tuple(args for args, _ in calls) == test_case.expected_installs - assert all(env["ADBC_DRIVER_PATH"] == str(driver_path) for _, env in calls) - - -@pytest.mark.parametrize( - "test_case", - ( - IngestrDriverFailureTestCase( - description="failed install names the duckdb version", - stderr="registry unreachable", - expected_message="duckdb 1.5.2: registry unreachable", - ), - ), - ids=lambda case: case.description, -) -def test_given_failing_driver_install_when_ensuring_then_names_the_duckdb_version( - test_case: IngestrDriverFailureTestCase, - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - monkeypatch.setattr(f"{_RUNNER}.sys.prefix", str(tmp_path)) - monkeypatch.setattr(f"{_RUNNER}.environ", {}) - monkeypatch.setattr(f"{_RUNNER}.metadata.version", lambda name: "1.5.2") - monkeypatch.setattr(f"{_RUNNER}._dbc_executable", lambda: "dbc") - monkeypatch.setattr( - f"{_RUNNER}.subprocess.run", - lambda args, **kwargs: subprocess.CompletedProcess(args, 1, "", test_case.stderr), - ) - - with pytest.raises(IngestrIntegrationError, match=test_case.expected_message): - ensure_duckdb_driver() diff --git a/tests/unit/src/sqlbuild/runtime/observability/_test_types.py b/tests/unit/src/sqlbuild/runtime/observability/_test_types.py index 64a82652bc..ef1011353f 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/observability/_test_types.py @@ -128,6 +128,14 @@ class OpaqueRoundTripCase: expected_raw: Mapping[str, JSONValue] +@dataclass(frozen=True) +class RetiredOperationCase: + description: str + operation_kind: str + operation_name: str + expected_operation_name: str + + @dataclass(frozen=True) class JsonErrorCase: description: str diff --git a/tests/unit/src/sqlbuild/runtime/observability/test_operation_lifecycle.py b/tests/unit/src/sqlbuild/runtime/observability/test_operation_lifecycle.py index 4c41b0d0b0..79fcc95e58 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/test_operation_lifecycle.py +++ b/tests/unit/src/sqlbuild/runtime/observability/test_operation_lifecycle.py @@ -319,7 +319,7 @@ def test_given_failing_subscriber_when_operation_runs_then_exact_result_is_prese with invocation_scope("inv-subscriber"), dispatcher_scope(dispatcher): result: object = OperationLifecycle( - operation_kind="loader", operation_name="external_source_load" + operation_kind="loader", operation_name="managed_source_load" ).run(lambda: expected) assert result is expected diff --git a/tests/unit/src/sqlbuild/runtime/observability/test_serialization.py b/tests/unit/src/sqlbuild/runtime/observability/test_serialization.py index 8504bc5fbc..4fe58e6704 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/test_serialization.py +++ b/tests/unit/src/sqlbuild/runtime/observability/test_serialization.py @@ -24,6 +24,7 @@ JsonErrorCase, MetadataBoundaryCase, OpaqueRoundTripCase, + RetiredOperationCase, SchemaVersionCase, StatementRoundTripCase, ) @@ -149,6 +150,47 @@ def test_given_known_event_when_serializing_and_decoding_then_round_trips( assert decoded.event_type == test_case.expected_event_type +@pytest.mark.parametrize( + "test_case", + [ + RetiredOperationCase( + "a stored dlt or callable load", + "loader", + "external_source_load", + "external_source_load", + ), + RetiredOperationCase( + "a stored ingestr subprocess", "subprocess", "ingestr_command", "ingestr_command" + ), + ], + ids=lambda case: case.description, +) +def test_given_operation_from_a_removed_integration_when_decoding_history_then_it_reads_back( + test_case: RetiredOperationCase, +) -> None: + stored: str = json.dumps( + { + "event_id": "evt-retired", + "event_type": "operation_started", + "schema_version": 1, + "producer": "sqlbuild", + "producer_version": "0.238.0", + "occurred_at": "2026-08-31T12:34:56.123456Z", + "invocation_id": "inv-1", + "operation_id": "op-1", + "payload": { + "operation_kind": test_case.operation_kind, + "operation_name": test_case.operation_name, + }, + } + ) + + decoded: LifecycleEvent | OpaqueLifecycleEvent = lifecycle_event_from_json(stored) + + assert isinstance(decoded, LifecycleEvent) + assert decoded.payload["operation_name"] == test_case.expected_operation_name + + @pytest.mark.parametrize( "test_case", [ diff --git a/uv.lock b/uv.lock index 955fa3f474..7e36904427 100644 --- a/uv.lock +++ b/uv.lock @@ -26,155 +26,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/53/89b197cb472a3175d73384761a3413fd58e6b65a794c1102d148b8de87bd/agate-1.9.1-py2.py3-none-any.whl", hash = "sha256:1cf329510b3dde07c4ad1740b7587c9c679abc3dcd92bb1107eabc10c2e03c50", size = 95085, upload-time = "2023-12-21T20:05:21.954Z" }, ] -[[package]] -name = "aiobotocore" -version = "3.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "aioitertools" }, - { name = "botocore" }, - { name = "jmespath" }, - { name = "multidict" }, - { name = "python-dateutil" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e7/75/42cce839c2ec263ff74b10b650fe36b066fbb124cbee6f247eac0983e1ab/aiobotocore-3.7.0.tar.gz", hash = "sha256:c64d871ed5491a6571948dd48eabd185b46c6c23b64e3afd0c059fc7593ada30", size = 127054, upload-time = "2026-05-09T10:02:52.332Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/5f/85535dfb3cfd6442d66d1df1694062c5d6df02f895329e7e120b2a3d2b8b/aiobotocore-3.7.0-py3-none-any.whl", hash = "sha256:680bde7c64679a821a9312641b759d9497f790ba8b2e88c6959e6273ee765b8e", size = 89539, upload-time = "2026-05-09T10:02:50.389Z" }, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.6.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/33/c6/61a2d7b7572279226bb2e7f61d7a19ca7c90da0329c93fa0d560cbf288d8/aiohappyeyeballs-2.6.2.tar.gz", hash = "sha256:e202810ee718bd01fc6ef49e8ea53d023d5cb6b581076d7925aa499fa55dbe64", size = 22591, upload-time = "2026-05-20T15:12:24.631Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/fc/a7bf5b6e4e617b45f90f2d9d2a68519c249c81dd4fc2658c7a2a61c4f4b7/aiohappyeyeballs-2.6.2-py3-none-any.whl", hash = "sha256:4708045e2d7a6c6bdf8aafa8ed39649eaf926a4543b54560659129e3365953c4", size = 15062, upload-time = "2026-05-20T15:12:23.328Z" }, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohappyeyeballs" }, - { name = "aiosignal" }, - { name = "attrs" }, - { name = "frozenlist" }, - { name = "multidict" }, - { name = "propcache" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, - { name = "yarl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/21/151624b51cd92553d95424daf4bf19f19ce9be9002d19253e7e7ce67197b/aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480", size = 757402, upload-time = "2026-06-07T21:06:40.311Z" }, - { url = "https://files.pythonhosted.org/packages/c2/82/280619e0bd7bf2454987e19282616e84762255dd9c8468f62382e8c191f1/aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d", size = 512310, upload-time = "2026-06-07T21:06:42.207Z" }, - { url = "https://files.pythonhosted.org/packages/55/b2/2aac325583aaa1353045f96dffa586d8a34e8322e14a7ba49cffeb103ab4/aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2", size = 512448, upload-time = "2026-06-07T21:06:43.813Z" }, - { url = "https://files.pythonhosted.org/packages/8a/72/a60607cb849faa8af8a356c9329ea2eb6f395d49e82cc82ccba1fd8deb8f/aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2", size = 1766854, upload-time = "2026-06-07T21:06:45.391Z" }, - { url = "https://files.pythonhosted.org/packages/b5/d3/d9fe1c9ec7557ab4d0d82bebaa728c6418f0b93295ec2f4ab015f7710cc7/aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a", size = 1740884, upload-time = "2026-06-07T21:06:47.413Z" }, - { url = "https://files.pythonhosted.org/packages/c1/dc/f2cecfaf9337ba3e63f181500814ff502aa3d00d9c7ec93a9d23d10a27b2/aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264", size = 1810034, upload-time = "2026-06-07T21:06:50.165Z" }, - { url = "https://files.pythonhosted.org/packages/66/d7/2ff65c5e65c0d7476daf7e15c032e0805e36811185b9623e3238ad6c763e/aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842", size = 1904054, upload-time = "2026-06-07T21:06:52.035Z" }, - { url = "https://files.pythonhosted.org/packages/20/9c/d445818389df371f56d141d881153ba23183c4735a03f7356ffb43f7757d/aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c", size = 1790278, upload-time = "2026-06-07T21:06:54.049Z" }, - { url = "https://files.pythonhosted.org/packages/4d/aa/bf04cb4d865fc6101c2229a294ad744973b72e513fdc5a6b791e6983d72a/aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95", size = 1591795, upload-time = "2026-06-07T21:06:55.911Z" }, - { url = "https://files.pythonhosted.org/packages/dc/b4/4dac0038960427ba832f6609dfb4ea5437d7fd80c72001b9e48f834f428b/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199", size = 1728397, upload-time = "2026-06-07T21:06:57.777Z" }, - { url = "https://files.pythonhosted.org/packages/2b/f9/7cd4e8ad7aa3b75f17d56bb5498dd604a93d4e6eece822ba0568c413fff0/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817", size = 1766504, upload-time = "2026-06-07T21:07:00.009Z" }, - { url = "https://files.pythonhosted.org/packages/f9/df/fc01d9fcad0f73fed3f3d361f1f94f975947b50dff82919f6dc2bf4316cc/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a", size = 1777806, upload-time = "2026-06-07T21:07:02.064Z" }, - { url = "https://files.pythonhosted.org/packages/41/09/47e2d090bddcc8fb4ccb4c314aadc32d7c5d9bb55f50f6ad1c92fc15d501/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4", size = 1580707, upload-time = "2026-06-07T21:07:03.942Z" }, - { url = "https://files.pythonhosted.org/packages/3d/36/f1a4ce904ae0b6930cfe9afc96d0896f7ec1a620c400405d63783bb95a9c/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087", size = 1798121, upload-time = "2026-06-07T21:07:05.987Z" }, - { url = "https://files.pythonhosted.org/packages/70/0a/e0075ce9ca0279ee1d4f0c0b85f54fea02ebc83c3007651a72bece658fec/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3", size = 1767580, upload-time = "2026-06-07T21:07:07.873Z" }, - { url = "https://files.pythonhosted.org/packages/3e/61/a0c0a8f327a9c52095cdd8e312391b00d3ed64ab6c72bb5c33d8ec251cf7/aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4", size = 452771, upload-time = "2026-06-07T21:07:09.669Z" }, - { url = "https://files.pythonhosted.org/packages/df/d9/ea367c75f16ac9c6cdc8febb25e8318fa21a2b1bc8d6514d4b2d890bface/aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271", size = 479873, upload-time = "2026-06-07T21:07:11.538Z" }, - { url = "https://files.pythonhosted.org/packages/03/64/8d96784a7851156db8a4c6c3f6f91042fdf39fb15a4cc38c8b3c14833c45/aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847", size = 448073, upload-time = "2026-06-07T21:07:13.637Z" }, - { url = "https://files.pythonhosted.org/packages/bc/97/bd137012dd97e1649162b099135a80e1fd59aaa807b2430fc448d1029aff/aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178", size = 506882, upload-time = "2026-06-07T21:07:15.501Z" }, - { url = "https://files.pythonhosted.org/packages/ef/79/e5cc690e9d922a66887ceeaca53a8ffd5a7b0be3816142b7abc433742d89/aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf", size = 515270, upload-time = "2026-06-07T21:07:17.53Z" }, - { url = "https://files.pythonhosted.org/packages/fe/22/a73ccbf9dbd6e26dda0b24d5fd5db7da92ee3383a79f47677ffb834c5c5b/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd", size = 485841, upload-time = "2026-06-07T21:07:19.555Z" }, - { url = "https://files.pythonhosted.org/packages/3b/b9/57ed8eaf596321c2ad747bd480fb1700dbd7177c60dfc9e4c187f629662e/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe", size = 492088, upload-time = "2026-06-07T21:07:21.581Z" }, - { url = "https://files.pythonhosted.org/packages/78/c0/5ebe5270a7c140d7c6f79dcb018640225f14d406c149e4eec04a7d82fe71/aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4", size = 501564, upload-time = "2026-06-07T21:07:23.388Z" }, - { url = "https://files.pythonhosted.org/packages/75/7f/8cdaa24fc7983865e0915153b96a9ac5bcdd3548d64c5a27d17cecccad2d/aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876", size = 751998, upload-time = "2026-06-07T21:07:25.046Z" }, - { url = "https://files.pythonhosted.org/packages/b2/f4/c4227aacfacc5cb0cc2d119b65301d177912a6842cd64e120c47af76064f/aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da", size = 510918, upload-time = "2026-06-07T21:07:27.28Z" }, - { url = "https://files.pythonhosted.org/packages/ab/01/a2d5f96cd4e74424864d30bc0a7e44d0a12dacdcfa91b5b2d1bd3dca6bf3/aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6", size = 508657, upload-time = "2026-06-07T21:07:29.252Z" }, - { url = "https://files.pythonhosted.org/packages/e8/ed/3c0fb5c500fdd8e7ebc10d1889c04384fffa1a9163eac1356088ca9da1b1/aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96", size = 1757907, upload-time = "2026-06-07T21:07:31.03Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ab/d4c924d9bd5be3050c226612413ce68cb54c70d2c31b661bfc8d9a5b6a70/aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f", size = 1737565, upload-time = "2026-06-07T21:07:33.031Z" }, - { url = "https://files.pythonhosted.org/packages/19/2a/37326821ff779084020cdc33224d20b19f42f4183a500ff92022a739eda7/aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296", size = 1799018, upload-time = "2026-06-07T21:07:35.003Z" }, - { url = "https://files.pythonhosted.org/packages/b3/4f/6e947ba73e4ce09070761c05ed3a8ceb7c21f5e46798671d8b2aac0e4626/aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa", size = 1894416, upload-time = "2026-06-07T21:07:36.956Z" }, - { url = "https://files.pythonhosted.org/packages/9d/6e/dbf1d0625dc711fb2851f4f3c3055c39ed58bae92082d8c627dbe6013736/aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451", size = 1783881, upload-time = "2026-06-07T21:07:39.063Z" }, - { url = "https://files.pythonhosted.org/packages/44/c2/5e25098a67268ed369483ae7d1a58bd0a13d03aab860d2a0e4a6eb25b046/aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c", size = 1587572, upload-time = "2026-06-07T21:07:41.058Z" }, - { url = "https://files.pythonhosted.org/packages/2a/bd/cf9cee17e140f942a3de73e658a543aa8fbf35a5fc67a9d2538d52d77f0b/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca", size = 1722137, upload-time = "2026-06-07T21:07:43.014Z" }, - { url = "https://files.pythonhosted.org/packages/89/6d/5684f8c59045c96f81a18cefbc1fbbd79d25b88f1c622f2a5c5c08fcb632/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09", size = 1755953, upload-time = "2026-06-07T21:07:45.933Z" }, - { url = "https://files.pythonhosted.org/packages/a8/40/35caf3170f8359760740a7d9aa0fff2e344bef98e1d1186f5a0f6dec17e6/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397", size = 1766479, upload-time = "2026-06-07T21:07:48.047Z" }, - { url = "https://files.pythonhosted.org/packages/6d/a1/b0c61e7a137f0d81de49a82023a6df73c3c16d6fefb0f8e4a93d21639002/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080", size = 1580077, upload-time = "2026-06-07T21:07:50.069Z" }, - { url = "https://files.pythonhosted.org/packages/0b/41/194ea4623693009fcefebef7aef63c141754f153e9cd0d39d3b9e36c175c/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345", size = 1791688, upload-time = "2026-06-07T21:07:52.106Z" }, - { url = "https://files.pythonhosted.org/packages/ba/45/4de841f005cfe1fd63e2a2fe011262c515e2a62aa6994b15947e7d717ac9/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588", size = 1761094, upload-time = "2026-06-07T21:07:54.113Z" }, - { url = "https://files.pythonhosted.org/packages/e4/ae/dbce10533d3896d544d5053939ed75b7dc31a1b0973d959b1b5ae21028d6/aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780", size = 452662, upload-time = "2026-06-07T21:07:56.06Z" }, - { url = "https://files.pythonhosted.org/packages/7b/d9/0bf1a19362c32f06229da5e7ddfcec91f93474d6307f7a2d3135e9c674dc/aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a", size = 479748, upload-time = "2026-06-07T21:07:58.319Z" }, - { url = "https://files.pythonhosted.org/packages/22/0a/62e7232dc9484fbec112ceb32efb6a624cc7994ec6e2b019286f17c4e8f2/aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8", size = 447723, upload-time = "2026-06-07T21:08:00.154Z" }, - { url = "https://files.pythonhosted.org/packages/c4/a1/5fafa04e1ca91ddb47608699d60649c1c6db3cf41c99e78fc4056f9513db/aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15", size = 508531, upload-time = "2026-06-07T21:08:02.093Z" }, - { url = "https://files.pythonhosted.org/packages/fa/2e/bfa02f699d87ffc86d5959270b28f1cb410add3ccaced8ed2e0b8a5238fc/aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8", size = 514718, upload-time = "2026-06-07T21:08:04.476Z" }, - { url = "https://files.pythonhosted.org/packages/85/a5/9594ad6289eebbc97d167c44213d557807f90e59115caad24de21ad2c3b1/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3", size = 487918, upload-time = "2026-06-07T21:08:06.377Z" }, - { url = "https://files.pythonhosted.org/packages/b4/61/16a32c36c3c49edec122a3dc811f2057df2f94d3b14aa107c8017d981618/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba", size = 494014, upload-time = "2026-06-07T21:08:08.263Z" }, - { url = "https://files.pythonhosted.org/packages/9b/89/3ebcf96ed99c05bec9c434aaac6963fd3cbab4a786ae739908a144d9ce44/aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397", size = 502398, upload-time = "2026-06-07T21:08:10.244Z" }, - { url = "https://files.pythonhosted.org/packages/fd/3d/b74870a0c2d40c355928cd5b96c7a11fa821b8a40fc41365e64479b151fb/aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448", size = 758018, upload-time = "2026-06-07T21:08:12.447Z" }, - { url = "https://files.pythonhosted.org/packages/d3/66/f42f5c984d99e49c6cff5f26f590750f2e2f7ef1fcfb99966ab5be1b632e/aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004", size = 512462, upload-time = "2026-06-07T21:08:14.624Z" }, - { url = "https://files.pythonhosted.org/packages/e9/a7/248e1aebe0c7810b0271e021a0f2a5eb6e78a051885b3c9df49f42a5802d/aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983", size = 512824, upload-time = "2026-06-07T21:08:16.572Z" }, - { url = "https://files.pythonhosted.org/packages/26/97/2aa0e5ba0727dc3bd5aaebb7ccbc510f7dfb7fb961ec87497cd496635ab1/aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe", size = 1749898, upload-time = "2026-06-07T21:08:18.635Z" }, - { url = "https://files.pythonhosted.org/packages/00/8d/e97f6c96c891d457c8479d92a514ba194d0412f981d72c70341ee18488ed/aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333", size = 1710114, upload-time = "2026-06-07T21:08:20.892Z" }, - { url = "https://files.pythonhosted.org/packages/6f/e6/aa8d7e863048c8fceb5cd6ce74017311cec3ead07847387e12265fb4444e/aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0", size = 1802541, upload-time = "2026-06-07T21:08:23.044Z" }, - { url = "https://files.pythonhosted.org/packages/83/a8/72193137de57fda4ebfae4563182d082c8856e3b6e9871d0b46f028fb369/aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a", size = 1875776, upload-time = "2026-06-07T21:08:25.288Z" }, - { url = "https://files.pythonhosted.org/packages/a0/18/938441025db6769a3464596b2410af3afde0b21eb2f204c6f766f68af4bd/aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602", size = 1760329, upload-time = "2026-06-07T21:08:27.363Z" }, - { url = "https://files.pythonhosted.org/packages/60/29/bf2496b4065e76e09fe48015aaffe5ce161d8f089b06ac6982070f653076/aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca", size = 1587293, upload-time = "2026-06-07T21:08:29.805Z" }, - { url = "https://files.pythonhosted.org/packages/49/a2/2136674d52123b1354bd05dd5753c318db47dc0c927cc70b27bab3755456/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35", size = 1714756, upload-time = "2026-06-07T21:08:32.094Z" }, - { url = "https://files.pythonhosted.org/packages/a7/b9/e5fd2e6f915503081c0f9b1e8540947037929c70c191da2e4d54b31a21a1/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844", size = 1721052, upload-time = "2026-06-07T21:08:34.167Z" }, - { url = "https://files.pythonhosted.org/packages/63/5a/2833e324a2263e104e31e2e91bc5bbee81bc499afd32203faee048a883f0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496", size = 1766888, upload-time = "2026-06-07T21:08:36.95Z" }, - { url = "https://files.pythonhosted.org/packages/57/fa/dea6511870913162f3b2e8c42a7614eb203a4540b8c2da43e0bfb0548f3c/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5", size = 1581679, upload-time = "2026-06-07T21:08:39.292Z" }, - { url = "https://files.pythonhosted.org/packages/14/bd/3cf0d55e71784b33534e9710a67d382d900598b4787fbce6cc7317f8c42a/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95", size = 1782021, upload-time = "2026-06-07T21:08:41.407Z" }, - { url = "https://files.pythonhosted.org/packages/c1/af/14bb5843eccbe234f4dfb78ab73e549d99727247e62ae5d62cbd22eaf5b0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444", size = 1742574, upload-time = "2026-06-07T21:08:43.795Z" }, - { url = "https://files.pythonhosted.org/packages/f2/1e/fbeb7af9210a67ac0f9c9bec0f8f4568497924e33137a3d5b48e1cf85f3f/aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0", size = 457773, upload-time = "2026-06-07T21:08:46.168Z" }, - { url = "https://files.pythonhosted.org/packages/f0/2b/13e8d741a9ec5db7d900c060554cf8352ab85e44e2a4469ebb9d377bda17/aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719", size = 485001, upload-time = "2026-06-07T21:08:48.401Z" }, - { url = "https://files.pythonhosted.org/packages/df/30/491acfa2c4d6c3ff59c49a14fc1b50be3241e25bbb0c84c09e2da4d11395/aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec", size = 453809, upload-time = "2026-06-07T21:08:50.7Z" }, - { url = "https://files.pythonhosted.org/packages/34/e3/19dbe1a1f4cc6230eb9e314de7fe68053b0992f9302b27d12141a0b5db53/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2", size = 793320, upload-time = "2026-06-07T21:08:52.775Z" }, - { url = "https://files.pythonhosted.org/packages/7f/20/1b7182219ba1b108430d6e4dc53d25ae02dcfcf5a045b33af4e8c5167527/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340", size = 529077, upload-time = "2026-06-07T21:08:55Z" }, - { url = "https://files.pythonhosted.org/packages/b9/c8/14ce60ec31a2e5f5274bb17d383a6f7a3aabca31ac04eee05585bbadab16/aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d", size = 532476, upload-time = "2026-06-07T21:08:57.176Z" }, - { url = "https://files.pythonhosted.org/packages/7e/02/9ac85e081e53da2e061b02fa7758fe0a12d17b8ce2d1f5e6c7cb76730328/aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94", size = 1922347, upload-time = "2026-06-07T21:08:59.563Z" }, - { url = "https://files.pythonhosted.org/packages/c0/3e/d3ba07a0ab38b5389e10bec4362d21e10a4f667cba2d79ba30837b3a5059/aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc", size = 1786465, upload-time = "2026-06-07T21:09:01.909Z" }, - { url = "https://files.pythonhosted.org/packages/0b/cb/e2ee978a00cfb2df829704a69528b18154eba5939f45bc1efa8f33aee4c5/aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251", size = 1909423, upload-time = "2026-06-07T21:09:04.357Z" }, - { url = "https://files.pythonhosted.org/packages/73/5d/1430334858b1022b58ae50399a918f0bd6fe8fa7fa183598d657ff61e040/aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1", size = 2001906, upload-time = "2026-06-07T21:09:06.722Z" }, - { url = "https://files.pythonhosted.org/packages/66/4e/560c7472d3d198a23aa5c8b19a5115bf6a9b77b7d3e4bb363da320430ad2/aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3", size = 1877095, upload-time = "2026-06-07T21:09:09.011Z" }, - { url = "https://files.pythonhosted.org/packages/0d/f1/4745806578d447db4a784a8591e2dae3afdfc2bcb96f8f81271b13df6543/aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c", size = 1676222, upload-time = "2026-06-07T21:09:11.461Z" }, - { url = "https://files.pythonhosted.org/packages/6a/c9/48255813cca749a229ef0ab476004ec623728ad79a9c0840616f6c076325/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203", size = 1842922, upload-time = "2026-06-07T21:09:14.118Z" }, - { url = "https://files.pythonhosted.org/packages/3d/c0/bbd054e2bee909f529523a5af3891052606af5143c09f5f183ec3b234676/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1", size = 1825035, upload-time = "2026-06-07T21:09:16.447Z" }, - { url = "https://files.pythonhosted.org/packages/a8/ae/90395d4376deceb74e09ec26b6adf7d2015a6f8802d6d84446af860fef04/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665", size = 1849512, upload-time = "2026-06-07T21:09:18.742Z" }, - { url = "https://files.pythonhosted.org/packages/93/bd/fb25f3049957553d4ce0ba6ae480aa2f592a6985497fca590837d16c1be0/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1", size = 1668571, upload-time = "2026-06-07T21:09:21.458Z" }, - { url = "https://files.pythonhosted.org/packages/3f/22/7f73303d64dd567ff3addca90b556690ed1233a47b8f55d242fb90af3681/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d", size = 1881159, upload-time = "2026-06-07T21:09:23.813Z" }, - { url = "https://files.pythonhosted.org/packages/44/be/0474c5a8b5640e1e4aa1923430a91f4151be82e511373fe764189b89aef5/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c", size = 1841409, upload-time = "2026-06-07T21:09:26.207Z" }, - { url = "https://files.pythonhosted.org/packages/7b/3c/bb4a7cba26956cb3da4553cc2056cf67be5b5ff6e6d8fa4fbdff73bfb7ae/aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365", size = 494166, upload-time = "2026-06-07T21:09:28.505Z" }, - { url = "https://files.pythonhosted.org/packages/8a/84/ec80c2c1f66a952555a9f86df6b33af65108a6febfa0471b69013a12f807/aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9", size = 530255, upload-time = "2026-06-07T21:09:30.843Z" }, - { url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" }, -] - -[[package]] -name = "aioitertools" -version = "0.13.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fd/3c/53c4a17a05fb9ea2313ee1777ff53f5e001aefd5cc85aa2f4c2d982e1e38/aioitertools-0.13.0.tar.gz", hash = "sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c", size = 19322, upload-time = "2025-11-06T22:17:07.609Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/a1/510b0a7fadc6f43a6ce50152e69dbd86415240835868bb0bd9b5b88b1e06/aioitertools-0.13.0-py3-none-any.whl", hash = "sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be", size = 24182, upload-time = "2025-11-06T22:17:06.502Z" }, -] - -[[package]] -name = "aiosignal" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "frozenlist" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, -] - [[package]] name = "alembic" version = "1.18.4" @@ -916,18 +767,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/7a/797ed371bde520223e49dbbe0f8762fa4cc724d0e21fc6af944bb8f82150/db_dtypes-1.7.0-py3-none-any.whl", hash = "sha256:c80a1d9bc7dda3cc63638a3f2cf4ec27af50b809ec2d92b8a94cb9d8438cdc76", size = 17740, upload-time = "2026-06-03T14:51:15.287Z" }, ] -[[package]] -name = "dbc" -version = "0.3.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/10/c595fd891a8a5415378d40fd4d428b72c7266dd5cbf8c04ea5f16279f323/dbc-0.3.1-py3-none-macosx_12_0_arm64.whl", hash = "sha256:9b55f649516055c121d80ebf189ec47451a4f30a3e66577dfb89d4c92c104524", size = 3994959, upload-time = "2026-10-01T16:00:28.743Z" }, - { url = "https://files.pythonhosted.org/packages/ee/38/abf425442f73343e0f27fd32451fdabedd403317d6fc3ec6fe423742cc1f/dbc-0.3.1-py3-none-macosx_12_0_x86_64.whl", hash = "sha256:4d6ac7ed02572a55ce8312ef510367155cbb0fee529512684a460895dfb991f5", size = 4429256, upload-time = "2026-10-01T16:00:30.927Z" }, - { url = "https://files.pythonhosted.org/packages/f1/59/f07789d4db5a8e3f1eb3afd8524d099e048ccb75a4f854ece6ef0cdb402f/dbc-0.3.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bbb1f137c75d4ac18b1e29f662ddbb06d636cb455a9b76a6fcc9bf4dcf2a8b3a", size = 4288844, upload-time = "2026-10-01T16:00:33.016Z" }, - { url = "https://files.pythonhosted.org/packages/ee/80/55ad27b5b6f21c0af04acb611d9a48da07fc0bb0a287f03515ee4b34aaf3/dbc-0.3.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a880d10ccb2c4816262ccd4cf11aca0db259bc36a34eced34b533a404ba029", size = 3865144, upload-time = "2026-10-01T16:00:34.848Z" }, - { url = "https://files.pythonhosted.org/packages/27/b9/904c2249a1bfd5a6f34011f33db01b113af0ca7685b1a1218a20b5e8838a/dbc-0.3.1-py3-none-win_amd64.whl", hash = "sha256:5ae67a0d0c9e028a24ed3cd802d85c184f73794e90405c59984471d0e3a3bd1b", size = 4415639, upload-time = "2026-10-01T16:00:36.529Z" }, -] - [[package]] name = "dbt-adapters" version = "1.24.2" @@ -1227,55 +1066,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] -[[package]] -name = "dlt" -version = "1.28.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "fsspec" }, - { name = "gitpython" }, - { name = "giturlparse" }, - { name = "humanize" }, - { name = "jsonpath-ng" }, - { name = "orjson", marker = "(python_full_version >= '3.14' and platform_python_implementation == 'PyPy') or (python_full_version >= '3.14' and sys_platform == 'emscripten') or (platform_python_implementation != 'PyPy' and sys_platform != 'emscripten')" }, - { name = "packaging" }, - { name = "pathvalidate" }, - { name = "pendulum" }, - { name = "pluggy" }, - { name = "pytz" }, - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "requirements-parser" }, - { name = "rich-argparse" }, - { name = "semver" }, - { name = "setuptools" }, - { name = "simplejson" }, - { name = "sqlglot" }, - { name = "tenacity" }, - { name = "tomlkit" }, - { name = "typing-extensions" }, - { name = "tzdata" }, - { name = "win-precise-time", marker = "python_full_version < '3.13' and os_name == 'nt'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a2/91/d1ba83149b894628fffbfab47509627e42625e562f01c892d238d02d9501/dlt-1.28.1.tar.gz", hash = "sha256:152adddcda24b57d6321100170c0aa11dfb45dd3cd71723b616d5c9bc46d4326", size = 1085998, upload-time = "2026-06-19T19:50:34.869Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/eb/cc49acdfe2beb1628b1b29bc93e6c5e78eee8641acfd5970b9d4f475c91c/dlt-1.28.1-py3-none-any.whl", hash = "sha256:d60751df25efb5c2243491b84d93ac179fa28b4766c05b3200be2caae54b5096", size = 1363623, upload-time = "2026-06-19T19:50:31.478Z" }, -] - -[package.optional-dependencies] -duckdb = [ - { name = "duckdb" }, -] -filesystem = [ - { name = "botocore" }, - { name = "s3fs" }, -] -sql-database = [ - { name = "sqlalchemy" }, -] - [[package]] name = "docker" version = "7.1.0" @@ -1379,95 +1169,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, ] -[[package]] -name = "frozenlist" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, - { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, - { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, - { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, - { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, - { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, - { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, - { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, - { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, - { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, - { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, - { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, - { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, - { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, - { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, - { url = "https://files.pythonhosted.org/packages/2d/40/0832c31a37d60f60ed79e9dfb5a92e1e2af4f40a16a29abcc7992af9edff/frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a", size = 85717, upload-time = "2025-10-06T05:36:27.341Z" }, - { url = "https://files.pythonhosted.org/packages/30/ba/b0b3de23f40bc55a7057bd38434e25c34fa48e17f20ee273bbde5e0650f3/frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7", size = 49651, upload-time = "2025-10-06T05:36:28.855Z" }, - { url = "https://files.pythonhosted.org/packages/0c/ab/6e5080ee374f875296c4243c381bbdef97a9ac39c6e3ce1d5f7d42cb78d6/frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40", size = 49417, upload-time = "2025-10-06T05:36:29.877Z" }, - { url = "https://files.pythonhosted.org/packages/d5/4e/e4691508f9477ce67da2015d8c00acd751e6287739123113a9fca6f1604e/frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027", size = 234391, upload-time = "2025-10-06T05:36:31.301Z" }, - { url = "https://files.pythonhosted.org/packages/40/76/c202df58e3acdf12969a7895fd6f3bc016c642e6726aa63bd3025e0fc71c/frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822", size = 233048, upload-time = "2025-10-06T05:36:32.531Z" }, - { url = "https://files.pythonhosted.org/packages/f9/c0/8746afb90f17b73ca5979c7a3958116e105ff796e718575175319b5bb4ce/frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121", size = 226549, upload-time = "2025-10-06T05:36:33.706Z" }, - { url = "https://files.pythonhosted.org/packages/7e/eb/4c7eefc718ff72f9b6c4893291abaae5fbc0c82226a32dcd8ef4f7a5dbef/frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5", size = 239833, upload-time = "2025-10-06T05:36:34.947Z" }, - { url = "https://files.pythonhosted.org/packages/c2/4e/e5c02187cf704224f8b21bee886f3d713ca379535f16893233b9d672ea71/frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e", size = 245363, upload-time = "2025-10-06T05:36:36.534Z" }, - { url = "https://files.pythonhosted.org/packages/1f/96/cb85ec608464472e82ad37a17f844889c36100eed57bea094518bf270692/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11", size = 229314, upload-time = "2025-10-06T05:36:38.582Z" }, - { url = "https://files.pythonhosted.org/packages/5d/6f/4ae69c550e4cee66b57887daeebe006fe985917c01d0fff9caab9883f6d0/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1", size = 243365, upload-time = "2025-10-06T05:36:40.152Z" }, - { url = "https://files.pythonhosted.org/packages/7a/58/afd56de246cf11780a40a2c28dc7cbabbf06337cc8ddb1c780a2d97e88d8/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1", size = 237763, upload-time = "2025-10-06T05:36:41.355Z" }, - { url = "https://files.pythonhosted.org/packages/cb/36/cdfaf6ed42e2644740d4a10452d8e97fa1c062e2a8006e4b09f1b5fd7d63/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8", size = 240110, upload-time = "2025-10-06T05:36:42.716Z" }, - { url = "https://files.pythonhosted.org/packages/03/a8/9ea226fbefad669f11b52e864c55f0bd57d3c8d7eb07e9f2e9a0b39502e1/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed", size = 233717, upload-time = "2025-10-06T05:36:44.251Z" }, - { url = "https://files.pythonhosted.org/packages/1e/0b/1b5531611e83ba7d13ccc9988967ea1b51186af64c42b7a7af465dcc9568/frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496", size = 39628, upload-time = "2025-10-06T05:36:45.423Z" }, - { url = "https://files.pythonhosted.org/packages/d8/cf/174c91dbc9cc49bc7b7aab74d8b734e974d1faa8f191c74af9b7e80848e6/frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231", size = 43882, upload-time = "2025-10-06T05:36:46.796Z" }, - { url = "https://files.pythonhosted.org/packages/c1/17/502cd212cbfa96eb1388614fe39a3fc9ab87dbbe042b66f97acb57474834/frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62", size = 39676, upload-time = "2025-10-06T05:36:47.8Z" }, - { url = "https://files.pythonhosted.org/packages/d2/5c/3bbfaa920dfab09e76946a5d2833a7cbdf7b9b4a91c714666ac4855b88b4/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94", size = 89235, upload-time = "2025-10-06T05:36:48.78Z" }, - { url = "https://files.pythonhosted.org/packages/d2/d6/f03961ef72166cec1687e84e8925838442b615bd0b8854b54923ce5b7b8a/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c", size = 50742, upload-time = "2025-10-06T05:36:49.837Z" }, - { url = "https://files.pythonhosted.org/packages/1e/bb/a6d12b7ba4c3337667d0e421f7181c82dda448ce4e7ad7ecd249a16fa806/frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52", size = 51725, upload-time = "2025-10-06T05:36:50.851Z" }, - { url = "https://files.pythonhosted.org/packages/bc/71/d1fed0ffe2c2ccd70b43714c6cab0f4188f09f8a67a7914a6b46ee30f274/frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51", size = 284533, upload-time = "2025-10-06T05:36:51.898Z" }, - { url = "https://files.pythonhosted.org/packages/c9/1f/fb1685a7b009d89f9bf78a42d94461bc06581f6e718c39344754a5d9bada/frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65", size = 292506, upload-time = "2025-10-06T05:36:53.101Z" }, - { url = "https://files.pythonhosted.org/packages/e6/3b/b991fe1612703f7e0d05c0cf734c1b77aaf7c7d321df4572e8d36e7048c8/frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82", size = 274161, upload-time = "2025-10-06T05:36:54.309Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ec/c5c618767bcdf66e88945ec0157d7f6c4a1322f1473392319b7a2501ded7/frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714", size = 294676, upload-time = "2025-10-06T05:36:55.566Z" }, - { url = "https://files.pythonhosted.org/packages/7c/ce/3934758637d8f8a88d11f0585d6495ef54b2044ed6ec84492a91fa3b27aa/frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d", size = 300638, upload-time = "2025-10-06T05:36:56.758Z" }, - { url = "https://files.pythonhosted.org/packages/fc/4f/a7e4d0d467298f42de4b41cbc7ddaf19d3cfeabaf9ff97c20c6c7ee409f9/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506", size = 283067, upload-time = "2025-10-06T05:36:57.965Z" }, - { url = "https://files.pythonhosted.org/packages/dc/48/c7b163063d55a83772b268e6d1affb960771b0e203b632cfe09522d67ea5/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51", size = 292101, upload-time = "2025-10-06T05:36:59.237Z" }, - { url = "https://files.pythonhosted.org/packages/9f/d0/2366d3c4ecdc2fd391e0afa6e11500bfba0ea772764d631bbf82f0136c9d/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e", size = 289901, upload-time = "2025-10-06T05:37:00.811Z" }, - { url = "https://files.pythonhosted.org/packages/b8/94/daff920e82c1b70e3618a2ac39fbc01ae3e2ff6124e80739ce5d71c9b920/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0", size = 289395, upload-time = "2025-10-06T05:37:02.115Z" }, - { url = "https://files.pythonhosted.org/packages/e3/20/bba307ab4235a09fdcd3cc5508dbabd17c4634a1af4b96e0f69bfe551ebd/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41", size = 283659, upload-time = "2025-10-06T05:37:03.711Z" }, - { url = "https://files.pythonhosted.org/packages/fd/00/04ca1c3a7a124b6de4f8a9a17cc2fcad138b4608e7a3fc5877804b8715d7/frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b", size = 43492, upload-time = "2025-10-06T05:37:04.915Z" }, - { url = "https://files.pythonhosted.org/packages/59/5e/c69f733a86a94ab10f68e496dc6b7e8bc078ebb415281d5698313e3af3a1/frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888", size = 48034, upload-time = "2025-10-06T05:37:06.343Z" }, - { url = "https://files.pythonhosted.org/packages/16/6c/be9d79775d8abe79b05fa6d23da99ad6e7763a1d080fbae7290b286093fd/frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042", size = 41749, upload-time = "2025-10-06T05:37:07.431Z" }, - { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127, upload-time = "2025-10-06T05:37:08.438Z" }, - { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698, upload-time = "2025-10-06T05:37:09.48Z" }, - { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749, upload-time = "2025-10-06T05:37:10.569Z" }, - { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298, upload-time = "2025-10-06T05:37:11.993Z" }, - { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015, upload-time = "2025-10-06T05:37:13.194Z" }, - { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038, upload-time = "2025-10-06T05:37:14.577Z" }, - { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130, upload-time = "2025-10-06T05:37:15.781Z" }, - { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845, upload-time = "2025-10-06T05:37:17.037Z" }, - { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131, upload-time = "2025-10-06T05:37:18.221Z" }, - { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542, upload-time = "2025-10-06T05:37:19.771Z" }, - { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308, upload-time = "2025-10-06T05:37:20.969Z" }, - { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210, upload-time = "2025-10-06T05:37:22.252Z" }, - { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972, upload-time = "2025-10-06T05:37:23.5Z" }, - { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536, upload-time = "2025-10-06T05:37:25.581Z" }, - { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330, upload-time = "2025-10-06T05:37:26.928Z" }, - { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627, upload-time = "2025-10-06T05:37:28.075Z" }, - { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238, upload-time = "2025-10-06T05:37:29.373Z" }, - { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738, upload-time = "2025-10-06T05:37:30.792Z" }, - { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739, upload-time = "2025-10-06T05:37:32.127Z" }, - { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186, upload-time = "2025-10-06T05:37:33.21Z" }, - { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196, upload-time = "2025-10-06T05:37:36.107Z" }, - { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830, upload-time = "2025-10-06T05:37:37.663Z" }, - { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289, upload-time = "2025-10-06T05:37:39.261Z" }, - { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318, upload-time = "2025-10-06T05:37:43.213Z" }, - { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814, upload-time = "2025-10-06T05:37:45.337Z" }, - { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762, upload-time = "2025-10-06T05:37:46.657Z" }, - { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470, upload-time = "2025-10-06T05:37:47.946Z" }, - { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042, upload-time = "2025-10-06T05:37:49.499Z" }, - { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148, upload-time = "2025-10-06T05:37:50.745Z" }, - { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676, upload-time = "2025-10-06T05:37:52.222Z" }, - { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451, upload-time = "2025-10-06T05:37:53.425Z" }, - { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507, upload-time = "2025-10-06T05:37:54.513Z" }, - { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, -] - [[package]] name = "fsspec" version = "2026.4.0" @@ -1477,39 +1178,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, ] -[[package]] -name = "gitdb" -version = "4.0.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "smmap" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, -] - -[[package]] -name = "gitpython" -version = "3.1.50" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "gitdb" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, -] - -[[package]] -name = "giturlparse" -version = "0.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/8a/b70b84cc78f9059d627f09560c81a11e8f046570d220a24e169489cad6c9/giturlparse-0.15.0.tar.gz", hash = "sha256:9af3f1fd5c4a0cac94ddb283593635005646393ee0debbe330d1bdff8866bf2c", size = 16138, upload-time = "2026-06-16T07:28:56.021Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/96/147a2771ab655b9353781fb2f95c94eaf1d8576dccc991c1a61d0d355067/giturlparse-0.15.0-py2.py3-none-any.whl", hash = "sha256:76d2e6983b037356ab99b30683e533ac3db96409b68e2163a20fc3aff6446f10", size = 16683, upload-time = "2026-06-16T07:28:55.184Z" }, -] - [[package]] name = "google-api-core" version = "2.30.3" @@ -2006,15 +1674,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794, upload-time = "2021-09-17T21:40:39.897Z" }, ] -[[package]] -name = "humanize" -version = "4.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/66/a3921783d54be8a6870ac4ccffcd15c4dc0dd7fcce51c6d63b8c63935276/humanize-4.15.0.tar.gz", hash = "sha256:1dd098483eb1c7ee8e32eb2e99ad1910baefa4b75c3aff3a82f4d78688993b10", size = 83599, upload-time = "2025-12-20T20:16:13.19Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/7b/bca5613a0c3b542420cf92bd5e5fb8ebd5435ce1011a091f66bb7693285e/humanize-4.15.0-py3-none-any.whl", hash = "sha256:b1186eb9f5a9749cd9cb8565aee77919dd7c8d076161cf44d70e59e3301e1769", size = 132203, upload-time = "2025-12-20T20:16:11.67Z" }, -] - [[package]] name = "identify" version = "2.6.19" @@ -2045,14 +1704,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7d/f9/97f2ca8bb3ec6e4b1d64f983ebe98b9a192faddff67fac3d6303a537e670/importlib_metadata-8.9.0-py3-none-any.whl", hash = "sha256:e0f761b6ea91ced3b0844c14c9d955224d538105921f8e6754c00f6ca79fba7f", size = 27220, upload-time = "2026-03-20T16:56:25.07Z" }, ] -[[package]] -name = "ingestr" -version = "1.1.73" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/c4/ec864b94260069470c94f0e6e0a84a10f7838bcc0a21e8ceb7f5196f81d7/ingestr-1.1.73-py3-none-any.whl", hash = "sha256:dad15bc1b0c0c1eaef746f885c89a4986c66a64941660a0e3971f106ee221db8", size = 18387, upload-time = "2026-10-21T16:03:12.817Z" }, -] - [[package]] name = "iniconfig" version = "2.3.0" @@ -2134,18 +1785,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] -[[package]] -name = "jsonpath-ng" -version = "1.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ply" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/86/08646239a313f895186ff0a4573452038eed8c86f54380b3ebac34d32fb2/jsonpath-ng-1.7.0.tar.gz", hash = "sha256:f6f5f7fd4e5ff79c785f1573b394043b39849fb2bb47bcead935d12b00beab3c", size = 37838, upload-time = "2024-10-11T15:41:42.404Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/35/5a/73ecb3d82f8615f32ccdadeb9356726d6cae3a4bbc840b437ceb95708063/jsonpath_ng-1.7.0-py3-none-any.whl", hash = "sha256:f3d7f9e848cba1b6da28c55b1c26ff915dc9e0b1ba7e752a53d6da8d5cbd00b6", size = 30105, upload-time = "2024-11-20T17:58:30.418Z" }, -] - [[package]] name = "jsonschema" version = "4.26.0" @@ -2883,58 +2522,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, ] -[[package]] -name = "pathvalidate" -version = "3.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fa/2a/52a8da6fe965dea6192eb716b357558e103aea0a1e9a8352ad575a8406ca/pathvalidate-3.3.1.tar.gz", hash = "sha256:b18c07212bfead624345bb8e1d6141cdcf15a39736994ea0b94035ad2b1ba177", size = 63262, upload-time = "2025-06-15T09:07:20.736Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/70/875f4a23bfc4731703a5835487d0d2fb999031bd415e7d17c0ae615c18b7/pathvalidate-3.3.1-py3-none-any.whl", hash = "sha256:5263baab691f8e1af96092fa5137ee17df5bdfbd6cff1fcac4d6ef4bc2e1735f", size = 24305, upload-time = "2025-06-15T09:07:19.117Z" }, -] - -[[package]] -name = "pendulum" -version = "3.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-dateutil" }, - { name = "tzdata" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cb/72/9a51afa0a822b09e286c4cb827ed7b00bc818dac7bd11a5f161e493a217d/pendulum-3.2.0.tar.gz", hash = "sha256:e80feda2d10fa3ff8b1526715f7d33dcb7e08494b3088f2c8a3ac92d4a4331ce", size = 86912, upload-time = "2026-01-30T11:22:24.093Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/56/dd0ea9f97d25a0763cda09e2217563b45714786118d8c68b0b745395d6eb/pendulum-3.2.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bf0b489def51202a39a2a665dcc4162d5e46934a740fe4c4fe3068979610156c", size = 337830, upload-time = "2026-01-30T11:21:08.298Z" }, - { url = "https://files.pythonhosted.org/packages/cf/98/83d62899bf7226fc12396de4bc1fb2b5da27e451c7c60790043aaf8b4731/pendulum-3.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:937a529aa302efa18dcf25e53834964a87ffb2df8f80e3669ab7757a6126beaf", size = 327574, upload-time = "2026-01-30T11:21:09.715Z" }, - { url = "https://files.pythonhosted.org/packages/76/fa/ff2aa992b23f0543c709b1a3f3f9ed760ec71fd02c8bb01f93bf008b52e4/pendulum-3.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85c7689defc65c4dc29bf257f7cca55d210fabb455de9476e1748d2ab2ae80d7", size = 339891, upload-time = "2026-01-30T11:21:11.089Z" }, - { url = "https://files.pythonhosted.org/packages/c5/4e/25b4fa11d19503d50d7b52d7ef943c0f20fd54422aaeb9e38f588c815c50/pendulum-3.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5e216e5a412563ea2ecf5de467dcf3d02717947fcdabe6811d5ee360726b02b", size = 373726, upload-time = "2026-01-30T11:21:12.493Z" }, - { url = "https://files.pythonhosted.org/packages/4f/30/0acad6396c4e74e5c689aa4f0b0c49e2ecdcfce368e7b5bf35ca1c0fc61a/pendulum-3.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a2af22eeec438fbaac72bb7fba783e0950a514fba980d9a32db394b51afccec", size = 379827, upload-time = "2026-01-30T11:21:14.08Z" }, - { url = "https://files.pythonhosted.org/packages/3a/f7/e6a2fdf2a23d59b4b48b8fa89e8d4bf2dd371aea2c6ba8fcecec20a4acb9/pendulum-3.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3159cceb54f5aa8b85b141c7f0ce3fac8bdd1ffdc7c79e67dca9133eac7c4d11", size = 348921, upload-time = "2026-01-30T11:21:15.816Z" }, - { url = "https://files.pythonhosted.org/packages/7f/f2/c15fa7f9ad4e181aa469b6040b574988bd108ccdf4ae509ad224f9e4db44/pendulum-3.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c39ea5e9ffa20ea8bae986d00e0908bd537c8468b71d6b6503ab0b4c3d76e0ea", size = 517188, upload-time = "2026-01-30T11:21:17.835Z" }, - { url = "https://files.pythonhosted.org/packages/47/c7/5f80b12ee88ec26e930c3a5a602608a63c29cf60c81a0eb066d583772550/pendulum-3.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e5afc753e570cce1f44197676371f68953f7d4f022303d141bb09f804d5fe6d7", size = 561833, upload-time = "2026-01-30T11:21:19.232Z" }, - { url = "https://files.pythonhosted.org/packages/90/15/1ac481626cb63db751f6281e294661947c1f0321ebe5d1c532a3b51a8006/pendulum-3.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:fd55c12560816d9122ca2142d9e428f32c0c083bf77719320b1767539c7a3a3b", size = 258725, upload-time = "2026-01-30T11:21:20.558Z" }, - { url = "https://files.pythonhosted.org/packages/40/ae/50b0398d7d027eb70a3e1e336de7b6e599c6b74431cb7d3863287e1292bb/pendulum-3.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:faef52a7ed99729f0838353b956f3fabf6c550c062db247e9e2fc2b48fcb9457", size = 253089, upload-time = "2026-01-30T11:21:22.497Z" }, - { url = "https://files.pythonhosted.org/packages/27/8c/400c8b8dbd7524424f3d9902ded64741e82e5e321d1aabbd68ade89e71cf/pendulum-3.2.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:addb0512f919fe5b70c8ee534ee71c775630d3efe567ea5763d92acff857cfc3", size = 337820, upload-time = "2026-01-30T11:21:24.305Z" }, - { url = "https://files.pythonhosted.org/packages/59/38/7c16f26cc55d9206d71da294ce6857d0da381e26bc9e0c2a069424c2b173/pendulum-3.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3aaa50342dc174acebdc21089315012e63789353957b39ac83cac9f9fc8d1075", size = 327551, upload-time = "2026-01-30T11:21:25.747Z" }, - { url = "https://files.pythonhosted.org/packages/0b/cd/f36ec5d56d55104232380fdbf84ff53cc05607574af3cbdc8a43991ac8a7/pendulum-3.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:927e9c9ab52ff68e71b76dd410e5f1cd78f5ea6e7f0a9f5eb549aea16a4d5354", size = 339894, upload-time = "2026-01-30T11:21:27.229Z" }, - { url = "https://files.pythonhosted.org/packages/aa/4e/b9a1e546519c3a92d5bc17787cea925e06a20def2ae344fa136d2fc40338/pendulum-3.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:249d18f5543c9f43aba3bd77b34864ec8cf6f64edbead405f442e23c94fce63d", size = 373766, upload-time = "2026-01-30T11:21:28.642Z" }, - { url = "https://files.pythonhosted.org/packages/ea/a6/6471ab87ae2260594501f071586a765fc894817043b7d2d4b04e2eff4f31/pendulum-3.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c644cc15eec5fb02291f0f193195156780fd5a0affd7a349592403826d1a35e", size = 379837, upload-time = "2026-01-30T11:21:30.637Z" }, - { url = "https://files.pythonhosted.org/packages/0d/79/0ba0c14e862388f7b822626e6e989163c23bebe7f96de5ec4b207cbe7c3d/pendulum-3.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:063ab61af953bb56ad5bc8e131fd0431c915ed766d90ccecd7549c8090b51004", size = 348904, upload-time = "2026-01-30T11:21:32.436Z" }, - { url = "https://files.pythonhosted.org/packages/17/34/df922c7c0b12719589d4954bfa5bdca9e02bcde220f5c5c1838a87118960/pendulum-3.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:26a3ae26c9dd70a4256f1c2f51addc43641813574c0db6ce5664f9861cd93621", size = 517173, upload-time = "2026-01-30T11:21:34.428Z" }, - { url = "https://files.pythonhosted.org/packages/87/ec/3b9e061eeee97b72a47c1434ee03f6d85f0284d9285d92b12b0fff2d19ac/pendulum-3.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:2b10d91dc00f424444a42f47c69e6b3bfd79376f330179dc06bc342184b35f9a", size = 561744, upload-time = "2026-01-30T11:21:35.861Z" }, - { url = "https://files.pythonhosted.org/packages/fd/7e/f12fdb6070b7975c1fcfa5685dbe4ab73c788878a71f4d1d7e3c87979e37/pendulum-3.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:63070ff03e30a57b16c8e793ee27da8dac4123c1d6e0cf74c460ce9ee8a64aa4", size = 258746, upload-time = "2026-01-30T11:21:37.782Z" }, - { url = "https://files.pythonhosted.org/packages/c9/b8/5abd872056357f069ae34a9b24a75ac58e79092d16201d779a8dd31386bb/pendulum-3.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:c8dde63e2796b62070a49ce813ce200aba9186130307f04ec78affcf6c2e8122", size = 253028, upload-time = "2026-01-30T11:21:39.381Z" }, - { url = "https://files.pythonhosted.org/packages/82/99/5b9cc823862450910bcb2c7cdc6884c0939b268639146d30e4a4f55eb1f1/pendulum-3.2.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c17ac069e88c5a1e930a5ae0ef17357a14b9cc5a28abadda74eaa8106d241c8e", size = 338281, upload-time = "2026-01-30T11:21:40.812Z" }, - { url = "https://files.pythonhosted.org/packages/cd/3a/64a35260f6ac36c0ad50eeb5f1a465b98b0d7603f79a5c2077c41326d639/pendulum-3.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e1fbb540edecb21f8244aebfb05a1f2333ddc6c7819378c099d4a61cc91ae93c", size = 328030, upload-time = "2026-01-30T11:21:42.778Z" }, - { url = "https://files.pythonhosted.org/packages/da/6b/1140e09310035a2afb05bb90a2b8fbda9d3222e03b92de9533123afe6b65/pendulum-3.2.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8c67fb9a1fe8fc1adae2cc01b0c292b268c12475b4609ff4aed71c9dd367b4d", size = 340206, upload-time = "2026-01-30T11:21:44.148Z" }, - { url = "https://files.pythonhosted.org/packages/52/4a/a493de56cbc24a64b21ac6ba98513a9ec5c67daa3dba325e39a8e53f30d8/pendulum-3.2.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:baa9a66c980defda6cfe1275103a94b22e90d83ebd7a84cc961cee6cbd25a244", size = 373976, upload-time = "2026-01-30T11:21:45.56Z" }, - { url = "https://files.pythonhosted.org/packages/3c/4c/f083c4fd1a161d4ab218680cc906338c541497b3098373f2241f58c429cb/pendulum-3.2.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef8f783fa7a14973b0596d8af2a5b2d90858a55030e9b4c6885eb4284b88314f", size = 380075, upload-time = "2026-01-30T11:21:46.959Z" }, - { url = "https://files.pythonhosted.org/packages/57/b6/333a0fcb33bf15eb879a46a11ce6300c1698a141e689665fe430783ff8d6/pendulum-3.2.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7d2e9bfb065727d8676e7ada3793b47a24349500a5e9637404355e482c822be", size = 349026, upload-time = "2026-01-30T11:21:48.271Z" }, - { url = "https://files.pythonhosted.org/packages/43/1a/dfb526ec0cba1e7cd6a5e4f4dd64a6ada7428d1449c54b15f7b295f6e122/pendulum-3.2.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:55d7ba6bb74171c3ee409bf30076ee3a259a3c2bb147ac87ebb76aaa3cf5d3a2", size = 517395, upload-time = "2026-01-30T11:21:49.643Z" }, - { url = "https://files.pythonhosted.org/packages/c9/37/b4f2b5f1200351c4869b8b46ad5c21019e3dbe0417f5867ae969fad7b5fe/pendulum-3.2.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:a50d8cf42f06d3d8c3f8bb2a7ac47fa93b5145e69de6a7209be6a47afdd9cf76", size = 561926, upload-time = "2026-01-30T11:21:51.698Z" }, - { url = "https://files.pythonhosted.org/packages/a0/9e/567376582da58f5fe8e4f579db2bcfbf243cf619a5825bdf1023ad1436b3/pendulum-3.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:e5bbb92b155cd5018b3cf70ee49ed3b9c94398caaaa7ed97fe41e5bb5a968418", size = 258817, upload-time = "2026-01-30T11:21:53.074Z" }, - { url = "https://files.pythonhosted.org/packages/95/67/dfffd7eb50d67fa821cd4d92cf71575ead6162930202bc40dfcedf78c38c/pendulum-3.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:d53134418e04335c3029a32e9341cccc9b085a28744fb5ee4e6a8f5039363b1a", size = 253292, upload-time = "2026-01-30T11:21:54.484Z" }, - { url = "https://files.pythonhosted.org/packages/02/fb/d65db067a67df7252f18b0cb7420dda84078b9e8bfb375215469c14a50be/pendulum-3.2.0-py3-none-any.whl", hash = "sha256:f3a9c18a89b4d9ef39c5fa6a78722aaff8d5be2597c129a3b16b9f40a561acf3", size = 114111, upload-time = "2026-01-30T11:22:22.361Z" }, -] - [[package]] name = "platformdirs" version = "4.9.6" @@ -2953,15 +2540,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] -[[package]] -name = "ply" -version = "3.11" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e5/69/882ee5c9d017149285cab114ebeab373308ef0f874fcdac9beb90e0ac4da/ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3", size = 159130, upload-time = "2018-02-15T19:01:31.097Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce", size = 49567, upload-time = "2018-02-15T19:01:27.172Z" }, -] - [[package]] name = "polyglot-sql-chio" version = "0.13.10" @@ -3808,18 +3386,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, ] -[[package]] -name = "requirements-parser" -version = "0.13.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/89/1a/5f3c22d38bf1d87d1f4a961489d9eba35c4370a21395562d94410cdd0e73/requirements_parser-0.13.1.tar.gz", hash = "sha256:78811383b2089b6c5197a1431bc2c12ff950245edca39a23eea3460782038dd3", size = 22783, upload-time = "2026-06-18T07:52:25.291Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/f9/15b44d5e4401b0013bbcefe3c09d7bfddcce28cc3d41b1d3077bcedf5b1f/requirements_parser-0.13.1-py3-none-any.whl", hash = "sha256:6e385663eb32589d16e5b22bb6e5251a57908e73803ffff438b53cd6ea2056e0", size = 14926, upload-time = "2026-06-18T07:52:24.171Z" }, -] - [[package]] name = "rich" version = "15.0.0" @@ -3833,18 +3399,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, ] -[[package]] -name = "rich-argparse" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "rich" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6a/e5/1064c43203a357d668cd42435f7a15fe6af51512d85b2104fecb937aa861/rich_argparse-1.8.0.tar.gz", hash = "sha256:679df3d832fa94ad6e4bdb07ded088cd7ea2dddc58ae9b2b46346a40b06cbc0c", size = 38940, upload-time = "2026-05-01T15:18:43.604Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/35/1cceccc5fcb50fa2ed53e2aa278cd032f3902682a73e763fb1ac3be8e6fa/rich_argparse-1.8.0-py3-none-any.whl", hash = "sha256:d2a3ce7854654e2253c578763ab0a32f05016f23a55fadba7b9a91b6c0e92142", size = 25616, upload-time = "2026-05-01T15:18:42.395Z" }, -] - [[package]] name = "rivers" version = "0.1.4" @@ -3972,20 +3526,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/b6/aeadee5443e49baa2facd51131159fd6301cc4ccfc1541e4df7b021c37dd/ruff-0.15.11-py3-none-win_arm64.whl", hash = "sha256:063fed18cc1bbe0ee7393957284a6fe8b588c6a406a285af3ee3f46da2391ee4", size = 11032614, upload-time = "2026-04-16T18:46:34.487Z" }, ] -[[package]] -name = "s3fs" -version = "2026.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiobotocore" }, - { name = "aiohttp" }, - { name = "fsspec" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cb/d8/76f3dc1558bdf4494b117a9f7a9cc0a5d9d34edadc9e5d7ceabc5a6a7c37/s3fs-2026.4.0.tar.gz", hash = "sha256:5bdce0abb00b0435ee150807a45fea727451dbc22de4cbc116464f8504ab9d37", size = 85986, upload-time = "2026-04-29T20:52:51.748Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a4/9d1ea10ebc9e028a289a72fec84da170689549a8102c8aacfcad26bc5035/s3fs-2026.4.0-py3-none-any.whl", hash = "sha256:de0d2a1f33cdf03831fd2382d278c6e4e31fe57c3bf2f703c61f8aec6b703e2a", size = 32392, upload-time = "2026-04-29T20:52:50.295Z" }, -] - [[package]] name = "s3transfer" version = "0.16.0" @@ -4011,15 +3551,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, ] -[[package]] -name = "semver" -version = "3.0.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/d1/d3159231aec234a59dd7d601e9dd9fe96f3afff15efd33c1070019b26132/semver-3.0.4.tar.gz", hash = "sha256:afc7d8c584a5ed0a11033af086e8af226a9c0b206f313e0301f8dd7b6b589602", size = 269730, upload-time = "2025-01-24T13:19:27.617Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl", hash = "sha256:9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746", size = 17912, upload-time = "2025-01-24T13:19:24.949Z" }, -] - [[package]] name = "setuptools" version = "82.0.1" @@ -4038,59 +3569,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] -[[package]] -name = "simplejson" -version = "4.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0e/2a/54837395a3487c725669428d513293612a48d82b95a0642c936932e5d898/simplejson-4.1.1.tar.gz", hash = "sha256:c08eb9f7a90f77ae470e19a07472e9a79ebc0d1c2315d86a72767665bd5ba79f", size = 118860, upload-time = "2026-04-24T19:24:59.819Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/60/25/e90998fe8e480eb43b966c09e835379887d427567ebd496563d3b1e16b19/simplejson-4.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:19040a17154dc03d289bab68d73ce0a6a0be01de30c584bbdd93490bead14b22", size = 112414, upload-time = "2026-04-24T19:23:06.084Z" }, - { url = "https://files.pythonhosted.org/packages/9c/a0/abd4785f36c3400f1fbb21f517be39295a750a714f04b7ee175adf6ef580/simplejson-4.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a94ebaecdbaa80d9551a3ec6bf0c9302fc8b53ab6c1b2bfd498a1df4cb28158d", size = 91120, upload-time = "2026-04-24T19:23:07.877Z" }, - { url = "https://files.pythonhosted.org/packages/b8/78/fc060d2e3b13c6ec59288574b8efac64075e316b2afba4396a56b2422f78/simplejson-4.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:67341c95c0a168ab4a6d1e807e50463f1c8da932c3286d81e201266c427061fa", size = 91055, upload-time = "2026-04-24T19:23:09.264Z" }, - { url = "https://files.pythonhosted.org/packages/0c/b6/156a8de1e1b47694f0e7de6675866936608d45dc68388fd017d36f8693be/simplejson-4.1.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:45ec18e337fec538b7e902d489505c450b2454653d1290f3f50385e6fd8aa607", size = 190297, upload-time = "2026-04-24T19:23:11.226Z" }, - { url = "https://files.pythonhosted.org/packages/86/1c/e4d0eab695be3eb21d0f46bce820752031f03e7113f9c80a9b3c73ee7157/simplejson-4.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:820c69a4710400e9b248d5670647d60be58824369282d3925e516b3ff1a7cd82", size = 187002, upload-time = "2026-04-24T19:23:12.982Z" }, - { url = "https://files.pythonhosted.org/packages/76/0e/7f5a59d29426b062d5928fb88b403c3f797129d53be7102f955dbe51aa44/simplejson-4.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e708d373a10e4378ef2d59f8361850c7150fd907ed49efe49bc5492160476d1", size = 195146, upload-time = "2026-04-24T19:23:14.517Z" }, - { url = "https://files.pythonhosted.org/packages/78/18/9943db224dd4d5fa3c090c3e56a94c37b254338c83995ec5680285111c40/simplejson-4.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:980fc33353f81fd12d8c49d44f8c2760d1dc8192285e627c5180d141035b228a", size = 183931, upload-time = "2026-04-24T19:23:16.742Z" }, - { url = "https://files.pythonhosted.org/packages/c2/08/9a690da9a766161c06c627d805362cf159f1abe480969372b2897649b955/simplejson-4.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:de2ed102fff88dacf543699f53ee3a533cc11539a39baa176b7e09dd783069d6", size = 192228, upload-time = "2026-04-24T19:23:18.33Z" }, - { url = "https://files.pythonhosted.org/packages/05/88/bd8aad36b451ffb0e0a3f721d695a88befa6d1ac7d1e02ae788ca7ff4029/simplejson-4.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2785ff8edc0e28bf773a32543a6bbed46351453c997b3f6709c744e3c2f7eabb", size = 187808, upload-time = "2026-04-24T19:23:21.165Z" }, - { url = "https://files.pythonhosted.org/packages/04/ee/14f91db0d1f481533b651dafbf8cd0da088d9817f7af30c68f7f19f9c847/simplejson-4.1.1-cp312-cp312-win32.whl", hash = "sha256:2e0d5ead6d14610467ec356ec1f6b5d8a56aa216abaad8d41c8b873b16cf313f", size = 88512, upload-time = "2026-04-24T19:23:22.764Z" }, - { url = "https://files.pythonhosted.org/packages/b9/c4/90de06b2d8737c68c05ff9274113f854dbf6a5f28b7a955212111672cb57/simplejson-4.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:63a5451f557d6be48a231bae932458655c620902b868170b2f1c8afed496f6b4", size = 90748, upload-time = "2026-04-24T19:23:24.494Z" }, - { url = "https://files.pythonhosted.org/packages/37/a9/47b445eeb559c9593453a0648e0fd6d08e8adff64dd5e5ced66726da8a09/simplejson-4.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dff52fc7af272e84fc21cc5a06c927c823ca6ae00af14f3b0d7707b42775ed98", size = 113160, upload-time = "2026-04-24T19:23:26.033Z" }, - { url = "https://files.pythonhosted.org/packages/4c/65/cb72db31523c164dea5dc55b02dad065a40c478856bc7534b279d2b51906/simplejson-4.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:971aed0647ad6e840a3943bec812fcda5f2d26a5497a4981d1fb49aa4f9a396c", size = 91521, upload-time = "2026-04-24T19:23:27.572Z" }, - { url = "https://files.pythonhosted.org/packages/9a/e5/54cb7c50ad5fdc1e0a86b7df4b135c2cbd5c4623605aa94466659098e8da/simplejson-4.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:249e2e220aa6d9b9d936bde84eb7bf79d5b6c5a8273c6e411f8b1635a9073f2d", size = 91407, upload-time = "2026-04-24T19:23:28.991Z" }, - { url = "https://files.pythonhosted.org/packages/38/2e/21a3ede87f0bf82d6c7bcb90480d50a6490eb974c6ab20881188e440957c/simplejson-4.1.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e5cdd6a5d52299f345c15ab5678cc4249e24f383f361d986afbc3c7072a6b6b", size = 192451, upload-time = "2026-04-24T19:23:30.56Z" }, - { url = "https://files.pythonhosted.org/packages/59/df/9903edd3102bf0b5984edfcb90c88612330996efa3b4fbf8a971d6e17839/simplejson-4.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:642cec364e0676e2d5a73fa4d31d0c7c55886997caa2fde24e8292ca44d32728", size = 189015, upload-time = "2026-04-24T19:23:32.647Z" }, - { url = "https://files.pythonhosted.org/packages/98/cd/33230927a780e1398b857e3944abb914556994d252b1d765ae40d112cb25/simplejson-4.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:76fe296ca1df23d290033f10aaacf534fd1b3e3007e7f9ff8aa68b21413aaa78", size = 196658, upload-time = "2026-04-24T19:23:34.563Z" }, - { url = "https://files.pythonhosted.org/packages/cd/84/2c5a7444eb53e9a86d3738299bffddd9f53aeed799ded2f45368221fdb19/simplejson-4.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f0ad25b7dc4e0fb23858355819f2e994f1a5badcdcde8737eac7921c2f1ed2a", size = 185967, upload-time = "2026-04-24T19:23:36.191Z" }, - { url = "https://files.pythonhosted.org/packages/d3/68/454378e06d059cd412a7ed5d87fb6d29fd5b60f13a4d89fc1f764ff434df/simplejson-4.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a59ebd0533f03fd06ff0c42ba0f02d93cbcdd7944922bf3b93911327a95b901f", size = 193940, upload-time = "2026-04-24T19:23:38.151Z" }, - { url = "https://files.pythonhosted.org/packages/d5/d5/a15bf915f623a2c5a079d6e3be8256fdb8ef06f110669493a09b9d6933e0/simplejson-4.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bccbf4419676b517939852e5aeff2af6aee4dc046881c67a1581fa6f1cb01abd", size = 189795, upload-time = "2026-04-24T19:23:40.139Z" }, - { url = "https://files.pythonhosted.org/packages/d2/c9/37212ae7dc4b607f0978c408e8633f05c810884e054c33113184c6c2c8a2/simplejson-4.1.1-cp313-cp313-win32.whl", hash = "sha256:6c845363eb5fd166fb7c72243da38f4fcfde666ede7fdf2cc6fd7762894626f7", size = 88773, upload-time = "2026-04-24T19:23:41.754Z" }, - { url = "https://files.pythonhosted.org/packages/fe/a5/c7a0a47883a9015b54c9d8a4b62f2aba17bd4335b1787b9b8a0fc2fa6d52/simplejson-4.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:104d8324c34f25b4b90800bc5fa363780cbc3d8496aef061cba7ce1af9162270", size = 90888, upload-time = "2026-04-24T19:23:43.11Z" }, - { url = "https://files.pythonhosted.org/packages/d3/18/4a118a6a92eb33bb08c8e2fe7ec85cb96f0673491bb2b829930831ee4fbe/simplejson-4.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:ed7473602b6625de793b6acba49aa949f144a475f538792067e4cf2fda2071f5", size = 110492, upload-time = "2026-04-24T19:23:44.957Z" }, - { url = "https://files.pythonhosted.org/packages/07/f4/84d160e9fa8cada1e0a9381cae4fa81eecd573577a5b34366d8ced59bdf7/simplejson-4.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:225c9caa324c5b554d009fb9cac22aee7711e71bd96f487938c659af467e828e", size = 90152, upload-time = "2026-04-24T19:23:46.355Z" }, - { url = "https://files.pythonhosted.org/packages/68/31/9a5432c433a7671107182cdc9a20ea78a70f99c4e5334aa54b6d4d0d79ed/simplejson-4.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:95407269340c7f22f09776ea7b717a52cf56cfcf119b5e45f66faa4a26445bea", size = 90115, upload-time = "2026-04-24T19:23:47.743Z" }, - { url = "https://files.pythonhosted.org/packages/78/91/3635cdb13318cb0a328abaa69e2b91251caad39d6779aa308098f341f6cb/simplejson-4.1.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3851658d642c1184d2023f0e6c9ce44a21eb1629e74e7c84ef956b128841fe12", size = 184036, upload-time = "2026-04-24T19:23:49.472Z" }, - { url = "https://files.pythonhosted.org/packages/fa/ba/149b6ec5393f6849d98c59cadba888b710a8ef4b805ab91e11a566960d40/simplejson-4.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:95a3bb0f78e85f4937f99092239f2011ce06f0f2d803df5c299cc05abbeae008", size = 180543, upload-time = "2026-04-24T19:23:51.023Z" }, - { url = "https://files.pythonhosted.org/packages/df/7c/a5d968d0b527a748b667e62bea94309ccbcb1e2b108e8f0cf8547efaa12b/simplejson-4.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bbfdaa7c0603f75b7b14b211b7f2be44696d4e26833ad2d91d5c87bf5fb9a920", size = 188725, upload-time = "2026-04-24T19:23:52.995Z" }, - { url = "https://files.pythonhosted.org/packages/db/e3/6a8d11181d587ef00e2db9112357e6832111e56dd56b01b5c11758a1965d/simplejson-4.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:39e3c584071dced8c21b4689f0254303521daeb9b5bc1f4289755d71fa3cb0d3", size = 177492, upload-time = "2026-04-24T19:23:54.581Z" }, - { url = "https://files.pythonhosted.org/packages/67/e3/8b0eb8b06e8198cfbd1270487da163d0093df05cc4f557350cd65e2f7e79/simplejson-4.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:036a27bd0469b9d79557cbddb392969f876cd7f278cfbd0fba81534927a06575", size = 185281, upload-time = "2026-04-24T19:23:56.13Z" }, - { url = "https://files.pythonhosted.org/packages/dc/5f/64990f07ec9e2cb1a814c674e2e21b5693207f74ac70eb72151b847ea4e6/simplejson-4.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b70bfd2f67f3351baba08aa3ae9233c83f21fd95ae5e6b3d0ecb8c647929112f", size = 181848, upload-time = "2026-04-24T19:23:57.92Z" }, - { url = "https://files.pythonhosted.org/packages/61/a5/bbc1bc0447f339f79f99ab8c37f7f037cb2f1f93af75d6a4d553096bb0c3/simplejson-4.1.1-cp314-cp314-win32.whl", hash = "sha256:37233c72ce88d06acb92747347742b3c07871eba6789f060c179c9302dde8efe", size = 88761, upload-time = "2026-04-24T19:23:59.397Z" }, - { url = "https://files.pythonhosted.org/packages/18/72/ec1b5cbdcb140c132e6c7bdf99bd73e4f675439e77126c88f472fcffa09c/simplejson-4.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:cc0442dea71cd9cbf30a0b8b9929ab5aa6c02c0443a3d977351e6ec5bada4388", size = 91018, upload-time = "2026-04-24T19:24:00.85Z" }, - { url = "https://files.pythonhosted.org/packages/3d/97/4fa437f68ff72219bac3bf3d050de9c6265691f3a170e16954bd69d7cddd/simplejson-4.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:c996a4d38290c515af347740659ce095b425449c164a5c9fa3977caa6eff5dbe", size = 113919, upload-time = "2026-04-24T19:24:02.287Z" }, - { url = "https://files.pythonhosted.org/packages/c2/83/59de041d09eb4a9577f7015d7263c32095dfb7fde49717dff62145d89809/simplejson-4.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c65c763fb20d7ca113c1c14dce2fc04a0fc3a57aceff533d6fdac707c7bffb40", size = 91904, upload-time = "2026-04-24T19:24:03.812Z" }, - { url = "https://files.pythonhosted.org/packages/03/8e/46bb345d540f6eb31427d984a4e518cdb182d0621814fee4fee045e8815b/simplejson-4.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0da5c9f57206ee7ef280ff7f1d924937b0a64f9a271a5ef371a2ecdbebba7421", size = 91752, upload-time = "2026-04-24T19:24:05.622Z" }, - { url = "https://files.pythonhosted.org/packages/83/e2/1b2ce97f068835eb3d253c116a4df7a3f436b7bf2fb5ff1ba29287e8b0ec/simplejson-4.1.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ea3426e786425d10e9e82f8a6eda74a7d6eb10d99165ac3d0d3bbcb65c0ea343", size = 214021, upload-time = "2026-04-24T19:24:07.447Z" }, - { url = "https://files.pythonhosted.org/packages/48/70/d93e556df6a0786298644a7c08304fcbeddc248325f23f38acbebeb21165/simplejson-4.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d75cea7a1025edd7e439b2966b3d977c45b5b899e2adaf422811b3ac702ed9fb", size = 213530, upload-time = "2026-04-24T19:24:09.289Z" }, - { url = "https://files.pythonhosted.org/packages/1b/a5/c93bf305b9f00d7259e09e713d60e75bd0f7f53da970f716ab90491770e7/simplejson-4.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63c2ada8e58f266491f19eed2eeeb7c25c6141e52f8f9e820f6bb94156cf8dbc", size = 218282, upload-time = "2026-04-24T19:24:10.991Z" }, - { url = "https://files.pythonhosted.org/packages/0c/20/a9b5d2e27ec44b069ee251bd55544fc76929a067107b1050001566ba86f3/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d1fffb56305c5b475ee746cf9e04f97423ba5aaacd292dc1255bd75b1d3b124b", size = 209249, upload-time = "2026-04-24T19:24:12.662Z" }, - { url = "https://files.pythonhosted.org/packages/97/e4/e06ee682ed5df67592181f5ecb062e35878967e27f5b6e087237d4548d95/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:a6525ec733f43d0541206cffa64fd2aad5a7ae3eb76566aff49cd4db6382209a", size = 213963, upload-time = "2026-04-24T19:24:14.302Z" }, - { url = "https://files.pythonhosted.org/packages/9c/9f/1e160e4cd8cdbf062bf6a454cdf814dc7a48eb47e566fdb8f80ccb202605/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:861e393260508efa64d8805a8e49c416c3484907e3f146ce966c69552b49b9a3", size = 210474, upload-time = "2026-04-24T19:24:15.917Z" }, - { url = "https://files.pythonhosted.org/packages/7a/e6/cecd913df322df5bbe7ebb8ba39e0708e505a165553900da8a7761026d6f/simplejson-4.1.1-cp314-cp314t-win32.whl", hash = "sha256:d083b89d30948a751d3d97476c2ed91e4caaa24a1a1459bdbadb8876242c71fe", size = 91134, upload-time = "2026-04-24T19:24:17.635Z" }, - { url = "https://files.pythonhosted.org/packages/97/73/f540dde99cc1d393bd062ab3b5735b777561a5d8f8a5f2e241164444d77a/simplejson-4.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4cbb299d0528ec0447fe366d8c9641860e28f997a62730690fef905f1f41046e", size = 94467, upload-time = "2026-04-24T19:24:19.109Z" }, - { url = "https://files.pythonhosted.org/packages/ce/6a/8b74c52ffd33dbbde00fe7251fee6a0acdc8cea33f7a43805aed258fb79b/simplejson-4.1.1-py3-none-any.whl", hash = "sha256:2ce92b3748f02423e26d2bfb636fb9d7a8f67c8f5854dcae69d350d123b2eee2", size = 69195, upload-time = "2026-04-24T19:24:57.962Z" }, -] - [[package]] name = "six" version = "1.17.0" @@ -4100,15 +3578,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] -[[package]] -name = "smmap" -version = "5.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1f/ea/49c993d6dfdd7338c9b1000a0f36817ed7ec84577ae2e52f890d1a4ff909/smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c", size = 22506, upload-time = "2026-03-09T03:43:26.1Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, -] - [[package]] name = "sniffio" version = "1.3.1" @@ -4264,13 +3733,6 @@ databricks = [ dbt = [ { name = "jinja2" }, ] -dlt = [ - { name = "dlt", extra = ["duckdb", "filesystem", "sql-database"] }, -] -ingestr = [ - { name = "dbc" }, - { name = "ingestr" }, -] postgres = [ { name = "psycopg", extra = ["binary"] }, ] @@ -4287,7 +3749,6 @@ sqlserver = [ [package.dev-dependencies] dev = [ { name = "commitizen" }, - { name = "dbc" }, { name = "dbt-bigquery" }, { name = "dbt-core" }, { name = "dbt-databricks" }, @@ -4295,9 +3756,7 @@ dev = [ { name = "dbt-postgres" }, { name = "dbt-snowflake" }, { name = "dbt-sqlserver" }, - { name = "dlt", extra = ["duckdb", "filesystem", "sql-database"] }, { name = "fensu" }, - { name = "ingestr" }, { name = "jsonschema" }, { name = "maturin" }, { name = "pre-commit" }, @@ -4315,12 +3774,9 @@ requires-dist = [ { name = "dagster", marker = "extra == 'dagster'", specifier = ">=1.13.0" }, { name = "dagster-webserver", marker = "extra == 'dagster'", specifier = ">=1.13.0" }, { name = "databricks-sql-connector", extras = ["pyarrow"], marker = "extra == 'databricks'", specifier = ">=4.1.2" }, - { name = "dbc", marker = "extra == 'ingestr'", specifier = ">=0.3.1" }, - { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], marker = "extra == 'dlt'", specifier = ">=1.28.0,<2" }, { name = "duckdb", specifier = ">=2.0.0" }, { name = "filelock", specifier = ">=3.20.3" }, { name = "google-cloud-bigquery", marker = "extra == 'bigquery'", specifier = ">=3.37.0" }, - { name = "ingestr", marker = "extra == 'ingestr'", specifier = ">=1.1.73" }, { name = "jinja2", marker = "extra == 'dbt'", specifier = ">=3.1" }, { name = "orjson", specifier = ">=3.11.7" }, { name = "polyglot-sql-chio", specifier = "==0.13.10" }, @@ -4332,12 +3788,11 @@ requires-dist = [ { name = "snowflake-connector-python", marker = "extra == 'snowflake'", specifier = ">=4.7.1,<5" }, { name = "sqlbuild-cli", directory = "crates/sqlbuild-cli" }, ] -provides-extras = ["dbt", "snowflake", "bigquery", "databricks", "dagster", "postgres", "sqlserver", "clickhouse", "ingestr", "dlt", "rivers"] +provides-extras = ["dbt", "snowflake", "bigquery", "databricks", "dagster", "postgres", "sqlserver", "clickhouse", "rivers"] [package.metadata.requires-dev] dev = [ { name = "commitizen", specifier = ">=4.9.1" }, - { name = "dbc", specifier = ">=0.3.1" }, { name = "dbt-bigquery", specifier = ">=1.9.0,<1.11" }, { name = "dbt-core", specifier = ">=1.11.6,<1.12" }, { name = "dbt-databricks", specifier = ">=1.9.0,<1.11" }, @@ -4345,9 +3800,7 @@ dev = [ { name = "dbt-postgres", specifier = ">=1.9.0,<1.11" }, { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, - { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, { name = "fensu", specifier = ">=0.31.1" }, - { name = "ingestr", specifier = ">=1.1.73" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" }, { name = "pre-commit", specifier = ">=4.3.0" }, @@ -4363,15 +3816,6 @@ dev = [ name = "sqlbuild-cli" source = { directory = "crates/sqlbuild-cli" } -[[package]] -name = "sqlglot" -version = "30.11.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/de/1542b04dcac0f85706fb8ce1ce7d2abcc230cdf10ea9e3aa7345393e5a90/sqlglot-30.11.0.tar.gz", hash = "sha256:1a23c6e2adb41da61fda46b1848d2fa26341d447fc0f0cd5ca21160362100991", size = 5893125, upload-time = "2026-06-11T17:11:37.845Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/86/53edf106e8cd3c883ccd0c6b470bf00ddf877a86e667665343b2d597329d/sqlglot-30.11.0-py3-none-any.whl", hash = "sha256:cffdee57d1f2f5472dc9f13087e618cf795841172b7d5ef78b63a051a52d2710", size = 698721, upload-time = "2026-06-11T17:11:35.737Z" }, -] - [[package]] name = "sqlparams" version = "6.2.0" @@ -4867,16 +4311,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" }, ] -[[package]] -name = "win-precise-time" -version = "1.4.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9e/b0/21547e16a47206ccdd15769bf65e143ade1ffae67f0881c855f76e44e9fa/win-precise-time-1.4.2.tar.gz", hash = "sha256:89274785cbc5f2997e01675206da3203835a442c60fd97798415c6b3c179c0b9", size = 7982, upload-time = "2023-10-08T17:08:18.618Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/96/55a14b5c0e90439951f4a72672223bba81a5f882033c5850f8a6c7f4308b/win_precise_time-1.4.2-cp312-cp312-win32.whl", hash = "sha256:0210dcea88a520c91de1708ae4c881e3c0ddc956daa08b9eabf2b7c35f3109f5", size = 14694, upload-time = "2023-10-08T17:08:09.275Z" }, - { url = "https://files.pythonhosted.org/packages/17/19/7ea9a22a69fc23d5ca02e8edf65e4a335a210497794af1af0ef8fda91fa0/win_precise_time-1.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:85670f77cc8accd8f1e6d05073999f77561c23012a9ee988cbd44bb7ce655062", size = 14913, upload-time = "2023-10-08T17:08:10.677Z" }, -] - [[package]] name = "wrapt" version = "2.1.2" diff --git a/website/public/dlt-declarative.png b/website/public/dlt-declarative.png deleted file mode 100644 index 795e810d23..0000000000 Binary files a/website/public/dlt-declarative.png and /dev/null differ diff --git a/website/public/ingestr-example.png b/website/public/ingestr-example.png deleted file mode 100644 index f23f075765..0000000000 Binary files a/website/public/ingestr-example.png and /dev/null differ diff --git a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx index c8401bcb8c..921d16dac0 100644 --- a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx +++ b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx @@ -85,6 +85,3 @@ model SQL as written: rejects later non-null values. Cast it, for example `CAST(NULL AS DECIMAL(18, 2)) AS discount`. - Integer division `//` and modulo `%` by zero raise an error instead of returning `NULL`. Wrap the expression in `TRY(...)` to get `NULL`. - -dlt's dataset read API calls a DuckDB method that DuckDB 2.0 removed when it reads a loaded table -back as one whole Arrow table, so that read fails. Dataframe and chunked Arrow reads still work. Loading data with dlt is not affected. diff --git a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx index 594b94f6fe..bbde5ffe3b 100644 --- a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx +++ b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx @@ -96,7 +96,7 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly | `sqb diff`, `sqb clone` | Supported; clones are physical copies | | Scenarios, `sqb scenario capture`, `--local` replay | Supported; see below | | Statement time limits | Connection interrupt | -| `sqb load`: Python, dlt and ingestr loaders | Supported; dlt writes through its `sqlalchemy` destination | +| `sqb load`: Python loaders | Supported | ## Scenarios diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index 29ae89ce89..ce135807da 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -63,7 +63,7 @@ error[P010]: model 'orders_summary' has no description | Model | `MODEL (description "...")`, or the `description` of its `model_schema` | | Scenario | `SCENARIO (description "...")` | | Seed | `description:` on its entry in `seeds/*.yml` | -| Source | `description:` on its entry in `sources/*.yml` (or on the dlt resource) | +| Source | `description:` on its entry in `sources/*.yml` | | SQL function or table function | `FUNCTION (description "...")` | | Python UDF | `@udf(description="...")` or the function docstring | | Named SQL hook | `HOOK (description "...")` | diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index 124ad222de..df12388f1b 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -217,7 +217,7 @@ sources: type: VARCHAR ``` -Self-managed loaders must not declare a `write_strategy`. They are useful when you want to use adapter-specific SQL (e.g. `COPY INTO`, external tables), call an external ingestion tool like [dlt](/docs/integrations/dlt), or handle writes in a way that doesn't fit the dict-return pattern. +Self-managed loaders must not declare a `write_strategy`. They are useful when you want to use adapter-specific SQL (e.g. `COPY INTO`, external tables), call an external ingestion tool, or handle writes in a way that doesn't fit the dict-return pattern. ## Loader context diff --git a/website/src/content/docs/docs/concepts/sources.mdx b/website/src/content/docs/docs/concepts/sources.mdx index 7c143b2ac4..172f718ea5 100644 --- a/website/src/content/docs/docs/concepts/sources.mdx +++ b/website/src/content/docs/docs/concepts/sources.mdx @@ -114,13 +114,6 @@ Managed sources support incremental write strategies (`table`, `append`, `delete See [Loaders](/docs/concepts/python-nodes/loaders) for the full guide on writing loader functions, write strategies, the loader context API, and auto-load behavior during builds. -### Declarative integrations (no Python) - -For common ingestion you can declare the source entirely in YAML, with no `@loader` function. SQLBuild generates the loader for you and runs it during the build: - -- [dlt](/docs/integrations/dlt) - declare `dlt_sources` for `rest_api`, `sql_database`, and `filesystem` sources. -- [ingestr](/docs/integrations/ingestr) - add an `ingestr` block to a source to pull from 50+ sources. - ## Source freshness Source freshness lets SQLBuild observe whether a source's data changed between runs. This feeds into @@ -280,11 +273,10 @@ without building. | `audits` | Source-level audits | | `meta` | Free-form metadata mapping | | `load_batch_size` | Positive row batch size for managed loaders | -| `ingestr` | Declarative ingestr loader config (see [ingestr](/docs/integrations/ingestr)) | -A sources file contains only `sources` and `dlt_sources`. Unknown keys in a sources file, a source, -a source column, `freshness`, `freshness.age_policy`, or `ingestr` fail discovery with `D006`, -naming the key and suggesting the nearest supported key. +A sources file contains only `sources`. Unknown keys in a sources file, a source, a source column, +`freshness`, or `freshness.age_policy` fail discovery with `D006`, naming the key and suggesting the +nearest supported key. ## YAML that SQLBuild reads diff --git a/website/src/content/docs/docs/feature-comparison.mdx b/website/src/content/docs/docs/feature-comparison.mdx index 71b4674b3d..7e6778372b 100644 --- a/website/src/content/docs/docs/feature-comparison.mdx +++ b/website/src/content/docs/docs/feature-comparison.mdx @@ -104,7 +104,6 @@ SQLBuild, dbt, and SQLMesh are all SQL pipeline frameworks. They share common gr | Feature | SQLBuild | dbt | SQLMesh | |---------|----------|-----|---------| | Source loaders | Python `@loader` functions with table/append/delete_insert/merge strategies | No (external to dbt) | No (external to SQLMesh) | -| Declarative ingestion | dlt and ingestr integrations - YAML-only source config, no Python | No | No | | Auto-load during builds | Managed sources loaded before dependent models | No | No | | Source deferral | `--defer-sources-to` reads source data from another target | No | No | diff --git a/website/src/content/docs/docs/integrations/dlt.mdx b/website/src/content/docs/docs/integrations/dlt.mdx deleted file mode 100644 index 75cf8e5121..0000000000 --- a/website/src/content/docs/docs/integrations/dlt.mdx +++ /dev/null @@ -1,232 +0,0 @@ ---- -title: "dlt" -description: "Declarative dlt sources in YAML, or full dlt pipelines inside Python source loaders." ---- - -import Frame from '../../../../components/mintlify/Frame.astro'; - -[dlt](https://dlthub.com) is an open-source Python library for loading data from APIs, databases, cloud storage, and more. SQLBuild integrates with dlt two ways: - -- **Declarative dlt sources** - configure dlt directly in your source YAML, no Python code. SQLBuild runs the dlt pipeline as part of the build lifecycle. Best for the common REST API, SQL database, and filesystem cases. -- **dlt inside a Python loader** - wrap any `dlt.pipeline(...)` in a SQLBuild [source loader](/docs/concepts/python-nodes/loaders) for full control when you need dlt features beyond the declarative surface. - -## Install - -```bash -pip install 'sqlbuild[dlt]' -# or -uv pip install 'sqlbuild[dlt]' -``` - -This installs dlt with the `duckdb`, `filesystem`, and `sql_database` extras alongside SQLBuild. - -## Declarative dlt sources (YAML) - -Declare a `dlt_sources` list in a `sources/*.yml` file. Each entry has a `type`, a `config` block passed to the dlt source, and a list of `resources` that become SQLBuild managed sources. SQLBuild generates a synthetic loader per resource and writes into the database its adapter manages, no Python and no destination setup required. - -Supported source types: `rest_api`, `sql_database`, `filesystem`. - - - sqb build loading three declarative dlt sources (raw_customers, raw_orders, raw_payments) as external (dlt) sources, each showing dlt pipeline extract/normalize/load progress before the models build - - -### REST API - -```yaml -# sources/github.yml -dlt_sources: - - type: rest_api - config: - client: - base_url: https://api.github.com/ - headers: - Authorization: "Bearer ${github_token}" - paginator: - type: header_link - resources: - - name: raw_github_issues - description: Issues from the GitHub API - write_disposition: append - endpoint: - path: "repos/${github_owner}/${github_repo}/issues" - params: - state: all - per_page: 100 -``` - -`rest_api` requires `client.base_url` in `config`. Each resource needs an `endpoint` mapping; the dlt resource name comes from `endpoint.name` if present, otherwise the resource `name`. - -### SQL database - -```yaml -# sources/postgres.yml -dlt_sources: - - type: sql_database - config: - credentials: "${postgres_connection_string}" - resources: - - name: raw_customers - description: Customers from the CRM database - table: customers - write_disposition: merge - primary_key: id -``` - -`sql_database` requires `credentials` in `config`, and each resource requires a `table`. - -### Filesystem - -```yaml -# sources/files.yml -dlt_sources: - - type: filesystem - config: - bucket_url: "s3://my-bucket/events/" - resources: - - name: raw_events - description: Events from the event export files - reader: csv - write_disposition: append -``` - -`filesystem` requires `bucket_url` in `config`. - -### Resource options - -| Key | Description | -|-----|-------------| -| `name` | Source name SQLBuild exposes (referenced via `__source("name")`). Required. | -| `description` | What the loaded rows are. Required ([Descriptions](/docs/concepts/models/configuration#descriptions)). | -| `table` | Source table to replicate (`sql_database` only). Required for that type. | -| `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | -| `reader` | File reader: `csv`, `jsonl`, or `parquet` (`filesystem` only). Required for that type. | -| `write_disposition` | dlt write disposition: `replace`, `append`, or `merge`. `merge` requires `primary_key`. | -| `primary_key` / `merge_key` | dlt keys for `merge`. A `merge_key` replaces every row whose key value arrives, like a SQL model's `delete_insert` with [`replace_key`](/docs/concepts/incremental#delete_insert). | -| `incremental` | dlt incremental config mapping (e.g. cursor column and start value). | -| `schema` | Per-resource schema override (falls back to the group `schema`). | - -A `dlt_sources` entry accepts only `type`, `config`, `destination`, `schema`, and `resources`, and each resource accepts only the keys above for its source type. Any other key fails compilation with the nearest supported key as a hint. - -Use SQLBuild's `write_disposition` (dlt's term), not `write_strategy`. `delete_insert` is not available declaratively; use a Python loader for that. Config values support SQLBuild's [interpolation](/docs/concepts/interpolation): `${name}` for a project variable, `${ENV:NAME}` for an environment variable, and `${CTX:...}` for context, resolved at load time, so credentials stay out of the YAML. - -### Destination - -By default each resource loads into the database SQLBuild's adapter manages, with the dataset/schema derived from your target. On SQLite, dlt writes through its `sqlalchemy` destination into the project's database file (dataset `main`); the `sqlbuild[dlt]` extra installs the SQLAlchemy it needs. An optional `destination` mapping passes extra settings to the dlt destination, but it cannot set `credentials`, `dataset_name`, or `default_schema_name` (SQLBuild manages those): - -```yaml -dlt_sources: - - type: rest_api - destination: - loader_file_format: parquet - config: - client: - base_url: https://api.example.com/ - resources: - - name: raw_widgets - endpoint: - path: widgets -``` - -### Reference in models - -Declared resources are managed sources, reference them like any other source: - -```sql -SELECT id, title, state FROM __source("raw_github_issues") -``` - -## dlt inside a Python loader - -When you need dlt capabilities beyond the declarative surface (custom transforms, `delete_insert`, sources not covered above), wrap a dlt pipeline in a [source loader](/docs/concepts/python-nodes/loaders). The loader calls `dlt.pipeline(...).run(...)` and returns `None`; dlt handles the writes and SQLBuild treats the source as loaded. - -```python -# python/loaders/github_sources.py -import dlt -from dlt.sources.rest_api import rest_api_source -from sqlbuild.loaders import loader -from sqlbuild.executor.load.models import LoaderContext - -@loader -def raw_github_issues(ctx: LoaderContext): - """Github issues.""" - source = rest_api_source({ - "client": { - "base_url": "https://api.github.com/", - "headers": {"Authorization": f"Bearer {ctx.vars['github_token']}"}, - "paginator": {"type": "header_link"}, - }, - "resources": [{ - "name": "issues", - "endpoint": { - "path": "repos/{owner}/{repo}/issues", - "params": { - "owner": ctx.vars["github_owner"], - "repo": ctx.vars["github_repo"], - "state": "all", - "per_page": 100, - }, - }, - }], - }) - - pipeline = dlt.pipeline( - pipeline_name="github_issues", - destination=dlt.destinations.duckdb(ctx.connection), - dataset_name=ctx.destination_schema or "main", - ) - pipeline.run(source) -``` - -Bind it to a source with `managed: true` in `sources/*.yml`: - -```yaml -# sources/github.yml -sources: - - name: raw_github_issues - description: Github issues - managed: true - table: issues - columns: - - name: id - type: INTEGER - - name: title - type: VARCHAR -``` - -### DuckDB connection sharing - -With the DuckDB adapter, pass `ctx.connection` to dlt's DuckDB destination to reuse SQLBuild's open connection, so dlt writes into the same database without a separate connection string: - -```python -pipeline = dlt.pipeline( - pipeline_name="my_pipeline", - destination=dlt.destinations.duckdb(ctx.connection), - dataset_name=ctx.destination_schema or "main", -) -``` - -### Warehouse destinations - -For Snowflake, BigQuery, or Databricks, configure dlt with its own credentials. dlt writes directly to the warehouse, and SQLBuild reads the resulting tables as sources: - -```python -pipeline = dlt.pipeline( - pipeline_name="api_data", - destination="snowflake", - dataset_name=ctx.destination_schema or "public", -) -``` - -Configure dlt credentials via its own `secrets.toml` or environment variables, as in the [dlt documentation](https://dlthub.com/docs/general-usage/credentials/setup). - -## Build integration - -Whether declarative or Python, loaders run automatically during `sqb build` (when `auto_load_sources` is enabled), so dlt pipelines execute as part of the normal build lifecycle: - -```bash -sqb build # loaders run, then models build -sqb build --no-load # skip loading, use existing source data -sqb load # run loaders standalone -``` - -See [Loaders](/docs/concepts/python-nodes/loaders) for write strategies, the loader context API, auto-load behavior, and source deferral. diff --git a/website/src/content/docs/docs/integrations/ingestr.mdx b/website/src/content/docs/docs/integrations/ingestr.mdx deleted file mode 100644 index 7b4f1327cc..0000000000 --- a/website/src/content/docs/docs/integrations/ingestr.mdx +++ /dev/null @@ -1,167 +0,0 @@ ---- -title: "ingestr" -description: "Declarative data ingestion from 50+ sources using ingestr." ---- - -import Frame from '../../../../components/mintlify/Frame.astro'; - -[ingestr](https://github.com/bruin-data/ingestr) is an open-source CLI tool by [Bruin](https://github.com/bruin-data) that copies data from any source to any destination using a single command. SQLBuild integrates with ingestr as a declarative source loader - you configure the ingestion directly in your source YAML, and SQLBuild handles execution as part of the build lifecycle. - -## Install - -```bash -pip install 'sqlbuild[ingestr]' -# or -uv pip install 'sqlbuild[ingestr]' -``` - -This installs `ingestr` alongside SQLBuild. The `ingestr` CLI must be available on `PATH`. - -ingestr reads and writes DuckDB through an ADBC driver. Before the first ingestr run, SQLBuild -installs the DuckDB driver that matches its own `duckdb` package (with `dbc`, included in the extra) -into `/etc/adbc/drivers`, so the files ingestr writes stay readable by SQLBuild. Set -`ADBC_DRIVER_PATH` to manage the driver yourself. - -## How it works - -1. Declare an `ingestr` block on a source in `sources/*.yml` -2. SQLBuild generates a synthetic loader that calls `ingestr ingest` as a subprocess -3. ingestr reads from the configured source and writes directly to the SQLBuild adapter's database -4. The destination URI is built automatically from your SQLBuild connection config - no manual destination setup - -No Python loader code is needed. The YAML declaration is the entire configuration. - - - sqb build loading three external (ingestr) sources (raw_customers, raw_orders, raw_payments); the raw_customers step shows the ingestr ingest command, pipeline source/destination, and an ingestion metrics table before the models build - - -## Example: PostgreSQL to DuckDB - -Replicate a table from PostgreSQL into your local DuckDB project: - -```yaml -# sources/raw.yml -sources: - - name: raw_orders - description: Orders - table: orders - ingestr: - source_uri: "postgresql://user:pass@host:5432/mydb" - source_table: "public.orders" -``` - -That's it. Run `sqb load` or `sqb build` and ingestr copies the `orders` table into your project. - -## Example: Stripe with incremental merge - -Load Stripe charges with incremental merge on a primary key: - -```yaml -sources: - - name: raw_stripe_charges - description: Stripe charges - table: charges - ingestr: - source_uri: "stripe://${stripe_api_key}" - source_table: "charges" - strategy: merge - primary_key: id - incremental_key: created -``` - -On subsequent runs, ingestr merges new and updated records based on the `id` column, using `created` to detect changes. - -## Configuration reference - -The `ingestr` block on a source supports the following fields: - -| Field | Required | Description | -|-------|----------|-------------| -| `source_uri` | Yes | ingestr source connection URI (e.g. `postgresql://...`, `stripe://...`, `shopify://...`) | -| `source_table` | Yes | Source table or resource name to ingest | -| `strategy` | No | Ingestion strategy: `replace`, `append`, `merge`, `delete+insert`, or `truncate+insert` | -| `incremental_key` | No | Column used for incremental change detection | -| `primary_key` | No | Primary key column(s) for merge strategy (string or list) | -| `columns` | No | Comma-separated column list to select from the source | -| `extra_args` | No | Additional CLI arguments passed to `ingestr ingest` (list of strings) | - -### Strategy mapping - -| Strategy | Behavior | -|----------|----------| -| `replace` | Drop and recreate the destination table (default when no strategy is set) | -| `append` | Insert all rows without deduplication | -| `merge` | Upsert based on `primary_key`, using `incremental_key` for change detection | -| `delete+insert` | Delete matching rows by `incremental_key` range, then insert replacements. A SQL model does the same with `delete_insert` and a `cursor`; use [`replace_key`](/docs/concepts/incremental#delete_insert) to replace groups of rows by key value instead | -| `truncate+insert` | Truncate destination table, then insert all rows | - -## Template variables - -All string fields in the `ingestr` block support SQLBuild project variable substitution and context templates: - -```yaml -sources: - - name: raw_orders - description: Orders - ingestr: - source_uri: "postgresql://${pg_user}:${pg_password}@${pg_host}:5432/${pg_database}" - source_table: "public.orders" -``` - -Variables are resolved from the project's merged variable config (project + target + local). - -Set sensitive values in `sqlbuild_local.toml` (gitignored): - -```toml -[vars] -pg_user = "readonly" -pg_password = "secret" -pg_host = "prod-db.example.com" -pg_database = "analytics" -stripe_api_key = "sk_live_..." -``` - -## Destination URI - -SQLBuild automatically builds the ingestr destination URI from your adapter connection config. All supported adapters work without manual destination configuration: - -| Adapter | Destination URI format | -|---------|----------------------| -| DuckDB | `duckdb:///path/to/db.duckdb` | -| SQLite | `sqlite:///path/to/db.sqlite` | -| MotherDuck | `motherduck://database` | -| PostgreSQL | `postgresql://user:pass@host:port/db` | -| Snowflake | `snowflake://user:pass@account/db/schema` | -| BigQuery | `bigquery://project` | -| Databricks | `databricks://token:...@host` | -| SQL Server | `mssql://user:pass@host:port/db` | - -## Reload - -When `--reload` is passed, ingestr runs with `--full-refresh`, forcing a complete reload regardless of the configured strategy: - -```bash -sqb load --reload -sqb build --reload -``` - -## Build integration - -ingestr sources are managed sources - they participate in the same lifecycle as Python loaders: - -```bash -# ingestr runs automatically before dependent models -sqb build - -# run ingestr sources standalone -sqb load - -# skip loading -sqb build --no-load -``` - -See [Loaders](/docs/concepts/python-nodes/loaders) for details on auto-load behavior, source deferral, and the `--load` / `--no-load` / `--reload` flags. - -## Supported sources - -ingestr supports 50+ sources including databases, SaaS APIs, and file systems. See the [ingestr documentation](https://bruin-data.github.io/ingestr/) for the full list of supported sources and their URI formats. diff --git a/website/src/navigation.mjs b/website/src/navigation.mjs index 7f0fafd3fb..d1ccded6da 100644 --- a/website/src/navigation.mjs +++ b/website/src/navigation.mjs @@ -6,5 +6,7 @@ export const redirects = { "/docs/concepts/rules/overview": "/docs/concepts/rules", "/docs/concepts/rules/custom-rules": "/docs/concepts/rules/custom-rules/overview", "/docs/concepts/planning/cascade-propagation": "/docs/concepts/planning/replay-decisions", - "/docs/concepts/enums-and-constants": "/docs/concepts/enums" + "/docs/concepts/enums-and-constants": "/docs/concepts/enums", + "/docs/integrations/dlt": "/docs/concepts/python-nodes/loaders", + "/docs/integrations/ingestr": "/docs/concepts/python-nodes/loaders" }; diff --git a/website/src/sidebar.json b/website/src/sidebar.json index f9f4f6563e..812f9e0b65 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -216,9 +216,7 @@ "docs/integrations/dagster-reference" ] }, - "docs/integrations/rivers", - "docs/integrations/dlt", - "docs/integrations/ingestr" + "docs/integrations/rivers" ] }, {