diff --git a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/sql_adapter.rs b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/sql_adapter.rs index 414bb35ea..be9ab7a54 100644 --- a/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/sql_adapter.rs +++ b/crates/sqlload/src/adapters/clickhouse/_helpers/dialect/sql_adapter.rs @@ -42,6 +42,11 @@ impl SqlAdapter for ClickHouseSql { Warehouse::ClickHouse } + /// `length` counts bytes here. + fn render_text_length(&self, expression: &str) -> String { + format!("lengthUTF8({expression})") + } + fn max_identifier_length(&self) -> Option { Some(255) } diff --git a/crates/sqlload/src/adapters/contract/_helpers/queries.rs b/crates/sqlload/src/adapters/contract/_helpers/queries.rs index 2cf0f2b81..2b120bf0a 100644 --- a/crates/sqlload/src/adapters/contract/_helpers/queries.rs +++ b/crates/sqlload/src/adapters/contract/_helpers/queries.rs @@ -288,18 +288,33 @@ pub(crate) fn check_query( "SELECT DISTINCT {} AS example FROM {from} ORDER BY 1 LIMIT {limit}", examples(&names_of(columns), true) ), - CheckQuery::KeySources { + CheckQuery::KeyFiles { key, - staged, file_column, - file, + held_by, } => { - let values = examples(&names_of(key), true); + let values = key_text(adapter, key); let column = adapter.render_identifier(file_column); + let length = adapter.render_text_cast(&adapter.render_text_length(&column)); + let held = held_by.map_or_else(String::new, |held_by| { + format!( + " WHERE {values} IN (SELECT {values} FROM {})", + adapter.render_qualified_name(held_by) + ) + }); format!( - "SELECT DISTINCT {column} AS example FROM {from} WHERE {values} IN (SELECT {values} FROM {}) AND {column} <> {} ORDER BY 1", - adapter.render_qualified_name(staged), - adapter.render_loader_value_literal(file) + "SELECT DISTINCT {length} || ':' || {column} || {values} AS example FROM {from}{held} ORDER BY 1" + ) + } + CheckQuery::NullKeyFiles(columns, file_column) => { + let missing: Vec = names_of(columns) + .iter() + .map(|name| format!("{name} IS NULL")) + .collect(); + format!( + "SELECT DISTINCT {} AS example FROM {from} WHERE {} ORDER BY 1", + adapter.render_identifier(file_column), + missing.join(" OR ") ) } CheckQuery::NullKeyExamples(columns) => { @@ -313,3 +328,41 @@ pub(crate) fn check_query( } } } + +/// `key`'s columns of a row as one text, each `NULL` for a NULL, joined by `, `. +fn key_text(adapter: &A, key: &[String]) -> String { + let texts: Vec = key + .iter() + .map(|column| { + let name = adapter.render_identifier(column); + format!("COALESCE({}, 'NULL')", adapter.render_text_cast(&name)) + }) + .collect(); + texts.join(" || ', ' || ") +} + +/// `DELETE FROM staging` of each row whose `key` values (converted to the table's types as the commit's replace step matches them, a NULL matching a NULL) a row of a later one of `files` (its `file_column`, in order) holds too: the rows ranked by a window over the key, which the warehouse can sort or hash, then found by file and `row_column`, which numbers each file's rows. +pub(crate) fn superseded_delete( + adapter: &A, + staging: &TableName, + (key, file_column, row_column): (&[&DesiredColumn], &str, &str), + files: &[&str], +) -> String { + let column = adapter.render_identifier(file_column); + let row = adapter.render_identifier(row_column); + let listed: Vec = files + .iter() + .enumerate() + .map(|(at, file)| format!("({}, {at})", adapter.render_loader_value_literal(file))) + .collect(); + let staged = adapter.render_qualified_name(staging); + let names: Vec = key + .iter() + .map(|column| adapter.render_source_expression_cast(&column.name, &column.column_type)) + .collect(); + format!( + "DELETE FROM {staged} AS __sqlload_earlier WHERE EXISTS (SELECT 1 FROM (SELECT {column} AS __sqlload_of, {row} AS __sqlload_at, __sqlload_order, MAX(__sqlload_order) OVER (PARTITION BY {}) AS __sqlload_newest FROM {staged} JOIN (VALUES {}) AS __sqlload_files (__sqlload_file, __sqlload_order) ON __sqlload_file = {column}) AS __sqlload_ranked WHERE __sqlload_of = __sqlload_earlier.{column} AND __sqlload_at = __sqlload_earlier.{row} AND __sqlload_order < __sqlload_newest)", + names.join(", "), + listed.join(", ") + ) +} diff --git a/crates/sqlload/src/adapters/contract/_helpers/write.rs b/crates/sqlload/src/adapters/contract/_helpers/write.rs index d2839c7c3..8697a03ff 100644 --- a/crates/sqlload/src/adapters/contract/_helpers/write.rs +++ b/crates/sqlload/src/adapters/contract/_helpers/write.rs @@ -59,7 +59,7 @@ pub(crate) fn write_statements( .. } => { let spared = |table: &str| { - request.spared_load.map(|(column, load_id)| { + request.spared_load.map(|(column, load_id, _)| { format!( "{table}{} IS DISTINCT FROM {}", adapter.render_identifier(column), @@ -78,10 +78,18 @@ pub(crate) fn write_statements( } }; let keyed = (!replace_key.is_empty()).then(|| { - let (keys, join) = replaced_keys(adapter, request, &values, replace_key); - let spared = spared("__target.") - .map(|condition| format!(" AND {condition}")) - .unwrap_or_default(); + let (mut keys, join) = replaced_keys(adapter, request, &values, replace_key); + let by_file = request.spared_load.and_then(|(_, _, file)| file); + let by_file = by_file.map(|file| adapter.render_identifier(file)); + if let Some(file) = &by_file { + keys.push_str(&format!(", {file} AS __sqlload_spared_file")); + } + let spared = spared("__target.").map_or_else(String::new, |condition| match &by_file { + Some(file) => format!( + " AND ({condition} OR __target.{file} IS DISTINCT FROM __source.__sqlload_spared_file)" + ), + None => format!(" AND {condition}"), + }); format!( "DELETE FROM {target} AS __target USING (SELECT DISTINCT {keys} FROM {source}) AS __source WHERE {join}{spared}" ) diff --git a/crates/sqlload/src/adapters/contract/models.rs b/crates/sqlload/src/adapters/contract/models.rs index c8e542147..a3bb039f2 100644 --- a/crates/sqlload/src/adapters/contract/models.rs +++ b/crates/sqlload/src/adapters/contract/models.rs @@ -165,8 +165,8 @@ pub struct WriteStatementsRequest<'a> { pub hashed: &'a [String], /// The batches went straight into `staging`, built in the table's shape: the write only swaps it in, so no row is written twice, then re-creates these indexes. pub built: Option<&'a [String]>, - /// The load id column and this load's id, whose rows a `delete_insert`'s deletes keep: the load committed some in pieces before. - pub spared_load: Option<(&'a str, &'a str)>, + /// The load id column and this load's id, whose rows a `delete_insert`'s deletes keep: the load committed some in pieces before. With a file column (a files source under `on_overlapping_files = "last_wins"`), a `replace_key` delete keeps only those of the same file as the staged rows of their value, so a later file replaces an earlier one's rows its earlier pieces committed. + pub spared_load: Option<(&'a str, &'a str, Option<&'a str>)>, } /// What a table rebuilt beside itself must keep, where the warehouse cannot copy it (DuckDB). @@ -322,13 +322,14 @@ pub enum CheckQuery<'a> { KeyDuplicateExamples(&'a [String]), /// Up to five values of these columns, in order, of rows holding a NULL in any of them, each as its columns' text (`NULL` for a NULL) joined by `, `. NullKeyExamples(&'a [String]), + /// The distinct values of the column named second among rows holding a NULL in any of the first, in order: the files whose rows have NULL keys. + NullKeyFiles(&'a [String], &'a str), /// Up to this many distinct values of these columns, in order, each as its columns' text (`NULL` for a NULL) joined by `, `. KeyValues(&'a [String], u64), - /// The distinct values of `file_column` among the relation's rows whose `key` values (as `KeyValues` gives them) a row of `staged` holds, other than `file`, in order: the files a commit's key values already come from. - KeySources { + /// The distinct pairs of `file_column` and `key` values (as `KeyValues` gives them) among the relation's rows, only those whose values a row of `held_by` holds when set, in order, each as the file's length in characters, `:`, the file, then the values: which files hold which key values. + KeyFiles { key: &'a [String], - staged: &'a TableName, file_column: &'a str, - file: &'a str, + held_by: Option<&'a TableName>, }, } diff --git a/crates/sqlload/src/adapters/contract/types.rs b/crates/sqlload/src/adapters/contract/types.rs index 80c3e1f4b..9ee1bec93 100644 --- a/crates/sqlload/src/adapters/contract/types.rs +++ b/crates/sqlload/src/adapters/contract/types.rs @@ -14,7 +14,7 @@ use crate::adapters::contract::_helpers::promote::{ }; use crate::adapters::contract::_helpers::queries::{ check_query, child_tables_query, cursor_query, insert_rows, loaded_files_query, - staging_tables_query, staging_time, stale_staging, + staging_tables_query, staging_time, stale_staging, superseded_delete, }; use crate::adapters::contract::_helpers::recreate::recreate_statements; use crate::adapters::contract::_helpers::schema::{ @@ -273,6 +273,26 @@ pub trait SqlAdapter: Sync + std::fmt::Debug { check_query(self, relation, check) } + /// Whether [`render_superseded_delete`](Self::render_superseded_delete) runs here, so a files source's `delete_insert` by `replace_key` under `on_overlapping_files = "last_wins"` can commit many files at once; elsewhere it commits each file on its own. + fn deletes_superseded_rows(&self) -> bool { + false + } + + /// A statement deleting from `staging` each row whose `key` values (converted as the table takes them, a NULL matching a NULL) a row of a later one of `files` (`file_column`, in order) holds too; `row_column` numbers each file's rows. + fn render_superseded_delete( + &self, + staging: &TableName, + (key, file_column, row_column): (&[&DesiredColumn], &str, &str), + files: &[&str], + ) -> String { + superseded_delete(self, staging, (key, file_column, row_column), files) + } + + /// The length of text `expression` in characters. + fn render_text_length(&self, expression: &str) -> String { + format!("length({expression})") + } + /// A query returning a row when `table` holds one. fn render_any_row_query(&self, table: &TableName) -> String { format!( @@ -543,7 +563,7 @@ pub trait SqlAdapter: Sync + std::fmt::Debug { false } - /// Whether a files source may `merge` into this warehouse, file by file with a run record in each file's commit (tested on DuckDB and Postgres so far). + /// Whether a files source may `merge` into this warehouse, in file order with a run record naming each commit's files (tested on DuckDB and Postgres so far). fn merges_files(&self) -> bool { false } diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs b/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs index 0d9dfd677..d5f072b8a 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs @@ -29,7 +29,7 @@ pub(crate) struct PulledFile { pub constants: Vec<(String, Option)>, } -/// `INSERT INTO into` every one of `columns` from the Parquet `files`, matched by name so a file that lacks a column gives NULL. A column comes from the file's constants when it has one of that name (a list of every file's value, indexed by the row's `file_index`: no join, so DuckDB streams the rows into the table instead of holding them), otherwise from the files when at least one of them holds it (`in_files`, each column with its name in the files, which `naming`, `rename` or numbering may have changed: `f."Order ID"` loads into `order_id`), and is cast to its type; `_sqlload_source_row` counts each file's rows from 1; the load id and load time are the batch's; any other column is NULL. Every file must carry the same constant columns. Into MotherDuck (`remote`) the files are read on its servers. +/// `INSERT INTO into` every one of `columns` from the Parquet `files`, matched by name so a file that lacks a column gives NULL. A column comes from the file's constants when it has one of that name (a list of every file's value, indexed by the row's `file_index`: no join, so DuckDB streams the rows into the table instead of holding them), otherwise from the files when at least one of them holds it (`in_files`, each column with its name in the files, which `naming`, `rename` or numbering may have changed: `f."Order ID"` loads into `order_id`), and is cast to its type; `_sqlload_source_row` counts each file's rows from 1, and a merge's `_sqlload_row` numbers the rows in file order, then by row, so the last row of a key is the newest file's; the load id and load time are the batch's; any other column is NULL. Every file must carry the same constant columns. Into MotherDuck (`remote`) the files are read on its servers. pub(crate) fn pull_insert( into: &TableName, (columns, in_files): (&[DesiredColumn], &[(String, String)]), @@ -57,6 +57,12 @@ pub(crate) fn pull_insert( (Some(MetadataRole::LoadId), _) => cast(literal(load_id), column), (Some(MetadataRole::LoadedAt), _) => cast(literal(loaded_at), column), (None, SOURCE_ROW_COLUMN) => cast("f.file_row_number + 1".to_owned(), column), + (None, MERGE_ROW_COLUMN) => cast( + format!( + "CAST(f.file_index AS BIGINT) * {UNLOADED_FILE_ROWS} + f.file_row_number" + ), + column, + ), (_, name) if files .first() diff --git a/crates/sqlload/src/adapters/duckdb/_helpers/sql_adapter.rs b/crates/sqlload/src/adapters/duckdb/_helpers/sql_adapter.rs index ae19afc49..4722cab44 100644 --- a/crates/sqlload/src/adapters/duckdb/_helpers/sql_adapter.rs +++ b/crates/sqlload/src/adapters/duckdb/_helpers/sql_adapter.rs @@ -44,6 +44,10 @@ impl SqlAdapter for DuckDbSql { true } + fn deletes_superseded_rows(&self) -> bool { + true + } + fn parses_variant_pins(&self) -> bool { true } diff --git a/crates/sqlload/src/adapters/postgres/_helpers/sql_adapter.rs b/crates/sqlload/src/adapters/postgres/_helpers/sql_adapter.rs index a49bbd529..ea9869756 100644 --- a/crates/sqlload/src/adapters/postgres/_helpers/sql_adapter.rs +++ b/crates/sqlload/src/adapters/postgres/_helpers/sql_adapter.rs @@ -124,6 +124,10 @@ impl SqlAdapter for PostgresSql { staging_type(column_type) } + fn deletes_superseded_rows(&self) -> bool { + true + } + /// JSON, UUIDs, arrays and native types are converted from their staged text by their type's own input, which refuses what it cannot read; the rest by the table's assignment casts, which refuse a value that does not fit. fn render_source_expression_cast(&self, name: &str, column_type: &ColumnType) -> String { let column = self.render_identifier(name); diff --git a/crates/sqlload/src/batch_plan/main/mod.rs b/crates/sqlload/src/batch_plan/main/mod.rs index 6b79bf0cd..817058ac2 100644 --- a/crates/sqlload/src/batch_plan/main/mod.rs +++ b/crates/sqlload/src/batch_plan/main/mod.rs @@ -4,6 +4,7 @@ pub mod file_groups; pub mod ignored_batch; pub mod ignored_concurrency; pub mod last_value; +pub mod ordered_groups; pub mod outside_text; pub mod read_at_once; pub mod read_in_order; diff --git a/crates/sqlload/src/batch_plan/main/ordered_groups.rs b/crates/sqlload/src/batch_plan/main/ordered_groups.rs new file mode 100644 index 000000000..e39afde06 --- /dev/null +++ b/crates/sqlload/src/batch_plan/main/ordered_groups.rs @@ -0,0 +1,33 @@ +//! Cut a files source's files into consecutive batches, keeping their order. + +use crate::batch_plan::constants::{BATCH_BYTES, MAX_BATCH_FILES}; +use crate::batch_plan::models::FileGroup; +use crate::files_source::models::DataFile; + +/// `files` in consecutive batches, in the order given, each closed at 1,000 files or about 1 GB of files, whichever comes first (a single larger file is a batch by itself): a files source's merge, or `delete_insert` by `replace_key`, which applies the files in order, the newest file winning a key repeated across them. +pub fn ordered_groups(files: &[DataFile]) -> Vec { + let mut groups: Vec = Vec::new(); + let mut current: Vec = Vec::new(); + let mut bytes: usize = 0; + for (index, data) in files.iter().enumerate() { + let file_bytes = data.file.size as usize; + if !current.is_empty() + && (current.len() == MAX_BATCH_FILES || bytes + file_bytes > BATCH_BYTES) + { + groups.push(FileGroup { + files: std::mem::take(&mut current), + values: Vec::new(), + }); + bytes = 0; + } + current.push(index); + bytes += file_bytes; + } + if !current.is_empty() { + groups.push(FileGroup { + files: current, + values: Vec::new(), + }); + } + groups +} diff --git a/crates/sqlload/src/contract/_helpers/toml_columns.rs b/crates/sqlload/src/contract/_helpers/toml_columns.rs index ae61085cf..df4fc1634 100644 --- a/crates/sqlload/src/contract/_helpers/toml_columns.rs +++ b/crates/sqlload/src/contract/_helpers/toml_columns.rs @@ -5,23 +5,7 @@ use toml_edit::{DocumentMut, InlineTable, Item, Table, Value}; use crate::contract::constants::{MISSING_DECLARATION, MISSING_PHYSICAL_COLUMN, TYPE_MISMATCH}; use crate::contract::models::{Adoption, ContractFinding}; -/// Whether writing `adoption` would create a `[sources.*.tables.*]` entry for a database table with no description to give it, which would then block the table. -pub(crate) fn lacks_description(root: &Item, adoption: &Adoption) -> bool { - let (source, table) = &adoption.toml_key; - let exists = root - .get("sources") - .and_then(|sources| sources.get(source.as_str())) - .and_then(|source| source.get("tables")) - .and_then(|tables| tables.get(table.as_str())) - .is_some(); - !exists - && adoption - .description - .as_deref() - .is_some_and(|text| text.trim().is_empty()) -} - -/// `document` with `adoption` written into the `columns` of `[sources..tables.]`, adding the tables on the way when missing, and a new database table entry's description; the source must be in the file. +/// `document` with `adoption` written into the `columns` of `[sources..tables.
]`, adding the tables on the way when missing, and a new database table entry's description when it has one; the source must be in the file. pub(crate) fn write_adoption( mut document: DocumentMut, adoption: &Adoption, @@ -37,7 +21,7 @@ pub(crate) fn write_adoption( let description = adoption .description .as_deref() - .filter(|_| tables.get(table.as_str()).is_none()); + .filter(|text| !text.trim().is_empty() && tables.get(table.as_str()).is_none()); let entry = child(tables, table, true)?; if let Some((text, entry)) = description.zip(entry.as_table_like_mut()) { entry.insert("description", Item::Value(Value::from(text))); diff --git a/crates/sqlload/src/contract/main/write_contracts.rs b/crates/sqlload/src/contract/main/write_contracts.rs index b65cde6bf..1c35c922a 100644 --- a/crates/sqlload/src/contract/main/write_contracts.rs +++ b/crates/sqlload/src/contract/main/write_contracts.rs @@ -2,10 +2,10 @@ use toml_edit::DocumentMut; -use crate::contract::_helpers::toml_columns::{lacks_description, resolved, write_adoption}; +use crate::contract::_helpers::toml_columns::{resolved, write_adoption}; use crate::contract::models::ContractReport; -/// `text` (sqlload.toml) with each adoption of a table declared there written into its `[sources..tables.
]` `columns`, which are created when missing, keeping the rest of the file as it was (`None` when nothing changes); and the report listing only the findings left, without the adoptions it wrote. A Python table's adoption is left to paste, and a database table that would need a new entry but has no description is a problem instead. +/// `text` (sqlload.toml) with each adoption of a table declared there written into its `[sources..tables.
]` `columns`, which are created when missing, keeping the rest of the file as it was (`None` when nothing changes); and the report listing only the findings left, without the adoptions it wrote. A Python table's adoption is left to paste. pub fn write_contracts( text: &str, mut report: ContractReport, @@ -21,13 +21,6 @@ pub fn write_contracts( else { continue; }; - if lacks_description(document.as_item(), &adoption) { - report.problems.push(format!( - "{}: columns not written: a table entry needs a description and this table has none; add description = \"...\" there, then rerun contract generate --write", - adoption.location - )); - continue; - } document = write_adoption(document, &adoption)?; resource .findings diff --git a/crates/sqlload/src/contract/tests/test_write.rs b/crates/sqlload/src/contract/tests/test_write.rs index 60bd29412..8b8e54b85 100644 --- a/crates/sqlload/src/contract/tests/test_write.rs +++ b/crates/sqlload/src/contract/tests/test_write.rs @@ -29,12 +29,11 @@ fn given_a_project_file_when_writing_adoptions_then_only_the_tables_columns_chan ], }, WriteTestCase { - description: "a table with no entry and no description is not written, since the entry would block it", + description: "a table with no entry and no description gets an entry holding only its columns", text: "[sources.app]\nkind = \"postgres\"\ninclude = [\"public.*\"]\n", table_description: "", expected_lines: &[ - "", - "[sources.app.tables.\"public.orders\"]: columns not written: a table entry needs a description and this table has none; add description = \"...\" there, then rerun contract generate --write", + "[sources.app]\nkind = \"postgres\"\ninclude = [\"public.*\"]\n\n[sources.app.tables.\"public.orders\".columns]\namount = \"float\"\nstatus = \"text\"\n", ], }, WriteTestCase { diff --git a/crates/sqlload/src/http_source/_helpers/job.rs b/crates/sqlload/src/http_source/_helpers/job.rs index 2dfbef0a9..40bbe2651 100644 --- a/crates/sqlload/src/http_source/_helpers/job.rs +++ b/crates/sqlload/src/http_source/_helpers/job.rs @@ -185,6 +185,7 @@ fn request_problems(request: &Request) -> Vec { let mut problems: Vec = Vec::new(); problems.extend(selector(request.select.as_deref()).err()); problems.extend(paginator_problem(&request.paginator)); + problems.extend(page_size_twice(request)); problems.extend(body_problems(request)); problems.extend(request.cells.as_ref().and_then(cells_problem)); problems.extend(request.error_path.as_deref().and_then(error_path_problem)); @@ -221,6 +222,21 @@ fn error_path_problem(path: &str) -> Option { } } +/// Why `params` can't set the query parameter a page-number paginator sends `page_size` as: it would go twice. +fn page_size_twice(request: &Request) -> Option { + match &request.paginator { + Paginator::PageNumber { + page_size: Some(_), + page_size_param: Some(name), + location: Location::Query, + .. + } if request.params.contains_key(name) => Some(format!( + "params sets {name}, which page_size sends; leave it out of params" + )), + _ => None, + } +} + /// Why the paginator cannot work, if it cannot: a pointer that is not a plain path, or an offset limit of 0. fn paginator_problem(paginator: &Paginator) -> Option { match paginator { @@ -714,7 +730,7 @@ fn request_calls( in_flight, retries: shared.retries.unwrap_or(DEFAULT_RETRIES), on_status, - full_page_seen: std::sync::Arc::default(), + last_page_below: std::sync::Arc::default(), source_part, stop: tokio_util::sync::CancellationToken::new(), }) diff --git a/crates/sqlload/src/http_source/_helpers/pages.rs b/crates/sqlload/src/http_source/_helpers/pages.rs index af687a2e7..28913cfe1 100644 --- a/crates/sqlload/src/http_source/_helpers/pages.rs +++ b/crates/sqlload/src/http_source/_helpers/pages.rs @@ -28,7 +28,7 @@ use crate::http_source::models::{ }; use crate::python_sources::errors::ReadError; -/// Send every page of `request` at `path` (its placeholders filled) to `pages`, with where its rows are; a page whose selected node is null has no rows and is not sent, and one missing it fails, naming what the response holds, unless the request reads a missing node as no rows (`select_missing`). An offset run that would end after a first page shorter than its limit asks for the next offset once, and fails when rows come back, as the API then caps its pages lower than the limit. With `read_ahead`, offset and page-number pages after a full first page are requested two at a time, then four, eight and so on up to that many (fewer under a memory budget: only as many as pages the size of the largest yet fit in a quarter of each table's share), each through the shared rate limit and retries in a task of its own (so it keeps reading while the pages' receiver holds this loop up), and handled strictly in page order, so the run still ends at the first short page, dropping the requests past it. Stops quietly once `pages` closes. A next position (cursor, link or count) already requested for this request fails before it is requested again, naming the value and the page that returned it, and a counted page repeating the previous response fails, rather than loop. With `error_path`, an error a page reports there fails the table before any of that page's rows load. +/// Send every page of `request` at `path` (its placeholders filled) to `pages`, with where its rows are; a page whose selected node is null has no rows and is not sent, and one missing it fails, naming what the response holds, unless the request reads a missing node as no rows (`select_missing`). An offset run that would end after a first page shorter than its limit asks for the next offset once, unless the job has shown the API sends pages of more rows, and fails when rows come back, as the API then caps its pages lower than the limit. A page-number run with `page_size` ends at a short page in the same way, and where it can't yet tell asks for the next page, paging on to an empty page when rows come back (an API capping its pages lower). With `read_ahead`, offset and page-number pages after a full first page are requested two at a time, then four, eight and so on up to that many (fewer under a memory budget: only as many as pages the size of the largest yet fit in a quarter of each table's share), each through the shared rate limit and retries in a task of its own (so it keeps reading while the pages' receiver holds this loop up), and handled strictly in page order, so the run still ends at the first short page, dropping the requests past it. Stops quietly once `pages` closes. A next position (cursor, link or count) already requested for this request fails before it is requested again, naming the value and the page that returned it, and a counted page repeating the previous response fails, rather than loop. With `error_path`, an error a page reports there fails the table before any of that page's rows load. pub(crate) async fn paginate( calls: &Calls, (request, context): (&Request, &RowContext), @@ -55,6 +55,7 @@ pub(crate) async fn paginate( let mut queued: u64 = 1; let mut largest: u64 = 0; let (mut ramp, mut ramped): (u64, u64) = (1, 1); + let mut checking: Option = None; for step in 1_u64.. { let shown = redacted( &format!("{} {}", call.method.as_str(), call.url), @@ -130,6 +131,14 @@ pub(crate) async fn paginate( request, (&first, &call), (&fetched, document.as_ref(), step), + calls.last_page_below.load(Ordering::Relaxed), + ) + .map_err(|error| format!("{shown}: {error}"))?; + learn_page_size( + calls, + request, + (document.as_ref(), next.is_some()), + (&mut checking, &shown), ) .map_err(|error| format!("{shown}: {error}"))?; if let Some((next, position)) = &next @@ -141,14 +150,8 @@ pub(crate) async fn paginate( ) .into()); } - match (step, &next, &document) { - (1, None, Some(document)) => { - capped_pages(calls, request, (&first, document), &shown).await? - } - (_, Some(_), _) if matches!(request.paginator, Paginator::Offset { .. }) => { - calls.full_page_seen.store(true, Ordering::Relaxed); - } - _ => {} + if let (1, None, Some(document)) = (step, &next, &document) { + capped_pages(calls, request, (&first, document), &shown).await?; } let has_rows = document.as_ref().is_none_or(|document| { selected_node(document, &rows).is_some_and(|node| !node.is_null()) @@ -195,6 +198,43 @@ pub(crate) async fn paginate( Ok(()) } +/// What this page (`document`, `continues` when paging goes on past it) shows about the API's page size, kept in [`Calls::last_page_below`] for every key of the job: a full page shows the paginator's whole page; the page after a short one (`checking`, that page's rows) shows those rows were all the API sends at once, whether it is empty (the short page was the last) or holds more, which shows a cap and is warned about the first time. A short page-number page that paging goes on past is checked by the next. +fn learn_page_size( + calls: &Calls, + request: &Request, + (document, continues): (Option<&Value>, bool), + (checking, shown): (&mut Option, &str), +) -> Result<(), String> { + let full = match &request.paginator { + Paginator::Offset { limit, .. } => Some(*limit), + Paginator::PageNumber { page_size, .. } => *page_size, + Paginator::Single + | Paginator::NextLink { .. } + | Paginator::LinkHeader + | Paginator::Cursor { .. } => None, + }; + let (Some(full), Some(document)) = (full, document) else { + return Ok(()); + }; + let read = page_rows(document, request)?; + if let Some(short) = checking.take() { + let before = calls.last_page_below.fetch_max(short, Ordering::Relaxed); + if read > 0 && before < short { + tracing::warn!( + "{shown}: the page before held {short} rows, fewer than page_size {full}, but more followed, so the API caps its pages at {short} rows; set page_size={short} to end each run at its last page" + ); + } + } + match (read >= full, read > 0 && continues) { + (true, _) => { + calls.last_page_below.fetch_max(full, Ordering::Relaxed); + } + (false, true) => *checking = Some(read), + (false, false) => {} + } + Ok(()) +} + /// Let the pages still read ahead when paging ends finish their requests, so their connections go back to the pool, instead of aborting them with the paging, as a failure or a closed `pages` does; the paging's stop token keeps them from waiting for a turn or retrying. fn release(in_flight: VecDeque>) { for handle in in_flight { @@ -233,7 +273,7 @@ fn reported_error( Err(format!("{shown}: the API reported an error at {path}: {text}").into()) } -/// For offset paging whose first page (`document`, from the request `shown`) held some rows but fewer than its limit: the page at the next offset, which an API capping its pages below the limit fills, so rows there fail with the page size to set. An empty or missing page is the real end, and so are the first page's rows again, from an API that rounds the offset down to a page boundary. Skipped once a page of the request held its whole limit in this job, which shows the API does not cap it (see `Calls::full_page_seen`). +/// For offset paging whose first page (`document`, from the request `shown`) held some rows but fewer than its limit: the page at the next offset, which an API capping its pages below the limit fills, so rows there fail with the page size to set. An empty or missing page is the real end, and so are the first page's rows again, from an API that rounds the offset down to a page boundary. Skipped when the job has already shown the API sends pages of more rows than this one held (see [`Calls::last_page_below`]); an empty next page shows it for later keys. async fn capped_pages( calls: &Calls, request: &Request, @@ -250,7 +290,7 @@ async fn capped_pages( return Ok(()); }; let read = page_rows(document, request).map_err(|error| format!("{shown}: {error}"))?; - if read == 0 || read >= *limit || calls.full_page_seen.load(Ordering::Relaxed) { + if read == 0 || read >= *limit || read < calls.last_page_below.load(Ordering::Relaxed) { return Ok(()); } let probe = page_call( @@ -281,7 +321,10 @@ async fn capped_pages( None => 0, }; match more { - 0 => Ok(()), + 0 => { + calls.last_page_below.fetch_max(read, Ordering::Relaxed); + Ok(()) + } _ => Err(format!( "{shown}: the first page held {read} rows, fewer than limit {limit}, but the API returned {more} more at offset {read}, so it caps its pages at {read} rows; set limit={read}" ) @@ -383,8 +426,8 @@ fn counted_call(request: &Request, first: &Url, step: u64) -> Result { - vec![(param.clone(), Value::from(start + step - 1))] + Paginator::PageNumber { start, .. } => { + page_number_values(&request.paginator, start + step - 1) } Paginator::Single | Paginator::NextLink { .. } @@ -406,7 +449,7 @@ fn start_values(paginator: &Paginator) -> Vec<(String, Value)> { (param.clone(), Value::from(0_u64)), (limit_param.clone(), Value::from(*limit)), ], - Paginator::PageNumber { param, start, .. } => vec![(param.clone(), Value::from(*start))], + Paginator::PageNumber { start, .. } => page_number_values(paginator, *start), Paginator::Single | Paginator::NextLink { .. } | Paginator::LinkHeader @@ -414,6 +457,24 @@ fn start_values(paginator: &Paginator) -> Vec<(String, Value)> { } } +/// A page-number paginator's values for page `page`: its number, then its `page_size` where it is sent. +fn page_number_values(paginator: &Paginator, page: u64) -> Vec<(String, Value)> { + let Paginator::PageNumber { + param, + page_size, + page_size_param, + .. + } = paginator + else { + return Vec::new(); + }; + let mut values = vec![(param.clone(), Value::from(page))]; + if let (Some(size), Some(name)) = (page_size, page_size_param) { + values.push((name.clone(), Value::from(*size))); + } + values +} + /// Where the paginator's values go. fn location(paginator: &Paginator) -> Location { match paginator { @@ -486,11 +547,12 @@ fn param_text(value: &Value) -> String { } } -/// The request after `fetched` (the `step`th page, sent as `current`) with the position it goes to, such as `cursor abc`, or `None` at the end; links resolve against the URL the response came from, after any redirect, and keep the current body; `document` is the parsed response whenever the paginator reads the body. +/// The request after `fetched` (the `step`th page, sent as `current`) with the position it goes to, such as `cursor abc`, or `None` at the end; links resolve against the URL the response came from, after any redirect, and keep the current body; `document` is the parsed response whenever the paginator reads the body. A page-number page with `page_size` ends the run when it is empty, or short and below `last_page_below` (see [`Calls::last_page_below`]). fn next_call( request: &Request, (first, current): (&Url, &PageCall), (fetched, document, step): (&Fetched, Option<&Value>, u64), + last_page_below: u64, ) -> Result, String> { let body = || document.ok_or_else(|| "the paginator needs a JSON response".to_owned()); let joined = |link: String, what: &str| { @@ -543,13 +605,22 @@ fn next_call( ), false => Ok(None), }, - Paginator::PageNumber { param, start, .. } => match page_rows(body()?, request)? > 0 { - true => next( - vec![(param.clone(), Value::from(start + step))], - format!("page number {}", start + step), - ), - false => Ok(None), - }, + Paginator::PageNumber { + start, page_size, .. + } => { + let read = page_rows(body()?, request)?; + let more = match page_size { + None => read > 0, + Some(size) => read >= *size || (read > 0 && read >= last_page_below), + }; + match more { + true => next( + page_number_values(&request.paginator, start + step), + format!("page number {}", start + step), + ), + false => Ok(None), + } + } } } diff --git a/crates/sqlload/src/http_source/models.rs b/crates/sqlload/src/http_source/models.rs index 96031be93..310c65f6e 100644 --- a/crates/sqlload/src/http_source/models.rs +++ b/crates/sqlload/src/http_source/models.rs @@ -365,10 +365,16 @@ pub enum Paginator { #[serde(default)] read_ahead: Option, }, - /// `param` counts pages from `start`; ends at an empty page. + /// `param` counts pages from `start`; ends at an empty page, or with `page_size` at a page holding fewer rows once the API has shown it doesn't cap its pages lower (see [`Calls::last_page_below`]). PageNumber { param: String, start: u64, + /// The rows a full page holds. + #[serde(default)] + page_size: Option, + /// Where `page_size` is sent with every page; `None` sends nothing, for an API whose page size is fixed. + #[serde(default)] + page_size_param: Option, #[serde(default)] location: Location, /// As `Offset`'s. @@ -465,8 +471,8 @@ pub struct Calls { pub in_flight: Option>, pub retries: u32, pub on_status: BTreeMap, - /// Whether an offset page of this request held its whole limit in this job, which shows the API does not cap its pages lower, so a short first page needs no check; shared by its clones, the keys of a fan-out. - pub full_page_seen: std::sync::Arc, + /// A page of this request holding fewer rows than this is its run's last, as this job has shown the API sends that many when it has them: the paginator's whole page once a page held it, or the rows of a short page that the next request showed was the last, or that more pages followed (the API's cap); 0 until then. A short page that isn't below it is checked with one more request. Shared by its clones, the keys of a fan-out. + pub last_page_below: std::sync::Arc, /// The most the pages `read_ahead` requests may hold at once: a quarter of the table's share of the run's memory budget; `None` without a budget. pub source_part: Option, /// Cancelled once the pages read ahead are no longer wanted: after paging ends normally a request already sent still finishes, so its connection goes back to the pool, but no wait for a turn, retry or other attempt starts. diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs index cc8d7a079..4a6ef0965 100644 --- a/crates/sqlload/src/loader/_helpers/publish.rs +++ b/crates/sqlload/src/loader/_helpers/publish.rs @@ -77,8 +77,8 @@ use crate::loader::_helpers::run_log::ensure_runs_table; use crate::loader::_helpers::run_log::fold_maximum; use crate::loader::_helpers::run_log::logs_files; use crate::loader::_helpers::run_log::{ - commit_transform, in_file, listed_sources, merge_rebuild, merged_file_text, merged_name, - merged_record, swap_rebuild, with_file_keys, with_transform, + checked_file_keys, commit_transform, in_file, merge_rebuild, merged_file_text, merged_record, + swap_rebuild, with_transform, }; use crate::loader::_helpers::run_log::{record_numbered, recorded_names, transformed}; use crate::loader::_helpers::sources::ReadPart; @@ -122,8 +122,7 @@ use crate::masking::main::mask_batch::mask_batch; use crate::memory_budget::constants::TABLE_SHARE_PARTS; use crate::number_text::main::thousands::thousands; use crate::postgres_source::errors::PostgresError; -use crate::project::models::{NullCursor, NullKeys}; -use crate::publish_checks::main::key_sources::key_sources; +use crate::project::models::{NullCursor, NullKeys, OnOverlappingFiles}; use crate::publish_checks::main::marked_rows::marked_rows; use crate::publish_checks::main::older_rows::older_rows; use crate::publish_checks::main::refuse_null_keys::refuse_null_keys; @@ -157,10 +156,9 @@ use arrow_array::Int64Array; use arrow_array::RecordBatch; use arrow_array::StringArray; use arrow_array::TimestampMicrosecondArray; -use arrow_array::cast::AsArray; use arrow_array::new_null_array; use arrow_cast::display::{ArrayFormatter, FormatOptions}; -use arrow_cast::{CastOptions, cast, cast_with_options}; +use arrow_cast::{CastOptions, cast_with_options}; use arrow_schema::Field; use arrow_schema::Schema; use chrono::DateTime; @@ -325,34 +323,17 @@ fn row_ids(table: &TablePlan, batch: &RecordBatch) -> Result> key_row_ids(batch, &key).map(Some) } -/// Under a merge, the ordinal each of `batch`'s rows is staged with, which its child rows carry so a dropped repeated row's leave with it: a files source's `_sqlload_source_row` when the merge ranks by it, otherwise from `first`, as `_sqlload_row` numbers them; `None` without a merge. +/// Under a merge, the ordinal each of `batch`'s rows is staged with, which its child rows carry so a dropped repeated row's leave with it: from `first`, as `_sqlload_row` numbers them; `None` without a merge. fn root_ordinals( table: &TablePlan, batch: &RecordBatch, first: i64, ) -> Result>, String> { - let WriteMode::Merge { dedupe, .. } = &table.mode else { + let WriteMode::Merge { .. } = &table.mode else { return Ok(None); }; let rows = i64::try_from(batch.num_rows()).unwrap_or(i64::MAX); - let Some(dedupe) = dedupe - .as_ref() - .filter(|dedupe| dedupe.ordinal != MERGE_ROW_COLUMN) - else { - return Ok(Some((first..first + rows).collect())); - }; - let column = batch - .column_by_name(&dedupe.ordinal) - .ok_or_else(|| format!("the staged rows have no {} column", dedupe.ordinal))?; - let ordinals = - cast(column, &arrow_schema::DataType::Int64).map_err(|error| error.to_string())?; - Ok(Some( - ordinals - .as_primitive::() - .iter() - .map(|ordinal| ordinal.unwrap_or_default()) - .collect(), - )) + Ok(Some((first..first + rows).collect())) } /// `batch` with `_sqlload_root_row` added last: each row's root row's ordinal, `roots`. @@ -666,7 +647,7 @@ impl Publisher { committed_state, batch_bytes: options.batch_bytes.unwrap_or(BATCH_BYTES), files, - merged_file: None, + merged_files: Vec::new(), replaced: None, marked: None, older: 0, @@ -1179,17 +1160,17 @@ impl Publisher { }; let warehouse = spec.warehouse; let runs = runs_table(&spec.destination); - let merging_file = self.merged_file.is_some(); - let file = self.merged_file.clone(); + let merging_file = self.merged_files.len() == 1; + let files = self.merged_files.clone(); let (ran, taken) = match self.pieces.piece { - true => (None, None), + true => (None, Vec::new()), false => ( commit_transform(&self.table, self.transform.as_mut()), - self.merged_file.take(), + std::mem::take(&mut self.merged_files), ), }; - let mut merged = taken.map(|file| { - let record = (file, self.waiting_rows); + let mut merged = (!taken.is_empty()).then(|| { + let record = (taken, self.waiting_rows); with_transform( &self.table, merged_record(spec, (&self.load_id, self.commits + 1), record), @@ -1223,6 +1204,8 @@ impl Publisher { spared_load: (self.pieces.committed > 0).then_some(( metadata_name(spec.compat, MetadataRole::LoadId), self.load_id.as_str(), + (spec.on_overlapping_files == OnOverlappingFiles::LastWins) + .then_some(SOURCE_FILE_COLUMN), )), }, &staging, @@ -1238,7 +1221,7 @@ impl Publisher { committed: self.committed_rows, }; if let (Some((key, setting)), NullKeys::Fail) = (checked_key(spec, &mode), spec.null_keys) { - refuse_null_keys(&checked, (key, setting), merging_file).await?; + refuse_null_keys(&checked, (key, setting), &files).await?; } if let WriteMode::Merge { unique_key, @@ -1257,21 +1240,13 @@ impl Publisher { self.older += older_rows(&checked, &mode).await?; let replaced = match &mode { WriteMode::DeleteInsert { replace_key, .. } if !replace_key.is_empty() => { - let limit = file - .as_ref() - .map_or(REPLACED_KEYS_SHOWN as u64 + 1, |_| i64::MAX as u64); + let limit = REPLACED_KEYS_SHOWN as u64 + 1; let mut values = staged_keys(&checked, (replace_key.as_slice(), limit)).await?; - if let Some(file) = file.as_deref() { + if !files.is_empty() { let earlier = std::mem::take(&mut self.run_keys); - self.run_keys = with_file_keys(&self.table, (file, &values), earlier) - .map_err(LoaderError::Invalid)?; - let named = ( - replace_key.as_slice(), - SOURCE_FILE_COLUMN, - merged_name(file), - ); - let sources = key_sources(&checked, named).await?; - listed_sources(&self.table, file, &sources).map_err(LoaderError::Invalid)?; + self.run_keys = + checked_file_keys(&self.table, (&checked, replace_key), (&files, earlier)) + .await?; } values.truncate(REPLACED_KEYS_SHOWN + 1); Some((replace_key.clone(), values)) @@ -1675,8 +1650,11 @@ async fn feed( .iter() .filter_map(|index| planned.and_then(|files| files.get(*index)).cloned()) .collect(); - let merged = merging.then(|| merged_file_text(table, &files)); - publisher.merged_file = merged.clone(); + let merged = match merging { + true => merged_file_text(table, &files), + false => Vec::new(), + }; + publisher.merged_files = merged.clone(); if table.spec.after_load != AfterLoad::Keep { publisher.fed_files.extend(unarchived(table, &files)); } @@ -1724,9 +1702,9 @@ async fn feed( .await?; publisher.cut(range).await }; - fed_group.await.map_err(|error| match &merged { - Some(name) => in_file(name, error), - None => error, + fed_group.await.map_err(|error| match merged.as_slice() { + [name] => in_file(name, error), + _ => error, })?; } } @@ -1815,7 +1793,7 @@ async fn feed( (publisher, fed) } -/// Whether `table` loads a files source's files one commit and run record per file: a merge, or `delete_insert` by `replace_key` (see [`logs_files`]). +/// Whether `table` loads a files source's files in order, each commit's run record naming its files: a merge, or `delete_insert` by `replace_key` (see [`logs_files`]). pub(crate) fn records_files(table: &TablePlan) -> bool { table.files.is_some() && logs_files(&table.spec) } diff --git a/crates/sqlload/src/loader/_helpers/render.rs b/crates/sqlload/src/loader/_helpers/render.rs index ae44a9ff5..c6939ab53 100644 --- a/crates/sqlload/src/loader/_helpers/render.rs +++ b/crates/sqlload/src/loader/_helpers/render.rs @@ -831,7 +831,7 @@ pub(crate) fn batch_line(table: &TablePlan) -> String { format!("{windows} of {step}, {commits}{}{end}", read_at_once(table)) } (CommitRule::FileGroups(groups), _) if records_files(table) => { - recorded_commits(groups.len(), spec) + recorded_commits(groups, spec) } (CommitRule::FileGroups(groups), _) => { format!( diff --git a/crates/sqlload/src/loader/_helpers/resolve.rs b/crates/sqlload/src/loader/_helpers/resolve.rs index 0ef9f99ad..1f8309e29 100644 --- a/crates/sqlload/src/loader/_helpers/resolve.rs +++ b/crates/sqlload/src/loader/_helpers/resolve.rs @@ -22,7 +22,6 @@ use crate::loader::_helpers::destination::folder_claims; use crate::loader::_helpers::run_log::kept_numbered; use crate::loader::_helpers::sources::scoped_tables; use crate::loader::_helpers::specs::ColumnSelection; -use crate::loader::_helpers::specs::missing_description; use crate::loader::_helpers::specs::table_name; use crate::loader::_helpers::specs::{ ColumnChoices, SourceContext, build_spec, column_selection, layered, pinned_types, @@ -1148,7 +1147,7 @@ pub(crate) fn spec_names(spec: &TableSpec) -> [String; 3] { ] } -/// `planned`, the selected tables of one source, added to `plan`, which is returned (`unselected` and `found`, the destinations and options, tell whether this is the source's first load): a table that appeared after the source's first load (`first_load`: none of its tables, selected or not, existed yet) and is not named exactly in `include` is skipped or fails as `new_tables` says, and a table without a description is blocked. +/// `planned`, the selected tables of one source, added to `plan`, which is returned (`unselected` and `found`, the destinations and options, tell whether this is the source's first load): a table that appeared after the source's first load (`first_load`: none of its tables, selected or not, existed yet) and is not named exactly in `include` is skipped or fails as `new_tables` says. pub(crate) async fn admit_tables( mut plan: ProjectPlan, (source_name, source): (&String, &SourceConfig), @@ -1162,7 +1161,7 @@ pub(crate) async fn admit_tables( table.schema = blocked_plan(vec![problem.clone()]); } } - for mut table in planned { + for table in planned { let spec = &table.spec; let qualified = spec.source_table.qualified_name(); let is_new = !first_load && !table.exists && !spec.approved; @@ -1191,13 +1190,7 @@ pub(crate) async fn admit_tables( )); plan.problems.push(problem); } - _ => { - if let Some(problem) = missing_description(source_name, source, &spec.source_table) - { - table.schema = blocked_plan(vec![problem]); - } - plan.tables.push(table); - } + _ => plan.tables.push(table), } } Ok(plan) diff --git a/crates/sqlload/src/loader/_helpers/run_log.rs b/crates/sqlload/src/loader/_helpers/run_log.rs index d612eeb31..4e7e42997 100644 --- a/crates/sqlload/src/loader/_helpers/run_log.rs +++ b/crates/sqlload/src/loader/_helpers/run_log.rs @@ -15,13 +15,15 @@ use crate::adapters::dispatch::main::session::live_columns::live_columns; use crate::adapters::dispatch::main::session::text_values::text_values; use crate::adapters::dispatch::models::Destination; use crate::adapters::files::models::PendingPhase; +use crate::batch_plan::constants::MAX_BATCH_FILES; +use crate::batch_plan::models::FileGroup; use crate::column_types::models::ColumnType; use crate::file_store::main::list_files::list_files; use crate::file_store::main::open_folder::open_folder; use crate::file_store::main::read_file::read_file; use crate::file_store::main::write_file::write_file; use crate::files_source::constants::{ - SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN, VERSION_SEPARATOR, + SOURCE_FILE_COLUMN, SOURCE_ROW_COLUMN, SOURCE_VERSION_COLUMN, VERSION_SEPARATOR, }; use crate::files_source::main::merge_order::merge_order; use crate::files_source::models::{DataFile, FilesOrder, LoadedFile}; @@ -33,7 +35,10 @@ use crate::loader::errors::LoaderError; use crate::loader::models::{ MergeRebuild, ProjectPlan, PythonLoad, RunEntry, TablePlan, TableSpec, TransformRun, }; +use crate::number_text::main::thousands::thousands; use crate::project::models::{OnOverlappingFiles, WriteStrategy}; +use crate::publish_checks::main::key_sources::key_sources; +use crate::publish_checks::models::CheckedCommit; use crate::row_batches::models::Unnest; use crate::run_log::constants::RECORD_FORMAT_VERSION; use crate::run_log::main::cursor_order::cursor_order; @@ -49,7 +54,7 @@ use crate::run_log::main::select_runs::select_runs; use crate::run_log::models::{ DecodedRuns, RecordTable, RunEvent, RunOutcome, RunRecord, StateEvent, UnreadableRecord, }; -use crate::schema_plan::models::{LiveColumn, TableAction}; +use crate::schema_plan::models::{DesiredColumn, LiveColumn, TableAction}; use crate::store_paths::models::Location; use crate::time_text::main::now_utc_text::now_utc_text; @@ -419,11 +424,11 @@ pub(crate) fn unchanged_files( .collect() } -/// The record a files source's merge commits with `file` (`name|version`), its `batch`th commit of run `run_id`, which wrote `rows`. +/// The record a files source's merge commits with `files` (each `name|version`, in the order merged), its `batch`th commit of run `run_id`, which wrote `rows`. pub(crate) fn merged_record( spec: &TableSpec, (run_id, batch): (&str, u64), - (file, rows): (String, u64), + (files, rows): (Vec, u64), ) -> RunRecord { let about = record_table(spec); RunRecord { @@ -438,7 +443,7 @@ pub(crate) fn merged_record( cursor_after: None, state: None, row_count: rows, - files: vec![file], + files, note: Some( match spec.strategy { WriteStrategy::Merge => "files source merge", @@ -799,8 +804,8 @@ fn record_json(record: &RunRecord) -> Result { serde_json::to_string(record).map_err(|error| LoaderError::Invalid(error.to_string())) } -/// The file a group of a files merge or `delete_insert` by `replace_key` reads (it holds one), as its run record names it: `name|version`. -pub(crate) fn merged_file_text(table: &TablePlan, files: &[DataFile]) -> String { +/// The files a group of a files merge or `delete_insert` by `replace_key` reads, in order, as its run record names them: `name|version`. +pub(crate) fn merged_file_text(table: &TablePlan, files: &[DataFile]) -> Vec { let location = table .spec .files @@ -816,8 +821,7 @@ pub(crate) fn merged_file_text(table: &TablePlan, files: &[DataFile]) -> String data.file.version() ) }) - .collect::>() - .join(", ") + .collect() } /// The name of the merged file `merged` (`name|version`). @@ -837,6 +841,61 @@ pub(crate) fn in_file(merged: &str, error: LoaderError) -> LoaderError { } } +/// `earlier` (as [`with_file_keys`] takes it) with the `replace_key` values of each of `files` (`name|version`, in the order the commit loads them) added, as each file's staged rows in `checked` hold them; or why the commit cannot replace them, checking each file as a commit of its own would: a value an earlier file of the run holds ([`with_file_keys`]), or one the table holds from a file still listed ([`listed_sources`]). Under `on_overlapping_files = "last_wins"` a file's staged rows of a value a later file of the commit holds (compared as typed) are deleted first, so the later file replaces it. +pub(crate) async fn checked_file_keys( + table: &TablePlan, + (checked, replace_key): (&CheckedCommit<'_>, &[String]), + (files, mut earlier): (&[String], BTreeMap), +) -> Result, LoaderError> { + let named = (replace_key, SOURCE_FILE_COLUMN); + let staged = key_sources(checked, (checked.staging, None), named).await?; + let held = key_sources(checked, (checked.table, Some(checked.staging)), named).await?; + for file in files { + let name = merged_name(file); + let values: Vec = staged + .iter() + .filter(|(of, _)| of == name) + .map(|(_, value)| value.clone()) + .collect(); + let named_file = |reason: String| match files.len() { + 1 => LoaderError::Invalid(reason), + _ => in_file(file, LoaderError::Invalid(reason)), + }; + earlier = with_file_keys(table, (file, &values), earlier).map_err(named_file)?; + let sources: BTreeSet = held + .iter() + .filter(|(of, value)| of != name && values.contains(value)) + .map(|(of, _)| of.clone()) + .collect(); + let sources: Vec = sources.into_iter().collect(); + listed_sources(table, file, &sources).map_err(named_file)?; + } + if files.len() > 1 && table.spec.on_overlapping_files == OnOverlappingFiles::LastWins { + let names: Vec<&str> = files.iter().map(|file| merged_name(file)).collect(); + let key = (replace_key.iter()) + .map(|name| written_column(checked.columns, name)) + .collect::>>() + .ok_or_else(|| { + LoaderError::Invalid(format!( + "a replace_key column of {} isn't one the table writes", + replace_key.join(", ") + )) + })?; + let delete = sql_adapter(checked.warehouse).render_superseded_delete( + checked.staging, + (&key, SOURCE_FILE_COLUMN, SOURCE_ROW_COLUMN), + &names, + ); + execute(checked.destination, vec![delete]).await?; + } + Ok(earlier) +} + +/// The column of `columns` named `name`. +fn written_column<'a>(columns: &'a [DesiredColumn], name: &str) -> Option<&'a DesiredColumn> { + columns.iter().find(|column| column.name == name) +} + /// `earlier` (each `replace_key` value this run's commits of a files source's `delete_insert` into `table` inserted, with the file it came from, named below the source's location) with the key `values` of the commit of `file` (`name|version`) added; or why that commit cannot replace them: one of them came in an earlier commit of the same run, so this commit would delete that file's rows. A value must come whole in one file, unless `on_overlapping_files = "last_wins"`, where the later file replaces it; [`listed_sources`] checks the files of earlier runs. pub(crate) fn with_file_keys( table: &TablePlan, @@ -1004,7 +1063,7 @@ pub(crate) fn overlapping_files_problem( (true, _, true) => None, (false, _, _) => Some(format!("{FOR}; this source reads no files, so remove it")), (true, WriteStrategy::Merge, _) => Some(format!( - "{FOR}; a merge applies the files one at a time, so the last file already wins each unique_key; remove it" + "{FOR}; a merge already lets the last file win each unique_key; remove it" )), (true, WriteStrategy::DeleteInsert, false) => Some(format!( "{FOR}; this delete_insert replaces a cursor range, not key values, so remove it, or set replace_key" @@ -1015,11 +1074,21 @@ pub(crate) fn overlapping_files_problem( } } -/// How `plan` describes the commits of a files table whose `groups` files are each recorded in the run log: their order, and what `on_overlapping_files = "last_wins"` adds. -pub(crate) fn recorded_commits(groups: usize, spec: &TableSpec) -> String { - let files = match groups { +/// How `plan` describes the commits of a files table whose files, in `groups`, are each recorded in the run log: how many commits, their order, and what `on_overlapping_files = "last_wins"` adds. +pub(crate) fn recorded_commits(groups: &[FileGroup], spec: &TableSpec) -> String { + let files = match groups.iter().map(|group| group.files.len()).sum::() { 1 => "1 file".to_owned(), - count => format!("{count} files"), + count => format!("{} files", thousands(count as u64)), + }; + let commits = match groups.len() { + _ if groups.iter().all(|group| group.files.len() == 1) && groups.len() > 1 => { + "one commit each".to_owned() + } + 1 => "1 commit".to_owned(), + count => format!( + "{count} commits of up to {} files or about 1 GB", + thousands(MAX_BATCH_FILES as u64) + ), }; let order = match spec.files_order { FilesOrder::Modified => "oldest modified first (ties by path)", @@ -1031,7 +1100,7 @@ pub(crate) fn recorded_commits(groups: usize, spec: &TableSpec) -> String { } OnOverlappingFiles::Fail => "", }; - format!("{files}, one commit each, {order}, each recorded in the run log{last_wins}") + format!("{files}, {commits}, {order}, each recorded in the run log{last_wins}") } /// The files a merge already merged into `request`'s table at the version listed now, which it skips (one rewritten in place since is merged again, as the newest), and under `--reload` none, with every file the table's merges recorded instead (oldest first), whose order the reload keeps; or why a files source cannot load into it: only into a DuckDB or Postgres table, never with `state = "logged"`, nor `--reload` with `unnest` or of a `delete_insert` (one commit would lose the files' order); none while the table does not exist. @@ -1057,7 +1126,7 @@ pub(crate) async fn merge_ledger( } if request.options.reload && spec.strategy != WriteStrategy::Merge { return Ok(Err(format!( - "--reload would {what} every file in one commit and lose their order; drop the table and load again to {what} every file anew" + "--reload rebuilds only a merge's table in file order, so it would {what} every file in one commit and lose their order; drop the table and load again to {what} every file anew" ))); } if request.options.reload && spec.unnest != Unnest::None { diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs index 36b144ef0..24b248adb 100644 --- a/crates/sqlload/src/loader/_helpers/specs.rs +++ b/crates/sqlload/src/loader/_helpers/specs.rs @@ -320,7 +320,7 @@ pub(crate) fn build_spec( && !(strategy == WriteStrategy::Merge || (strategy == WriteStrategy::DeleteInsert && !replace_key.is_empty())) { - problems.push(format!("order = \"path\" orders the files a merge or a delete_insert by replace_key applies one at a time; {strategy} loads them without an order, so remove it")); + problems.push(format!("order = \"path\" orders the files a merge or a delete_insert by replace_key applies in turn; {strategy} loads them without an order, so remove it")); } let own_strategy = strategy_origin == SettingOrigin::Table && !keyed; let overlapping = layered( @@ -775,50 +775,6 @@ fn cdc_key_problems(table: &SourceTable, unique_key: &[String]) -> Vec { } } -/// Why a table someone wrote is blocked without a description they wrote, with where to give it; `None` when it has one, or when it was discovered (matched by an `include` pattern, not named), which takes the source's own table comment or none (D203). Python tables, files sources (described by the source's own `description`), `[sources.*.tables]` entries and tables named exactly in `include` are written. -pub(crate) fn missing_description( - source_name: &str, - source: &SourceConfig, - table: &SourceTable, -) -> Option { - let qualified = table.qualified_name(); - let entry = source.tables.get(&qualified); - let written = - !source.discovers_tables() || entry.is_some() || source.include.contains(&qualified); - let described = [ - entry.and_then(|entry| entry.description.as_deref()), - source - .description - .as_deref() - .filter(|_| matches!(source.kind, SourceKind::Files | SourceKind::GoogleSheets)), - ] - .into_iter() - .flatten() - .any(|text| !text.trim().is_empty()); - let fix = match source.kind { - SourceKind::Python => { - "no description; add description=\"...\" to its table declaration, or a docstring to its function".to_owned() - } - SourceKind::Files => { - format!("no description; add description = \"...\" under [sources.{source_name}]") - } - SourceKind::GoogleSheets => format!( - "no description; add description = \"...\" under [sources.{source_name}.tables.{qualified}], or under [sources.{source_name}] for every table" - ), - SourceKind::Postgres - | SourceKind::PostgresCdc - | SourceKind::Snowflake - | SourceKind::Odbc - | SourceKind::BigQuery - | SourceKind::Databricks - | SourceKind::DuckDb - | SourceKind::MotherDuck => format!( - "no description; add description = \"...\" under [sources.{source_name}.tables.\"{qualified}\"]" - ), - }; - (written && !described).then_some(fix) -} - fn glob(pattern: &str) -> Result { Glob::new(pattern).map_err(|error| format!("invalid column pattern {pattern:?}: {error}")) } diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index 97aa6c732..aa148e24e 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -35,6 +35,7 @@ use crate::adapters::snowflake::models::StageRoute; use crate::batch_plan::main::file_groups::file_groups; use crate::batch_plan::main::ignored_batch::ignored_batch; use crate::batch_plan::main::ignored_concurrency::ignored_concurrency; +use crate::batch_plan::main::ordered_groups::ordered_groups; use crate::batch_plan::main::split_windows::split_windows; use crate::batch_plan::models::{FileGroup, WindowSplit}; use crate::bigquery_source::constants::BIGQUERY_SOURCE; @@ -90,7 +91,7 @@ use crate::project::models::BatchStep; use crate::project::models::MaskMethod; use crate::project::models::WriteStrategy; use crate::project::models::{FilesFormat, OutputEncoding}; -use crate::project::models::{NullCursor, OnDuplicateKeys}; +use crate::project::models::{NullCursor, OnDuplicateKeys, OnOverlappingFiles}; use crate::publish_checks::main::check_column_problems::check_column_problems; use crate::routing::main::plan_route::plan_route; use crate::routing::models::FileFormat; @@ -306,56 +307,6 @@ fn unwritten_column( .next() } -/// `mode` and `schema` once `unordered_merge` checked how a files merge orders its rows: its warning added, or the plan blocked by its problem. -fn ordered_mode(mode: WriteMode, mut schema: SchemaPlan) -> (WriteMode, SchemaPlan) { - match unordered_merge(mode, &schema) { - Ok((mode, warning)) => { - schema.warnings.extend(warning); - (mode, schema) - } - Err(problem) => (WriteMode::Append, blocked_plan(vec![problem])), - } -} - -/// `mode` for a files merge whose rows are ranked by `_sqlload_source_row` when that column is not written (excluded, or left out by the contract), so no row is known to be last: without `newest_by`, repeated keys fail as before 0.39.0, with a warning saying why; with it, the problem, as ties need that order. -fn unordered_merge( - mode: WriteMode, - schema: &SchemaPlan, -) -> Result<(WriteMode, Option), String> { - let WriteMode::Merge { - unique_key, - dedupe: Some(dedupe), - not_null_keys, - } = &mode - else { - return Ok((mode, None)); - }; - if schema.blocked.is_some() { - return Ok((mode, None)); - } - let ordered = dedupe.ordinal != SOURCE_ROW_COLUMN - || schema - .write_columns - .iter() - .any(|column| column.name == SOURCE_ROW_COLUMN); - match (ordered, dedupe.newest_by.is_empty()) { - (true, _) => Ok((mode, None)), - (false, false) => Err(format!( - "newest_by breaks ties by row order, which a files merge takes from {SOURCE_ROW_COLUMN}; stop excluding it" - )), - (false, true) => Ok(( - WriteMode::Merge { - unique_key: unique_key.clone(), - dedupe: None, - not_null_keys: not_null_keys.clone(), - }, - Some(format!( - "{SOURCE_ROW_COLUMN} is not written, so a file's rows have no order to pick the last of a repeated key by: a file repeating its unique_key fails; write {SOURCE_ROW_COLUMN} to keep its last row instead" - )), - )), - } -} - /// A table's columns and schema plan once more during its load, when its source shows new columns; nothing else about its plan is read or checked again. `known` is the live columns when they are known to be current (a Python table's first batch, just after plan); otherwise one read of them, except that a files target has no SQL to read them with and keeps `stale`, what plan saw. pub(crate) async fn replan_columns( spec: &TableSpec, @@ -864,7 +815,7 @@ async fn plan_table_once( }; let source_filter = source_filter.filter(|_| spec.files.is_none()); schema.warnings.extend(resnapshot_warning(spec, options)); - let (mode, mut schema) = ordered_mode(captured_mode(spec, mode), schema); + let mode = captured_mode(spec, mode); let exists = live.is_some(); schema = checked_columns(spec, schema); if schema.blocked.is_none() { @@ -913,6 +864,7 @@ async fn plan_table_once( read_columns: &read_columns, files: files.as_ref(), mode: &mode, + write_columns: &schema.write_columns, }); let commits = match commits { Ok(commits) => commits, @@ -1138,7 +1090,7 @@ pub(crate) fn reader_offer( } } -/// One commit per file, in the order planned: a files source's merge, or `delete_insert` by `replace_key`, applies each file on its own, so a key repeated across files ends with the newest file's value (or rows). +/// One commit per file, in the order planned: a files source's merge under `on_duplicate_keys = "fail"`, which refuses a key repeated within one file but not across files, checks each file on its own; and a `delete_insert` by `replace_key` under `on_overlapping_files = "last_wins"` without `_sqlload_source_row`, whose rows of one file a later file of a batch replaces can't be told apart from its other rows by number. fn one_file_each(count: usize) -> Vec { (0..count) .map(|index| FileGroup { @@ -1251,10 +1203,7 @@ fn write_mode( unique_key: spec.unique_key.clone(), not_null_keys: spec.not_null_keys.clone(), dedupe: (spec.on_duplicate_keys == OnDuplicateKeys::Dedupe).then(|| MergeDedupe { - ordinal: match spec.files.is_some() { - true => SOURCE_ROW_COLUMN.to_owned(), - false => MERGE_ROW_COLUMN.to_owned(), - }, + ordinal: MERGE_ROW_COLUMN.to_owned(), newest_by: spec.newest_by.clone(), }), }, @@ -1680,6 +1629,8 @@ pub(crate) struct CommitRequest<'a> { pub(crate) read_columns: &'a [ReadColumn], pub(crate) files: Option<&'a FilesRead>, pub(crate) mode: &'a WriteMode, + /// The columns the table writes, which say whether a files source's rows carry their number in their file. + pub(crate) write_columns: &'a [DesiredColumn], } /// The commits of a table's load: a files source's batches of whole partitions; a cursor table's `batch_size` windows; an ordered Postgres read (an index leads with a non-text cursor) committing along the way; the whole read for any other cursor table, since committing part of an unordered read could skip rows the next run never reads, for `batch_size = "none"`, and for `append` without a cursor or logged state, which would have nowhere to resume from; about every 1 GB for other tables without a cursor. @@ -1694,9 +1645,19 @@ pub(crate) fn commit_rule(request: &CommitRequest<'_>) -> Result Some(cursor_column.as_str()), _ => None, }; - if matches!(request.mode, WriteMode::Merge { .. }) || keyed { + let numbered = request + .write_columns + .iter() + .any(|column| column.name == SOURCE_ROW_COLUMN); + let unbatched_last_wins = keyed + && spec.on_overlapping_files == OnOverlappingFiles::LastWins + && !(numbered && sql_adapter(spec.warehouse).deletes_superseded_rows()); + if matches!(request.mode, WriteMode::Merge { dedupe: None, .. }) || unbatched_last_wins { return Ok(CommitRule::FileGroups(one_file_each(files.files.len()))); } + if matches!(request.mode, WriteMode::Merge { .. }) || keyed { + return Ok(CommitRule::FileGroups(ordered_groups(&files.files))); + } return Ok(CommitRule::FileGroups(file_groups(&files.files, by_value))); } if keyed { diff --git a/crates/sqlload/src/loader/main/_plan_files.rs b/crates/sqlload/src/loader/main/_plan_files.rs index ef6d4ed1c..c33bbaf1f 100644 --- a/crates/sqlload/src/loader/main/_plan_files.rs +++ b/crates/sqlload/src/loader/main/_plan_files.rs @@ -80,6 +80,7 @@ pub(in crate::loader) async fn plan_files_table( read_columns: &read_columns, files: files.as_ref(), mode: &mode, + write_columns: &schema.write_columns, }); let commits = match commits { Ok(_) if destination.encoding.single_file => CommitRule::Whole(WholeReason::SingleFile), diff --git a/crates/sqlload/src/loader/models.rs b/crates/sqlload/src/loader/models.rs index 4df35dfc5..cc07586ce 100644 --- a/crates/sqlload/src/loader/models.rs +++ b/crates/sqlload/src/loader/models.rs @@ -1171,8 +1171,8 @@ pub struct Publisher { pub batch_bytes: usize, /// A files target's folder and the commit being written. pub files: Option, - /// A files source's merge, or `delete_insert` by `replace_key`: the file (`name|version`) the next commit loads, recorded in the run log with it. - pub merged_file: Option, + /// A files source's merge, or `delete_insert` by `replace_key`: the files (`name|version`) the next commit loads, in order, recorded in the run log with it; empty for other tables. + pub merged_files: Vec, /// The `replace_key` values committed so far; `None` without a `replace_key`. pub replaced: Option, /// The rows a key scan's commit marked deleted; `None` for other loads. diff --git a/crates/sqlload/src/loader/tests/helpers.rs b/crates/sqlload/src/loader/tests/helpers.rs index 3ac0492ec..0ec01cee1 100644 --- a/crates/sqlload/src/loader/tests/helpers.rs +++ b/crates/sqlload/src/loader/tests/helpers.rs @@ -264,7 +264,7 @@ pub(crate) fn merged_file_round_trip( let record = crate::loader::_helpers::run_log::merged_record( spec, ("load-1", 1), - (format!("{name}|{version}"), 2), + (vec![format!("{name}|{version}")], 2), ); let text = serde_json::to_string(&record).map_err(|error| error.to_string())?; let decoded = crate::run_log::main::decode_runs::decode_runs(&[text]); diff --git a/crates/sqlload/src/publish_checks/main/key_sources.rs b/crates/sqlload/src/publish_checks/main/key_sources.rs index 1d8883a29..3752d4811 100644 --- a/crates/sqlload/src/publish_checks/main/key_sources.rs +++ b/crates/sqlload/src/publish_checks/main/key_sources.rs @@ -1,24 +1,39 @@ -//! The files a commit's key values already come from. +//! Which files hold which key values, among a commit's staged rows or its table's. -use crate::adapters::contract::models::CheckQuery; +use crate::adapters::contract::models::{CheckQuery, TableName}; use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter; use crate::adapters::dispatch::main::session::text_values::text_values; use crate::loader::errors::LoaderError; use crate::publish_checks::models::CheckedCommit; -/// The distinct values of `file_column` among the rows of `commit`'s table whose `key` values its staged rows hold, other than `file`, in order: the files a `delete_insert` by `replace_key` would delete rows of. +/// The distinct `(file, key values)` pairs of `relation`'s rows (`file_column`, and the `key` values as `KeyValues` gives them), only those whose values a row of `held_by` holds when set: with `relation` the staged rows, which file of a commit holds which values; with the table and `held_by` the staged rows, the files a `delete_insert` by `replace_key` would delete rows of. pub async fn key_sources( commit: &CheckedCommit<'_>, - (key, file_column, file): (&[String], &str, &str), -) -> Result, LoaderError> { + (relation, held_by): (&TableName, Option<&TableName>), + (key, file_column): (&[String], &str), +) -> Result, LoaderError> { let query = sql_adapter(commit.warehouse).render_check_query( - commit.table, - CheckQuery::KeySources { + relation, + CheckQuery::KeyFiles { key, - staged: commit.staging, file_column, - file, + held_by, }, ); - text_values(commit.destination, query).await + let texts = text_values(commit.destination, query).await?; + texts + .iter() + .map(|text| { + let (length, rest) = text.split_once(':').unwrap_or_default(); + let length: usize = length.parse().map_err(|_| { + LoaderError::Invalid(format!("the key's file is unreadable: {text}")) + })?; + let split = rest + .char_indices() + .nth(length) + .map_or(rest.len(), |(at, _)| at); + let (file, values) = rest.split_at(split); + Ok((file.to_owned(), values.to_owned())) + }) + .collect() } diff --git a/crates/sqlload/src/publish_checks/main/refuse_null_keys.rs b/crates/sqlload/src/publish_checks/main/refuse_null_keys.rs index b4c0dcf7f..477fdb7ff 100644 --- a/crates/sqlload/src/publish_checks/main/refuse_null_keys.rs +++ b/crates/sqlload/src/publish_checks/main/refuse_null_keys.rs @@ -1,23 +1,53 @@ //! Refuse a merge or `delete_insert` batch holding a NULL in its key. +use crate::adapters::contract::models::CheckQuery; +use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter; +use crate::adapters::dispatch::main::session::text_values::text_values; +use crate::files_source::constants::{SOURCE_FILE_COLUMN, VERSION_SEPARATOR}; use crate::loader::errors::LoaderError; use crate::publish_checks::_helpers::failures::null_keys; use crate::publish_checks::models::CheckedCommit; -/// Fail `commit` when a staged row holds a NULL in a column of `unique_key` (the key setting `setting` names: `unique_key`, or `replace_key`), naming the columns and example keys, so a merge or `delete_insert` under `null_keys = "fail"` never writes a NULL key; `file` says the rows are one source file. +/// Fail `commit` when a staged row holds a NULL in a column of `unique_key` (the key setting `setting` names: `unique_key`, or `replace_key`), naming the columns and example keys, so a merge or `delete_insert` under `null_keys = "fail"` never writes a NULL key. `files` are the source files of the commit (`name|version`, in order), if any: with one, the rows are that file's; with more, the error names the first holding such a row, when the table writes `_sqlload_source_file`. pub async fn refuse_null_keys( commit: &CheckedCommit<'_>, (unique_key, setting): (&[String], &str), - file: bool, + files: &[String], ) -> Result<(), LoaderError> { - let subject = match file { - true => "the file holds", - false => "the loaded rows hold", + let subject = match files.len() { + 0 => "the loaded rows hold", + _ => "the file holds", }; - match null_keys(commit, unique_key).await? { - None => Ok(()), - Some(detail) => Err(LoaderError::Invalid(format!( - "{subject} a NULL in its {setting}: {detail}; null_keys = \"fail\" refuses a NULL in a key column, so fill it in the source, or set null_keys = \"match\" to match it with a NULL" - ))), - } + let Some(detail) = null_keys(commit, unique_key).await? else { + return Ok(()); + }; + let names_files = commit + .columns + .iter() + .any(|column| column.name == SOURCE_FILE_COLUMN); + let named = match files.len() { + _ if !names_files => String::new(), + 0 | 1 => String::new(), + _ => { + let query = sql_adapter(commit.warehouse).render_check_query( + commit.staging, + CheckQuery::NullKeyFiles(unique_key, SOURCE_FILE_COLUMN), + ); + let holding = text_values(commit.destination, query).await?; + let mut named = String::new(); + for file in files { + let name = file + .rsplit_once(VERSION_SEPARATOR) + .map_or(file.as_str(), |(name, _)| name); + if holding.contains(&name.to_owned()) { + named = format!("{name}: "); + break; + } + } + named + } + }; + Err(LoaderError::Invalid(format!( + "{named}{subject} a NULL in its {setting}: {detail}; null_keys = \"fail\" refuses a NULL in a key column, so fill it in the source, or set null_keys = \"match\" to match it with a NULL" + ))) } diff --git a/crates/sqlload/tests/loader.rs b/crates/sqlload/tests/loader.rs index 29dfa343f..706d0f704 100644 --- a/crates/sqlload/tests/loader.rs +++ b/crates/sqlload/tests/loader.rs @@ -57,6 +57,8 @@ mod test_debug_project; mod test_duckdb_source; #[path = "loader/test_file_store.rs"] mod test_file_store; +#[path = "loader/test_files_batches.rs"] +mod test_files_batches; #[path = "loader/test_files_round_trip.rs"] mod test_files_round_trip; #[path = "loader/test_files_to_clickhouse.rs"] diff --git a/crates/sqlload/tests/loader/merge/helpers.rs b/crates/sqlload/tests/loader/merge/helpers.rs index 7aedf4a63..87dbe6cce 100644 --- a/crates/sqlload/tests/loader/merge/helpers.rs +++ b/crates/sqlload/tests/loader/merge/helpers.rs @@ -1,6 +1,10 @@ //! What the merge loader tests share: a Postgres table with repeated keys loaded into any target. use std::collections::BTreeMap; +use std::path::Path; +use std::sync::Arc; + +use arrow_array::{ArrayRef, Int64Array, RecordBatch, StringArray}; use sqlload::adapters::dispatch::main::session::execute::execute; use sqlload::adapters::dispatch::main::session::single_text::single_text; @@ -16,8 +20,8 @@ use sqlload_testkit::postgres::constants::{PASSWORD, USER}; use crate::helpers::{described_project, load_selected, words}; use crate::test_types::{ - DedupeRule, DedupeTargetTestCase, FilesMergeTestCase, MergeFile, MergeStep, - NullKeyTargetTestCase, ReplaceKeyTargetTestCase, SyncDeletesTestCase, + DedupeRule, DedupeTargetTestCase, FileBatchTestCase, FilesMergeTestCase, ManyFilesTestCase, + MergeFile, MergeStep, NullKeyTargetTestCase, ReplaceKeyTargetTestCase, SyncDeletesTestCase, }; /// `public.readings`: `s1` and `s2` repeat with their `seen_at` order unlike their `reading_id` order, `s2`'s last row has no `seen_at`, and two rows have no sensor (nor `seen_at`), a repeated NULL key. @@ -796,3 +800,254 @@ pub(crate) const fn load_step( expected_plan, } } + +/// The `(id, v, seen)` rows of file `index` of [`run_many_files_case`]: its first and last rows share an id, which other files hold too, and `bad` gives its second row a `seen` that isn't a number. +fn generated_rows(index: usize, bad: bool) -> Vec<(i64, String, String)> { + let at = i64::try_from(index).unwrap_or(i64::MAX); + [(at * 3) % 50, (at * 7 + 1) % 50, (at * 3) % 50] + .into_iter() + .enumerate() + .map(|(row, id)| { + let seen = [at.to_string(), "oops".to_owned()]; + let seen = seen[usize::from(bad && row == 1)].clone(); + (id, format!("f{index}r{row}"), seen) + }) + .collect() +} + +/// Write `rows` as `folder/key.`, modified at `modified_secs`. +fn write_batch_file( + folder: &Path, + (format, key): (&str, &str), + rows: &[(i64, String, String)], + modified_secs: u64, +) -> Result<(), String> { + let path = folder.join(format!("{key}.{format}")); + let writers: [(&str, FileBytes); 2] = [("parquet", parquet_bytes), ("csv", csv_bytes)]; + let (_, writer) = writers + .iter() + .find(|(name, _)| *name == format) + .ok_or_else(|| format!("no writer for {format} files"))?; + let bytes = writer(rows)?; + std::fs::write(&path, bytes).map_err(|error| error.to_string())?; + let modified = std::time::UNIX_EPOCH + std::time::Duration::from_secs(modified_secs); + std::fs::File::options() + .write(true) + .open(&path) + .and_then(|opened| opened.set_modified(modified)) + .map_err(|error| error.to_string()) +} + +/// Writes a batched test file's rows as one format's bytes. +type FileBytes = fn(&[(i64, String, String)]) -> Result, String>; + +/// `rows` as one Parquet file of `id` and `seen` integers and `v` text. +fn parquet_bytes(rows: &[(i64, String, String)]) -> Result, String> { + let seen: Vec = rows + .iter() + .map(|row| row.2.parse::().map_err(|error| error.to_string())) + .collect::>()?; + let columns: Vec<(&str, ArrayRef)> = vec![ + ( + "id", + Arc::new(Int64Array::from_iter_values(rows.iter().map(|row| row.0))), + ), + ( + "v", + Arc::new(StringArray::from_iter_values( + rows.iter().map(|row| row.1.as_str()), + )), + ), + ("seen", Arc::new(Int64Array::from(seen))), + ]; + let batch = RecordBatch::try_from_iter(columns).map_err(|error| error.to_string())?; + let mut buffer: Vec = Vec::new(); + let mut writer = parquet::arrow::ArrowWriter::try_new(&mut buffer, batch.schema(), None) + .map_err(|error| error.to_string())?; + writer.write(&batch).map_err(|error| error.to_string())?; + writer.close().map_err(|error| error.to_string())?; + Ok(buffer) +} + +/// `rows` as a CSV file with a header line. +fn csv_bytes(rows: &[(i64, String, String)]) -> Result, String> { + let lines: Vec = rows + .iter() + .map(|(id, v, seen)| format!("{id},{v},{seen}\n")) + .collect(); + Ok(format!("id,v,seen\n{}", lines.concat()).into_bytes()) +} + +/// A project whose files connection is rooted at `root`, loading into `target` (its TOML lines) under `schema`, with one files source per `(name, folder, settings)`. +fn batch_project( + root: &Path, + (target, schema): (&str, &str), + format: &str, + sources: &[(&str, &str, &str)], +) -> String { + let sources: Vec = sources + .iter() + .map(|(name, folder, settings)| { + format!( + "[sources.{name}]\nkind = \"files\"\ndescription = \"Test table.\"\nconnection = \"files\"\npath = \"{folder}/\"\nformat = \"{format}\"\n{}\n", + settings.replace("{name}", name) + ) + }) + .collect(); + format!( + "[connections.files]\nkind = \"filesystem\"\nroot = \"{}\"\n\n[connections.target]\n{target}\n\n[targets.dev]\nconnection = \"target\"\nloader_schema = \"{schema}\"\n\n{}", + root.display(), + sources.concat() + ) +} + +/// How many `run` records the run log under `schema` holds for `table`, as text. +fn commits(schema: &str, table: &str) -> String { + format!( + "(SELECT CAST(count(*) AS VARCHAR) FROM {schema}._sqlload_runs WHERE table_name = '{table}' AND event = 'run')" + ) +} + +/// The rows of `left` not in `right` (each a query of the same columns), counted, as text. +fn missing(left: &str, right: &str) -> String { + format!("CAST((SELECT count(*) FROM ({left} EXCEPT ALL {right}) AS missing) AS VARCHAR)") +} + +/// Load `test_case.files` into `target` under `schema` as table `batched` in one run, then into table `single` one file per run; the comparison its `expected_result` describes. +pub(crate) async fn run_batch_case( + (target, schema): (&str, &str), + test_case: &FileBatchTestCase, +) -> Result, String> { + let directory = tempfile::tempdir().map_err(|error| error.to_string())?; + let text = batch_project( + directory.path(), + (target, schema), + test_case.format, + &[ + ("batched", "all", test_case.settings), + ("single", "one", test_case.settings), + ], + ); + let project = described_project(&text).await?; + let destinations = connect_targets(&project) + .await + .map_err(|error| error.to_string())?; + let destination = destinations + .get("dev") + .ok_or_else(|| "no dev destination".to_owned())?; + for folder in ["all", "one"] { + std::fs::create_dir_all(directory.path().join(folder)) + .map_err(|error| error.to_string())?; + } + let rows_of = |rows: &[(i64, &str, i64)]| -> Vec<(i64, String, String)> { + rows.iter() + .map(|(id, v, seen)| (*id, (*v).to_owned(), seen.to_string())) + .collect() + }; + for file in test_case.files { + write_batch_file( + &directory.path().join("all"), + (test_case.format, file.key), + &rows_of(file.rows), + file.modified_secs, + )?; + } + load_selected(&project, &destinations, &["batched"]).await?; + for file in test_case.files { + write_batch_file( + &directory.path().join("one"), + (test_case.format, file.key), + &rows_of(file.rows), + file.modified_secs, + )?; + load_selected(&project, &destinations, &["single"]).await?; + } + let rows = |table: &str| { + format!( + "SELECT id, v, seen, regexp_replace(_sqlload_source_file, '^.*/', '') FROM {schema}.{table}" + ) + }; + let (batched, single) = (rows("batched"), rows("single")); + let query = format!( + "SELECT {} || '|' || {} || '|' || (SELECT string_agg(CAST(id AS VARCHAR) || ':' || CAST(v AS VARCHAR), ',' ORDER BY id, v) FROM {schema}.batched) || '|' || {} || '|' || {}", + missing(&batched, &single), + missing(&single, &batched), + commits(schema, "batched"), + commits(schema, "single") + ); + single_text(destination, query) + .await + .map_err(|error| error.to_string()) +} + +/// Load `test_case.files` generated files into DuckDB as table `batched` (with a bad file first, then fixed, when it has one; after the failed load, the files its run log records) and once more with nothing new, then as table `appended`, every row; the two checks its expected fields describe. +pub(crate) async fn run_many_files_case( + test_case: &ManyFilesTestCase, +) -> Result<(String, Option), String> { + let directory = tempfile::tempdir().map_err(|error| error.to_string())?; + let folder = directory.path().join("all"); + std::fs::create_dir_all(&folder).map_err(|error| error.to_string())?; + let text = batch_project( + directory.path(), + ("kind = \"duckdb\"\ndatabase = \":memory:\"", "raw"), + test_case.format, + &[ + ("batched", "all", test_case.settings), + ("appended", "all", "write_strategy = \"append\""), + ], + ); + let project = described_project(&text).await?; + let destinations = connect_targets(&project) + .await + .map_err(|error| error.to_string())?; + let destination = destinations + .get("dev") + .ok_or_else(|| "no dev destination".to_owned())?; + let write = |index: usize, bad: bool| { + write_batch_file( + &folder, + (test_case.format, &format!("f{index:05}")), + &generated_rows(index, bad), + 1_000_000 + index as u64, + ) + }; + for index in 0..test_case.files { + write(index, test_case.bad_files.contains(&index))?; + } + let mut after_error = String::new(); + for bad in test_case.bad_files { + let error = load_selected(&project, &destinations, &["batched"]) + .await + .err() + .unwrap_or_default(); + assert!( + error.contains(test_case.expected_error), + "{}: {error}", + test_case.description + ); + let state = format!( + "SELECT (SELECT CAST(sum(json_array_length(record_json, '$.files')) AS VARCHAR) FROM raw._sqlload_runs WHERE table_name = 'batched' AND event = 'run') || '|' || {}", + commits("raw", "batched") + ); + after_error = single_text(destination, state) + .await + .map_err(|error| error.to_string())? + .unwrap_or_default(); + write(*bad, false)?; + } + load_selected(&project, &destinations, &["batched"]).await?; + load_selected(&project, &destinations, &["batched"]).await?; + load_selected(&project, &destinations, &["appended"]).await?; + let merged = "SELECT id, v, seen FROM raw.batched"; + let oracle = "SELECT id, v, seen FROM (SELECT id, v, seen, row_number() OVER (PARTITION BY id ORDER BY _sqlload_source_file DESC, _sqlload_source_row DESC) AS newest FROM raw.appended) AS ranked WHERE newest = 1"; + let query = format!( + "SELECT {} || '|' || {} || '|' || (SELECT CAST(count(*) AS VARCHAR) FROM raw.batched) || '|' || {} || '|' || (SELECT CAST(sum(CAST(row_count AS BIGINT)) AS VARCHAR) FROM raw._sqlload_runs WHERE table_name = 'batched' AND event = 'run')", + missing(merged, oracle), + missing(oracle, merged), + commits("raw", "batched") + ); + let result = single_text(destination, query) + .await + .map_err(|error| error.to_string())?; + Ok((after_error, result)) +} diff --git a/crates/sqlload/tests/loader/test_files_batches.rs b/crates/sqlload/tests/loader/test_files_batches.rs new file mode 100644 index 000000000..56b01ab21 --- /dev/null +++ b/crates/sqlload/tests/loader/test_files_batches.rs @@ -0,0 +1,273 @@ +use sqlload_testkit::postgres::constants::PASSWORD; +use sqlload_testkit::postgres::main::connect::connect; +use sqlload_testkit::postgres::main::start_postgres::start_postgres; + +use crate::helpers::run_async; +use crate::merge_helpers::{run_batch_case, run_many_files_case}; +use crate::test_types::{BatchFile, FileBatchTestCase, ManyFilesTestCase}; + +const MERGE: &str = "write_strategy = \"merge\"\nunique_key = [\"id\"]\n"; + +const NEWEST_BY: &str = "write_strategy = \"merge\"\nunique_key = [\"id\"]\nnewest_by = \"seen\"\n"; + +const REPLACE_LAST_WINS: &str = "write_strategy = \"delete_insert\"\nreplace_key = [\"id\"]\non_overlapping_files = \"last_wins\"\n"; + +const REPLACE_UNNUMBERED: &str = "write_strategy = \"delete_insert\"\nreplace_key = [\"id\"]\non_overlapping_files = \"last_wins\"\nexclude_columns = [\"_sqlload_source_row\"]\n"; + +const REPLACE_BY_V: &str = "write_strategy = \"delete_insert\"\nreplace_key = [\"v\"]\non_overlapping_files = \"last_wins\"\n"; + +/// Keys repeated inside a file (1 in `c`, 4 in `a`, 2 in `d`) and across files, whose names sort unlike their modified times. +const REPEATS: &[BatchFile] = &[ + BatchFile { + key: "c", + rows: &[(1, "a1", 1), (2, "a2", 1), (1, "a1b", 1), (3, "a3", 1)], + modified_secs: 1_000, + }, + BatchFile { + key: "a", + rows: &[(2, "b2", 2), (4, "b4", 2), (4, "b4b", 2)], + modified_secs: 2_000, + }, + BatchFile { + key: "b", + rows: &[(1, "c1", 3), (5, "c5", 3)], + modified_secs: 3_000, + }, + BatchFile { + key: "d", + rows: &[(3, "d3", 4), (2, "d2", 4), (2, "d2b", 4)], + modified_secs: 4_000, + }, +]; + +/// Under `newest_by = "seen"`: a later file's older row of 1 loses, its tie with the first file's row of 1 wins, and a tie inside a file goes to its last row. +const NEWEST: &[BatchFile] = &[ + BatchFile { + key: "c", + rows: &[(1, "a1", 5), (2, "a2", 1)], + modified_secs: 1_000, + }, + BatchFile { + key: "a", + rows: &[(1, "b1", 3), (2, "b2", 1), (2, "b2b", 1)], + modified_secs: 2_000, + }, + BatchFile { + key: "b", + rows: &[(1, "c1", 5), (3, "c3", 1)], + modified_secs: 3_000, + }, +]; + +/// Values of `replace_key` restated by later files: 2 by `a`, then 1 by `b`, while `c` keeps both rows of its own 1 until then. +const RESTATED: &[BatchFile] = &[ + BatchFile { + key: "c", + rows: &[(1, "a1", 1), (1, "a1b", 1), (2, "a2", 1)], + modified_secs: 1_000, + }, + BatchFile { + key: "a", + rows: &[(2, "b2", 2), (3, "b3", 2)], + modified_secs: 2_000, + }, + BatchFile { + key: "b", + rows: &[(1, "c1", 3)], + modified_secs: 3_000, + }, +]; + +const REPLACE_BY_UUID: &str = "write_strategy = \"delete_insert\"\nreplace_key = [\"v\"]\non_overlapping_files = \"last_wins\"\n[sources.{name}.tables.{name}]\ncolumns = { v = \"uuid\" }\n"; + +/// One `uuid` value written in capitals, then by a later file in small letters: one value, which only differs as text. +const UUID_CASES: &[BatchFile] = &[ + BatchFile { + key: "c", + rows: &[(1, "A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11", 1)], + modified_secs: 1_000, + }, + BatchFile { + key: "a", + rows: &[(2, "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", 2)], + modified_secs: 2_000, + }, +]; + +/// A row without a `v`, then a later file's row whose `v` is the text `NULL`: two values, which only look alike as text. +const NULL_TEXT: &[BatchFile] = &[ + BatchFile { + key: "c", + rows: &[(1, "", 1)], + modified_secs: 1_000, + }, + BatchFile { + key: "a", + rows: &[(2, "NULL", 2)], + modified_secs: 2_000, + }, +]; + +#[test] +fn given_files_loaded_in_one_commit_when_loading_into_duckdb_then_the_rows_match_loading_them_file_by_file() +-> Result<(), String> { + run_async(|| async move { + let test_cases = [ + FileBatchTestCase { + description: "a CSV merge keeps the newest file's row of each key, and the last row of a key a file repeats", + settings: MERGE, + format: "csv", + files: REPEATS, + expected_result: "0|0|1:c1,2:d2b,3:d3,4:b4b,5:c5|1|4", + }, + FileBatchTestCase { + description: "a Parquet merge, which DuckDB reads itself, keeps the same rows", + settings: MERGE, + format: "parquet", + files: REPEATS, + expected_result: "0|0|1:c1,2:d2b,3:d3,4:b4b,5:c5|1|4", + }, + FileBatchTestCase { + description: "a merge with newest_by keeps the newest row of each key, a tie going to the later file", + settings: NEWEST_BY, + format: "csv", + files: NEWEST, + expected_result: "0|0|1:c1,2:b2b,3:c3|1|3", + }, + FileBatchTestCase { + description: "a CSV delete_insert by replace_key under last_wins keeps each value's rows from the last file holding it", + settings: REPLACE_LAST_WINS, + format: "csv", + files: RESTATED, + expected_result: "0|0|1:c1,2:b2,3:b3|1|3", + }, + FileBatchTestCase { + description: "under last_wins a later file replaces only its own values, so a NULL value stays beside the text NULL", + settings: REPLACE_BY_V, + format: "csv", + files: NULL_TEXT, + expected_result: "0|0|2:NULL|1|2", + }, + FileBatchTestCase { + description: "under last_wins a later file replaces a uuid value however its letters are written, as the table compares them", + settings: REPLACE_BY_UUID, + format: "csv", + files: UUID_CASES, + expected_result: "0|0|2:a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11|1|2", + }, + FileBatchTestCase { + description: "without _sqlload_source_row, a delete_insert by replace_key under last_wins commits each file, keeping the same rows", + settings: REPLACE_UNNUMBERED, + format: "csv", + files: RESTATED, + expected_result: "0|0|1:c1,2:b2,3:b3|3|3", + }, + FileBatchTestCase { + description: "a Parquet delete_insert by replace_key under last_wins keeps the same rows", + settings: REPLACE_LAST_WINS, + format: "parquet", + files: RESTATED, + expected_result: "0|0|1:c1,2:b2,3:b3|1|3", + }, + ]; + for test_case in &test_cases { + let result = run_batch_case( + ("kind = \"duckdb\"\ndatabase = \":memory:\"", "raw"), + test_case, + ) + .await?; + assert_eq!( + result.as_deref(), + Some(test_case.expected_result), + "{}", + test_case.description + ); + } + Ok(()) + }) +} + +#[test] +fn given_files_loaded_in_one_commit_when_loading_into_postgres_then_the_rows_match_loading_them_file_by_file() +-> Result<(), String> { + run_async(|| async move { + let postgres = start_postgres().await?; + let client = connect(&postgres).await?; + let target = format!( + "kind = \"postgres\"\nhost = \"127.0.0.1\"\nport = {}\nuser = \"sqlload\"\npassword = \"{PASSWORD}\"\ndatabase = \"app\"", + postgres.port + ); + let test_cases = [ + FileBatchTestCase { + description: "a merge keeps the newest file's row of each key, and the last row of a key a file repeats", + settings: MERGE, + format: "csv", + files: REPEATS, + expected_result: "0|0|1:c1,2:d2b,3:d3,4:b4b,5:c5|1|4", + }, + FileBatchTestCase { + description: "a delete_insert by replace_key under last_wins keeps each value's rows from the last file holding it", + settings: REPLACE_LAST_WINS, + format: "csv", + files: RESTATED, + expected_result: "0|0|1:c1,2:b2,3:b3|1|3", + }, + ]; + for test_case in &test_cases { + client + .batch_execute("DROP SCHEMA IF EXISTS raw CASCADE") + .await + .map_err(|error| error.to_string())?; + let result = run_batch_case((&target, "raw"), test_case).await?; + assert_eq!( + result.as_deref(), + Some(test_case.expected_result), + "{}", + test_case.description + ); + } + Ok(()) + }) +} + +#[test] +fn given_more_than_1000_files_when_merging_into_duckdb_then_they_commit_in_batches_resuming_after_a_bad_file() +-> Result<(), String> { + run_async(|| async move { + let test_cases = [ + ManyFilesTestCase { + description: "1,030 Parquet files, which DuckDB reads itself, merge in 2 commits, each key holding its newest file's last row", + settings: MERGE, + format: "parquet", + files: 1_030, + bad_files: &[], + expected_error: "", + expected_after_error: "", + expected_result: "0|0|50|2|3090", + }, + ManyFilesTestCase { + description: "a bad CSV file in the second batch fails it, naming the file, after the first batch committed; once fixed, the next load merges the second batch, and each file once", + settings: MERGE, + format: "csv", + files: 1_030, + bad_files: &[1_005], + expected_error: "f01005.csv", + expected_after_error: "1000|1", + expected_result: "0|0|50|2|3090", + }, + ]; + for test_case in &test_cases { + let (after_error, result) = run_many_files_case(test_case).await?; + assert_eq!( + (after_error.as_str(), result.as_deref()), + ( + test_case.expected_after_error, + Some(test_case.expected_result) + ), + "{}", + test_case.description + ); + } + Ok(()) + }) +} diff --git a/crates/sqlload/tests/loader/test_files_to_duckdb.rs b/crates/sqlload/tests/loader/test_files_to_duckdb.rs index 0af93982a..c812ea6c0 100644 --- a/crates/sqlload/tests/loader/test_files_to_duckdb.rs +++ b/crates/sqlload/tests/loader/test_files_to_duckdb.rs @@ -8,7 +8,7 @@ use crate::helpers::{ int96_in_a_later_batch, local_store, run_async, run_file_set, run_files_scenario, run_large_partition, run_replaced_keys, seaweedfs_store, }; -use crate::merge_helpers::run_merge_case; +use crate::merge_helpers::{load_step, run_merge_case}; use crate::test_types::{ AfterLoadStep, AfterLoadTestCase, ArchivedReloadTestCase, FileSetTestCase, }; @@ -20,13 +20,13 @@ use crate::test_types::{PeakMemoryTestCase, SourceRowUpgradeTestCase}; pub(crate) const MERGE_CHECK: &str = "SELECT (SELECT string_agg(CAST(id AS VARCHAR) || ':' || v, ',' ORDER BY id) FROM raw.scores) || '|' || (SELECT CAST(count(*) AS VARCHAR) FROM raw._sqlload_runs WHERE table_name = 'scores' AND event = 'run')"; /// Every row as `id:v`, a NULL id as `NULL`, then how many merged files the run log records. -const NULL_MERGE_CHECK: &str = "SELECT (SELECT string_agg(COALESCE(CAST(id AS VARCHAR), 'NULL') || ':' || v, ',' ORDER BY id NULLS LAST) FROM raw.scores) || '|' || (SELECT CAST(count(*) AS VARCHAR) FROM raw._sqlload_runs WHERE table_name = 'scores' AND event = 'run')"; +const NULL_MERGE_CHECK: &str = "SELECT (SELECT COALESCE(string_agg(COALESCE(CAST(id AS VARCHAR), 'NULL') || ':' || v, ',' ORDER BY id NULLS LAST), '') FROM raw.scores) || '|' || (SELECT CAST(count(*) AS VARCHAR) FROM raw._sqlload_runs WHERE table_name = 'scores' AND event = 'run')"; const NULL_FAIL_SETTINGS: &str = "write_strategy = \"merge\"\nunique_key = [\"id\"]\nnull_keys = \"fail\"\n"; /// Every row as `id:v`, the run log's events in order (each run with its file), and how many staging or build tables are left. -const RELOAD_CHECK: &str = "SELECT (SELECT string_agg(CAST(id AS VARCHAR) || ':' || v, ',' ORDER BY id) FROM raw.scores) || '|' || (SELECT string_agg(event || coalesce(':' || regexp_extract(json_extract_string(record_json, '$.files[0]'), '([a-z]+[.]csv)', 1), ''), ',' ORDER BY recorded_at) FROM raw._sqlload_runs WHERE table_name = 'scores') || '|' || (SELECT CAST(count(*) AS VARCHAR) FROM information_schema.tables WHERE table_name LIKE '%sqlload_staging%')"; +const RELOAD_CHECK: &str = "SELECT (SELECT string_agg(CAST(id AS VARCHAR) || ':' || v, ',' ORDER BY id) FROM raw.scores) || '|' || (SELECT string_agg(event || coalesce(':' || nullif(array_to_string(list_transform(json_extract_string(record_json, '$.files[*]'), lambda file: regexp_extract(file, '([a-z]+[.]csv)', 1)), '+'), ''), ''), ',' ORDER BY recorded_at) FROM raw._sqlload_runs WHERE table_name = 'scores') || '|' || (SELECT CAST(count(*) AS VARCHAR) FROM information_schema.tables WHERE table_name LIKE '%sqlload_staging%')"; pub(crate) const MERGE_SETTINGS: &str = "write_strategy = \"merge\"\nunique_key = [\"id\"]\n"; @@ -35,7 +35,27 @@ pub(crate) const MERGE_FAIL_SETTINGS: &str = const UNORDERED_MERGE_SETTINGS: &str = "write_strategy = \"merge\"\nunique_key = [\"id\"]\nexclude_columns = [\"_sqlload_source_row\"]\n"; -const UNORDERED_SORT_SETTINGS: &str = "write_strategy = \"merge\"\nunique_key = [\"id\"]\nexclude_columns = [\"_sqlload_source_row\"]\nnewest_by = \"v\"\n"; +/// Two files merged in two runs, the second repeating a key. +const UNORDERED_STEPS: &[MergeStep] = &[ + load_step( + &[MergeFile { + key: "a.csv", + contents: "id,v\n1,a\n2,a\n", + modified_secs: 1_000, + }], + "", + "", + ), + load_step( + &[MergeFile { + key: "b.csv", + contents: "id,v\n1,b\n1,bb\n", + modified_secs: 2_000, + }], + "", + "", + ), +]; /// Monday's customers file: 42 once, and 43 twice, its newest row read first. pub(crate) const MONDAY: &str = "id,v,day\n42,mon,2026-11-02\n43,a,2026-11-02\n43,b,2026-11-01\n"; @@ -135,7 +155,7 @@ const PATH_CHECK: &str = "SELECT string_agg(id || ':' || score || ':' || region const PATH_RESULT: &str = "1:1.0:eu:2026-11-03:main:1:2026-11-08,2:1.0:eu:2026-11-03:main:1:2026-11-08,3:2.0:eu:2026-11-04:main:1:2026-11-08,4:2.0:eu:2026-11-04:main:1:2026-11-08"; /// Every row as `le_version:line:amount` (a NULL version as `NULL`), in order, then how many files the run log records. -const LE_CHECK: &str = "SELECT (SELECT string_agg(COALESCE(le_version, 'NULL') || ':' || line || ':' || CAST(amount AS VARCHAR), ',' ORDER BY le_version NULLS LAST, line) FROM raw.scores) || '|' || (SELECT CAST(count(*) AS VARCHAR) FROM raw._sqlload_runs WHERE table_name = 'scores' AND event = 'run')"; +const LE_CHECK: &str = "SELECT (SELECT COALESCE(string_agg(COALESCE(le_version, 'NULL') || ':' || line || ':' || CAST(amount AS VARCHAR), ',' ORDER BY le_version NULLS LAST, line), '') FROM raw.scores) || '|' || (SELECT CAST(count(*) AS VARCHAR) FROM raw._sqlload_runs WHERE table_name = 'scores' AND event = 'run')"; /// Every row as `le_version:line:amount`, in order, for a table without a run log. const LE_ROWS: &str = "SELECT string_agg(le_version || ':' || line || ':' || CAST(amount AS VARCHAR), ',' ORDER BY le_version, line) FROM raw.scores"; @@ -595,10 +615,10 @@ fn given_csv_files_when_merging_into_duckdb_then_the_newest_modified_file_wins_a removed: &[], loads: true, reload: false, - expected_plan: "3 files, one commit each, oldest modified first (ties by path)", + expected_plan: "3 files, 1 commit, oldest modified first (ties by path)", }], check_query: MERGE_CHECK, - expected_result: "1:b,2:c,3:a|3", + expected_result: "1:b,2:c,3:a|1", }, FilesMergeTestCase { description: "files modified at the same time merge in path order", @@ -624,7 +644,7 @@ fn given_csv_files_when_merging_into_duckdb_then_the_newest_modified_file_wins_a expected_plan: "", }], check_query: MERGE_CHECK, - expected_result: "1:b|2", + expected_result: "1:b|1", }, FilesMergeTestCase { description: "with order = \"path\" files merge in name order whatever their modified times, so a key repeated across files ends with the last name's value", @@ -652,10 +672,10 @@ fn given_csv_files_when_merging_into_duckdb_then_the_newest_modified_file_wins_a removed: &[], loads: true, reload: false, - expected_plan: "3 files, one commit each, in path order", + expected_plan: "3 files, 1 commit, in path order", }], check_query: MERGE_CHECK, - expected_result: "1:wed,2:mon,3:wed|3", + expected_result: "1:wed,2:mon,3:wed|1", }, FilesMergeTestCase { description: "order = \"path\" on an append table, which loads files without an order, is refused at plan", @@ -670,7 +690,7 @@ fn given_csv_files_when_merging_into_duckdb_then_the_newest_modified_file_wins_a removed: &[], loads: true, reload: false, - expected_error: "order = \"path\" orders the files a merge or a delete_insert by replace_key applies one at a time; append loads them without an order, so remove it", + expected_error: "order = \"path\" orders the files a merge or a delete_insert by replace_key applies in turn; append loads them without an order, so remove it", expected_plan: "", }], check_query: "SELECT CAST(count(*) AS VARCHAR) FROM information_schema.tables WHERE table_name = 'scores'", @@ -743,57 +763,11 @@ fn given_csv_files_when_merging_into_duckdb_then_the_newest_modified_file_wins_a expected_result: "42:tue,43:a|3", }, FilesMergeTestCase { - description: "without _sqlload_source_row a file's rows have no order, so files merge as in 0.38.0: one without repeats merges, one repeating a key fails, and plan says why", + description: "excluding _sqlload_source_row leaves the merge's order alone: a file repeating a key keeps its last row", settings: UNORDERED_MERGE_SETTINGS, - steps: &[ - MergeStep { - before: &[], - files: &[MergeFile { - key: "a.csv", - contents: "id,v\n1,a\n2,a\n", - modified_secs: 1_000, - }], - removed: &[], - loads: true, - reload: false, - expected_error: "", - expected_plan: "_sqlload_source_row is not written, so a file's rows have no order to pick the last of a repeated key by", - }, - MergeStep { - before: &[], - files: &[MergeFile { - key: "b.csv", - contents: "id,v\n1,b\n1,bb\n", - modified_secs: 2_000, - }], - removed: &[], - loads: true, - reload: false, - expected_error: "drop/b.csv: the file repeats its unique_key: 1 value of (id) appears in more than one row, such as (1)", - expected_plan: "", - }, - ], + steps: UNORDERED_STEPS, check_query: MERGE_CHECK, - expected_result: "1:a,2:a|1", - }, - FilesMergeTestCase { - description: "newest_by without _sqlload_source_row is refused at plan, as ties need the file's row order", - settings: UNORDERED_SORT_SETTINGS, - steps: &[MergeStep { - before: &[], - files: &[MergeFile { - key: "a.csv", - contents: "id,v\n1,a\n", - modified_secs: 1_000, - }], - removed: &[], - loads: true, - reload: false, - expected_error: "newest_by breaks ties by row order, which a files merge takes from _sqlload_source_row; stop excluding it", - expected_plan: "", - }], - check_query: "SELECT CAST(count(*) AS VARCHAR) FROM information_schema.tables WHERE table_name = 'scores'", - expected_result: "0", + expected_result: "1:bb,2:a|2", }, FilesMergeTestCase { description: "with on_duplicate_keys = \"fail\" a file repeating a key no row has yet fails before it merges, naming the key", @@ -909,11 +883,11 @@ fn given_csv_files_when_merging_into_duckdb_then_the_newest_modified_file_wins_a loads: true, reload: false, expected_error: "", - expected_plan: "2 files, one commit each", + expected_plan: "2 files, 1 commit", }, ], check_query: MERGE_CHECK, - expected_result: "1:b|4", + expected_result: "1:b|2", }, FilesMergeTestCase { description: "a file changed back to a version merged before merges again, as the newest", @@ -943,7 +917,7 @@ fn given_csv_files_when_merging_into_duckdb_then_the_newest_modified_file_wins_a loads: true, reload: false, expected_error: "", - expected_plan: "1 file, one commit each", + expected_plan: "1 file, 1 commit", }, MergeStep { before: &[], @@ -956,7 +930,7 @@ fn given_csv_files_when_merging_into_duckdb_then_the_newest_modified_file_wins_a loads: true, reload: false, expected_error: "", - expected_plan: "1 file, one commit each", + expected_plan: "1 file, 1 commit", }, ], check_query: MERGE_CHECK, @@ -997,11 +971,11 @@ fn given_csv_files_when_merging_into_duckdb_then_the_newest_modified_file_wins_a removed: &[], loads: true, reload: false, - expected_plan: "1 file, one commit each", + expected_plan: "1 file, 1 commit", }, ], check_query: MERGE_CHECK, - expected_result: "1:a2|3", + expected_result: "1:a2|2", }, FilesMergeTestCase { description: "a row without an id in one file is updated by the next file's row without one, not inserted again, as a NULL in the key matches a NULL", @@ -1027,10 +1001,10 @@ fn given_csv_files_when_merging_into_duckdb_then_the_newest_modified_file_wins_a expected_plan: "merge (unique_key: id; NULLs in keys match)", }], check_query: NULL_MERGE_CHECK, - expected_result: "1:a,NULL:bb|2", + expected_result: "1:a,NULL:bb|1", }, FilesMergeTestCase { - description: "with null_keys = \"fail\" a file holding a row without an id fails before it merges, naming the key, and the file before it stays merged", + description: "with null_keys = \"fail\" a file holding a row without an id fails before it merges, naming it and the key, and the file committed with it rolls back", settings: NULL_FAIL_SETTINGS, steps: &[MergeStep { before: &[], @@ -1053,7 +1027,7 @@ fn given_csv_files_when_merging_into_duckdb_then_the_newest_modified_file_wins_a expected_plan: "", }], check_query: NULL_MERGE_CHECK, - expected_result: "1:a|1", + expected_result: "|0", }, ]; for test_case in &test_cases { @@ -1126,7 +1100,7 @@ fn given_a_merge_files_table_when_reloading_then_every_file_merges_again_in_orde }, ], check_query: RELOAD_CHECK, - expected_result: "1:c,2:a|reset,run:a.csv,run:b.csv,reset,run:a.csv,run:b.csv,run:c.csv|0", + expected_result: "1:c,2:a|reset,run:a.csv+b.csv,reset,run:a.csv+b.csv+c.csv|0", }, FilesMergeTestCase { description: "a --reload failing on its third file leaves the old table, its run log and no build table", @@ -1201,7 +1175,7 @@ fn given_a_merge_files_table_when_reloading_then_every_file_merges_again_in_orde }, ], check_query: RELOAD_CHECK, - expected_result: "1:b|reset,run:export.csv,reset,run:export.csv,run:export.csv|0", + expected_result: "1:b|reset,run:export.csv,reset,run:export.csv+export.csv|0", }, ]; for test_case in &test_cases { @@ -1440,7 +1414,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h loads: true, reload: false, expected_error: "", - expected_plan: "1 file, one commit each, oldest modified first (ties by path), each recorded in the run log", + expected_plan: "1 file, 1 commit, oldest modified first (ties by path), each recorded in the run log", }, MergeStep { before: &[], @@ -1465,7 +1439,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h expected_result: "LE-FY27-P01:A:100,LE-FY27-P01:B:200,LE-FY27-P02:A:10,LE-FY27-P02:B:20|3", }, FilesMergeTestCase { - description: "in one run, a corrected P01 modified after the first P01 fails, naming both files, as the run would keep only one of them; the files before it stay", + description: "in one run, a corrected P01 modified after the first P01 fails, naming both files, as the run would keep only one of them; the files committed with it roll back", settings: LE_SETTINGS, steps: &[MergeStep { before: &[], @@ -1477,7 +1451,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h expected_plan: "", }], check_query: LE_CHECK, - expected_result: "LE-FY27-P01:A:1,LE-FY27-P01:B:2,LE-FY27-P01:C:3,LE-FY27-P02:A:10,LE-FY27-P02:B:20|2", + expected_result: "|0", }, FilesMergeTestCase { description: "a version split across two part files of one run fails on the later modified, naming both, whatever their names, instead of keeping only its lines", @@ -1503,7 +1477,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h expected_plan: "", }], check_query: LE_CHECK, - expected_result: "LE-FY27-P05:A:1,LE-FY27-P05:B:2,LE-FY27-P05:C:3|1", + expected_result: "|0", }, FilesMergeTestCase { description: "a later run's file whose rows have no version replaces the rows without one, as a NULL in the key matches a NULL by default, once the file they came from has left the folder", @@ -1540,7 +1514,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h expected_result: "LE-FY27-P01:A:5,NULL:C:3|2", }, FilesMergeTestCase { - description: "with null_keys = \"fail\" a file holding a row without a version fails before it loads, naming replace_key, and the file before it stays", + description: "with null_keys = \"fail\" a file holding a row without a version fails before it loads, naming it and replace_key, and the file committed with it rolls back", settings: "write_strategy = \"delete_insert\"\nreplace_key = [\"le_version\"]\nnull_keys = \"fail\"\n", steps: &[MergeStep { before: &[], @@ -1559,10 +1533,10 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h expected_plan: "", }], check_query: LE_CHECK, - expected_result: "LE-FY27-P02:A:10,LE-FY27-P02:B:20|1", + expected_result: "|0", }, FilesMergeTestCase { - description: "a version split across two part files fails on the second in the first run, and again when the run is retried, which loads it alone, keeping the first part's rows", + description: "a version split across two part files fails on the second, rolling back both, and again in the same way when the run is retried", settings: LE_SETTINGS, steps: &[ MergeStep { @@ -1580,12 +1554,12 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h removed: &[], loads: true, reload: false, - expected_error: "drop/part-0001.csv: the table holds rows of its replace_key (le_version) values from part-0000.csv, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by part-0001.csv would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if newer files restate older ones, set on_overlapping_files = \"last_wins\", so the file last in order replaces the values, or remove the older of part-0000.csv and part-0001.csv from the folder by hand and set after_load = { move_to = ... } (or \"delete\"), so that each file leaves the folder once it loads", + expected_error: "drop/part-0001.csv: it holds the replace_key (le_version) value (LE-FY27-P01), which part-0000.csv also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if newer files restate older ones, set on_overlapping_files = \"last_wins\", so the file last in order replaces the values, or remove the older of part-0000.csv and part-0001.csv from the folder by hand and set after_load = { move_to = ... } (or \"delete\"), so that each file leaves the folder once it loads", expected_plan: "", }, ], check_query: LE_CHECK, - expected_result: "LE-FY27-P01:A:1,LE-FY27-P01:B:2|1", + expected_result: "|0", }, FilesMergeTestCase { description: "a corrected P01 under another name fails while the old P01 is still in the folder, naming both, and replaces P01 whole once the old file is removed", @@ -1620,7 +1594,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h }, ], check_query: LE_CHECK, - expected_result: "LE-FY27-P01:A:100,LE-FY27-P01:B:200,LE-FY27-P02:A:10,LE-FY27-P02:B:20|3", + expected_result: "LE-FY27-P01:A:100,LE-FY27-P01:B:200,LE-FY27-P02:A:10,LE-FY27-P02:B:20|2", }, FilesMergeTestCase { description: "with after_load moving each file once it loads, P01 sent again into the folder still replaces P01's rows", @@ -1646,7 +1620,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h }, ], check_query: LE_CHECK, - expected_result: "LE-FY27-P01:A:100,LE-FY27-P01:B:200,LE-FY27-P02:A:10,LE-FY27-P02:B:20|3", + expected_result: "LE-FY27-P01:A:100,LE-FY27-P01:B:200,LE-FY27-P02:A:10,LE-FY27-P02:B:20|2", }, ]; for test_case in &test_cases { diff --git a/crates/sqlload/tests/loader/test_loader_edge_cases.rs b/crates/sqlload/tests/loader/test_loader_edge_cases.rs index fd39bd847..6a41cdc03 100644 --- a/crates/sqlload/tests/loader/test_loader_edge_cases.rs +++ b/crates/sqlload/tests/loader/test_loader_edge_cases.rs @@ -195,26 +195,20 @@ fn given_edge_cases_when_loading_postgres_into_duckdb_then_tables_stay_correct() expected_result: "app__noted=Noted it's here.,app__plain=null", }, ScenarioTestCase { - description: "a table entry without a description fails the plan with where to add one", + description: "a table entry whose description is empty loads, with no comment, even over the source table's own", setup: "CREATE TABLE public.listed (id int8 PRIMARY KEY); COMMENT ON TABLE public.listed IS 'Only the source says so.';", settings: "write_strategy = \"replace\"\n[sources.app.tables.\"public.listed\"]\ndescription = \"\"\n", - runs: &[ScenarioRun { - expected_error: "app__listed\n └── reason no description; add description = \"...\" under [sources.app.tables.\"public.listed\"]", - ..OK - }], - check_query: "SELECT CAST(count(*) AS VARCHAR) FROM information_schema.tables WHERE table_schema = 'raw'", - expected_result: "0", + runs: &[OK], + check_query: "SELECT string_agg(table_name || '=' || coalesce(comment, 'null'), ',' ORDER BY table_name) FROM duckdb_tables() WHERE schema_name = 'raw'", + expected_result: "app__listed=null", }, ScenarioTestCase { - description: "a table named in include without a description fails the plan", - setup: "CREATE TABLE public.named (id int8 PRIMARY KEY);", - settings: "include = [\"public.named\"]\nwrite_strategy = \"replace\"\n", - runs: &[ScenarioRun { - expected_error: "app__named\n └── reason no description; add description = \"...\" under [sources.app.tables.\"public.named\"]", - ..OK - }], - check_query: "SELECT CAST(count(*) AS VARCHAR) FROM information_schema.tables WHERE table_schema = 'raw'", - expected_result: "0", + description: "tables named in include need no table entry: one without a comment loads with none, one with its source comment carries it", + setup: "CREATE TABLE public.named (id int8 PRIMARY KEY); CREATE TABLE public.noted (id int8 PRIMARY KEY); COMMENT ON TABLE public.noted IS 'Noted here.';", + settings: "include = [\"public.named\", \"public.noted\"]\nwrite_strategy = \"replace\"\n", + runs: &[OK], + check_query: "SELECT string_agg(table_name || '=' || coalesce(comment, 'null'), ',' ORDER BY table_name) FROM duckdb_tables() WHERE schema_name = 'raw'", + expected_result: "app__named=null,app__noted=Noted here.", }, ScenarioTestCase { description: "--reload rebuilds a table whose source type changed incompatibly", diff --git a/crates/sqlload/tests/loader/test_memory_budget.rs b/crates/sqlload/tests/loader/test_memory_budget.rs index 12b11c0d2..75080a1b6 100644 --- a/crates/sqlload/tests/loader/test_memory_budget.rs +++ b/crates/sqlload/tests/loader/test_memory_budget.rs @@ -1,12 +1,13 @@ //! Rows of 1 MiB loaded into a DuckDB file under a memory budget that gives DuckDB a `memory_limit` of 256MB, which one commit of 300 of them overflows. -use crate::helpers::run_async; +use crate::helpers::{duckdb_text, run_async}; use crate::memory_helpers::{ - FEW_WIDE_ROWS, KILLED, LOAD, NEW_BODY, NEW_BODY_KILLED, NEW_BODY_LATER, NEW_BODY_TRIPPED, OFF, - OFF_DUCKDB_2TB, OFF_DUCKDB_256MB, OFF_PLAN, OFF_TRIPPED, OLDER_MARKER, PLAN, REPEATED_KEY, - SMALL_ROWS, WIDE_THEN_SMALL, expected_loads, missing_phrases, run_wide_load, wide_postgres, + BUDGET, FEW_WIDE_ROWS, KILLED, LOAD, NEW_BODY, NEW_BODY_KILLED, NEW_BODY_LATER, + NEW_BODY_TRIPPED, OFF, OFF_DUCKDB_2TB, OFF_DUCKDB_256MB, OFF_PLAN, OFF_TRIPPED, OLDER_MARKER, + PLAN, REPEATED_KEY, SMALL_ROWS, WIDE_THEN_SMALL, expected_loads, missing_phrases, + run_wide_load, wide_postgres, }; -use crate::test_types::WideLoadTestCase; +use crate::test_types::{RestatedPiecesTestCase, WideLoadTestCase}; #[test] fn given_300_rows_of_1_mib_when_loading_them_with_a_256mb_duckdb_limit_then_every_strategy_loads_them_in_pieces() @@ -500,3 +501,76 @@ fn given_memory_limit_off_when_loading_1_mib_rows_then_they_commit_whole_without Ok(()) }) } + +#[test] +fn given_a_later_file_restating_a_value_an_earlier_piece_committed_when_loading_under_a_budget_then_the_later_file_replaces_it() +-> Result<(), String> { + let test_cases = [ + RestatedPiecesTestCase { + description: "files committed together, cut into pieces, keep only the later file's row of 1", + settings: "", + expected_result: "70:1:new", + }, + RestatedPiecesTestCase { + description: "files committed one at a time, the later one cut into pieces, keep only its row of 1", + settings: "exclude_columns = [\"_sqlload_source_row\"]\n", + expected_result: "70:1:new", + }, + ]; + let body = "x".repeat(1 << 20); + let rows = |ids: std::ops::RangeInclusive| -> String { + ids.map(|id| format!("{id},{body}\n")).collect() + }; + let files = [ + ("c.csv", format!("id,v\n{}", rows(1..=40)), 1_000), + ("a.csv", format!("id,v\n{}1,new\n", rows(41..=70)), 2_000), + ]; + for test_case in &test_cases { + let directory = tempfile::tempdir().map_err(|error| error.to_string())?; + let folder = directory.path().join("in"); + std::fs::create_dir_all(&folder).map_err(|error| error.to_string())?; + for (name, text, modified_secs) in &files { + let path = folder.join(name); + std::fs::write(&path, text).map_err(|error| error.to_string())?; + let modified = std::time::UNIX_EPOCH + std::time::Duration::from_secs(*modified_secs); + std::fs::File::options() + .write(true) + .open(&path) + .and_then(|opened| opened.set_modified(modified)) + .map_err(|error| error.to_string())?; + } + let database = directory.path().join("restated.duckdb"); + let config = directory.path().join("sqlload.toml"); + std::fs::write( + &config, + format!( + "[connections.files]\nkind = \"filesystem\"\nroot = \"{}\"\n\n[connections.local]\nkind = \"duckdb\"\ndatabase = \"{}\"\n\n[targets.dev]\nconnection = \"local\"\nloader_schema = \"raw\"\n\n[sources.restated]\nkind = \"files\"\nconnection = \"files\"\npath = \"in/\"\nformat = \"csv\"\nwrite_strategy = \"delete_insert\"\nreplace_key = [\"id\"]\non_overlapping_files = \"last_wins\"\n{}", + directory.path().display(), + database.display(), + test_case.settings + ), + ) + .map_err(|error| error.to_string())?; + let output = std::process::Command::new(env!("CARGO_BIN_EXE_sqlload")) + .arg("--config") + .arg(&config) + .args(["--no-color", "load", "--memory-limit", BUDGET]) + .output() + .map_err(|error| error.to_string())?; + let output = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let result = duckdb_text( + &database, + "SELECT count(*) || ':' || count(*) FILTER (WHERE id = 1) || ':' || string_agg(v, ',') FILTER (WHERE length(v) < 10) FROM raw.restated", + )?; + assert_eq!( + result, test_case.expected_result, + "{}:\n{output}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlload/tests/loader/test_overlapping_files.rs b/crates/sqlload/tests/loader/test_overlapping_files.rs index f0899f507..51028bcd0 100644 --- a/crates/sqlload/tests/loader/test_overlapping_files.rs +++ b/crates/sqlload/tests/loader/test_overlapping_files.rs @@ -7,7 +7,7 @@ use crate::merge_helpers::{load_step, run_merge_case}; use crate::test_types::{FilesMergeTestCase, MergeFile, MergeStep}; /// Every row as `version:line:amount`, then how many file commits the run log records. -const CHECK: &str = "SELECT (SELECT string_agg(COALESCE(le_version, 'NULL') || ':' || line || ':' || CAST(amount AS VARCHAR), ',' ORDER BY le_version NULLS LAST, line) FROM raw.scores) || '|' || (SELECT CAST(count(*) AS VARCHAR) FROM raw._sqlload_runs WHERE table_name = 'scores' AND event = 'run')"; +const CHECK: &str = "SELECT (SELECT COALESCE(string_agg(COALESCE(le_version, 'NULL') || ':' || line || ':' || CAST(amount AS VARCHAR), ',' ORDER BY le_version NULLS LAST, line), '') FROM raw.scores) || '|' || (SELECT CAST(count(*) AS VARCHAR) FROM raw._sqlload_runs WHERE table_name = 'scores' AND event = 'run')"; /// No table is created when plan refuses the settings. const NO_TABLE: &str = @@ -33,7 +33,7 @@ const SECOND: MergeFile = MergeFile { modified_secs: 2_000, }; -const LAST_WINS_PLAN: &str = "2 files, one commit each, oldest modified first (ties by path), each recorded in the run log; the last file holding a replace_key value replaces it (on_overlapping_files = \"last_wins\")"; +const LAST_WINS_PLAN: &str = "2 files, 1 commit, oldest modified first (ties by path), each recorded in the run log; the last file holding a replace_key value replaces it (on_overlapping_files = \"last_wins\")"; const OUTSIDE: &str = "on_overlapping_files says which file wins when two files of a files source's folder hold one replace_key value"; @@ -123,7 +123,7 @@ fn given_files_holding_one_replace_key_value_when_loading_into_duckdb_then_on_ov settings: LAST_WINS, steps: ONE_RUN, check_query: CHECK, - expected_result: "P01:A:1,P01:B:2,P02:X:7,P02:Y:8,P03:Z:9|2", + expected_result: "P01:A:1,P01:B:2,P02:X:7,P02:Y:8,P03:Z:9|1", }, FilesMergeTestCase { description: "across runs, a file sent later replaces a value of a file still in the folder", @@ -137,14 +137,14 @@ fn given_files_holding_one_replace_key_value_when_loading_into_duckdb_then_on_ov settings: LAST_WINS_BY_PATH, steps: BY_PATH, check_query: CHECK, - expected_result: "P01:A:1,P02:X:7|2", + expected_result: "P01:A:1,P02:X:7|1", }, FilesMergeTestCase { description: "with on_overlapping_files = \"fail\", a value split across part files still fails on the later file, naming both and the setting", settings: FAILS, steps: SPLIT, check_query: CHECK, - expected_result: "P01:A:1,P01:B:2,P02:A:10|1", + expected_result: "|0", }, FilesMergeTestCase { description: "with order = \"path\", an older file sent after a newer one fails instead of taking back the newer file's value, which stays", @@ -158,7 +158,7 @@ fn given_files_holding_one_replace_key_value_when_loading_into_duckdb_then_on_ov settings: LAST_WINS_BY_PATH, steps: REWRITTEN, check_query: CHECK, - expected_result: "P01:A:5,P02:X:70,P03:Z:9|4", + expected_result: "P01:A:5,P02:X:70,P03:Z:9|2", }, FilesMergeTestCase { description: "set on the source, it leaves alone a table of the source that merges", @@ -222,7 +222,7 @@ fn given_files_holding_one_replace_key_value_when_loading_into_postgres_then_on_ settings: LAST_WINS, steps: ONE_RUN, check_query: CHECK, - expected_result: "P01:A:1,P01:B:2,P02:X:7,P02:Y:8,P03:Z:9|2", + expected_result: "P01:A:1,P01:B:2,P02:X:7,P02:Y:8,P03:Z:9|1", }, FilesMergeTestCase { description: "across runs, a file sent later replaces a value of a file still in the folder", @@ -236,7 +236,7 @@ fn given_files_holding_one_replace_key_value_when_loading_into_postgres_then_on_ settings: FAILS, steps: SPLIT, check_query: CHECK, - expected_result: "P01:A:1,P01:B:2,P02:A:10|1", + expected_result: "|0", }, ]; for test_case in &test_cases { diff --git a/crates/sqlload/tests/loader/test_postgres_target.rs b/crates/sqlload/tests/loader/test_postgres_target.rs index 86cbe1c09..d955c0b23 100644 --- a/crates/sqlload/tests/loader/test_postgres_target.rs +++ b/crates/sqlload/tests/loader/test_postgres_target.rs @@ -341,10 +341,10 @@ fn given_csv_files_when_merging_into_postgres_then_the_newest_modified_file_wins removed: &[], loads: true, reload: false, - expected_plan: "3 files, one commit each, oldest modified first (ties by path)", + expected_plan: "3 files, 1 commit, oldest modified first (ties by path)", }], check_query: MERGE_CHECK, - expected_result: "1:b,2:c,3:a|3", + expected_result: "1:b,2:c,3:a|1", }, FilesMergeTestCase { description: "files modified at the same time merge in path order", @@ -370,7 +370,7 @@ fn given_csv_files_when_merging_into_postgres_then_the_newest_modified_file_wins expected_plan: "", }], check_query: MERGE_CHECK, - expected_result: "1:b|2", + expected_result: "1:b|1", }, FilesMergeTestCase { description: "by default a file repeating a key merges its last row of that key", @@ -547,11 +547,11 @@ fn given_csv_files_when_merging_into_postgres_then_the_newest_modified_file_wins removed: &[], loads: true, reload: false, - expected_plan: "1 file, one commit each", + expected_plan: "1 file, 1 commit", }, ], check_query: MERGE_CHECK, - expected_result: "1:a2|3", + expected_result: "1:a2|2", }, FilesMergeTestCase { description: "an existing table whose key is a NOT NULL primary key merges a files source, whose columns may all be NULL, through that primary key, without asking for an index treating NULLs as equal", @@ -572,7 +572,7 @@ fn given_csv_files_when_merging_into_postgres_then_the_newest_modified_file_wins removed: &[], loads: true, reload: false, - expected_plan: "merge (unique_key: id; repeated keys fail; NULLs in keys match)", + expected_plan: "merge (unique_key: id; NULLs in keys match)", }, MergeStep { before: &[], @@ -585,7 +585,7 @@ fn given_csv_files_when_merging_into_postgres_then_the_newest_modified_file_wins removed: &[], loads: true, reload: false, - expected_plan: "1 file, one commit each", + expected_plan: "1 file, 1 commit", }, ], check_query: MERGE_CHECK, diff --git a/crates/sqlload/tests/loader/test_types.rs b/crates/sqlload/tests/loader/test_types.rs index 3f8989b86..fdee434f8 100644 --- a/crates/sqlload/tests/loader/test_types.rs +++ b/crates/sqlload/tests/loader/test_types.rs @@ -1692,3 +1692,44 @@ pub(crate) struct DuckDbSourceTestCase { /// The orders' rows, the target's columns, the view's row count and the query table's ids. pub(crate) expected_rows: &'static str, } + +/// A file a batched files load reads: its name below the folder, its rows as `(id, v, seen)`, and its modified time in seconds since 1970. +pub(crate) struct BatchFile { + pub(crate) key: &'static str, + pub(crate) rows: &'static [(i64, &'static str, i64)], + pub(crate) modified_secs: u64, +} + +/// `files` loaded with `settings` in one run, and again one file per run, into two tables of one target, compared. +pub(crate) struct FileBatchTestCase { + pub(crate) description: &'static str, + pub(crate) settings: &'static str, + /// `csv` or `parquet`. + pub(crate) format: &'static str, + pub(crate) files: &'static [BatchFile], + /// Rows of the batched table not in the file-by-file one, then the reverse, then its `id:v` rows, then each table's commits, joined by `|`. + pub(crate) expected_result: &'static str, +} + +/// Two CSV files of 1 MiB rows, the later restating the earlier's value 1 in a row of its own, loaded under a memory budget that cuts their commits into pieces. +pub(crate) struct RestatedPiecesTestCase { + pub(crate) description: &'static str, + pub(crate) settings: &'static str, + /// The table's rows, its rows of 1, then the values under 10 bytes, joined by `:`. + pub(crate) expected_result: &'static str, +} + +/// `files` generated files (each repeating a key, and repeating keys of other files), loaded with `settings` in one run; each of `bad_files` first holds a value its column can't take, fails the load, and is then fixed and loaded again. +pub(crate) struct ManyFilesTestCase { + pub(crate) description: &'static str, + pub(crate) settings: &'static str, + pub(crate) format: &'static str, + pub(crate) files: usize, + pub(crate) bad_files: &'static [usize], + /// Text the failed load's error holds; empty without `bad_files`. + pub(crate) expected_error: &'static str, + /// After the failed load: the files its run log records, then its commits. + pub(crate) expected_after_error: &'static str, + /// Rows of the table not in the newest-file-wins oracle, then the reverse, then its rows, its commits and the rows its commits recorded. + pub(crate) expected_result: &'static str, +} diff --git a/packages/sqlload/python/sqlload/_declare.py b/packages/sqlload/python/sqlload/_declare.py index 64ba7e703..d1b875da1 100644 --- a/packages/sqlload/python/sqlload/_declare.py +++ b/packages/sqlload/python/sqlload/_declare.py @@ -134,7 +134,7 @@ def table( **settings: Any, ) -> Table | Callable[[TableFunction], Table]: """Attach a table to this source: `@shop.table` or `@shop.table(unique_key="id", ...)`; its - description, which `plan` requires, is `description` or else the function's docstring.""" + description, the destination table's comment, is `description` or else the function's docstring.""" _check_settings(settings, TABLE_SETTINGS | TABLE_ONLY_SETTINGS, "table") def attach(table_function: TableFunction) -> Table: @@ -191,7 +191,7 @@ def table( **settings: Any, ) -> Table | Callable[[TableFunction], Table]: """A standalone table, which sets its own write_strategy: `@sl.table` or `@sl.table(...)`; its - description, which `plan` requires, is `description` or else the function's docstring.""" + description, the destination table's comment, is `description` or else the function's docstring.""" _check_settings(settings, TABLE_SETTINGS | TABLE_ONLY_SETTINGS, "table") def declare(table_function: TableFunction) -> Table: diff --git a/packages/sqlload/python/sqlload/_http.py b/packages/sqlload/python/sqlload/_http.py index f7908c65a..c921fca27 100644 --- a/packages/sqlload/python/sqlload/_http.py +++ b/packages/sqlload/python/sqlload/_http.py @@ -104,11 +104,34 @@ def offset( ) -def page_number(param: str = "page", *, start: int = 1, location: str = "query", read_ahead: int | None = None) -> Paginator: - """`param` counts pages from `start`; ends at an empty page. With `location="body"`, `param` is a path in the JSON body. +def page_number( + param: str = "page", + *, + start: int = 1, + page_size: int | None = None, + page_size_param: str | None = "page_size", + location: str = "query", + read_ahead: int | None = None, +) -> Paginator: + """`param` counts pages from `start`; ends at an empty page, or with `page_size=N` at a page with fewer than N rows. + `page_size` is sent as `page_size_param` with every page; `page_size_param=None` sends nothing, for an API whose + page size is fixed. A short page is checked with one more request until the API has shown it doesn't cap its pages + lower: an API that does still loads every row, paged to an empty page. + With `location="body"`, `param` and `page_size_param` are paths in the JSON body. With `read_ahead=N`, the pages after a full first page are requested N at a time and still loaded in page order.""" - return Paginator({"kind": "page_number", "param": param, "start": start, "location": _location(location), **_read_ahead(read_ahead)}) + sized: dict[str, Any] = {} + if page_size is not None: + if isinstance(page_size, bool) or not isinstance(page_size, int): + raise TypeError(f"sl.page_number page_size takes a whole number of rows, not {page_size!r}") + if page_size < 1: + raise ValueError(f"sl.page_number page_size must be at least 1, not {page_size}") + sized = {"page_size": page_size, "page_size_param": page_size_param} + elif page_size_param != "page_size": + raise ValueError("sl.page_number page_size_param needs page_size: the number of rows a page holds") + return Paginator( + {"kind": "page_number", "param": param, "start": start, **sized, "location": _location(location), **_read_ahead(read_ahead)} + ) def cursor(pointer: str, param: str, *, location: str = "query", read_ahead: int | None = None) -> Paginator: diff --git a/packages/sqlload/tests/test_http_page_size.py b/packages/sqlload/tests/test_http_page_size.py new file mode 100644 index 000000000..06241b1de --- /dev/null +++ b/packages/sqlload/tests/test_http_page_size.py @@ -0,0 +1,185 @@ +"""`sl.page_number(page_size=N)` ends a run at a page holding fewer than `N` rows, rather than asking for an empty page. +The stub API pages `/items/` (`page`, `per_page`), holding as many rows as the key names, records every request it +answers, and can cap its pages below what is asked for.""" + +import json +import os +import subprocess +import sys +import textwrap +import threading +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlparse + +import duckdb +import pytest + + +class Api(BaseHTTPRequestHandler): + """`/v1/items/?page=P&per_page=N`: `` rows in pages of `N` (or the server's `cap` when lower), page `P` + counted from 1; each request's key, page and query go in `self.server.requests`.""" + + def log_message(self, *args: Any) -> None: + pass + + def do_GET(self) -> None: + server: Any = self.server + url = urlparse(self.path) + query = {key: values[0] for key, values in parse_qs(url.query).items()} + total = int(url.path.rsplit("/", 1)[-1]) + page = int(query.get("page", 1)) + size = min(int(query.get("per_page", server.fixed)), server.cap) + with server.lock: + server.requests.append((total, page, query)) + rows = [{"key": total, "n": n} for n in range((page - 1) * size, min(page * size, total))] + data = json.dumps({"items": rows}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + +@pytest.fixture +def api() -> Iterator[ThreadingHTTPServer]: + server = ThreadingHTTPServer(("127.0.0.1", 0), Api) + server.daemon_threads = True + server.lock = threading.Lock() # type: ignore[attr-defined] + server.requests, server.cap, server.fixed = [], 10_000, 20 # type: ignore[attr-defined] + threading.Thread(target=server.serve_forever, daemon=True).start() + yield server + server.shutdown() + + +def make_project(tmp_path: Path, server: ThreadingHTTPServer, job: str) -> Path: + """A project whose one table `items` is `job`, built with `api` on the stub's connection.""" + host, port = server.server_address[:2] + (tmp_path / "sources").mkdir() + (tmp_path / "sqlload.toml").write_text( + textwrap.dedent(f"""\ + python_paths = ["sources"] + + [connections.wh] + kind = "duckdb" + database = "wh.duckdb" + + [connections.shop] + kind = "http" + base_url = "http://{host}:{port}/v1" + + [targets.dev] + connection = "wh" + loader_schema = "RAW" + """) + ) + module = "import sqlload as sl\n\nshop = sl.source('shop', write_strategy='replace', connection='shop')\napi = sl.http(shop)\n" + module += f"\n@shop.table(name='items', description='T.')\ndef items(ctx):\n return {job}\n" + (tmp_path / "sources" / "shop.py").write_text(module) + return tmp_path + + +def sqlload(project: Path, *args: str) -> subprocess.CompletedProcess[str]: + home = project / "home" + home.mkdir(exist_ok=True) + env = {**os.environ, "HOME": str(home), "XDG_CONFIG_HOME": str(home / ".config")} + env.pop("NO_COLOR", None) + script = "from sqlload._cli import main; main()" + return subprocess.run([sys.executable, "-c", script, *args], cwd=project, env=env, capture_output=True, text=True, timeout=120) + + +def loaded(project: Path) -> list[tuple[int, int]]: + with duckdb.connect(str(project / "wh.duckdb"), read_only=True) as connection: + return connection.execute('SELECT "key", n FROM "RAW".shop__items ORDER BY "key", n').fetchall() + + +def requests_per_key(server: ThreadingHTTPServer) -> dict[int, int]: + counts: dict[int, int] = {} + for key, _, _ in server.requests: # type: ignore[attr-defined] + counts[key] = counts.get(key, 0) + 1 + return counts + + +def fan_out(keys: list[int], paginator: str) -> str: + return f"api.fan_out('/items/{{rows}}', over={[{'rows': key} for key in keys]!r}, concurrency=1, paginator={paginator}, select='$.items')" + + +@pytest.mark.parametrize( + ("keys", "expected_requests"), + [ + ([20, 19, 5, 12, 18, 41], {20: 2, 19: 1, 5: 1, 12: 1, 18: 1, 41: 3}), + ([12, 5, 19, 18, 25, 8], {12: 2, 5: 1, 19: 2, 18: 1, 25: 2, 8: 1}), + ], + ids=["after-a-full-page", "before-any-full-page"], +) +def test_given_page_size_when_fanning_out_over_keys_then_a_short_page_ends_each_key_once_the_api_shows_it_does_not_cap_lower( + tmp_path: Path, api: ThreadingHTTPServer, keys: list[int], expected_requests: dict[int, int] +) -> None: + # A full page shows the API sends whole pages, so every short page after it ends its key in one request; a key with + # exactly page_size rows still needs the empty page after it. Before any full page, a short first page at least as + # long as any proven last page is checked with one more request, as the API might cap its pages there. + project = make_project(tmp_path, api, fan_out(keys, "sl.page_number('page', page_size=20, page_size_param='per_page')")) + done = sqlload(project, "load") + assert done.returncode == 0, done.stdout + done.stderr + assert loaded(project) == sorted((key, n) for key in keys for n in range(key)) + assert requests_per_key(api) == expected_requests + assert {query.get("per_page") for _, _, query in api.requests} == {"20"} # type: ignore[attr-defined] + + +def test_given_an_api_capping_its_pages_below_page_size_when_paging_then_every_row_still_loads_and_the_cap_is_named( + tmp_path: Path, api: ThreadingHTTPServer +) -> None: + api.cap = 8 # type: ignore[attr-defined] + project = make_project(tmp_path, api, fan_out([30, 5, 19], "sl.page_number('page', page_size=20, page_size_param='per_page')")) + done = sqlload(project, "load") + assert done.returncode == 0, done.stdout + done.stderr + assert loaded(project) == sorted((key, n) for key in (30, 5, 19) for n in range(key)) + # 30 rows: pages of 8, 8, 8 and 6, the second page showing the cap; then 5 rows in one page, and 19 in 8, 8 and 3. + assert requests_per_key(api) == {30: 4, 5: 1, 19: 3} + assert "set page_size=8" in done.stdout + done.stderr + + +def test_given_page_size_without_a_parameter_when_paging_then_it_only_describes_the_apis_fixed_page_size( + tmp_path: Path, api: ThreadingHTTPServer +) -> None: + project = make_project(tmp_path, api, "api.paginate('/items/45', paginator=sl.page_number('page', page_size=20, page_size_param=None), select='$.items')") + done = sqlload(project, "load") + assert done.returncode == 0, done.stdout + done.stderr + assert len(loaded(project)) == 45 + assert [(page, query) for _, page, query in api.requests] == [(1, {"page": "1"}), (2, {"page": "2"}), (3, {"page": "3"})] # type: ignore[attr-defined] + + +def test_given_page_size_and_read_ahead_when_the_last_page_is_short_then_paging_ends_there_with_every_row( + tmp_path: Path, api: ThreadingHTTPServer +) -> None: + project = make_project(tmp_path, api, "api.paginate('/items/93', paginator=sl.page_number('page', page_size=10, page_size_param='per_page', read_ahead=4), select='$.items')") + done = sqlload(project, "load") + assert done.returncode == 0, done.stdout + done.stderr + assert loaded(project) == [(93, n) for n in range(93)] + # Pages 1 to 10 hold the rows; pages read ahead past the short tenth are dropped, never an empty eleventh asked for first. + assert max(page for _, page, _ in api.requests) <= 13 # type: ignore[attr-defined] + + +@pytest.mark.parametrize( + ("paginator", "params", "expected_text"), + [ + ("sl.page_number('page', page_size=0)", "None", "sl.page_number page_size must be at least 1, not 0"), + ("sl.page_number('page', page_size_param='per_page')", "None", "sl.page_number page_size_param needs page_size"), + ( + "sl.page_number('page', page_size=20, page_size_param='per_page')", + "{'per_page': 20}", + "params sets per_page, which page_size sends; leave it out of params", + ), + ], + ids=["zero", "param-without-size", "param-twice"], +) +def test_given_a_wrong_page_size_when_loading_then_the_load_fails_before_any_request_saying_what_to_fix( + tmp_path: Path, api: ThreadingHTTPServer, paginator: str, params: str, expected_text: str +) -> None: + project = make_project(tmp_path, api, f"api.paginate('/items/5', paginator={paginator}, params={params}, select='$.items')") + done = sqlload(project, "load") + assert done.returncode == 1, done.stdout + done.stderr + assert expected_text in done.stdout + done.stderr + assert api.requests == [] # type: ignore[attr-defined] diff --git a/packages/sqlload/tests/test_python_sources.py b/packages/sqlload/tests/test_python_sources.py index f6b7b918b..d24502602 100644 --- a/packages/sqlload/tests/test_python_sources.py +++ b/packages/sqlload/tests/test_python_sources.py @@ -457,10 +457,6 @@ def test_given_python_tables_when_planning_then_the_plan_says_where_columns_come ("import sqlload as sl\n@sl.table(description='Test table.', write_strategy='upsert')\ndef t(ctx):\n yield {}\n", "unknown variant `upsert`"), ("import sqlload as sl\na = sl.source('a', write_strategy='replace')\nb = sl.source('b', write_strategy='replace')\na.table(description='Test table.', name='t')(lambda ctx: [])\nb.table(description='Test table.', name='t')(lambda ctx: [])\n", "Python table t is declared twice"), ("import sqlload as sl\nshop = sl.source('shop', write_strategy='replace', connection='nope')\n", "Python source shop (sources/shop.py) connection \"nope\" is not a [connections] entry"), - ( - "import sqlload as sl\n@sl.table(write_strategy='replace')\ndef t(ctx):\n yield {}\n", - "t\n └── reason no description; add description=\"...\" to its table declaration, or a docstring to its function", - ), ( "import sqlload as sl\nshop = sl.source('shop')\n@shop.table(description='Test table.')\ndef t(ctx):\n yield {}\n", 't (kind = "python"): no write_strategy is set on the table or its source, and SQLLoad doesn\'t choose one\n └── fix set write_strategy = "replace" in sl.source("shop", ...) for all its tables, or in @shop.table(...) for this one', @@ -470,7 +466,7 @@ def test_given_python_tables_when_planning_then_the_plan_says_where_columns_come 't (kind = "python"): unique_key = "{primary_key}" merges on the primary key the source reports, and a python source reports none\n └── fix name the key columns', ), ], - ids=["import error", "unknown setting", "bad value", "duplicate table", "unknown connection", "no description", "no write_strategy", "primary_key placeholder"], + ids=["import error", "unknown setting", "bad value", "duplicate table", "unknown connection", "no write_strategy", "primary_key placeholder"], ) def test_given_bad_python_sources_when_planning_then_the_project_fails_closed(tmp_path: Path, module: str, message: str) -> None: project = make_project(tmp_path, DUCKDB_PROJECT, "", module=module) @@ -1024,6 +1020,11 @@ def stores(ctx): def standalone(ctx): '''A table described by its docstring.''' yield {"id": 1} + + +@sl.table(write_strategy="replace") +def bare(ctx): + yield {"id": 1} """ @@ -1031,9 +1032,9 @@ def test_given_table_name_settings_when_loading_then_tables_are_named_from_them( project = make_project(tmp_path, DUCKDB_PROJECT, "", module=NAMED_MODULE) load(project) tables = query(project, "SELECT table_name FROM information_schema.tables ORDER BY table_name") - assert tables == [("api_orders",), ("shop_stores_v2",), ("standalone",)] + assert tables == [("api_orders",), ("bare",), ("shop_stores_v2",), ("standalone",)] comments = query(project, "SELECT table_name, comment FROM duckdb_tables() ORDER BY table_name") - assert comments == [("api_orders", "Test table."), ("shop_stores_v2", "Test table."), ("standalone", "A table described by its docstring.")] + assert comments == [("api_orders", "Test table."), ("bare", None), ("shop_stores_v2", "Test table."), ("standalone", "A table described by its docstring.")] @pytest.mark.parametrize("entry", [".", "..", "sources/../.."], ids=["root", "parent", "escape"]) diff --git a/packages/sqlload/tests/test_transforms.py b/packages/sqlload/tests/test_transforms.py index 0995ac0ac..d3f907e6a 100644 --- a/packages/sqlload/tests/test_transforms.py +++ b/packages/sqlload/tests/test_transforms.py @@ -180,7 +180,7 @@ def test_a_failing_transform_fails_its_file_which_neither_commits_nor_moves(tmp_ assert query(project, "SELECT id FROM RAW.metrics ORDER BY id") == [("1",)] -def test_a_merge_still_merges_files_oldest_first_with_a_record_each(tmp_path: Path) -> None: +def test_a_merge_still_merges_files_oldest_first_recording_each_file(tmp_path: Path) -> None: settings = 'write_strategy = "merge"\nunique_key = ["id"]\n' project = make_project(tmp_path, RETURNS["dicts"], settings=settings) now = time.time() @@ -189,8 +189,8 @@ def test_a_merge_still_merges_files_oldest_first_with_a_record_each(tmp_path: Pa give(project, "c.csv", "id,v\n1,middle\n3,c\n", modified=now - 100) load(project) assert query(project, "SELECT id, v FROM RAW.metrics ORDER BY id") == [(1, "newest"), (2, "b"), (3, "c")] - merged = query(project, "SELECT count(*) FROM RAW._sqlload_runs WHERE note = 'files source merge'") - assert merged == [(3,)] + merged = query(project, "SELECT count(*), sum(json_array_length(record_json, '$.files')) FROM RAW._sqlload_runs WHERE note = 'files source merge'") + assert merged == [(1, 3)] #: A second module registering a transform as `metric_rows` too, and one as `tidy`. diff --git a/website/public/index.html b/website/public/index.html index bc64ff272..3f5823636 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -159,12 +159,12 @@

See every change first

├── app postgres into raw │ └── orders merge on order_id updated_at: first run, everything └── events ./lake/events/ into raw - └── events delete_insert on dt dt: first run, everything · 3 new of 3 files + └── events delete_insert on dt 3 new of 3 files └── route DuckDB reads directly from ./lake/events/ (nothing through the loader) Errors (1) └── raw.app__customers - └── reason no description; add description = "..." under [sources.app.tables."public.customers"] + └── reason lifetime_value has type numeric(12,2), which cannot be read as int ✗ Plan failed 2 objects to create, 2 to load, 1 error diff --git a/website/src/content/docs/docs/cli/contract.mdx b/website/src/content/docs/docs/cli/contract.mdx index 8c916330d..b2a2d1e94 100644 --- a/website/src/content/docs/docs/cli/contract.mdx +++ b/website/src/content/docs/docs/cli/contract.mdx @@ -85,8 +85,6 @@ Run contract generate --write to update sqlload.toml. declared columns the table lacks. - Python tables are never edited: the report prints the whole `columns={...}` to paste into the decorator. -- A table found by an `include` pattern with no description isn't written, since its new entry - would need one. - A database or files table declares only columns its source has, spelled as the source spells them. Masked columns stay under `mask`. - Adopted columns become pinned types, so a CSV or JSON source converts them while reading. diff --git a/website/src/content/docs/docs/concepts/cursors.mdx b/website/src/content/docs/docs/concepts/cursors.mdx index e557ae6dc..63ce8359f 100644 --- a/website/src/content/docs/docs/concepts/cursors.mdx +++ b/website/src/content/docs/docs/concepts/cursors.mdx @@ -232,7 +232,7 @@ and `merge` publish once per run (last row below). `plan -v` of one table shows | `batch_size = "1d"` (date or timestamp cursor) or `batch_size = 1000` (integer cursor) | One cursor window at a time, up to 100,000 windows (32,767 for a Snowflake table read through an unload). | | A Postgres table whose `cursor_column` leads an index, without `batch_size` | About every 1 GB, reading in cursor order (NULLs first), only between distinct cursor values. `batch_size = "none"` opts out. | | Other cursor tables, text cursors, and `append` without a cursor or logged state | Once per read, since committing part of an unordered read could skip or duplicate rows. | -| Files sources | At 1,000 files or about 1 GB; under `delete_insert` a partition is never split. | +| Files sources | At 1,000 files or about 1 GB; under `delete_insert` a partition is never split. A `merge`, or `delete_insert` by `replace_key`, commits consecutive files together, in order, recording them in the run log; with `on_duplicate_keys = "fail"` a merge commits each file. | | `replace` and `--reload` | Once, at the end: the new table is swapped in. | | `delete_insert` into ClickHouse in windows or cursor order, and `append` and `merge` | Once, at the end: everything read is staged and published together. A merge also commits every 10 million waiting rows, and with `on_duplicate_keys = "fail"` it commits each batch. A stopped run publishes nothing after its last commit, so split a long backfill with `--end-cursor`. | diff --git a/website/src/content/docs/docs/concepts/projects.mdx b/website/src/content/docs/docs/concepts/projects.mdx index efadba865..d09f7818f 100644 --- a/website/src/content/docs/docs/concepts/projects.mdx +++ b/website/src/content/docs/docs/concepts/projects.mdx @@ -109,13 +109,12 @@ source's `schema`, or the target's `loader_schema`. ## Descriptions -Every table SQLLoad writes needs a description, which becomes the destination table's comment. -`plan` fails for a table without one and says where to add it: +A table's description becomes the destination table's comment. Descriptions are optional: a table +without one loads with no comment. | Table | Description from | |-------|------------------| -| A Postgres table with its own `[sources.app.tables."public.orders"]` entry, or named exactly in `include` | `description = "..."` in that entry | -| A Postgres table found by an `include` pattern | The source table's comment (`COMMENT ON TABLE`), if any; none is needed | +| A Postgres table | `description = "..."` in its `[sources.app.tables."public.orders"]` entry, or else the source table's comment (`COMMENT ON TABLE`) | | A files source | The source's own `description` | | A Python table | `description="..."`, or the function's docstring | diff --git a/website/src/content/docs/docs/concepts/write-strategies.mdx b/website/src/content/docs/docs/concepts/write-strategies.mdx index 6b7309d82..28434fadc 100644 --- a/website/src/content/docs/docs/concepts/write-strategies.mdx +++ b/website/src/content/docs/docs/concepts/write-strategies.mdx @@ -120,7 +120,7 @@ once, `merge` keeps one row for each value. The destination never gets the key t never fails on the warehouse's own error about it. - Without `newest_by`, the row read last wins, and every row overwrites the table's row of its - key: the last one the source returned, or the last row of a file. + key: the last one the source returned, or of files, the last row of the newest file holding it. - `newest_by = "updated_at"` names the column whose higher value is the newer version of a row. Of a commit's rows sharing a key, the newest wins; a NULL loses to any value, and a tie goes to the row read later. It also applies against the table: a row only overwrites the table's diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx index eff9922ff..9bfb4d0ef 100644 --- a/website/src/content/docs/docs/configuration.mdx +++ b/website/src/content/docs/docs/configuration.mdx @@ -310,7 +310,7 @@ http shop rate_limit 10/s, max_in_flight 4, retries 5, auth bearer | `partition_columns` | files | Hive `key=value` folders as typed columns, such as `{ dt = "date" }`. | | `path_columns` | files | A path template whose captures load as typed columns, such as `"{region}/{day:date}/*"`. See [Columns from paths](/docs/sources/files/#columns-from-paths). | | `unmatched_paths` | files | `fail` (default) or `skip`: what a file whose path doesn't match `path_columns` does. | -| `order` | files | `modified` (default) or `path`: the order a `merge` or `delete_insert` by `replace_key` applies files in, one commit each. See [Merging files](/docs/sources/files/#merging-files). | +| `order` | files | `modified` (default) or `path`: the order a `merge` or `delete_insert` by `replace_key` applies files in, the last file winning a key several hold. See [Merging files](/docs/sources/files/#merging-files). | | `on_empty` | files | `fail` (default) or `skip`: what a load does when the folder exists and holds no files. See [Empty folders](/docs/sources/files/#empty-folders). | | `copy_to_internal_stage` | files into Snowflake | Upload files no external stage covers to `@~`. Much slower. | | `[sources.NAME.tables."schema.table"]` | all | Per-table settings. A files source's table is named after the source. | @@ -349,7 +349,7 @@ Set on a source for all its tables, or in a `[sources.NAME.tables."schema.table" | `replace_key` | none | A column or list of columns whose values `delete_insert` replaces whole: each commit deletes the rows holding a value its rows hold, then inserts them. Not unique. Only for `delete_insert`, and not with a `cursor_column`: each load reads the whole source, or a files source's new and changed files. See [Replacing rows by a key](/docs/concepts/write-strategies/#replacing-rows-by-a-key). | | `deletes` | none | `soft` or `hard`: what `load --sync-deletes` does with the rows of a `merge` table whose key the source no longer holds (Postgres, ODBC, Snowflake and BigQuery sources, and Databricks sources, into DuckDB and Postgres targets, Snowflake, MotherDuck, and BigQuery and Databricks). `soft` sets `_sqlload_deleted_at`; `hard` removes the row. A `postgres_cdc` source applies captured deletes this way, `soft` by default. See [Rows deleted at the source](/docs/concepts/write-strategies/#rows-deleted-at-the-source). | | `deletes_from` | none | Tables only: `{ table = "audit.deleted_items", cursor_column = "deleted_at" }`, a source table recording deleted keys. Every load loads its new rows into `
__sqlload_deletes` and marks the rows they name, as `deletes` says. `lookback = "1h"` reads that far back again on every load, for tombstones written by long transactions. See [Deletes from a tombstone table](/docs/concepts/write-strategies/#deletes-from-a-tombstone-table). | -| `on_overlapping_files` | `fail` | What a files source's `delete_insert` by `replace_key` does when two files of its folder hold one key value: `fail` refuses the later file's commit, naming both; `last_wins` lets the file last in `order` replace the value. See [Replacing rows by a key](/docs/sources/files/#when-newer-files-restate-older-ones). | +| `on_overlapping_files` | `fail` | What a files source's `delete_insert` by `replace_key` does when two files of its folder hold one key value: `fail` refuses the commit holding the later file, naming both; `last_wins` lets the file last in `order` replace the value. See [Replacing rows by a key](/docs/sources/files/#when-newer-files-restate-older-ones). | | `newest_by` | none | A column, or a list compared in order, whose higher value is a row's newer version: `merge` keeps the newest of a commit's rows sharing a key, and never overwrites the table's row with an older one. Without it, the row read last wins. See [Repeated keys](/docs/concepts/write-strategies/#repeated-keys). | | `cursor_column` | none | The column runs continue from. A list such as `["updated_at", "created_at"]` reads the first non-NULL column of each row. See [Cursors](/docs/concepts/cursors/). | | `cursor_start` | none | Where the first run starts. | diff --git a/website/src/content/docs/docs/guides/from-dlt.mdx b/website/src/content/docs/docs/guides/from-dlt.mdx index fcfb44d2a..e9ebbc9d4 100644 --- a/website/src/content/docs/docs/guides/from-dlt.mdx +++ b/website/src/content/docs/docs/guides/from-dlt.mdx @@ -107,7 +107,7 @@ How `rest_api` settings map: OAuth2 password grant, which dlt's `rest_api` has no auth type for ([config_setup.py](https://github.com/dlt-hub/dlt/blob/90c0cea1f7b59795de91824be466c99efca0f574/dlt/sources/rest_api/config_setup.py#L89-L94)), is `{ kind = "oauth2_password", token_url = "..." }`; see [OAuth2](/docs/sources/python/#oauth2). - Paginators: `json_link` is `sl.next_link("$.next")`, `header_link` is `sl.link_header()`, - `offset` is `sl.offset(limit=...)`, `page_number` is `sl.page_number()`, `cursor` is + `offset` is `sl.offset(limit=...)`, `page_number` is `sl.page_number()` (add `page_size=` to end at a short page rather than an empty one), `cursor` is `sl.cursor(pointer, param)`, and `single_page` is `sl.single()`. dlt's `auto` guesses one ([config_setup.py](https://github.com/dlt-hub/dlt/blob/90c0cea1f7b59795de91824be466c99efca0f574/dlt/sources/rest_api/config_setup.py#L75-L87)); SQLLoad never does. - `data_selector` is `select`, as a path from `$`. diff --git a/website/src/content/docs/docs/guides/python.mdx b/website/src/content/docs/docs/guides/python.mdx index 8024e592b..30126830c 100644 --- a/website/src/content/docs/docs/guides/python.mdx +++ b/website/src/content/docs/docs/guides/python.mdx @@ -112,7 +112,8 @@ The table loads into `shop__orders`: `{source}__{table}`, unless the source or t `table_name`. A `@sl.table` declared without a source keeps its own name. Table names are unique across the project. -A table's description is `description=`, or else the function's docstring; `plan` requires one. +A table's description, its destination table's comment, is `description=`, or else the function's +docstring. Both are optional. Every declaring function and setting is under [Declaring](/docs/sources/python/#declaring) in the Python API. diff --git a/website/src/content/docs/docs/quickstart.mdx b/website/src/content/docs/docs/quickstart.mdx index bb320f5b2..04bd8977e 100644 --- a/website/src/content/docs/docs/quickstart.mdx +++ b/website/src/content/docs/docs/quickstart.mdx @@ -79,7 +79,7 @@ order = "path" - Each `[sources]` entry loads one folder of CSV files into one table. `merge` upserts each new file on its `unique_key`, one file at a time in path order, and later loads skip the files already merged. -- Each table SQLLoad writes needs a description, which becomes the table's comment. +- A table's `description`, when it has one, becomes the table's comment. ## Check, plan, apply and load @@ -273,7 +273,7 @@ description = "Customers, one row per customer." source sets `merge` for all of them, and `unique_key = "{primary_key}"` upserts on each table's primary key. - `orders` is incremental: each run continues from the highest `updated_at` already loaded. -- Each table SQLLoad writes needs a description, which becomes the table's comment. +- A table's `description`, when it has one, becomes the table's comment. ```bash export PGPASSWORD=secret diff --git a/website/src/content/docs/docs/sources/files.mdx b/website/src/content/docs/docs/sources/files.mdx index 430463293..2aacc6b84 100644 --- a/website/src/content/docs/docs/sources/files.mdx +++ b/website/src/content/docs/docs/sources/files.mdx @@ -383,12 +383,12 @@ old one in one commit at the end; until then readers see the old table. - `append`, `replace` and `delete_insert` tables read the files in batches, as a first load does. A `delete_insert` with a [`replace_key`](#replacing-rows-by-a-key) refuses `--reload`. - `merge` tables reload too, on DuckDB targets (Postgres targets refuse `--reload` for - every table). Each file merges into the new table on its own, as a load merges them: files merged + every table). The files merge into the new table in batches, as a load merges them: files merged before go in the order they merged, matched by name and version, or for a file under `move_to` by name, so a file `after_load` moved keeps its place; files never merged follow, oldest modified first, and so does a new file in `path` named as one `move_to` holds, whose rows then win over the older copy's. The new - table's file record (a reset, then one record per file) is written in the commit that swaps it + table's file record (a reset, then one record per batch, naming its files) is written in the commit that swaps it in, so a reload that fails part way leaves the old table and its file record as they were. After a reload, the next load reads only files added since. `plan` shows `merge (unique_key: …) into a new table, swapped in at the end (--reload)`. A `merge` table with `unnest` is refused, since its child tables aren't rebuilt: @@ -453,9 +453,9 @@ Without `pattern`, a folder of `.txt` files fails `plan` with the suffixes it lo |----------|-------| | `append` | Only files not loaded yet, by `_sqlload_source_file`. A file rewritten in place since it loaded fails the table, with the fix. So does a listed file loaded without its version (an empty `_sqlload_source_version`, as in rows loaded before SQLLoad recorded it), since whether it was rewritten can't be told: `--reload` loads the table again and records every file's version. `plan` refuses an `append` table that excludes `_sqlload_source_version`, as it refuses one excluding `_sqlload_source_file`. | | `delete_insert` with a partition `cursor_column` (from `partition_columns` or `path_columns`) | The newest loaded partition and everything after it, replacing those partitions whole. A file that changed since it loaded whose partition lies below that range fails the table, as does a new file below it in a partition the table holds; see below. | -| `delete_insert` with a `replace_key` | Only files not loaded yet, or changed since, one at a time, oldest modified first, each replacing the rows of the `replace_key` values it holds. | +| `delete_insert` with a `replace_key` | Only files not loaded yet, or changed since, oldest modified first, each replacing the rows of the `replace_key` values it holds. | | `replace` | Every file, swapped in as a new table. | -| `merge` with a `unique_key` | Only files not merged yet, one at a time, oldest modified first. Under `--reload` (DuckDB), every file again, one at a time, into a new table swapped in at the end; see [Reloading a files table](#reloading-a-files-table). | +| `merge` with a `unique_key` | Only files not merged yet, oldest modified first. Under `--reload` (DuckDB), every file again, in the same order, into a new table swapped in at the end; see [Reloading a files table](#reloading-a-files-table). | A files source needs a `write_strategy`, from the source or its table entry: `plan` refuses it otherwise. Files report no primary key, so a `merge` names its `unique_key` columns. @@ -475,7 +475,9 @@ loaded raw.events_log 1 row 0.0s ``` Large loads commit at 1,000 files or about 1 GB. Under `delete_insert` a partition is never split -across commits. +across commits. A `merge`, or `delete_insert` by `replace_key`, commits consecutive files together, +in order, and ends with the same rows as committing them one by one: the newest file wins a key +several of them hold. `delete_insert` on a partition cursor fails the table when a listed file changed since it loaded but its partition lies below the range the load reads, such as a monthly file sent again @@ -516,12 +518,14 @@ again as a matter of course, use a [`replace_key`](#replacing-rows-by-a-key). ### Replacing rows by a key `delete_insert` with a `replace_key` and no `cursor_column` loads each new or changed -file in its own commit, oldest modified first, then path. Each commit deletes the rows holding a -`replace_key` value the file holds, then inserts the file's rows. A file sent again corrected +file, oldest modified first, then path, up to 1,000 files or about 1 GB per commit. Each file +replaces the rows holding a `replace_key` value it holds with its own rows, as if it committed on +its own. A file sent again corrected replaces its version whole, lines it dropped included, and leaves other versions alone. One file -must hold all of a value's rows. Before each commit, SQLLoad finds the files the table's rows of -the commit's values came from, by `_sqlload_source_file`. When one of them is still in the folder, -or loaded earlier in the same run, the table fails, naming both files: two files of the folder +must hold all of a value's rows. Before each commit, SQLLoad finds, for each of its files, the files the table's +rows of the file's values came from, by `_sqlload_source_file`. When one of them is still in the +folder, or loaded earlier in the same run, the table fails, naming both files, and the commit, +with every file in it, rolls back: two files of the folder holding one value, such as the parts of an export split into part files, would leave only the later one's rows. The error names the fixes: put all of a value's rows in one file; or, when newer files restate older ones, set `on_overlapping_files = "last_wins"` (below), or remove the older file @@ -549,11 +553,11 @@ $ sqlload plan -v ├── source files lake/finance/le/ csv, compression by file name … ├── files 1 new of 1 listed - └── batch 1 file, one commit each, oldest modified first (ties by path), each recorded in the run log + └── batch 1 file, 1 commit, oldest modified first (ties by path), each recorded in the run log ``` -It picks files as [merging files](#merging-files) does: each commit records its file and version in -`_sqlload_runs`, and loads into DuckDB and Postgres targets so far. `--reload`, `state = "logged"` +It picks files as [merging files](#merging-files) does: each commit records its files and versions +in `_sqlload_runs`, and loads into DuckDB and Postgres targets so far. `--reload`, `state = "logged"` and a `cursor_column` are refused. `after_load` applies to it, as to `merge`. #### When newer files restate older ones @@ -561,7 +565,8 @@ and a `cursor_column` are refused. `after_load` applies to it, as to `merge`. A feed whose deliveries overlap, each restating the end of the period before, holds one value in two files on purpose. `on_overlapping_files = "last_wins"` (default `"fail"`) lets the file last in `order` replace the values it holds, even while an earlier file holding them is still in the -folder. Each file still loads in its own commit, so the earlier file's other values stay: +folder. Each file replaces only the values it holds, so the earlier file's other values stay, in +one commit or across several: ```text sales/ @@ -584,6 +589,9 @@ on_overlapping_files = "last_wins" ``` Set `order = "path"` when the names sort by date, as here; by default the file modified last wins. +Under `"last_wins"` files commit together into DuckDB and PostgreSQL; into other warehouses, or +for a table that excludes `_sqlload_source_row`, whose rows carry no number to tell which of them a +later file in the same commit replaces, each file loads in its own commit. A file that comes before one already loaded, such as an older file sent late, or with `order = "path"` one rewritten while the later file isn't, still fails rather than take back the later file's values. By modified time a rewritten file sorts last and replaces them, and files @@ -596,12 +604,16 @@ elsewhere: a [merge](#merging-files) already lets the last file win each key. ### Merging files `merge` upserts each new file on its `unique_key`. A load merges -its files in order of their modified time, then path, one commit each, so a key that several of them -carry ends with the value from the file modified last. A file that turns up after others merged, even -with an older modified time, merges after them. Each commit also records that file and its version in -`_sqlload_runs`, in the same transaction, and later loads skip the files recorded there. If a file -fails, the files before it stay merged, and the error names the file that failed. The next load -continues from that file. +its files in order of their modified time, then path, so a key that several of them carry ends with +the value from the file modified last. Consecutive files merge in one commit, up to 1,000 files or +about 1 GB, with the same rows as merging them one by one. A file that turns up after others +merged, even with an older modified time, merges after them. Each commit also records its files +and their versions in `_sqlload_runs`, in the same transaction, and later loads skip the files +recorded there. If a commit fails, it rolls back with all its files and the commits before it +stay merged; a file that fails to read or convert is named in the error. The next load continues +from that commit's first file. With +`on_duplicate_keys = "fail"`, each file merges in its own commit, so a key repeated across files +isn't refused. ```toml [sources.prices] @@ -622,8 +634,8 @@ unique_key = ["sku"] sorts last wins a repeated key whatever the modified times. Landing steps often rewrite modified times: unzipping (Python's `zipfile` stamps the time it extracts), `cp`, `gsutil` and sync tools all do. Use it when names carry a sortable date, such as `export_2026-11-14.csv`; dates need ISO - order and zero padding (`2026-11-09`, not `2026-11-9`). `plan` names it: `3 files, one commit - each, in path order`. It applies to `delete_insert` by `replace_key` too, and to `--reload`. + order and zero padding (`2026-11-09`, not `2026-11-9`). `plan` names it: `3 files, 1 commit, + in path order`. It applies to `delete_insert` by `replace_key` too, and to `--reload`. - Stores that keep modified times to the second, as SFTP does, and the members of one ZIP archive, which share the archive's time, fall back to path order for files from the same second. - A file that holds one `unique_key` value in more than one row @@ -631,11 +643,7 @@ unique_key = ["sku"] [Repeated keys](/docs/concepts/write-strategies/#repeated-keys). With `on_duplicate_keys = "fail"`, such a file fails instead, naming the key's columns and examples, before anything of it merges. Rows whose key holds NULLs are - collapsed too, a NULL matching a NULL. The - order of a file's rows comes from `_sqlload_source_row`: when `exclude_columns` drops it, a file - repeating a key fails instead, `plan` says so, and `newest_by` is refused. That - includes a file repeating a key with NULLs in it, even under the default - `on_duplicate_keys`. + collapsed too, a NULL matching a NULL. - Merging files loads into DuckDB and Postgres targets so far, and MotherDuck through its DuckDB connection, which isn't tested live. `state = "logged"` and a `cursor_column` are refused: the run log, not a cursor, picks the files. To merge every file again, drop the table (a table diff --git a/website/src/content/docs/docs/sources/python.mdx b/website/src/content/docs/docs/sources/python.mdx index f69e29ff0..cd452eaed 100644 --- a/website/src/content/docs/docs/sources/python.mdx +++ b/website/src/content/docs/docs/sources/python.mdx @@ -42,7 +42,8 @@ def orders(ctx: sl.Context): | `@sl.table` or `@sl.table(...)` | A standalone `Table`, loaded into the target's schema; it sets its own `write_strategy`. | | `@source.client` | Declares a client the source's tables share as `ctx.client`, built once per run. | -A table's description is `description=`, or else the function's docstring; `plan` requires one. +A table's description, its destination table's comment, is `description=`, or else the function's +docstring. Both are optional. Unknown settings raise `TypeError` when the module is imported. ### Settings @@ -350,8 +351,8 @@ thread-safe. Set `parallel_tables=True` on the source when its client is. | `sl.single()` | None: one request. | | `sl.next_link(pointer)` | The URL at `pointer` in the body, such as `"$.next"`. A link already followed fails the load; see [Repeated pages](#repeated-pages). | | `sl.link_header()` | The `Link` header's `rel="next"`. A link already followed fails the load. | -| `sl.offset(param="offset", *, limit, limit_param="limit", location="query", read_ahead=None)` | Counts rows by `limit`; ends at a short page. For `read_ahead`, see [Reading pages ahead](#reading-pages-ahead). When a first page holds rows but fewer than `limit`, one more request at the next offset tells an API that caps its pages below `limit` (rows come back: the load fails, naming the page size to set as `limit`) from a table that really ended (no rows: the run ends). | -| `sl.page_number(param="page", *, start=1, location="query", read_ahead=None)` | Counts pages; ends at an empty page. | +| `sl.offset(param="offset", *, limit, limit_param="limit", location="query", read_ahead=None)` | Counts rows by `limit`; ends at a short page. For `read_ahead`, see [Reading pages ahead](#reading-pages-ahead). When a first page holds rows but fewer than `limit`, one more request at the next offset tells an API that caps its pages below `limit` (rows come back: the load fails, naming the page size to set as `limit`) from a table that really ended (no rows: the run ends). A fan-out checks a key's short first page only until its requests have shown the API sends pages that long. | +| `sl.page_number(param="page", *, start=1, page_size=None, page_size_param="page_size", location="query", read_ahead=None)` | Counts pages; ends at an empty page, or with `page_size` at a short page. See [Ending at a short page](#ending-at-a-short-page). | | `sl.cursor(pointer, param, *, location="query")` | The token at `pointer` goes back as `param`. A token already sent fails the load. | | Cells | Columns | @@ -380,6 +381,28 @@ for each and the credentials it uses. `python_row_loop`); filter rows downstream, or reshape each page with [`transform`](#transforming-each-page). +### Ending at a short page + +`sl.page_number` without `page_size` ends only at an empty page, as it can't tell a short page from a full one. A +fan-out whose keys mostly fit in one page then sends two requests per key. With `page_size`, a page holding fewer +rows ends the run, as for `sl.offset`: + +```python +api.fan_out("/proofs/{id}/activities", over=ids, + paginator=sl.page_number("page", page_size=100, page_size_param="count"), select="$.items") +``` + +`page_size` goes with every request as `page_size_param`, so leave it out of `params`; `page_size_param=None` sends +nothing, for an API whose page size is fixed. An API can send fewer rows a page than asked, so a short page ends the +run only once the table's requests have shown that the API sends that many: + +- After a page holding the whole `page_size`, every short page is the last. +- Before that, a short page is checked with the next page. An empty one shows the short page was the last, and later + keys end at once on a shorter page. Rows there show the API caps its pages: paging goes on to an empty page, so + every row loads, and a warning names the `page_size` to set. + +A fan-out over 3,121 keys of about 19 rows each sends 3,153 requests rather than 6,127. + ### Reading pages ahead `read_ahead=N` on `sl.offset` or `sl.page_number` keeps up to `N` page requests open at once @@ -393,7 +416,7 @@ api.paginate("/invoices", paginator=sl.offset(limit=50, read_ahead=20)) The first page is read alone, so a short first page is checked as without `read_ahead`. After a full first page, the pages after it are requested two at a time, then four, eight and so on up to `N`, the open requests doubling each time the pages of the last round have loaded. Each goes through the connection's `rate_limit`, `max_in_flight` and retries, and their -rows load strictly in page order. The run ends at the first short (or, for `sl.page_number`, empty) page, and the +rows load strictly in page order. The run ends at the first short (or, for `sl.page_number` without `page_size`, empty) page, and the requests already sent past it are dropped, so a table or cursor window of a few pages sends only a few requests more than it loads, and a long one soon has `N` open. `sl.cursor`, `sl.next_link` and `sl.link_header` refuse `read_ahead`, as each page's address comes from the page @@ -516,7 +539,7 @@ loaded raw.shop__lines 6 rows 0.1s - A page's nested objects and lists arrive as JSON text, as `lines` does above: `json.loads` them, or pick their values with [`fields`](#fields) first. - Every page that's read goes through the function once, including an empty last page, such as the - one that ends `sl.page_number` paging. What it returns or yields loads in page order, batch by + one that ends `sl.page_number` paging without `page_size`. What it returns or yields loads in page order, batch by batch, and `flatten`, `columns` and `mask` apply to it. - A function that raises, or returns or yields something else, fails the table, naming the request: `GET .../regions/north/orders: transform order_lines failed: ...`. diff --git a/website/src/content/docs/docs/troubleshooting.mdx b/website/src/content/docs/docs/troubleshooting.mdx index e8c9c0ac3..0a9d76d4a 100644 --- a/website/src/content/docs/docs/troubleshooting.mdx +++ b/website/src/content/docs/docs/troubleshooting.mdx @@ -9,7 +9,6 @@ common ones; each message is copied from SQLLoad. | Message | Fix | |---------|-----| | `error: environment variable PGPASSWORD is not set` | Set the variable a `${ENV:NAME}` in `sqlload.toml` refers to. | -| `no description; add description = "..." under [sources.app.tables."public.orders"]` | Every table SQLLoad writes needs a description. Add one where the message says. | | `load_method is required: how should data get into Snowflake?` | Add a `load_method` to the Snowflake connection. See [Snowflake](/docs/targets/snowflake/). | | `n_free: numeric without a precision has no fixed scale; pin a type such as "decimal(38,10)" or "text"` | Pin the column in `columns`, such as `columns = { amount = "decimal(38,18)" }`. | | `columns "total" is not a column of the source table (names are case-sensitive); did you mean "email"?` | Fix the column name, or move the entry to the right table. |