Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ impl SqlAdapter for ClickHouseSql {
Warehouse::ClickHouse
}

/// `length` counts bytes here.
fn render_text_length(&self, expression: &str) -> String {
format!("lengthUTF8({expression})")
}

fn max_identifier_length(&self) -> Option<usize> {
Some(255)
}
Expand Down
67 changes: 60 additions & 7 deletions crates/sqlload/src/adapters/contract/_helpers/queries.rs
Original file line number Diff line number Diff line change
Expand Up @@ -288,18 +288,33 @@ pub(crate) fn check_query<A: SqlAdapter + ?Sized>(
"SELECT DISTINCT {} AS example FROM {from} ORDER BY 1 LIMIT {limit}",
examples(&names_of(columns), true)
),
CheckQuery::KeySources {
CheckQuery::KeyFiles {
key,
staged,
file_column,
file,
held_by,
} => {
let values = examples(&names_of(key), true);
let values = key_text(adapter, key);
let column = adapter.render_identifier(file_column);
let length = adapter.render_text_cast(&adapter.render_text_length(&column));
let held = held_by.map_or_else(String::new, |held_by| {
format!(
" WHERE {values} IN (SELECT {values} FROM {})",
adapter.render_qualified_name(held_by)
)
});
format!(
"SELECT DISTINCT {column} AS example FROM {from} WHERE {values} IN (SELECT {values} FROM {}) AND {column} <> {} ORDER BY 1",
adapter.render_qualified_name(staged),
adapter.render_loader_value_literal(file)
"SELECT DISTINCT {length} || ':' || {column} || {values} AS example FROM {from}{held} ORDER BY 1"
)
}
CheckQuery::NullKeyFiles(columns, file_column) => {
let missing: Vec<String> = names_of(columns)
.iter()
.map(|name| format!("{name} IS NULL"))
.collect();
format!(
"SELECT DISTINCT {} AS example FROM {from} WHERE {} ORDER BY 1",
adapter.render_identifier(file_column),
missing.join(" OR ")
)
}
CheckQuery::NullKeyExamples(columns) => {
Expand All @@ -313,3 +328,41 @@ pub(crate) fn check_query<A: SqlAdapter + ?Sized>(
}
}
}

/// `key`'s columns of a row as one text, each `NULL` for a NULL, joined by `, `.
fn key_text<A: SqlAdapter + ?Sized>(adapter: &A, key: &[String]) -> String {
let texts: Vec<String> = key
.iter()
.map(|column| {
let name = adapter.render_identifier(column);
format!("COALESCE({}, 'NULL')", adapter.render_text_cast(&name))
})
.collect();
texts.join(" || ', ' || ")
}

/// `DELETE FROM staging` of each row whose `key` values (converted to the table's types as the commit's replace step matches them, a NULL matching a NULL) a row of a later one of `files` (its `file_column`, in order) holds too: the rows ranked by a window over the key, which the warehouse can sort or hash, then found by file and `row_column`, which numbers each file's rows.
pub(crate) fn superseded_delete<A: SqlAdapter + ?Sized>(
adapter: &A,
staging: &TableName,
(key, file_column, row_column): (&[&DesiredColumn], &str, &str),
files: &[&str],
) -> String {
let column = adapter.render_identifier(file_column);
let row = adapter.render_identifier(row_column);
let listed: Vec<String> = files
.iter()
.enumerate()
.map(|(at, file)| format!("({}, {at})", adapter.render_loader_value_literal(file)))
.collect();
let staged = adapter.render_qualified_name(staging);
let names: Vec<String> = key
.iter()
.map(|column| adapter.render_source_expression_cast(&column.name, &column.column_type))
.collect();
format!(
"DELETE FROM {staged} AS __sqlload_earlier WHERE EXISTS (SELECT 1 FROM (SELECT {column} AS __sqlload_of, {row} AS __sqlload_at, __sqlload_order, MAX(__sqlload_order) OVER (PARTITION BY {}) AS __sqlload_newest FROM {staged} JOIN (VALUES {}) AS __sqlload_files (__sqlload_file, __sqlload_order) ON __sqlload_file = {column}) AS __sqlload_ranked WHERE __sqlload_of = __sqlload_earlier.{column} AND __sqlload_at = __sqlload_earlier.{row} AND __sqlload_order < __sqlload_newest)",
names.join(", "),
listed.join(", ")
)
}
18 changes: 13 additions & 5 deletions crates/sqlload/src/adapters/contract/_helpers/write.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ pub(crate) fn write_statements<A: SqlAdapter + ?Sized>(
..
} => {
let spared = |table: &str| {
request.spared_load.map(|(column, load_id)| {
request.spared_load.map(|(column, load_id, _)| {
format!(
"{table}{} IS DISTINCT FROM {}",
adapter.render_identifier(column),
Expand All @@ -78,10 +78,18 @@ pub(crate) fn write_statements<A: SqlAdapter + ?Sized>(
}
};
let keyed = (!replace_key.is_empty()).then(|| {
let (keys, join) = replaced_keys(adapter, request, &values, replace_key);
let spared = spared("__target.")
.map(|condition| format!(" AND {condition}"))
.unwrap_or_default();
let (mut keys, join) = replaced_keys(adapter, request, &values, replace_key);
let by_file = request.spared_load.and_then(|(_, _, file)| file);
let by_file = by_file.map(|file| adapter.render_identifier(file));
if let Some(file) = &by_file {
keys.push_str(&format!(", {file} AS __sqlload_spared_file"));
}
let spared = spared("__target.").map_or_else(String::new, |condition| match &by_file {
Some(file) => format!(
" AND ({condition} OR __target.{file} IS DISTINCT FROM __source.__sqlload_spared_file)"
),
None => format!(" AND {condition}"),
});
format!(
"DELETE FROM {target} AS __target USING (SELECT DISTINCT {keys} FROM {source}) AS __source WHERE {join}{spared}"
)
Expand Down
13 changes: 7 additions & 6 deletions crates/sqlload/src/adapters/contract/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -165,8 +165,8 @@ pub struct WriteStatementsRequest<'a> {
pub hashed: &'a [String],
/// The batches went straight into `staging`, built in the table's shape: the write only swaps it in, so no row is written twice, then re-creates these indexes.
pub built: Option<&'a [String]>,
/// The load id column and this load's id, whose rows a `delete_insert`'s deletes keep: the load committed some in pieces before.
pub spared_load: Option<(&'a str, &'a str)>,
/// The load id column and this load's id, whose rows a `delete_insert`'s deletes keep: the load committed some in pieces before. With a file column (a files source under `on_overlapping_files = "last_wins"`), a `replace_key` delete keeps only those of the same file as the staged rows of their value, so a later file replaces an earlier one's rows its earlier pieces committed.
pub spared_load: Option<(&'a str, &'a str, Option<&'a str>)>,
}

/// What a table rebuilt beside itself must keep, where the warehouse cannot copy it (DuckDB).
Expand Down Expand Up @@ -322,13 +322,14 @@ pub enum CheckQuery<'a> {
KeyDuplicateExamples(&'a [String]),
/// Up to five values of these columns, in order, of rows holding a NULL in any of them, each as its columns' text (`NULL` for a NULL) joined by `, `.
NullKeyExamples(&'a [String]),
/// The distinct values of the column named second among rows holding a NULL in any of the first, in order: the files whose rows have NULL keys.
NullKeyFiles(&'a [String], &'a str),
/// Up to this many distinct values of these columns, in order, each as its columns' text (`NULL` for a NULL) joined by `, `.
KeyValues(&'a [String], u64),
/// The distinct values of `file_column` among the relation's rows whose `key` values (as `KeyValues` gives them) a row of `staged` holds, other than `file`, in order: the files a commit's key values already come from.
KeySources {
/// The distinct pairs of `file_column` and `key` values (as `KeyValues` gives them) among the relation's rows, only those whose values a row of `held_by` holds when set, in order, each as the file's length in characters, `:`, the file, then the values: which files hold which key values.
KeyFiles {
key: &'a [String],
staged: &'a TableName,
file_column: &'a str,
file: &'a str,
held_by: Option<&'a TableName>,
},
}
24 changes: 22 additions & 2 deletions crates/sqlload/src/adapters/contract/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use crate::adapters::contract::_helpers::promote::{
};
use crate::adapters::contract::_helpers::queries::{
check_query, child_tables_query, cursor_query, insert_rows, loaded_files_query,
staging_tables_query, staging_time, stale_staging,
staging_tables_query, staging_time, stale_staging, superseded_delete,
};
use crate::adapters::contract::_helpers::recreate::recreate_statements;
use crate::adapters::contract::_helpers::schema::{
Expand Down Expand Up @@ -273,6 +273,26 @@ pub trait SqlAdapter: Sync + std::fmt::Debug {
check_query(self, relation, check)
}

/// Whether [`render_superseded_delete`](Self::render_superseded_delete) runs here, so a files source's `delete_insert` by `replace_key` under `on_overlapping_files = "last_wins"` can commit many files at once; elsewhere it commits each file on its own.
fn deletes_superseded_rows(&self) -> bool {
false
}

/// A statement deleting from `staging` each row whose `key` values (converted as the table takes them, a NULL matching a NULL) a row of a later one of `files` (`file_column`, in order) holds too; `row_column` numbers each file's rows.
fn render_superseded_delete(
&self,
staging: &TableName,
(key, file_column, row_column): (&[&DesiredColumn], &str, &str),
files: &[&str],
) -> String {
superseded_delete(self, staging, (key, file_column, row_column), files)
}

/// The length of text `expression` in characters.
fn render_text_length(&self, expression: &str) -> String {
format!("length({expression})")
}

/// A query returning a row when `table` holds one.
fn render_any_row_query(&self, table: &TableName) -> String {
format!(
Expand Down Expand Up @@ -543,7 +563,7 @@ pub trait SqlAdapter: Sync + std::fmt::Debug {
false
}

/// Whether a files source may `merge` into this warehouse, file by file with a run record in each file's commit (tested on DuckDB and Postgres so far).
/// Whether a files source may `merge` into this warehouse, in file order with a run record naming each commit's files (tested on DuckDB and Postgres so far).
fn merges_files(&self) -> bool {
false
}
Expand Down
8 changes: 7 additions & 1 deletion crates/sqlload/src/adapters/duckdb/_helpers/pull_sql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ pub(crate) struct PulledFile {
pub constants: Vec<(String, Option<String>)>,
}

/// `INSERT INTO into` every one of `columns` from the Parquet `files`, matched by name so a file that lacks a column gives NULL. A column comes from the file's constants when it has one of that name (a list of every file's value, indexed by the row's `file_index`: no join, so DuckDB streams the rows into the table instead of holding them), otherwise from the files when at least one of them holds it (`in_files`, each column with its name in the files, which `naming`, `rename` or numbering may have changed: `f."Order ID"` loads into `order_id`), and is cast to its type; `_sqlload_source_row` counts each file's rows from 1; the load id and load time are the batch's; any other column is NULL. Every file must carry the same constant columns. Into MotherDuck (`remote`) the files are read on its servers.
/// `INSERT INTO into` every one of `columns` from the Parquet `files`, matched by name so a file that lacks a column gives NULL. A column comes from the file's constants when it has one of that name (a list of every file's value, indexed by the row's `file_index`: no join, so DuckDB streams the rows into the table instead of holding them), otherwise from the files when at least one of them holds it (`in_files`, each column with its name in the files, which `naming`, `rename` or numbering may have changed: `f."Order ID"` loads into `order_id`), and is cast to its type; `_sqlload_source_row` counts each file's rows from 1, and a merge's `_sqlload_row` numbers the rows in file order, then by row, so the last row of a key is the newest file's; the load id and load time are the batch's; any other column is NULL. Every file must carry the same constant columns. Into MotherDuck (`remote`) the files are read on its servers.
pub(crate) fn pull_insert(
into: &TableName,
(columns, in_files): (&[DesiredColumn], &[(String, String)]),
Expand Down Expand Up @@ -57,6 +57,12 @@ pub(crate) fn pull_insert(
(Some(MetadataRole::LoadId), _) => cast(literal(load_id), column),
(Some(MetadataRole::LoadedAt), _) => cast(literal(loaded_at), column),
(None, SOURCE_ROW_COLUMN) => cast("f.file_row_number + 1".to_owned(), column),
(None, MERGE_ROW_COLUMN) => cast(
format!(
"CAST(f.file_index AS BIGINT) * {UNLOADED_FILE_ROWS} + f.file_row_number"
),
column,
),
(_, name)
if files
.first()
Expand Down
4 changes: 4 additions & 0 deletions crates/sqlload/src/adapters/duckdb/_helpers/sql_adapter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@ impl SqlAdapter for DuckDbSql {
true
}

fn deletes_superseded_rows(&self) -> bool {
true
}

fn parses_variant_pins(&self) -> bool {
true
}
Expand Down
4 changes: 4 additions & 0 deletions crates/sqlload/src/adapters/postgres/_helpers/sql_adapter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,10 @@ impl SqlAdapter for PostgresSql {
staging_type(column_type)
}

fn deletes_superseded_rows(&self) -> bool {
true
}

/// JSON, UUIDs, arrays and native types are converted from their staged text by their type's own input, which refuses what it cannot read; the rest by the table's assignment casts, which refuse a value that does not fit.
fn render_source_expression_cast(&self, name: &str, column_type: &ColumnType) -> String {
let column = self.render_identifier(name);
Expand Down
1 change: 1 addition & 0 deletions crates/sqlload/src/batch_plan/main/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ pub mod file_groups;
pub mod ignored_batch;
pub mod ignored_concurrency;
pub mod last_value;
pub mod ordered_groups;
pub mod outside_text;
pub mod read_at_once;
pub mod read_in_order;
Expand Down
33 changes: 33 additions & 0 deletions crates/sqlload/src/batch_plan/main/ordered_groups.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
//! Cut a files source's files into consecutive batches, keeping their order.

use crate::batch_plan::constants::{BATCH_BYTES, MAX_BATCH_FILES};
use crate::batch_plan::models::FileGroup;
use crate::files_source::models::DataFile;

/// `files` in consecutive batches, in the order given, each closed at 1,000 files or about 1 GB of files, whichever comes first (a single larger file is a batch by itself): a files source's merge, or `delete_insert` by `replace_key`, which applies the files in order, the newest file winning a key repeated across them.
pub fn ordered_groups(files: &[DataFile]) -> Vec<FileGroup> {
let mut groups: Vec<FileGroup> = Vec::new();
let mut current: Vec<usize> = Vec::new();
let mut bytes: usize = 0;
for (index, data) in files.iter().enumerate() {
let file_bytes = data.file.size as usize;
if !current.is_empty()
&& (current.len() == MAX_BATCH_FILES || bytes + file_bytes > BATCH_BYTES)
{
groups.push(FileGroup {
files: std::mem::take(&mut current),
values: Vec::new(),
});
bytes = 0;
}
current.push(index);
bytes += file_bytes;
}
if !current.is_empty() {
groups.push(FileGroup {
files: current,
values: Vec::new(),
});
}
groups
}
20 changes: 2 additions & 18 deletions crates/sqlload/src/contract/_helpers/toml_columns.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,7 @@ use toml_edit::{DocumentMut, InlineTable, Item, Table, Value};
use crate::contract::constants::{MISSING_DECLARATION, MISSING_PHYSICAL_COLUMN, TYPE_MISMATCH};
use crate::contract::models::{Adoption, ContractFinding};

/// Whether writing `adoption` would create a `[sources.*.tables.*]` entry for a database table with no description to give it, which would then block the table.
pub(crate) fn lacks_description(root: &Item, adoption: &Adoption) -> bool {
let (source, table) = &adoption.toml_key;
let exists = root
.get("sources")
.and_then(|sources| sources.get(source.as_str()))
.and_then(|source| source.get("tables"))
.and_then(|tables| tables.get(table.as_str()))
.is_some();
!exists
&& adoption
.description
.as_deref()
.is_some_and(|text| text.trim().is_empty())
}

/// `document` with `adoption` written into the `columns` of `[sources.<source>.tables.<table>]`, adding the tables on the way when missing, and a new database table entry's description; the source must be in the file.
/// `document` with `adoption` written into the `columns` of `[sources.<source>.tables.<table>]`, adding the tables on the way when missing, and a new database table entry's description when it has one; the source must be in the file.
pub(crate) fn write_adoption(
mut document: DocumentMut,
adoption: &Adoption,
Expand All @@ -37,7 +21,7 @@ pub(crate) fn write_adoption(
let description = adoption
.description
.as_deref()
.filter(|_| tables.get(table.as_str()).is_none());
.filter(|text| !text.trim().is_empty() && tables.get(table.as_str()).is_none());
let entry = child(tables, table, true)?;
if let Some((text, entry)) = description.zip(entry.as_table_like_mut()) {
entry.insert("description", Item::Value(Value::from(text)));
Expand Down
11 changes: 2 additions & 9 deletions crates/sqlload/src/contract/main/write_contracts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@

use toml_edit::DocumentMut;

use crate::contract::_helpers::toml_columns::{lacks_description, resolved, write_adoption};
use crate::contract::_helpers::toml_columns::{resolved, write_adoption};
use crate::contract::models::ContractReport;

/// `text` (sqlload.toml) with each adoption of a table declared there written into its `[sources.<source>.tables.<table>]` `columns`, which are created when missing, keeping the rest of the file as it was (`None` when nothing changes); and the report listing only the findings left, without the adoptions it wrote. A Python table's adoption is left to paste, and a database table that would need a new entry but has no description is a problem instead.
/// `text` (sqlload.toml) with each adoption of a table declared there written into its `[sources.<source>.tables.<table>]` `columns`, which are created when missing, keeping the rest of the file as it was (`None` when nothing changes); and the report listing only the findings left, without the adoptions it wrote. A Python table's adoption is left to paste.
pub fn write_contracts(
text: &str,
mut report: ContractReport,
Expand All @@ -21,13 +21,6 @@ pub fn write_contracts(
else {
continue;
};
if lacks_description(document.as_item(), &adoption) {
report.problems.push(format!(
"{}: columns not written: a table entry needs a description and this table has none; add description = \"...\" there, then rerun contract generate --write",
adoption.location
));
continue;
}
document = write_adoption(document, &adoption)?;
resource
.findings
Expand Down
5 changes: 2 additions & 3 deletions crates/sqlload/src/contract/tests/test_write.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,11 @@ fn given_a_project_file_when_writing_adoptions_then_only_the_tables_columns_chan
],
},
WriteTestCase {
description: "a table with no entry and no description is not written, since the entry would block it",
description: "a table with no entry and no description gets an entry holding only its columns",
text: "[sources.app]\nkind = \"postgres\"\ninclude = [\"public.*\"]\n",
table_description: "",
expected_lines: &[
"",
"[sources.app.tables.\"public.orders\"]: columns not written: a table entry needs a description and this table has none; add description = \"...\" there, then rerun contract generate --write",
"[sources.app]\nkind = \"postgres\"\ninclude = [\"public.*\"]\n\n[sources.app.tables.\"public.orders\".columns]\namount = \"float\"\nstatus = \"text\"\n",
],
},
WriteTestCase {
Expand Down
Loading
Loading