Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 3 additions & 12 deletions crates/sqlload/src/loader/_helpers/render.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,12 @@ use crate::batch_plan::main::cursor_rules::cursor_rules;
use crate::batch_plan::main::read_at_once::read_at_once;
use crate::column_parse::main::parse_lines::parse_lines;
use crate::column_types::models::ColumnType;
use crate::files_source::models::{FilesOrder, SourceFormat};
use crate::files_source::models::SourceFormat;
use crate::load_naming::main::is_load_column::is_load_column;
use crate::loader::_helpers::publish::{published_once, publishes_once, records_files, send_lines};
use crate::loader::_helpers::python::{list_choices, unnested_table};
use crate::loader::_helpers::resolve::display_name;
use crate::loader::_helpers::run_log::recorded_commits;
use crate::loader::_helpers::sources::{readers_note, unarchived};
use crate::loader::_helpers::table_plan::{
archived_reason, is_direct, pulls_unloaded, selects_rows,
Expand Down Expand Up @@ -822,17 +823,7 @@ pub(crate) fn batch_line(table: &TablePlan) -> String {
format!("{windows} of {step}, {commits}{}{end}", read_at_once(table))
}
(CommitRule::FileGroups(groups), _) if records_files(table) => {
format!(
"{}, one commit each, {}, each recorded in the run log",
match groups.len() {
1 => "1 file".to_owned(),
count => format!("{count} files"),
},
match spec.files_order {
FilesOrder::Modified => "oldest modified first (ties by path)",
FilesOrder::Path => "in path order",
}
)
recorded_commits(groups.len(), spec)
}
(CommitRule::FileGroups(groups), _) => {
format!(
Expand Down
96 changes: 83 additions & 13 deletions crates/sqlload/src/loader/_helpers/run_log.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ use crate::loader::errors::LoaderError;
use crate::loader::models::{
MergeRebuild, ProjectPlan, PythonLoad, RunEntry, TablePlan, TableSpec, TransformRun,
};
use crate::project::models::WriteStrategy;
use crate::project::models::{OnOverlappingFiles, WriteStrategy};
use crate::row_batches::models::Unnest;
use crate::run_log::constants::RECORD_FORMAT_VERSION;
use crate::run_log::main::cursor_order::cursor_order;
Expand Down Expand Up @@ -837,7 +837,7 @@ pub(crate) fn in_file(merged: &str, error: LoaderError) -> LoaderError {
}
}

/// `earlier` (each `replace_key` value this run's commits of a files source's `delete_insert` into `table` inserted, with the file it came from, named below the source's location) with the key `values` of the commit of `file` (`name|version`) added; or why that commit cannot replace them: one of them came in an earlier commit of the same run, so this commit would delete that file's rows. A value must come whole in one file; [`listed_sources`] checks the files of earlier runs.
/// `earlier` (each `replace_key` value this run's commits of a files source's `delete_insert` into `table` inserted, with the file it came from, named below the source's location) with the key `values` of the commit of `file` (`name|version`) added; or why that commit cannot replace them: one of them came in an earlier commit of the same run, so this commit would delete that file's rows. A value must come whole in one file, unless `on_overlapping_files = "last_wins"`, where the later file replaces it; [`listed_sources`] checks the files of earlier runs.
pub(crate) fn with_file_keys(
table: &TablePlan,
(file, values): (&str, &[String]),
Expand All @@ -851,7 +851,9 @@ pub(crate) fn with_file_keys(
.unwrap_or_default();
let mut split: Vec<(&String, &String)> = Vec::new();
for value in values {
if let Some(first) = earlier.get(value) {
if let Some(first) = earlier.get(value)
&& table.spec.on_overlapping_files == OnOverlappingFiles::Fail
{
split.push((value, first));
}
}
Expand All @@ -864,7 +866,7 @@ pub(crate) fn with_file_keys(
count => format!(" (and {} more)", count - 1),
};
return Err(format!(
"it holds the replace_key ({columns}) value ({value}){more}, which {first} also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if one restates the other, remove the older of {first} and {this} from the folder by hand, and set after_load = {{ move_to = ... }} (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values"
"it holds the replace_key ({columns}) value ({value}){more}, which {first} also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if newer files restate older ones, set on_overlapping_files = \"last_wins\", so the file last in order replaces the values, or remove the older of {first} and {this} from the folder by hand and set after_load = {{ move_to = ... }} (or \"delete\"), so that each file leaves the folder once it loads"
));
}
earlier.extend(values.iter().map(|value| (value.clone(), this.to_owned())));
Expand Down Expand Up @@ -928,7 +930,7 @@ pub(crate) fn file_column_problem(
None
}

/// Why the commit of `file` (`name|version`), a files source's `delete_insert` by `replace_key` into `table`, cannot replace the rows of its key values: some of them come from `sources` (other files the table's rows of those values came from, by name) that are still listed in the source's folder, so two files of the folder hold one value and which is current can't be told. A file that has left the folder (deleted, or moved by `after_load`) is replaced.
/// Why the commit of `file` (`name|version`), a files source's `delete_insert` by `replace_key` into `table`, cannot replace the rows of its key values: some of them come from `sources` (other files the table's rows of those values came from, by name) that are still listed in the source's folder, so two files of the folder hold one value and which is current can't be told. A file that has left the folder (deleted, or moved by `after_load`) is replaced; under `on_overlapping_files = "last_wins"` so is one still listed that comes before `file` in the source's `order`, or that this run reads again (it loads in order, replacing the values back), but never another after it, whose values a file sent late or rewritten would otherwise take back.
pub(crate) fn listed_sources(
table: &TablePlan,
file: &str,
Expand All @@ -938,32 +940,100 @@ pub(crate) fn listed_sources(
return Ok(());
};
let location = source.location.to_string();
let listed: Vec<String> = source
let order = |data: &DataFile| match table.spec.files_order {
FilesOrder::Modified => (data.file.modified_ms, data.file.relative_key.clone()),
FilesOrder::Path => (0, data.file.relative_key.clone()),
};
let listed: Vec<(String, (i64, String))> = source
.table
.files
.iter()
.map(|data| format!("{location}{}", data.file.relative_key))
.map(|data| (format!("{location}{}", data.file.relative_key), order(data)))
.collect();
let held: Vec<&str> = sources
let read: Vec<String> = table
.files
.iter()
.filter(|name| listed.contains(name))
.map(|name| name.strip_prefix(location.as_str()).unwrap_or(name))
.flat_map(|read| &read.files)
.map(|data| format!("{location}{}", data.file.relative_key))
.collect();
let this_name = merged_name(file);
let this_order = listed
.iter()
.find(|(name, _)| name == this_name)
.map(|(_, at)| at.clone());
let last_wins = table.spec.on_overlapping_files == OnOverlappingFiles::LastWins;
let mut held: Vec<&str> = Vec::new();
for name in sources {
let Some((_, at)) = listed.iter().find(|(listed, _)| listed == name) else {
continue;
};
if !last_wins || (!read.contains(name) && this_order.as_ref().is_none_or(|this| at > this))
{
held.push(name.strip_prefix(location.as_str()).unwrap_or(name));
}
}
let Some(first) = held.first() else {
return Ok(());
};
let more = match held.len() {
1 => String::new(),
count => format!(" (and {} more)", count - 1),
};
let this = merged_name(file);
let this = this.strip_prefix(location.as_str()).unwrap_or(this);
let this = this_name
.strip_prefix(location.as_str())
.unwrap_or(this_name);
let columns = table.spec.replace_key.join(", ");
if last_wins {
return Err(format!(
"the table holds rows of its replace_key ({columns}) values from {first}{more}, which is still in the folder and comes after {this} in order; with on_overlapping_files = \"last_wins\" the file last in order keeps a value, so {this} would take back values a later file replaced. Fix: if {this} is a correction, give it a name (or, by modified time, a time) that sorts after {first}, or remove {first} from the folder"
));
}
Err(format!(
"the table holds rows of its replace_key ({columns}) values from {first}{more}, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by {this} would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if one restates the other, remove the older of {first} and {this} from the folder by hand, and set after_load = {{ move_to = ... }} (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values"
"the table holds rows of its replace_key ({columns}) values from {first}{more}, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by {this} would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if newer files restate older ones, set on_overlapping_files = \"last_wins\", so the file last in order replaces the values, or remove the older of {first} and {this} from the folder by hand and set after_load = {{ move_to = ... }} (or \"delete\"), so that each file leaves the folder once it loads"
))
}

/// Why `on_overlapping_files` does not apply to a table loaded by `strategy`: it is for a `delete_insert` by `replace_key` (`keyed`) of a source of `files`, where two files of the folder can hold one key value.
pub(crate) fn overlapping_files_problem(
files: bool,
strategy: WriteStrategy,
keyed: bool,
) -> Option<String> {
const FOR: &str = "on_overlapping_files says which file wins when two files of a files source's folder hold one replace_key value";
match (files, strategy, keyed) {
(true, _, true) => None,
(false, _, _) => Some(format!("{FOR}; this source reads no files, so remove it")),
(true, WriteStrategy::Merge, _) => Some(format!(
"{FOR}; a merge applies the files one at a time, so the last file already wins each unique_key; remove it"
)),
(true, WriteStrategy::DeleteInsert, false) => Some(format!(
"{FOR}; this delete_insert replaces a cursor range, not key values, so remove it, or set replace_key"
)),
(true, strategy, false) => Some(format!(
"{FOR}; {strategy} replaces no key values, so remove it"
)),
}
}

/// How `plan` describes the commits of a files table whose `groups` files are each recorded in the run log: their order, and what `on_overlapping_files = "last_wins"` adds.
pub(crate) fn recorded_commits(groups: usize, spec: &TableSpec) -> String {
let files = match groups {
1 => "1 file".to_owned(),
count => format!("{count} files"),
};
let order = match spec.files_order {
FilesOrder::Modified => "oldest modified first (ties by path)",
FilesOrder::Path => "in path order",
};
let last_wins = match spec.on_overlapping_files {
OnOverlappingFiles::LastWins => {
"; the last file holding a replace_key value replaces it (on_overlapping_files = \"last_wins\")"
}
OnOverlappingFiles::Fail => "",
};
format!("{files}, one commit each, {order}, each recorded in the run log{last_wins}")
}

/// The files a merge already merged into `request`'s table at the version listed now, which it skips (one rewritten in place since is merged again, as the newest), and under `--reload` none, with every file the table's merges recorded instead (oldest first), whose order the reload keeps; or why a files source cannot load into it: only into a DuckDB or Postgres table, never with `state = "logged"`, nor `--reload` with `unnest` or of a `delete_insert` (one commit would lose the files' order); none while the table does not exist.
pub(crate) async fn merge_ledger(
request: &FilesPlanRequest<'_>,
Expand Down
12 changes: 12 additions & 0 deletions crates/sqlload/src/loader/_helpers/specs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ use crate::load_naming::models::{Compat, LoadedNames, NameRules, Naming};
use crate::loader::_helpers::destination::read_as_text;
use crate::loader::_helpers::python::field_type;
use crate::loader::_helpers::resolve::{NameRule, ResolvedTarget};
use crate::loader::_helpers::run_log::overlapping_files_problem;
use crate::loader::_helpers::sources::sheet_rule;
use crate::loader::_helpers::table_plan::source_cursor;
use crate::loader::constants::{
Expand Down Expand Up @@ -321,6 +322,16 @@ pub(crate) fn build_spec(
{
problems.push(format!("order = \"path\" orders the files a merge or a delete_insert by replace_key applies one at a time; {strategy} loads them without an order, so remove it"));
}
let own_strategy = strategy_origin == SettingOrigin::Table && !keyed;
let overlapping = layered(
&settings.on_overlapping_files,
&defaults.on_overlapping_files,
);
let overlapping = overlapping
.filter(|(_, origin)| !own_strategy || *origin == SettingOrigin::Table)
.map(|(value, _)| value);
let files = context.source.kind == SourceKind::Files;
problems.extend(overlapping.and_then(|_| overlapping_files_problem(files, strategy, keyed)));
let matched_key = match keyed {
true => &replace_key,
false => &unique_key,
Expand Down Expand Up @@ -445,6 +456,7 @@ pub(crate) fn build_spec(
null_keys,
not_null_keys,
replace_key,
on_overlapping_files: overlapping.unwrap_or_default(),
newest_by,
cursor,
batch,
Expand Down
6 changes: 4 additions & 2 deletions crates/sqlload/src/loader/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ use crate::masking::models::MaskRules;
use crate::postgres_source::models::{ReadColumn, SourceTable};
use crate::project::models::{
BatchStep, Contract, Deletes, NewTables, NullCursor, NullKeys, OnCursorOutOfRange,
OnDuplicateKeys, OnSchemaChange, OutputEncoding, Policy, ProjectConfig, SendMode, TableChecks,
WriteStrategy,
OnDuplicateKeys, OnOverlappingFiles, OnSchemaChange, OutputEncoding, Policy, ProjectConfig,
SendMode, TableChecks, WriteStrategy,
};
use crate::python_sources::types::PythonHost;
use crate::routing::models::{Route, Store};
Expand Down Expand Up @@ -145,6 +145,8 @@ pub struct TableSpec {
pub not_null_keys: Vec<String>,
/// Under `delete_insert`, the columns whose values each commit replaces whole: the rows holding a value of them that the commit's rows hold are deleted first. Empty otherwise.
pub replace_key: Vec<String>,
/// What a files source's `delete_insert` by `replace_key` does when two files of its folder hold one key value.
pub on_overlapping_files: OnOverlappingFiles,
/// The columns, compared in order, whose higher values are a row's newer version (`newest_by`): a merge keeps the newest of the rows sharing a key and never overwrites a loaded row with an older one; empty, the last row read wins.
pub newest_by: Vec<String>,
pub cursor: Option<CursorSpec>,
Expand Down
1 change: 1 addition & 0 deletions crates/sqlload/src/project/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ pub const TABLE_SETTING_KEYS: &[&str] = &[
"on_duplicate_keys",
"null_keys",
"replace_key",
"on_overlapping_files",
"newest_by",
"naming",
"compat",
Expand Down
14 changes: 14 additions & 0 deletions crates/sqlload/src/project/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -901,6 +901,9 @@ pub struct TableSettings {
/// The columns whose values `delete_insert` replaces whole: each commit first deletes the rows holding a value of them that its rows hold.
#[serde(default, deserialize_with = "deserialize_optional_keys")]
pub replace_key: Option<Vec<String>>,
/// What a files source's `delete_insert` by `replace_key` does when two files of its folder hold one key value: fail (default) or let the file last in `order` replace it.
#[serde(default)]
pub on_overlapping_files: Option<OnOverlappingFiles>,
/// The column, or columns compared in order, whose higher value is the newer version of a row: of the rows sharing a `unique_key` a merge keeps the newest, and never overwrites a loaded row with an older one.
#[serde(default, deserialize_with = "deserialize_optional_keys")]
pub newest_by: Option<Vec<String>>,
Expand Down Expand Up @@ -955,6 +958,17 @@ pub enum OnDuplicateKeys {
Fail,
}

/// What a files source's `delete_insert` by `replace_key` does when two files of its folder hold one key value.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OnOverlappingFiles {
/// Fail the file's commit, naming both files: they may be parts of one export.
#[default]
Fail,
/// The file's commit replaces the value's rows, whichever file loaded them: the file last in `order` wins.
LastWins,
}

/// What a merge does with a row whose key the source no longer holds, found by `load --sync-deletes`, or a captured delete does to a merged row.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
Expand Down
2 changes: 2 additions & 0 deletions crates/sqlload/tests/loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ mod test_odbc_memory;
#[cfg(feature = "odbc")]
#[path = "loader/test_odbc_to_duckdb.rs"]
mod test_odbc_to_duckdb;
#[path = "loader/test_overlapping_files.rs"]
mod test_overlapping_files;
#[path = "loader/test_parsed_columns.rs"]
mod test_parsed_columns;
#[path = "loader/test_plan_progress.rs"]
Expand Down
21 changes: 19 additions & 2 deletions crates/sqlload/tests/loader/merge/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ use sqlload_testkit::postgres::constants::{PASSWORD, USER};

use crate::helpers::{described_project, load_selected, words};
use crate::test_types::{
DedupeRule, DedupeTargetTestCase, FilesMergeTestCase, NullKeyTargetTestCase,
ReplaceKeyTargetTestCase, SyncDeletesTestCase,
DedupeRule, DedupeTargetTestCase, FilesMergeTestCase, MergeFile, MergeStep,
NullKeyTargetTestCase, ReplaceKeyTargetTestCase, SyncDeletesTestCase,
};

/// `public.readings`: `s1` and `s2` repeat with their `seen_at` order unlike their `reading_id` order, `s2`'s last row has no `seen_at`, and two rows have no sensor (nor `seen_at`), a repeated NULL key.
Expand Down Expand Up @@ -779,3 +779,20 @@ pub(crate) fn hard_tombstone_steps() -> Vec<String> {
.map(str::to_owned)
.to_vec()
}

/// One load of a files merge case with `files` written first, no SQL before it and nothing removed, expecting `expected_error` (empty for none) and `expected_plan` in its plan.
pub(crate) const fn load_step(
files: &'static [MergeFile],
expected_error: &'static str,
expected_plan: &'static str,
) -> MergeStep {
MergeStep {
before: &[],
files,
removed: &[],
loads: true,
reload: false,
expected_error,
expected_plan,
}
}
Loading
Loading