From 63f2a8ef63b64fd4ee592e735182d62d88347254 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:23:30 +0000 Subject: [PATCH 1/5] wip: on_overlapping_files = "last_wins" lets a files source's later file replace a replace_key value an earlier one holds --- crates/sqlload/src/loader/_helpers/render.rs | 6 +- crates/sqlload/src/loader/_helpers/run_log.rs | 49 ++++- crates/sqlload/src/loader/_helpers/specs.rs | 12 ++ crates/sqlload/src/loader/models.rs | 6 +- crates/sqlload/src/project/constants.rs | 1 + crates/sqlload/src/project/models.rs | 14 ++ crates/sqlload/tests/loader.rs | 2 + crates/sqlload/tests/loader/merge/helpers.rs | 21 +- .../tests/loader/test_files_to_duckdb.rs | 4 +- .../tests/loader/test_overlapping_files.rs | 190 ++++++++++++++++++ .../src/content/docs/docs/configuration.mdx | 1 + .../src/content/docs/docs/sources/files.mdx | 43 +++- 12 files changed, 330 insertions(+), 19 deletions(-) create mode 100644 crates/sqlload/tests/loader/test_overlapping_files.rs diff --git a/crates/sqlload/src/loader/_helpers/render.rs b/crates/sqlload/src/loader/_helpers/render.rs index dcda32667..dbe2c5915 100644 --- a/crates/sqlload/src/loader/_helpers/render.rs +++ b/crates/sqlload/src/loader/_helpers/render.rs @@ -27,6 +27,7 @@ use crate::load_naming::main::is_load_column::is_load_column; use crate::loader::_helpers::publish::{published_once, records_files, send_lines}; use crate::loader::_helpers::python::{list_choices, unnested_table}; use crate::loader::_helpers::resolve::display_name; +use crate::loader::_helpers::run_log::last_wins_note; use crate::loader::_helpers::sources::{readers_note, unarchived}; use crate::loader::_helpers::table_plan::{ archived_reason, is_direct, publishes_once, pulls_unloaded, @@ -822,7 +823,7 @@ pub(crate) fn batch_line(table: &TablePlan) -> String { } (CommitRule::FileGroups(groups), _) if records_files(table) => { format!( - "{}, one commit each, {}, each recorded in the run log", + "{}, one commit each, {}, each recorded in the run log{}", match groups.len() { 1 => "1 file".to_owned(), count => format!("{count} files"), @@ -830,7 +831,8 @@ pub(crate) fn batch_line(table: &TablePlan) -> String { match spec.files_order { FilesOrder::Modified => "oldest modified first (ties by path)", FilesOrder::Path => "in path order", - } + }, + last_wins_note(spec.on_overlapping_files) ) } (CommitRule::FileGroups(groups), _) => { diff --git a/crates/sqlload/src/loader/_helpers/run_log.rs b/crates/sqlload/src/loader/_helpers/run_log.rs index 6d06afb4f..eafd09657 100644 --- a/crates/sqlload/src/loader/_helpers/run_log.rs +++ b/crates/sqlload/src/loader/_helpers/run_log.rs @@ -33,7 +33,7 @@ use crate::loader::errors::LoaderError; use crate::loader::models::{ MergeRebuild, ProjectPlan, PythonLoad, RunEntry, TablePlan, TableSpec, TransformRun, }; -use crate::project::models::WriteStrategy; +use crate::project::models::{OnOverlappingFiles, WriteStrategy}; use crate::row_batches::models::Unnest; use crate::run_log::constants::RECORD_FORMAT_VERSION; use crate::run_log::main::cursor_order::cursor_order; @@ -837,7 +837,7 @@ pub(crate) fn in_file(merged: &str, error: LoaderError) -> LoaderError { } } -/// `earlier` (each `replace_key` value this run's commits of a files source's `delete_insert` into `table` inserted, with the file it came from, named below the source's location) with the key `values` of the commit of `file` (`name|version`) added; or why that commit cannot replace them: one of them came in an earlier commit of the same run, so this commit would delete that file's rows. A value must come whole in one file; [`listed_sources`] checks the files of earlier runs. +/// `earlier` (each `replace_key` value this run's commits of a files source's `delete_insert` into `table` inserted, with the file it came from, named below the source's location) with the key `values` of the commit of `file` (`name|version`) added; or why that commit cannot replace them: one of them came in an earlier commit of the same run, so this commit would delete that file's rows. A value must come whole in one file, unless `on_overlapping_files = "last_wins"`, where the later file replaces it; [`listed_sources`] checks the files of earlier runs. pub(crate) fn with_file_keys( table: &TablePlan, (file, values): (&str, &[String]), @@ -851,7 +851,9 @@ pub(crate) fn with_file_keys( .unwrap_or_default(); let mut split: Vec<(&String, &String)> = Vec::new(); for value in values { - if let Some(first) = earlier.get(value) { + if let Some(first) = earlier.get(value) + && table.spec.on_overlapping_files == OnOverlappingFiles::Fail + { split.push((value, first)); } } @@ -864,7 +866,7 @@ pub(crate) fn with_file_keys( count => format!(" (and {} more)", count - 1), }; return Err(format!( - "it holds the replace_key ({columns}) value ({value}){more}, which {first} also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if one restates the other, remove the older of {first} and {this} from the folder by hand, and set after_load = {{ move_to = ... }} (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values" + "it holds the replace_key ({columns}) value ({value}){more}, which {first} also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if newer files restate older ones, set on_overlapping_files = \"last_wins\", so the file last in order replaces the values, or remove the older of {first} and {this} from the folder by hand and set after_load = {{ move_to = ... }} (or \"delete\"), so that each file leaves the folder once it loads" )); } earlier.extend(values.iter().map(|value| (value.clone(), this.to_owned()))); @@ -928,7 +930,7 @@ pub(crate) fn file_column_problem( None } -/// Why the commit of `file` (`name|version`), a files source's `delete_insert` by `replace_key` into `table`, cannot replace the rows of its key values: some of them come from `sources` (other files the table's rows of those values came from, by name) that are still listed in the source's folder, so two files of the folder hold one value and which is current can't be told. A file that has left the folder (deleted, or moved by `after_load`) is replaced. +/// Why the commit of `file` (`name|version`), a files source's `delete_insert` by `replace_key` into `table`, cannot replace the rows of its key values: some of them come from `sources` (other files the table's rows of those values came from, by name) that are still listed in the source's folder, so two files of the folder hold one value and which is current can't be told. A file that has left the folder (deleted, or moved by `after_load`) is replaced, and under `on_overlapping_files = "last_wins"` so is one still listed. pub(crate) fn listed_sources( table: &TablePlan, file: &str, @@ -937,6 +939,9 @@ pub(crate) fn listed_sources( let Some(source) = table.spec.files.as_ref() else { return Ok(()); }; + if table.spec.on_overlapping_files == OnOverlappingFiles::LastWins { + return Ok(()); + } let location = source.location.to_string(); let listed: Vec = source .table @@ -960,10 +965,42 @@ pub(crate) fn listed_sources( let this = this.strip_prefix(location.as_str()).unwrap_or(this); let columns = table.spec.replace_key.join(", "); Err(format!( - "the table holds rows of its replace_key ({columns}) values from {first}{more}, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by {this} would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if one restates the other, remove the older of {first} and {this} from the folder by hand, and set after_load = {{ move_to = ... }} (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values" + "the table holds rows of its replace_key ({columns}) values from {first}{more}, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by {this} would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if newer files restate older ones, set on_overlapping_files = \"last_wins\", so the file last in order replaces the values, or remove the older of {first} and {this} from the folder by hand and set after_load = {{ move_to = ... }} (or \"delete\"), so that each file leaves the folder once it loads" )) } +/// Why `on_overlapping_files` does not apply to a table loaded by `strategy`: it is for a `delete_insert` by `replace_key` (`keyed`) of a source of `files`, where two files of the folder can hold one key value. +pub(crate) fn overlapping_files_problem( + files: bool, + strategy: WriteStrategy, + keyed: bool, +) -> Option { + const FOR: &str = "on_overlapping_files says which file wins when two files of a files source's folder hold one replace_key value"; + match (files, strategy, keyed) { + (true, _, true) => None, + (false, _, _) => Some(format!("{FOR}; this source reads no files, so remove it")), + (true, WriteStrategy::Merge, _) => Some(format!( + "{FOR}; a merge applies the files one at a time, so the last file already wins each unique_key; remove it" + )), + (true, WriteStrategy::DeleteInsert, false) => Some(format!( + "{FOR}; this delete_insert replaces a cursor range, not key values, so remove it, or set replace_key" + )), + (true, strategy, false) => Some(format!( + "{FOR}; {strategy} replaces no key values, so remove it" + )), + } +} + +/// What `plan` adds to a files table's commits under `on_overlapping_files = "last_wins"`. +pub(crate) fn last_wins_note(on_overlapping_files: OnOverlappingFiles) -> &'static str { + match on_overlapping_files { + OnOverlappingFiles::LastWins => { + "; the last file holding a replace_key value replaces it (on_overlapping_files = \"last_wins\")" + } + OnOverlappingFiles::Fail => "", + } +} + /// The files a merge already merged into `request`'s table at the version listed now, which it skips (one rewritten in place since is merged again, as the newest), and under `--reload` none, with every file the table's merges recorded instead (oldest first), whose order the reload keeps; or why a files source cannot load into it: only into a DuckDB or Postgres table, never with `state = "logged"`, nor `--reload` with `unnest` or of a `delete_insert` (one commit would lose the files' order); none while the table does not exist. pub(crate) async fn merge_ledger( request: &FilesPlanRequest<'_>, diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs index e9e0d0ca8..d6f7dd5b7 100644 --- a/crates/sqlload/src/loader/_helpers/specs.rs +++ b/crates/sqlload/src/loader/_helpers/specs.rs @@ -30,6 +30,7 @@ use crate::load_naming::models::{Compat, LoadedNames, NameRules, Naming}; use crate::loader::_helpers::destination::read_as_text; use crate::loader::_helpers::python::field_type; use crate::loader::_helpers::resolve::{NameRule, ResolvedTarget}; +use crate::loader::_helpers::run_log::overlapping_files_problem; use crate::loader::_helpers::sources::sheet_rule; use crate::loader::_helpers::table_plan::source_cursor; use crate::loader::constants::{ @@ -321,6 +322,16 @@ pub(crate) fn build_spec( { problems.push(format!("order = \"path\" orders the files a merge or a delete_insert by replace_key applies one at a time; {strategy} loads them without an order, so remove it")); } + let on_overlapping_files = layered( + &settings.on_overlapping_files, + &defaults.on_overlapping_files, + ) + .map(|(value, _)| value); + if let Some(problem) = on_overlapping_files.and_then(|_| { + overlapping_files_problem(context.source.kind == SourceKind::Files, strategy, keyed) + }) { + problems.push(problem); + } let matched_key = match keyed { true => &replace_key, false => &unique_key, @@ -445,6 +456,7 @@ pub(crate) fn build_spec( null_keys, not_null_keys, replace_key, + on_overlapping_files: on_overlapping_files.unwrap_or_default(), newest_by, cursor, batch, diff --git a/crates/sqlload/src/loader/models.rs b/crates/sqlload/src/loader/models.rs index b24678bba..4089059f3 100644 --- a/crates/sqlload/src/loader/models.rs +++ b/crates/sqlload/src/loader/models.rs @@ -28,8 +28,8 @@ use crate::masking::models::MaskRules; use crate::postgres_source::models::{ReadColumn, SourceTable}; use crate::project::models::{ BatchStep, Contract, Deletes, NewTables, NullCursor, NullKeys, OnCursorOutOfRange, - OnDuplicateKeys, OnSchemaChange, OutputEncoding, Policy, ProjectConfig, SendMode, TableChecks, - WriteStrategy, + OnDuplicateKeys, OnOverlappingFiles, OnSchemaChange, OutputEncoding, Policy, ProjectConfig, + SendMode, TableChecks, WriteStrategy, }; use crate::python_sources::types::PythonHost; use crate::routing::models::{Route, Store}; @@ -145,6 +145,8 @@ pub struct TableSpec { pub not_null_keys: Vec, /// Under `delete_insert`, the columns whose values each commit replaces whole: the rows holding a value of them that the commit's rows hold are deleted first. Empty otherwise. pub replace_key: Vec, + /// What a files source's `delete_insert` by `replace_key` does when two files of its folder hold one key value. + pub on_overlapping_files: OnOverlappingFiles, /// The columns, compared in order, whose higher values are a row's newer version (`newest_by`): a merge keeps the newest of the rows sharing a key and never overwrites a loaded row with an older one; empty, the last row read wins. pub newest_by: Vec, pub cursor: Option, diff --git a/crates/sqlload/src/project/constants.rs b/crates/sqlload/src/project/constants.rs index c09fd198e..b7733f691 100644 --- a/crates/sqlload/src/project/constants.rs +++ b/crates/sqlload/src/project/constants.rs @@ -32,6 +32,7 @@ pub const TABLE_SETTING_KEYS: &[&str] = &[ "on_duplicate_keys", "null_keys", "replace_key", + "on_overlapping_files", "newest_by", "naming", "compat", diff --git a/crates/sqlload/src/project/models.rs b/crates/sqlload/src/project/models.rs index 2869c819f..549cc5984 100644 --- a/crates/sqlload/src/project/models.rs +++ b/crates/sqlload/src/project/models.rs @@ -901,6 +901,9 @@ pub struct TableSettings { /// The columns whose values `delete_insert` replaces whole: each commit first deletes the rows holding a value of them that its rows hold. #[serde(default, deserialize_with = "deserialize_optional_keys")] pub replace_key: Option>, + /// What a files source's `delete_insert` by `replace_key` does when two files of its folder hold one key value: fail (default) or let the file last in `order` replace it. + #[serde(default)] + pub on_overlapping_files: Option, /// The column, or columns compared in order, whose higher value is the newer version of a row: of the rows sharing a `unique_key` a merge keeps the newest, and never overwrites a loaded row with an older one. #[serde(default, deserialize_with = "deserialize_optional_keys")] pub newest_by: Option>, @@ -955,6 +958,17 @@ pub enum OnDuplicateKeys { Fail, } +/// What a files source's `delete_insert` by `replace_key` does when two files of its folder hold one key value. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OnOverlappingFiles { + /// Fail the file's commit, naming both files: they may be parts of one export. + #[default] + Fail, + /// The file's commit replaces the value's rows, whichever file loaded them: the file last in `order` wins. + LastWins, +} + /// What a merge does with a row whose key the source no longer holds, found by `load --sync-deletes`, or a captured delete does to a merged row. #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] #[serde(rename_all = "snake_case")] diff --git a/crates/sqlload/tests/loader.rs b/crates/sqlload/tests/loader.rs index 094767b60..8b642b76c 100644 --- a/crates/sqlload/tests/loader.rs +++ b/crates/sqlload/tests/loader.rs @@ -85,6 +85,8 @@ mod test_odbc_memory; #[cfg(feature = "odbc")] #[path = "loader/test_odbc_to_duckdb.rs"] mod test_odbc_to_duckdb; +#[path = "loader/test_overlapping_files.rs"] +mod test_overlapping_files; #[path = "loader/test_parsed_columns.rs"] mod test_parsed_columns; #[path = "loader/test_plan_progress.rs"] diff --git a/crates/sqlload/tests/loader/merge/helpers.rs b/crates/sqlload/tests/loader/merge/helpers.rs index 8801a8781..7aedf4a63 100644 --- a/crates/sqlload/tests/loader/merge/helpers.rs +++ b/crates/sqlload/tests/loader/merge/helpers.rs @@ -16,8 +16,8 @@ use sqlload_testkit::postgres::constants::{PASSWORD, USER}; use crate::helpers::{described_project, load_selected, words}; use crate::test_types::{ - DedupeRule, DedupeTargetTestCase, FilesMergeTestCase, NullKeyTargetTestCase, - ReplaceKeyTargetTestCase, SyncDeletesTestCase, + DedupeRule, DedupeTargetTestCase, FilesMergeTestCase, MergeFile, MergeStep, + NullKeyTargetTestCase, ReplaceKeyTargetTestCase, SyncDeletesTestCase, }; /// `public.readings`: `s1` and `s2` repeat with their `seen_at` order unlike their `reading_id` order, `s2`'s last row has no `seen_at`, and two rows have no sensor (nor `seen_at`), a repeated NULL key. @@ -779,3 +779,20 @@ pub(crate) fn hard_tombstone_steps() -> Vec { .map(str::to_owned) .to_vec() } + +/// One load of a files merge case with `files` written first, no SQL before it and nothing removed, expecting `expected_error` (empty for none) and `expected_plan` in its plan. +pub(crate) const fn load_step( + files: &'static [MergeFile], + expected_error: &'static str, + expected_plan: &'static str, +) -> MergeStep { + MergeStep { + before: &[], + files, + removed: &[], + loads: true, + reload: false, + expected_error, + expected_plan, + } +} diff --git a/crates/sqlload/tests/loader/test_files_to_duckdb.rs b/crates/sqlload/tests/loader/test_files_to_duckdb.rs index d42c3dadc..c5eb6f06c 100644 --- a/crates/sqlload/tests/loader/test_files_to_duckdb.rs +++ b/crates/sqlload/tests/loader/test_files_to_duckdb.rs @@ -1571,7 +1571,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h removed: &[], loads: true, reload: false, - expected_error: "drop/part-0001.csv: it holds the replace_key (le_version) value (LE-FY27-P01), which part-0000.csv also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if one restates the other, remove the older of part-0000.csv and part-0001.csv from the folder by hand, and set after_load = { move_to = ... } (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values", + expected_error: "drop/part-0001.csv: it holds the replace_key (le_version) value (LE-FY27-P01), which part-0000.csv also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if newer files restate older ones, set on_overlapping_files = \"last_wins\", so the file last in order replaces the values, or remove the older of part-0000.csv and part-0001.csv from the folder by hand and set after_load = { move_to = ... } (or \"delete\"), so that each file leaves the folder once it loads", expected_plan: "", }, MergeStep { @@ -1580,7 +1580,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h removed: &[], loads: true, reload: false, - expected_error: "drop/part-0001.csv: the table holds rows of its replace_key (le_version) values from part-0000.csv, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by part-0001.csv would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if one restates the other, remove the older of part-0000.csv and part-0001.csv from the folder by hand, and set after_load = { move_to = ... } (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values", + expected_error: "drop/part-0001.csv: the table holds rows of its replace_key (le_version) values from part-0000.csv, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by part-0001.csv would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if newer files restate older ones, set on_overlapping_files = \"last_wins\", so the file last in order replaces the values, or remove the older of part-0000.csv and part-0001.csv from the folder by hand and set after_load = { move_to = ... } (or \"delete\"), so that each file leaves the folder once it loads", expected_plan: "", }, ], diff --git a/crates/sqlload/tests/loader/test_overlapping_files.rs b/crates/sqlload/tests/loader/test_overlapping_files.rs new file mode 100644 index 000000000..eecad4856 --- /dev/null +++ b/crates/sqlload/tests/loader/test_overlapping_files.rs @@ -0,0 +1,190 @@ +use sqlload_testkit::postgres::constants::PASSWORD; +use sqlload_testkit::postgres::main::connect::connect; +use sqlload_testkit::postgres::main::start_postgres::start_postgres; + +use crate::helpers::run_async; +use crate::merge_helpers::{load_step, run_merge_case}; +use crate::test_types::{FilesMergeTestCase, MergeFile, MergeStep}; + +/// Every row as `version:line:amount`, then how many file commits the run log records. +const CHECK: &str = "SELECT (SELECT string_agg(COALESCE(le_version, 'NULL') || ':' || line || ':' || CAST(amount AS VARCHAR), ',' ORDER BY le_version NULLS LAST, line) FROM raw.scores) || '|' || (SELECT CAST(count(*) AS VARCHAR) FROM raw._sqlload_runs WHERE table_name = 'scores' AND event = 'run')"; + +/// No table is created when plan refuses the settings. +const NO_TABLE: &str = + "SELECT CAST(count(*) AS VARCHAR) FROM information_schema.tables WHERE table_name = 'scores'"; + +const LAST_WINS: &str = "write_strategy = \"delete_insert\"\nreplace_key = [\"le_version\"]\non_overlapping_files = \"last_wins\"\n"; + +const LAST_WINS_BY_PATH: &str = "write_strategy = \"delete_insert\"\nreplace_key = [\"le_version\"]\non_overlapping_files = \"last_wins\"\norder = \"path\"\n"; + +const FAILS: &str = "write_strategy = \"delete_insert\"\nreplace_key = [\"le_version\"]\non_overlapping_files = \"fail\"\n"; + +/// P01 and P02, sent first. +const FIRST: MergeFile = MergeFile { + key: "first.csv", + contents: "le_version,line,amount\nP01,A,1\nP01,B,2\nP02,A,10\n", + modified_secs: 1_000, +}; + +/// P02 restated with other lines, and the new P03, sent later. +const SECOND: MergeFile = MergeFile { + key: "second.csv", + contents: "le_version,line,amount\nP02,X,7\nP02,Y,8\nP03,Z,9\n", + modified_secs: 2_000, +}; + +const LAST_WINS_PLAN: &str = "2 files, one commit each, oldest modified first (ties by path), each recorded in the run log; the last file holding a replace_key value replaces it (on_overlapping_files = \"last_wins\")"; + +const OUTSIDE: &str = "on_overlapping_files says which file wins when two files of a files source's folder hold one replace_key value"; + +const ONE_RUN: &[MergeStep] = &[load_step(&[FIRST, SECOND], "", LAST_WINS_PLAN)]; + +const TWO_RUNS: &[MergeStep] = &[load_step(&[FIRST], "", ""), load_step(&[SECOND], "", "")]; + +/// The later of the two dated files was modified first. +const BY_PATH: &[MergeStep] = &[load_step( + &[ + MergeFile { + key: "sales_2026-11-14.csv", + contents: "le_version,line,amount\nP02,X,7\n", + modified_secs: 1_000, + }, + MergeFile { + key: "sales_2026-11-07.csv", + contents: "le_version,line,amount\nP01,A,1\nP02,A,10\n", + modified_secs: 2_000, + }, + ], + "", + "in path order, each recorded in the run log; the last file holding a replace_key value replaces it", +)]; + +const SPLIT: &[MergeStep] = &[load_step( + &[FIRST, SECOND], + "drop/second.csv: it holds the replace_key (le_version) value (P02), which first.csv also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows first.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if newer files restate older ones, set on_overlapping_files = \"last_wins\", so the file last in order replaces the values, or remove the older of first.csv and second.csv from the folder by hand and set after_load = { move_to = ... } (or \"delete\"), so that each file leaves the folder once it loads", + "", +)]; + +const REFUSED: &[MergeStep] = &[load_step(&[FIRST], OUTSIDE, "")]; + +#[test] +fn given_files_holding_one_replace_key_value_when_loading_into_duckdb_then_on_overlapping_files_says_which_wins() +-> Result<(), String> { + run_async(|| async move { + let test_cases = [ + FilesMergeTestCase { + description: "in one run, the later file replaces the value both hold, and the earlier file's other values stay", + settings: LAST_WINS, + steps: ONE_RUN, + check_query: CHECK, + expected_result: "P01:A:1,P01:B:2,P02:X:7,P02:Y:8,P03:Z:9|2", + }, + FilesMergeTestCase { + description: "across runs, a file sent later replaces a value of a file still in the folder", + settings: LAST_WINS, + steps: TWO_RUNS, + check_query: CHECK, + expected_result: "P01:A:1,P01:B:2,P02:X:7,P02:Y:8,P03:Z:9|2", + }, + FilesMergeTestCase { + description: "with order = \"path\", the file last by path wins, though it was modified first", + settings: LAST_WINS_BY_PATH, + steps: BY_PATH, + check_query: CHECK, + expected_result: "P01:A:1,P02:X:7|2", + }, + FilesMergeTestCase { + description: "with on_overlapping_files = \"fail\", a value split across part files still fails on the later file, naming both and the setting", + settings: FAILS, + steps: SPLIT, + check_query: CHECK, + expected_result: "P01:A:1,P01:B:2,P02:A:10|1", + }, + FilesMergeTestCase { + description: "plan refuses it on a merge, where the last file already wins each key", + settings: "write_strategy = \"merge\"\nunique_key = [\"le_version\", \"line\"]\non_overlapping_files = \"last_wins\"\n", + steps: REFUSED, + check_query: NO_TABLE, + expected_result: "0", + }, + FilesMergeTestCase { + description: "plan refuses it on an append", + settings: "write_strategy = \"append\"\non_overlapping_files = \"fail\"\n", + steps: REFUSED, + check_query: NO_TABLE, + expected_result: "0", + }, + FilesMergeTestCase { + description: "plan refuses it on a delete_insert by cursor", + settings: "write_strategy = \"delete_insert\"\ncursor_column = \"le_version\"\non_overlapping_files = \"last_wins\"\n", + steps: REFUSED, + check_query: NO_TABLE, + expected_result: "0", + }, + ]; + for test_case in &test_cases { + let result = run_merge_case( + ("kind = \"duckdb\"\ndatabase = \":memory:\"", "raw"), + test_case, + ) + .await?; + assert_eq!( + result.as_deref(), + Some(test_case.expected_result), + "{}", + test_case.description + ); + } + Ok(()) + }) +} + +#[test] +fn given_files_holding_one_replace_key_value_when_loading_into_postgres_then_on_overlapping_files_says_which_wins() +-> Result<(), String> { + run_async(|| async move { + let postgres = start_postgres().await?; + let client = connect(&postgres).await?; + let target = format!( + "kind = \"postgres\"\nhost = \"127.0.0.1\"\nport = {}\nuser = \"sqlload\"\npassword = \"{PASSWORD}\"\ndatabase = \"app\"", + postgres.port + ); + let test_cases = [ + FilesMergeTestCase { + description: "in one run, the later file replaces the value both hold, and the earlier file's other values stay", + settings: LAST_WINS, + steps: ONE_RUN, + check_query: CHECK, + expected_result: "P01:A:1,P01:B:2,P02:X:7,P02:Y:8,P03:Z:9|2", + }, + FilesMergeTestCase { + description: "across runs, a file sent later replaces a value of a file still in the folder", + settings: LAST_WINS, + steps: TWO_RUNS, + check_query: CHECK, + expected_result: "P01:A:1,P01:B:2,P02:X:7,P02:Y:8,P03:Z:9|2", + }, + FilesMergeTestCase { + description: "with on_overlapping_files = \"fail\", a value split across part files still fails on the later file", + settings: FAILS, + steps: SPLIT, + check_query: CHECK, + expected_result: "P01:A:1,P01:B:2,P02:A:10|1", + }, + ]; + for test_case in &test_cases { + client + .batch_execute("DROP SCHEMA IF EXISTS raw CASCADE") + .await + .map_err(|error| error.to_string())?; + let result = run_merge_case((&target, "raw"), test_case).await?; + assert_eq!( + result.as_deref(), + Some(test_case.expected_result), + "{}", + test_case.description + ); + } + Ok(()) + }) +} diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx index 4bbd4ba6d..433349e72 100644 --- a/website/src/content/docs/docs/configuration.mdx +++ b/website/src/content/docs/docs/configuration.mdx @@ -281,6 +281,7 @@ Set on a source for all its tables, or in a `[sources.NAME.tables."schema.table" | `replace_key` | none | A column or list of columns whose values `delete_insert` replaces whole: each commit deletes the rows holding a value its rows hold, then inserts them. Not unique. Only for `delete_insert`, and not with a `cursor_column`: each load reads the whole source, or a files source's new and changed files. See [Replacing rows by a key](/docs/concepts/write-strategies/#replacing-rows-by-a-key). | | `deletes` | none | `soft` or `hard`: what `load --sync-deletes` does with the rows of a `merge` table whose key the source no longer holds (Postgres, ODBC, Snowflake and BigQuery sources, and Databricks sources, into DuckDB and Postgres targets, Snowflake, MotherDuck, and BigQuery and Databricks). `soft` sets `_sqlload_deleted_at`; `hard` removes the row. A `postgres_cdc` source applies captured deletes this way, `soft` by default. See [Rows deleted at the source](/docs/concepts/write-strategies/#rows-deleted-at-the-source). | | `deletes_from` | none | Tables only: `{ table = "audit.deleted_items", cursor_column = "deleted_at" }`, a source table recording deleted keys. Every load loads its new rows into `__sqlload_deletes` and marks the rows they name, as `deletes` says. `lookback = "1h"` reads that far back again on every load, for tombstones written by long transactions. See [Deletes from a tombstone table](/docs/concepts/write-strategies/#deletes-from-a-tombstone-table). | +| `on_overlapping_files` | `fail` | What a files source's `delete_insert` by `replace_key` does when two files of its folder hold one key value: `fail` refuses the later file's commit, naming both; `last_wins` lets the file last in `order` replace the value. See [Replacing rows by a key](/docs/sources/files/#when-newer-files-restate-older-ones). | | `newest_by` | none | A column, or a list compared in order, whose higher value is a row's newer version: `merge` keeps the newest of a commit's rows sharing a key, and never overwrites the table's row with an older one. Without it, the row read last wins. See [Repeated keys](/docs/concepts/write-strategies/#repeated-keys). | | `cursor_column` | none | The column runs continue from. A list such as `["updated_at", "created_at"]` reads the first non-NULL column of each row. See [Cursors](/docs/concepts/cursors/). | | `cursor_start` | none | Where the first run starts. | diff --git a/website/src/content/docs/docs/sources/files.mdx b/website/src/content/docs/docs/sources/files.mdx index 19b356f1d..eb0c8d486 100644 --- a/website/src/content/docs/docs/sources/files.mdx +++ b/website/src/content/docs/docs/sources/files.mdx @@ -523,11 +523,11 @@ must hold all of a value's rows. Before each commit, SQLLoad finds the files the the commit's values came from, by `_sqlload_source_file`. When one of them is still in the folder, or loaded earlier in the same run, the table fails, naming both files: two files of the folder holding one value, such as the parts of an export split into part files, would leave only the -later one's rows. The error names the fixes: put all of a value's rows in one file, or, when one -file restates another, remove the older one from the folder by hand. A failed load moves nothing, -so `after_load` can't clear the failure, but `after_load = { move_to = ... }` (or `"delete"`, which -rules out `--reload`) keeps it from happening again: each file leaves the folder once it loads, and -a file sent later replaces its values. A file sent again under the same name, or one whose earlier +later one's rows. The error names the fixes: put all of a value's rows in one file; or, when newer +files restate older ones, set `on_overlapping_files = "last_wins"` (below), or remove the older file +from the folder by hand. A failed load moves nothing, so `after_load` can't clear the failure, but +`after_load = { move_to = ... }` (or `"delete"`, which rules out `--reload`) keeps it from happening +again: each file leaves the folder once it loads, and a file sent later replaces its values. A file sent again under the same name, or one whose earlier file has left the folder (deleted, or moved by `after_load`), replaces the value. A file's rows always commit together. `plan` refuses a table that excludes `_sqlload_source_file`. @@ -552,6 +552,39 @@ $ sqlload plan -v └── batch 1 file, one commit each, oldest modified first (ties by path), each recorded in the run log ``` +#### When newer files restate older ones + +A feed whose deliveries overlap, each restating the end of the period before, holds one value in +two files on purpose. `on_overlapping_files = "last_wins"` (default `"fail"`) lets the file last in +`order` replace the values it holds, even while an earlier file holding them is still in the +folder. Each file still loads in its own commit, so the earlier file's other values stay: + +```text +sales/ + sales_2026-11-07.csv business_date Nov 1 .. Nov 7 + sales_2026-11-14.csv business_date Nov 5 .. Nov 14 <- Nov 5 to 7 come from this file +``` + +```toml +[sources.sales] +kind = "files" +connection = "lake" +path = "sales/" +pattern = "sales_*.csv" +format = "csv" +description = "Daily sales, each delivery restating the days before it." +write_strategy = "delete_insert" +replace_key = ["business_date"] +order = "path" +on_overlapping_files = "last_wins" +``` + +Set `order = "path"` when the names sort by date, as here; by default the file modified last wins. +Leave it at `"fail"` when the files may be parts of one export, as `"last_wins"` would keep only +the last part's rows of a value split across them. `plan -v` says so on the table's `batch` line. +It applies only to a files source's `delete_insert` with a `replace_key`, and `plan` refuses it +elsewhere: a [merge](#merging-files) already lets the last file win each key. + It picks files as [merging files](#merging-files) does: each commit records its file and version in `_sqlload_runs`, and loads into DuckDB and Postgres targets so far. `--reload`, `state = "logged"` and a `cursor_column` are refused. `after_load` applies to it, as to `merge`. From 2c70429d8ee19daeefd0b40bd92e2631d5da5c90 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:43:14 +0000 Subject: [PATCH 2/5] wip: last_wins never lets an earlier file in order take back a later file's values; source-level setting skips other strategies --- crates/sqlload/src/loader/_helpers/run_log.rs | 35 ++++++++++++----- crates/sqlload/src/loader/_helpers/specs.rs | 3 ++ .../tests/loader/test_overlapping_files.rs | 39 +++++++++++++++++++ .../src/content/docs/docs/sources/files.mdx | 12 +++--- 4 files changed, 75 insertions(+), 14 deletions(-) diff --git a/crates/sqlload/src/loader/_helpers/run_log.rs b/crates/sqlload/src/loader/_helpers/run_log.rs index eafd09657..c8801e856 100644 --- a/crates/sqlload/src/loader/_helpers/run_log.rs +++ b/crates/sqlload/src/loader/_helpers/run_log.rs @@ -930,7 +930,7 @@ pub(crate) fn file_column_problem( None } -/// Why the commit of `file` (`name|version`), a files source's `delete_insert` by `replace_key` into `table`, cannot replace the rows of its key values: some of them come from `sources` (other files the table's rows of those values came from, by name) that are still listed in the source's folder, so two files of the folder hold one value and which is current can't be told. A file that has left the folder (deleted, or moved by `after_load`) is replaced, and under `on_overlapping_files = "last_wins"` so is one still listed. +/// Why the commit of `file` (`name|version`), a files source's `delete_insert` by `replace_key` into `table`, cannot replace the rows of its key values: some of them come from `sources` (other files the table's rows of those values came from, by name) that are still listed in the source's folder, so two files of the folder hold one value and which is current can't be told. A file that has left the folder (deleted, or moved by `after_load`) is replaced; under `on_overlapping_files = "last_wins"` so is one still listed that comes before `file` in the source's `order`, but never one after it, whose values a file sent late or rewritten would otherwise take back. pub(crate) fn listed_sources( table: &TablePlan, file: &str, @@ -939,19 +939,30 @@ pub(crate) fn listed_sources( let Some(source) = table.spec.files.as_ref() else { return Ok(()); }; - if table.spec.on_overlapping_files == OnOverlappingFiles::LastWins { - return Ok(()); - } let location = source.location.to_string(); - let listed: Vec = source + let order = |data: &DataFile| match table.spec.files_order { + FilesOrder::Modified => (data.file.modified_ms, data.file.relative_key.clone()), + FilesOrder::Path => (0, data.file.relative_key.clone()), + }; + let listed: Vec<(String, (i64, String))> = source .table .files .iter() - .map(|data| format!("{location}{}", data.file.relative_key)) + .map(|data| (format!("{location}{}", data.file.relative_key), order(data))) .collect(); + let this_name = merged_name(file); + let this_order = listed + .iter() + .find(|(name, _)| name == this_name) + .map(|(_, at)| at.clone()); + let last_wins = table.spec.on_overlapping_files == OnOverlappingFiles::LastWins; let held: Vec<&str> = sources .iter() - .filter(|name| listed.contains(name)) + .filter(|name| { + listed.iter().any(|(listed, at)| { + listed == *name && (!last_wins || this_order.as_ref().is_none_or(|this| at > this)) + }) + }) .map(|name| name.strip_prefix(location.as_str()).unwrap_or(name)) .collect(); let Some(first) = held.first() else { @@ -961,9 +972,15 @@ pub(crate) fn listed_sources( 1 => String::new(), count => format!(" (and {} more)", count - 1), }; - let this = merged_name(file); - let this = this.strip_prefix(location.as_str()).unwrap_or(this); + let this = this_name + .strip_prefix(location.as_str()) + .unwrap_or(this_name); let columns = table.spec.replace_key.join(", "); + if last_wins { + return Err(format!( + "the table holds rows of its replace_key ({columns}) values from {first}{more}, which is still in the folder and comes after {this} in order; with on_overlapping_files = \"last_wins\" the file last in order keeps a value, so {this} would take back values a later file replaced. Fix: if {this} is a correction, give it a name (or, by modified time, a time) that sorts after {first}, or remove {first} from the folder" + )); + } Err(format!( "the table holds rows of its replace_key ({columns}) values from {first}{more}, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by {this} would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if newer files restate older ones, set on_overlapping_files = \"last_wins\", so the file last in order replaces the values, or remove the older of {first} and {this} from the folder by hand and set after_load = {{ move_to = ... }} (or \"delete\"), so that each file leaves the folder once it loads" )) diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs index d6f7dd5b7..011542301 100644 --- a/crates/sqlload/src/loader/_helpers/specs.rs +++ b/crates/sqlload/src/loader/_helpers/specs.rs @@ -326,6 +326,9 @@ pub(crate) fn build_spec( &settings.on_overlapping_files, &defaults.on_overlapping_files, ) + .filter(|(_, origin)| { + *origin == SettingOrigin::Table || keyed || strategy_origin != SettingOrigin::Table + }) .map(|(value, _)| value); if let Some(problem) = on_overlapping_files.and_then(|_| { overlapping_files_problem(context.source.kind == SourceKind::Files, strategy, keyed) diff --git a/crates/sqlload/tests/loader/test_overlapping_files.rs b/crates/sqlload/tests/loader/test_overlapping_files.rs index eecad4856..be046c061 100644 --- a/crates/sqlload/tests/loader/test_overlapping_files.rs +++ b/crates/sqlload/tests/loader/test_overlapping_files.rs @@ -67,6 +67,31 @@ const SPLIT: &[MergeStep] = &[load_step( const REFUSED: &[MergeStep] = &[load_step(&[FIRST], OUTSIDE, "")]; +/// The newer file loads first, and the older one, by path, arrives in a later run. +const SENT_LATE: &[MergeStep] = &[ + load_step( + &[MergeFile { + key: "sales_2026-11-14.csv", + contents: "le_version,line,amount\nP02,X,7\n", + modified_secs: 1_000, + }], + "", + "", + ), + load_step( + &[MergeFile { + key: "sales_2026-11-07.csv", + contents: "le_version,line,amount\nP01,A,1\nP02,A,10\n", + modified_secs: 2_000, + }], + "drop/sales_2026-11-07.csv: the table holds rows of its replace_key (le_version) values from sales_2026-11-14.csv, which is still in the folder and comes after sales_2026-11-07.csv in order; with on_overlapping_files = \"last_wins\" the file last in order keeps a value, so sales_2026-11-07.csv would take back values a later file replaced", + "", + ), +]; + +/// One file of P01 and P02, merged by key. +const MERGED: &[MergeStep] = &[load_step(&[FIRST], "", "")]; + #[test] fn given_files_holding_one_replace_key_value_when_loading_into_duckdb_then_on_overlapping_files_says_which_wins() -> Result<(), String> { @@ -100,6 +125,20 @@ fn given_files_holding_one_replace_key_value_when_loading_into_duckdb_then_on_ov check_query: CHECK, expected_result: "P01:A:1,P01:B:2,P02:A:10|1", }, + FilesMergeTestCase { + description: "with order = \"path\", an older file sent after a newer one fails instead of taking back the newer file's value, which stays", + settings: LAST_WINS_BY_PATH, + steps: SENT_LATE, + check_query: CHECK, + expected_result: "P02:X:7|1", + }, + FilesMergeTestCase { + description: "set on the source, it leaves alone a table of the source that merges", + settings: "write_strategy = \"delete_insert\"\nreplace_key = [\"le_version\"]\non_overlapping_files = \"last_wins\"\n[sources.scores.tables.scores]\nwrite_strategy = \"merge\"\nunique_key = [\"le_version\", \"line\"]\n", + steps: MERGED, + check_query: CHECK, + expected_result: "P01:A:1,P01:B:2,P02:A:10|1", + }, FilesMergeTestCase { description: "plan refuses it on a merge, where the last file already wins each key", settings: "write_strategy = \"merge\"\nunique_key = [\"le_version\", \"line\"]\non_overlapping_files = \"last_wins\"\n", diff --git a/website/src/content/docs/docs/sources/files.mdx b/website/src/content/docs/docs/sources/files.mdx index eb0c8d486..fc7233e16 100644 --- a/website/src/content/docs/docs/sources/files.mdx +++ b/website/src/content/docs/docs/sources/files.mdx @@ -552,12 +552,18 @@ $ sqlload plan -v └── batch 1 file, one commit each, oldest modified first (ties by path), each recorded in the run log ``` +It picks files as [merging files](#merging-files) does: each commit records its file and version in +`_sqlload_runs`, and loads into DuckDB and Postgres targets so far. `--reload`, `state = "logged"` +and a `cursor_column` are refused. `after_load` applies to it, as to `merge`. + #### When newer files restate older ones A feed whose deliveries overlap, each restating the end of the period before, holds one value in two files on purpose. `on_overlapping_files = "last_wins"` (default `"fail"`) lets the file last in `order` replace the values it holds, even while an earlier file holding them is still in the -folder. Each file still loads in its own commit, so the earlier file's other values stay: +folder. Each file still loads in its own commit, so the earlier file's other values stay. A file +that comes before one already loaded, such as an older file rewritten or sent late, still fails +rather than take back the later file's values: ```text sales/ @@ -585,10 +591,6 @@ the last part's rows of a value split across them. `plan -v` says so on the tabl It applies only to a files source's `delete_insert` with a `replace_key`, and `plan` refuses it elsewhere: a [merge](#merging-files) already lets the last file win each key. -It picks files as [merging files](#merging-files) does: each commit records its file and version in -`_sqlload_runs`, and loads into DuckDB and Postgres targets so far. `--reload`, `state = "logged"` -and a `cursor_column` are refused. `after_load` applies to it, as to `merge`. - ### Merging files `merge` upserts each new file on its `unique_key`. A load merges From 6bcc0a436dcc046f94c8e9dc14dd148fd244c596 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:44:26 +0000 Subject: [PATCH 3/5] wip: listed_sources as a plain loop --- crates/sqlload/src/loader/_helpers/run_log.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/sqlload/src/loader/_helpers/run_log.rs b/crates/sqlload/src/loader/_helpers/run_log.rs index c8801e856..76dfd5ce6 100644 --- a/crates/sqlload/src/loader/_helpers/run_log.rs +++ b/crates/sqlload/src/loader/_helpers/run_log.rs @@ -956,15 +956,15 @@ pub(crate) fn listed_sources( .find(|(name, _)| name == this_name) .map(|(_, at)| at.clone()); let last_wins = table.spec.on_overlapping_files == OnOverlappingFiles::LastWins; - let held: Vec<&str> = sources - .iter() - .filter(|name| { - listed.iter().any(|(listed, at)| { - listed == *name && (!last_wins || this_order.as_ref().is_none_or(|this| at > this)) - }) - }) - .map(|name| name.strip_prefix(location.as_str()).unwrap_or(name)) - .collect(); + let mut held: Vec<&str> = Vec::new(); + for name in sources { + let Some((_, at)) = listed.iter().find(|(listed, _)| listed == name) else { + continue; + }; + if !last_wins || this_order.as_ref().is_none_or(|this| at > this) { + held.push(name.strip_prefix(location.as_str()).unwrap_or(name)); + } + } let Some(first) = held.first() else { return Ok(()); }; From dbc50c8e942838d0194371dbe0f771184354bebd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:08:37 +0000 Subject: [PATCH 4/5] wip: last_wins: a later file this run reads again doesn't block an earlier one; docs scope the rewritten case to order = path --- crates/sqlload/src/loader/_helpers/run_log.rs | 11 ++++++-- .../tests/loader/test_overlapping_files.rs | 28 +++++++++++++++++++ .../src/content/docs/docs/sources/files.mdx | 8 ++++-- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/crates/sqlload/src/loader/_helpers/run_log.rs b/crates/sqlload/src/loader/_helpers/run_log.rs index 76dfd5ce6..89a18461c 100644 --- a/crates/sqlload/src/loader/_helpers/run_log.rs +++ b/crates/sqlload/src/loader/_helpers/run_log.rs @@ -930,7 +930,7 @@ pub(crate) fn file_column_problem( None } -/// Why the commit of `file` (`name|version`), a files source's `delete_insert` by `replace_key` into `table`, cannot replace the rows of its key values: some of them come from `sources` (other files the table's rows of those values came from, by name) that are still listed in the source's folder, so two files of the folder hold one value and which is current can't be told. A file that has left the folder (deleted, or moved by `after_load`) is replaced; under `on_overlapping_files = "last_wins"` so is one still listed that comes before `file` in the source's `order`, but never one after it, whose values a file sent late or rewritten would otherwise take back. +/// Why the commit of `file` (`name|version`), a files source's `delete_insert` by `replace_key` into `table`, cannot replace the rows of its key values: some of them come from `sources` (other files the table's rows of those values came from, by name) that are still listed in the source's folder, so two files of the folder hold one value and which is current can't be told. A file that has left the folder (deleted, or moved by `after_load`) is replaced; under `on_overlapping_files = "last_wins"` so is one still listed that comes before `file` in the source's `order`, or that this run reads again (it loads in order, replacing the values back), but never another after it, whose values a file sent late or rewritten would otherwise take back. pub(crate) fn listed_sources( table: &TablePlan, file: &str, @@ -950,6 +950,12 @@ pub(crate) fn listed_sources( .iter() .map(|data| (format!("{location}{}", data.file.relative_key), order(data))) .collect(); + let read: Vec = table + .files + .iter() + .flat_map(|read| &read.files) + .map(|data| format!("{location}{}", data.file.relative_key)) + .collect(); let this_name = merged_name(file); let this_order = listed .iter() @@ -961,7 +967,8 @@ pub(crate) fn listed_sources( let Some((_, at)) = listed.iter().find(|(listed, _)| listed == name) else { continue; }; - if !last_wins || this_order.as_ref().is_none_or(|this| at > this) { + if !last_wins || (!read.contains(name) && this_order.as_ref().is_none_or(|this| at > this)) + { held.push(name.strip_prefix(location.as_str()).unwrap_or(name)); } } diff --git a/crates/sqlload/tests/loader/test_overlapping_files.rs b/crates/sqlload/tests/loader/test_overlapping_files.rs index be046c061..f0899f507 100644 --- a/crates/sqlload/tests/loader/test_overlapping_files.rs +++ b/crates/sqlload/tests/loader/test_overlapping_files.rs @@ -89,6 +89,27 @@ const SENT_LATE: &[MergeStep] = &[ ), ]; +/// Both files load, then both are rewritten in place before the next run, which reads them again in order. +const REWRITTEN: &[MergeStep] = &[ + load_step(&[FIRST, SECOND], "", ""), + load_step( + &[ + MergeFile { + key: "first.csv", + contents: "le_version,line,amount\nP01,A,5\nP02,A,11\n", + modified_secs: 3_000, + }, + MergeFile { + key: "second.csv", + contents: "le_version,line,amount\nP02,X,70\nP03,Z,9\n", + modified_secs: 4_000, + }, + ], + "", + "", + ), +]; + /// One file of P01 and P02, merged by key. const MERGED: &[MergeStep] = &[load_step(&[FIRST], "", "")]; @@ -132,6 +153,13 @@ fn given_files_holding_one_replace_key_value_when_loading_into_duckdb_then_on_ov check_query: CHECK, expected_result: "P02:X:7|1", }, + FilesMergeTestCase { + description: "with order = \"path\", both files rewritten in place reload in order, the later one, read again in the same run, not blocking the earlier", + settings: LAST_WINS_BY_PATH, + steps: REWRITTEN, + check_query: CHECK, + expected_result: "P01:A:5,P02:X:70,P03:Z:9|4", + }, FilesMergeTestCase { description: "set on the source, it leaves alone a table of the source that merges", settings: "write_strategy = \"delete_insert\"\nreplace_key = [\"le_version\"]\non_overlapping_files = \"last_wins\"\n[sources.scores.tables.scores]\nwrite_strategy = \"merge\"\nunique_key = [\"le_version\", \"line\"]\n", diff --git a/website/src/content/docs/docs/sources/files.mdx b/website/src/content/docs/docs/sources/files.mdx index fc7233e16..78bc68f51 100644 --- a/website/src/content/docs/docs/sources/files.mdx +++ b/website/src/content/docs/docs/sources/files.mdx @@ -561,9 +561,7 @@ and a `cursor_column` are refused. `after_load` applies to it, as to `merge`. A feed whose deliveries overlap, each restating the end of the period before, holds one value in two files on purpose. `on_overlapping_files = "last_wins"` (default `"fail"`) lets the file last in `order` replace the values it holds, even while an earlier file holding them is still in the -folder. Each file still loads in its own commit, so the earlier file's other values stay. A file -that comes before one already loaded, such as an older file rewritten or sent late, still fails -rather than take back the later file's values: +folder. Each file still loads in its own commit, so the earlier file's other values stay: ```text sales/ @@ -586,6 +584,10 @@ on_overlapping_files = "last_wins" ``` Set `order = "path"` when the names sort by date, as here; by default the file modified last wins. +A file that comes before one already loaded, such as an older file sent late, or with +`order = "path"` one rewritten while the later file isn't, still fails rather than take back the +later file's values. By modified time a rewritten file sorts last and replaces them, and files +rewritten together load again in order. Leave it at `"fail"` when the files may be parts of one export, as `"last_wins"` would keep only the last part's rows of a value split across them. `plan -v` says so on the table's `batch` line. It applies only to a files source's `delete_insert` with a `replace_key`, and `plan` refuses it From 9884152f42a0a132d3019c48942c7562640e52f6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:37:27 +0000 Subject: [PATCH 5/5] refactor: describe recorded file commits in run_log so render.rs and specs.rs stay under 2,000 lines --- crates/sqlload/src/loader/_helpers/render.rs | 17 +++------------ crates/sqlload/src/loader/_helpers/run_log.rs | 17 +++++++++++---- crates/sqlload/src/loader/_helpers/specs.rs | 21 ++++++++----------- 3 files changed, 25 insertions(+), 30 deletions(-) diff --git a/crates/sqlload/src/loader/_helpers/render.rs b/crates/sqlload/src/loader/_helpers/render.rs index f862ce350..cad82faad 100644 --- a/crates/sqlload/src/loader/_helpers/render.rs +++ b/crates/sqlload/src/loader/_helpers/render.rs @@ -22,12 +22,12 @@ use crate::batch_plan::main::cursor_rules::cursor_rules; use crate::batch_plan::main::read_at_once::read_at_once; use crate::column_parse::main::parse_lines::parse_lines; use crate::column_types::models::ColumnType; -use crate::files_source::models::{FilesOrder, SourceFormat}; +use crate::files_source::models::SourceFormat; use crate::load_naming::main::is_load_column::is_load_column; use crate::loader::_helpers::publish::{published_once, publishes_once, records_files, send_lines}; use crate::loader::_helpers::python::{list_choices, unnested_table}; use crate::loader::_helpers::resolve::display_name; -use crate::loader::_helpers::run_log::last_wins_note; +use crate::loader::_helpers::run_log::recorded_commits; use crate::loader::_helpers::sources::{readers_note, unarchived}; use crate::loader::_helpers::table_plan::{ archived_reason, is_direct, pulls_unloaded, selects_rows, @@ -823,18 +823,7 @@ pub(crate) fn batch_line(table: &TablePlan) -> String { format!("{windows} of {step}, {commits}{}{end}", read_at_once(table)) } (CommitRule::FileGroups(groups), _) if records_files(table) => { - format!( - "{}, one commit each, {}, each recorded in the run log{}", - match groups.len() { - 1 => "1 file".to_owned(), - count => format!("{count} files"), - }, - match spec.files_order { - FilesOrder::Modified => "oldest modified first (ties by path)", - FilesOrder::Path => "in path order", - }, - last_wins_note(spec.on_overlapping_files) - ) + recorded_commits(groups.len(), spec) } (CommitRule::FileGroups(groups), _) => { format!( diff --git a/crates/sqlload/src/loader/_helpers/run_log.rs b/crates/sqlload/src/loader/_helpers/run_log.rs index 89a18461c..d612eeb31 100644 --- a/crates/sqlload/src/loader/_helpers/run_log.rs +++ b/crates/sqlload/src/loader/_helpers/run_log.rs @@ -1015,14 +1015,23 @@ pub(crate) fn overlapping_files_problem( } } -/// What `plan` adds to a files table's commits under `on_overlapping_files = "last_wins"`. -pub(crate) fn last_wins_note(on_overlapping_files: OnOverlappingFiles) -> &'static str { - match on_overlapping_files { +/// How `plan` describes the commits of a files table whose `groups` files are each recorded in the run log: their order, and what `on_overlapping_files = "last_wins"` adds. +pub(crate) fn recorded_commits(groups: usize, spec: &TableSpec) -> String { + let files = match groups { + 1 => "1 file".to_owned(), + count => format!("{count} files"), + }; + let order = match spec.files_order { + FilesOrder::Modified => "oldest modified first (ties by path)", + FilesOrder::Path => "in path order", + }; + let last_wins = match spec.on_overlapping_files { OnOverlappingFiles::LastWins => { "; the last file holding a replace_key value replaces it (on_overlapping_files = \"last_wins\")" } OnOverlappingFiles::Fail => "", - } + }; + format!("{files}, one commit each, {order}, each recorded in the run log{last_wins}") } /// The files a merge already merged into `request`'s table at the version listed now, which it skips (one rewritten in place since is merged again, as the newest), and under `--reload` none, with every file the table's merges recorded instead (oldest first), whose order the reload keeps; or why a files source cannot load into it: only into a DuckDB or Postgres table, never with `state = "logged"`, nor `--reload` with `unnest` or of a `delete_insert` (one commit would lose the files' order); none while the table does not exist. diff --git a/crates/sqlload/src/loader/_helpers/specs.rs b/crates/sqlload/src/loader/_helpers/specs.rs index ffb867d77..1db158ec1 100644 --- a/crates/sqlload/src/loader/_helpers/specs.rs +++ b/crates/sqlload/src/loader/_helpers/specs.rs @@ -322,19 +322,16 @@ pub(crate) fn build_spec( { problems.push(format!("order = \"path\" orders the files a merge or a delete_insert by replace_key applies one at a time; {strategy} loads them without an order, so remove it")); } - let on_overlapping_files = layered( + let own_strategy = strategy_origin == SettingOrigin::Table && !keyed; + let overlapping = layered( &settings.on_overlapping_files, &defaults.on_overlapping_files, - ) - .filter(|(_, origin)| { - *origin == SettingOrigin::Table || keyed || strategy_origin != SettingOrigin::Table - }) - .map(|(value, _)| value); - if let Some(problem) = on_overlapping_files.and_then(|_| { - overlapping_files_problem(context.source.kind == SourceKind::Files, strategy, keyed) - }) { - problems.push(problem); - } + ); + let overlapping = overlapping + .filter(|(_, origin)| !own_strategy || *origin == SettingOrigin::Table) + .map(|(value, _)| value); + let files = context.source.kind == SourceKind::Files; + problems.extend(overlapping.and_then(|_| overlapping_files_problem(files, strategy, keyed))); let matched_key = match keyed { true => &replace_key, false => &unique_key, @@ -459,7 +456,7 @@ pub(crate) fn build_spec( null_keys, not_null_keys, replace_key, - on_overlapping_files: on_overlapping_files.unwrap_or_default(), + on_overlapping_files: overlapping.unwrap_or_default(), newest_by, cursor, batch,