From e4769d0adf4bab4b64a5880da5db040fa7a21bb0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:41:29 +0000 Subject: [PATCH 1/2] fix(files): name the fixes when two files of a folder hold one replace_key value The error now says what to do in each case: put a value's rows in one file when the files are parts of one export, or, when a newer file restates an older one, take the older file out of the folder by hand or with after_load. --- crates/sqlload/src/loader/_helpers/run_log.rs | 10 +++++----- crates/sqlload/tests/loader/test_files_to_duckdb.rs | 4 ++-- website/src/content/docs/docs/sources/files.mdx | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/crates/sqlload/src/loader/_helpers/run_log.rs b/crates/sqlload/src/loader/_helpers/run_log.rs index 75d1709fa..f460e07cd 100644 --- a/crates/sqlload/src/loader/_helpers/run_log.rs +++ b/crates/sqlload/src/loader/_helpers/run_log.rs @@ -856,18 +856,18 @@ pub(crate) fn with_file_keys( } } let columns = table.spec.replace_key.join(", "); + let this = merged_name(file); + let this = this.strip_prefix(location.as_str()).unwrap_or(this); if let Some((value, first)) = split.first() { let more = match split.len() { 1 => String::new(), count => format!(" (and {} more)", count - 1), }; return Err(format!( - "it holds the replace_key ({columns}) value ({value}){more}, which {first} also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: put all the rows of each ({columns}) value in one file, or remove whichever of the two files is outdated from the folder" + "it holds the replace_key ({columns}) value ({value}){more}, which {first} also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if {this} restates {first}, remove {first} from the folder, by hand or with after_load = \"delete\" or after_load = {{ move_to = ... }}, which take each file out of the folder once it loads" )); } - let name = merged_name(file); - let name = name.strip_prefix(location.as_str()).unwrap_or(name); - earlier.extend(values.iter().map(|value| (value.clone(), name.to_owned()))); + earlier.extend(values.iter().map(|value| (value.clone(), this.to_owned()))); Ok(earlier) } @@ -960,7 +960,7 @@ pub(crate) fn listed_sources( let this = this.strip_prefix(location.as_str()).unwrap_or(this); let columns = table.spec.replace_key.join(", "); Err(format!( - "the table holds rows of its replace_key ({columns}) values from {first}{more}, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by {this} would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: put all the rows of each ({columns}) value in one file, or remove the outdated file, {first} or {this}, from the folder" + "the table holds rows of its replace_key ({columns}) values from {first}{more}, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by {this} would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if {this} restates {first}, remove {first} from the folder, by hand or with after_load = \"delete\" or after_load = {{ move_to = ... }}, which take each file out of the folder once it loads" )) } diff --git a/crates/sqlload/tests/loader/test_files_to_duckdb.rs b/crates/sqlload/tests/loader/test_files_to_duckdb.rs index bb4bdba52..6986ad6c6 100644 --- a/crates/sqlload/tests/loader/test_files_to_duckdb.rs +++ b/crates/sqlload/tests/loader/test_files_to_duckdb.rs @@ -1571,7 +1571,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h removed: &[], loads: true, reload: false, - expected_error: "drop/part-0001.csv: it holds the replace_key (le_version) value (LE-FY27-P01), which part-0000.csv also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: put all the rows of each (le_version) value in one file, or remove whichever of the two files is outdated from the folder", + expected_error: "drop/part-0001.csv: it holds the replace_key (le_version) value (LE-FY27-P01), which part-0000.csv also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if part-0001.csv restates part-0000.csv, remove part-0000.csv from the folder, by hand or with after_load = \"delete\" or after_load = { move_to = ... }, which take each file out of the folder once it loads", expected_plan: "", }, MergeStep { @@ -1580,7 +1580,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h removed: &[], loads: true, reload: false, - expected_error: "drop/part-0001.csv: the table holds rows of its replace_key (le_version) values from part-0000.csv, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by part-0001.csv would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: put all the rows of each (le_version) value in one file, or remove the outdated file, part-0000.csv or part-0001.csv, from the folder", + expected_error: "drop/part-0001.csv: the table holds rows of its replace_key (le_version) values from part-0000.csv, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by part-0001.csv would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if part-0001.csv restates part-0000.csv, remove part-0000.csv from the folder, by hand or with after_load = \"delete\" or after_load = { move_to = ... }, which take each file out of the folder once it loads", expected_plan: "", }, ], diff --git a/website/src/content/docs/docs/sources/files.mdx b/website/src/content/docs/docs/sources/files.mdx index 0a8a8fc02..5a60c0d95 100644 --- a/website/src/content/docs/docs/sources/files.mdx +++ b/website/src/content/docs/docs/sources/files.mdx @@ -523,8 +523,9 @@ must hold all of a value's rows. Before each commit, SQLLoad finds the files the the commit's values came from, by `_sqlload_source_file`. When one of them is still in the folder, or loaded earlier in the same run, the table fails, naming both files: two files of the folder holding one value, such as the parts of an export split into part files, would leave only the -later one's rows. Put all of a value's rows in one file, or remove the outdated file from the -folder. A file sent again under the same name, or one whose earlier file has left the folder +later one's rows. The error names the fixes: put all of a value's rows in one file, or, when a +newer file restates an older one, remove the older file from the folder, by hand or with +`after_load = "delete"` or `after_load = { move_to = ... }`. A file sent again under the same name, or one whose earlier file has left the folder (deleted, or moved by `after_load`), replaces the value. A file's rows always commit together. `plan` refuses a table that excludes `_sqlload_source_file`. From b14d692a161ada12ef8c95f8ce6b277b936720e2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:45:39 +0000 Subject: [PATCH 2/2] fix(files): say after_load prevents a repeat of the overlap error, not that it clears one A failed load moves nothing, so the older file must be removed by hand; after_load keeps a later file from meeting it again. --- crates/sqlload/src/loader/_helpers/run_log.rs | 4 ++-- crates/sqlload/tests/loader/test_files_to_duckdb.rs | 4 ++-- website/src/content/docs/docs/sources/files.mdx | 12 +++++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/crates/sqlload/src/loader/_helpers/run_log.rs b/crates/sqlload/src/loader/_helpers/run_log.rs index f460e07cd..6d06afb4f 100644 --- a/crates/sqlload/src/loader/_helpers/run_log.rs +++ b/crates/sqlload/src/loader/_helpers/run_log.rs @@ -864,7 +864,7 @@ pub(crate) fn with_file_keys( count => format!(" (and {} more)", count - 1), }; return Err(format!( - "it holds the replace_key ({columns}) value ({value}){more}, which {first} also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if {this} restates {first}, remove {first} from the folder, by hand or with after_load = \"delete\" or after_load = {{ move_to = ... }}, which take each file out of the folder once it loads" + "it holds the replace_key ({columns}) value ({value}){more}, which {first} also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if one restates the other, remove the older of {first} and {this} from the folder by hand, and set after_load = {{ move_to = ... }} (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values" )); } earlier.extend(values.iter().map(|value| (value.clone(), this.to_owned()))); @@ -960,7 +960,7 @@ pub(crate) fn listed_sources( let this = this.strip_prefix(location.as_str()).unwrap_or(this); let columns = table.spec.replace_key.join(", "); Err(format!( - "the table holds rows of its replace_key ({columns}) values from {first}{more}, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by {this} would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if {this} restates {first}, remove {first} from the folder, by hand or with after_load = \"delete\" or after_load = {{ move_to = ... }}, which take each file out of the folder once it loads" + "the table holds rows of its replace_key ({columns}) values from {first}{more}, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by {this} would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if one restates the other, remove the older of {first} and {this} from the folder by hand, and set after_load = {{ move_to = ... }} (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values" )) } diff --git a/crates/sqlload/tests/loader/test_files_to_duckdb.rs b/crates/sqlload/tests/loader/test_files_to_duckdb.rs index 6986ad6c6..d42c3dadc 100644 --- a/crates/sqlload/tests/loader/test_files_to_duckdb.rs +++ b/crates/sqlload/tests/loader/test_files_to_duckdb.rs @@ -1571,7 +1571,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h removed: &[], loads: true, reload: false, - expected_error: "drop/part-0001.csv: it holds the replace_key (le_version) value (LE-FY27-P01), which part-0000.csv also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if part-0001.csv restates part-0000.csv, remove part-0000.csv from the folder, by hand or with after_load = \"delete\" or after_load = { move_to = ... }, which take each file out of the folder once it loads", + expected_error: "drop/part-0001.csv: it holds the replace_key (le_version) value (LE-FY27-P01), which part-0000.csv also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if one restates the other, remove the older of part-0000.csv and part-0001.csv from the folder by hand, and set after_load = { move_to = ... } (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values", expected_plan: "", }, MergeStep { @@ -1580,7 +1580,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h removed: &[], loads: true, reload: false, - expected_error: "drop/part-0001.csv: the table holds rows of its replace_key (le_version) values from part-0000.csv, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by part-0001.csv would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if part-0001.csv restates part-0000.csv, remove part-0000.csv from the folder, by hand or with after_load = \"delete\" or after_load = { move_to = ... }, which take each file out of the folder once it loads", + expected_error: "drop/part-0001.csv: the table holds rows of its replace_key (le_version) values from part-0000.csv, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by part-0001.csv would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if one restates the other, remove the older of part-0000.csv and part-0001.csv from the folder by hand, and set after_load = { move_to = ... } (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values", expected_plan: "", }, ], diff --git a/website/src/content/docs/docs/sources/files.mdx b/website/src/content/docs/docs/sources/files.mdx index 5a60c0d95..19b356f1d 100644 --- a/website/src/content/docs/docs/sources/files.mdx +++ b/website/src/content/docs/docs/sources/files.mdx @@ -523,11 +523,13 @@ must hold all of a value's rows. Before each commit, SQLLoad finds the files the the commit's values came from, by `_sqlload_source_file`. When one of them is still in the folder, or loaded earlier in the same run, the table fails, naming both files: two files of the folder holding one value, such as the parts of an export split into part files, would leave only the -later one's rows. The error names the fixes: put all of a value's rows in one file, or, when a -newer file restates an older one, remove the older file from the folder, by hand or with -`after_load = "delete"` or `after_load = { move_to = ... }`. A file sent again under the same name, or one whose earlier file has left the folder -(deleted, or moved by `after_load`), replaces the value. A file's rows always commit together. -`plan` refuses a table that excludes `_sqlload_source_file`. +later one's rows. The error names the fixes: put all of a value's rows in one file, or, when one +file restates another, remove the older one from the folder by hand. A failed load moves nothing, +so `after_load` can't clear the failure, but `after_load = { move_to = ... }` (or `"delete"`, which +rules out `--reload`) keeps it from happening again: each file leaves the folder once it loads, and +a file sent later replaces its values. A file sent again under the same name, or one whose earlier +file has left the folder (deleted, or moved by `after_load`), replaces the value. A file's rows +always commit together. `plan` refuses a table that excludes `_sqlload_source_file`. ```toml [sources.le]