diff --git a/crates/sqlload/src/loader/_helpers/run_log.rs b/crates/sqlload/src/loader/_helpers/run_log.rs index 75d1709f..6d06afb4 100644 --- a/crates/sqlload/src/loader/_helpers/run_log.rs +++ b/crates/sqlload/src/loader/_helpers/run_log.rs @@ -856,18 +856,18 @@ pub(crate) fn with_file_keys( } } let columns = table.spec.replace_key.join(", "); + let this = merged_name(file); + let this = this.strip_prefix(location.as_str()).unwrap_or(this); if let Some((value, first)) = split.first() { let more = match split.len() { 1 => String::new(), count => format!(" (and {} more)", count - 1), }; return Err(format!( - "it holds the replace_key ({columns}) value ({value}){more}, which {first} also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: put all the rows of each ({columns}) value in one file, or remove whichever of the two files is outdated from the folder" + "it holds the replace_key ({columns}) value ({value}){more}, which {first} also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if one restates the other, remove the older of {first} and {this} from the folder by hand, and set after_load = {{ move_to = ... }} (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values" )); } - let name = merged_name(file); - let name = name.strip_prefix(location.as_str()).unwrap_or(name); - earlier.extend(values.iter().map(|value| (value.clone(), name.to_owned()))); + earlier.extend(values.iter().map(|value| (value.clone(), this.to_owned()))); Ok(earlier) } @@ -960,7 +960,7 @@ pub(crate) fn listed_sources( let this = this.strip_prefix(location.as_str()).unwrap_or(this); let columns = table.spec.replace_key.join(", "); Err(format!( - "the table holds rows of its replace_key ({columns}) values from {first}{more}, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by {this} would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: put all the rows of each ({columns}) value in one file, or remove the outdated file, {first} or {this}, from the folder" + "the table holds rows of its replace_key ({columns}) values from {first}{more}, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by {this} would delete the rows {first} loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each ({columns}) value in one file; if one restates the other, remove the older of {first} and {this} from the folder by hand, and set after_load = {{ move_to = ... }} (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values" )) } diff --git a/crates/sqlload/tests/loader/test_files_to_duckdb.rs b/crates/sqlload/tests/loader/test_files_to_duckdb.rs index bb4bdba5..d42c3dad 100644 --- a/crates/sqlload/tests/loader/test_files_to_duckdb.rs +++ b/crates/sqlload/tests/loader/test_files_to_duckdb.rs @@ -1571,7 +1571,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h removed: &[], loads: true, reload: false, - expected_error: "drop/part-0001.csv: it holds the replace_key (le_version) value (LE-FY27-P01), which part-0000.csv also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: put all the rows of each (le_version) value in one file, or remove whichever of the two files is outdated from the folder", + expected_error: "drop/part-0001.csv: it holds the replace_key (le_version) value (LE-FY27-P01), which part-0000.csv also holds, loaded earlier in this run; each file replaces the rows of the values it holds, so it would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if one restates the other, remove the older of part-0000.csv and part-0001.csv from the folder by hand, and set after_load = { move_to = ... } (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values", expected_plan: "", }, MergeStep { @@ -1580,7 +1580,7 @@ fn given_le_files_when_delete_insert_replaces_by_replace_key_then_each_version_h removed: &[], loads: true, reload: false, - expected_error: "drop/part-0001.csv: the table holds rows of its replace_key (le_version) values from part-0000.csv, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by part-0001.csv would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: put all the rows of each (le_version) value in one file, or remove the outdated file, part-0000.csv or part-0001.csv, from the folder", + expected_error: "drop/part-0001.csv: the table holds rows of its replace_key (le_version) values from part-0000.csv, which is still in the folder; with two files of the folder holding one value, which of them is current can't be told, so replacing the value by part-0001.csv would delete the rows part-0000.csv loaded, as a value split across part files would keep only its last part. One file must hold all of a value's rows. Fix: if the files are parts of one export, put all the rows of each (le_version) value in one file; if one restates the other, remove the older of part-0000.csv and part-0001.csv from the folder by hand, and set after_load = { move_to = ... } (or \"delete\") so that each file leaves the folder once it loads and a file sent later replaces its values", expected_plan: "", }, ], diff --git a/website/src/content/docs/docs/sources/files.mdx b/website/src/content/docs/docs/sources/files.mdx index 0a8a8fc0..19b356f1 100644 --- a/website/src/content/docs/docs/sources/files.mdx +++ b/website/src/content/docs/docs/sources/files.mdx @@ -523,10 +523,13 @@ must hold all of a value's rows. Before each commit, SQLLoad finds the files the the commit's values came from, by `_sqlload_source_file`. When one of them is still in the folder, or loaded earlier in the same run, the table fails, naming both files: two files of the folder holding one value, such as the parts of an export split into part files, would leave only the -later one's rows. Put all of a value's rows in one file, or remove the outdated file from the -folder. A file sent again under the same name, or one whose earlier file has left the folder -(deleted, or moved by `after_load`), replaces the value. A file's rows always commit together. -`plan` refuses a table that excludes `_sqlload_source_file`. +later one's rows. The error names the fixes: put all of a value's rows in one file, or, when one +file restates another, remove the older one from the folder by hand. A failed load moves nothing, +so `after_load` can't clear the failure, but `after_load = { move_to = ... }` (or `"delete"`, which +rules out `--reload`) keeps it from happening again: each file leaves the folder once it loads, and +a file sent later replaces its values. A file sent again under the same name, or one whose earlier +file has left the folder (deleted, or moved by `after_load`), replaces the value. A file's rows +always commit together. `plan` refuses a table that excludes `_sqlload_source_file`. ```toml [sources.le]