diff --git a/docs/type-mapping.md b/docs/type-mapping.md index cff618d..df92427 100644 --- a/docs/type-mapping.md +++ b/docs/type-mapping.md @@ -67,6 +67,7 @@ runtime checks. | `UInt64` | `UInt64Policy::CheckedBigInt` | `bigint` | runtime check | runtime check | Values greater than `i64::MAX` are rejected. | | `Utf8`, `LargeUtf8` | `StringPolicy::NVarCharMax` | `nvarchar(max)` | yes | yes | Default. | | `Utf8`, `LargeUtf8` | `StringPolicy::NVarChar(n)` | `nvarchar(n)` | runtime check | runtime check | Runtime rejects values whose UTF-16 length exceeds `n`. | +| `Utf8`, `LargeUtf8`, `Utf8View` | `StringPolicy::AsciiVarChar(n)` | `varchar(n)` | runtime check | runtime check | `n` must be in `1..=8000`; runtime rejects non-ASCII values and values longer than `n` bytes. | | `Utf8`, `LargeUtf8` | `StringPolicy::ObservedNVarChar` | inferred `nvarchar(n)` | schema-only reject | schema-only reject | Requires observed values or statistics; schema-only planning currently rejects it. | | `Binary`, `LargeBinary` | `BinaryPolicy::VarBinaryMax` | `varbinary(max)` | yes | yes | Default. | | `Binary`, `LargeBinary` | `BinaryPolicy::VarBinary(n)` | `varbinary(n)` | runtime check | runtime check | Runtime rejects values whose byte length exceeds `n`. | diff --git a/src/conversion/arrow_to_mssql/variable_width.rs b/src/conversion/arrow_to_mssql/variable_width.rs index 7a0b3b4..0922eec 100644 --- a/src/conversion/arrow_to_mssql/variable_width.rs +++ b/src/conversion/arrow_to_mssql/variable_width.rs @@ -13,6 +13,8 @@ use crate::{ pub(crate) enum VariableWidthArrowToMssql { /// Arrow string family to SQL Server `nvarchar(n|max)`. StringToNVarChar { length: MssqlTypeLength }, + /// Arrow string family to SQL Server `varchar(n)` with ASCII-only values. + StringToAsciiVarChar { length: MssqlTypeLength }, /// Arrow binary family to SQL Server `varbinary(n|max)`. BytesToVarBinary { length: MssqlTypeLength }, } @@ -24,6 +26,9 @@ impl VariableWidthArrowToMssql { (data_type, MssqlType::NVarChar(length)) if is_arrow_string_family(data_type) => { Self::StringToNVarChar { length: *length } } + (data_type, MssqlType::VarChar(length)) if is_arrow_string_family(data_type) => { + Self::StringToAsciiVarChar { length: *length } + } (data_type, MssqlType::VarBinary(length)) if is_arrow_binary_family(data_type) => { Self::BytesToVarBinary { length: *length } } @@ -45,10 +50,13 @@ impl VariableWidthArrowToMssql { } } -/// Returns true when a planned mapping writes Arrow string-family values to `nvarchar`. -pub(crate) fn is_string_family_to_nvarchar(mapping: &SchemaMapping) -> bool { +/// Returns true when a planned mapping writes Arrow string-family values to a SQL text type. +pub(crate) fn is_string_family_to_sql_text(mapping: &SchemaMapping) -> bool { is_arrow_string_family(mapping.arrow().data_type()) - && matches!(mapping.mssql().ty(), MssqlType::NVarChar(_)) + && matches!( + mapping.mssql().ty(), + MssqlType::NVarChar(_) | MssqlType::VarChar(_) + ) } /// Returns true when a planned mapping writes Arrow binary-family values to `varbinary`. @@ -63,7 +71,7 @@ pub(crate) fn arrow_type_compatible_with_mapping( mapping: &SchemaMapping, ) -> bool { runtime == mapping.arrow().data_type() - || (is_arrow_string_family(runtime) && is_string_family_to_nvarchar(mapping)) + || (is_arrow_string_family(runtime) && is_string_family_to_sql_text(mapping)) || (is_arrow_binary_family(runtime) && is_binary_family_to_varbinary(mapping)) } @@ -115,6 +123,13 @@ mod tests { length: MssqlTypeLength::Bounded(32), }, ), + ( + DataType::Utf8, + MssqlType::VarChar(MssqlTypeLength::Bounded(32)), + VariableWidthArrowToMssql::StringToAsciiVarChar { + length: MssqlTypeLength::Bounded(32), + }, + ), ( DataType::LargeUtf8, MssqlType::NVarChar(MssqlTypeLength::Max), @@ -202,17 +217,17 @@ mod tests { } #[test] - fn accepts_string_family_runtime_types_for_nvarchar_mappings() { - for planned in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] { - let mapping = mapping( - 0, - "text", - planned, - MssqlType::NVarChar(MssqlTypeLength::Max), - ); + fn accepts_string_family_runtime_types_for_sql_text_mappings() { + for target in [ + MssqlType::NVarChar(MssqlTypeLength::Max), + MssqlType::VarChar(MssqlTypeLength::Bounded(32)), + ] { + for planned in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] { + let mapping = mapping(0, "text", planned, target.clone()); - for runtime in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] { - assert!(arrow_type_compatible_with_mapping(&runtime, &mapping)); + for runtime in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] { + assert!(arrow_type_compatible_with_mapping(&runtime, &mapping)); + } } } } diff --git a/src/mssql/cell.rs b/src/mssql/cell.rs index 69a905d..dd4e208 100644 --- a/src/mssql/cell.rs +++ b/src/mssql/cell.rs @@ -34,6 +34,8 @@ pub(crate) enum MssqlCell<'a> { Float(Option), /// SQL Server `nvarchar` cell. NVarChar(Option<&'a str>), + /// SQL Server `varchar` cell containing only ASCII characters. + VarChar(Option<&'a str>), /// SQL Server `varbinary` cell. VarBinary(Option<&'a [u8]>), } diff --git a/src/mssql/cell/from_arrow.rs b/src/mssql/cell/from_arrow.rs index 1fdb91c..9410f19 100644 --- a/src/mssql/cell/from_arrow.rs +++ b/src/mssql/cell/from_arrow.rs @@ -21,7 +21,7 @@ use temporal::{ mssql_time_value, null_datetime_cell, null_datetime2_cell, null_datetimeoffset_cell, null_time_cell, }; -use variable_width::{binary_cell, nvar_char_cell, var_binary_cell}; +use variable_width::{ascii_var_char_cell, binary_cell, nvar_char_cell, var_binary_cell}; /// Direction-specific runtime context for Arrow-to-MSSQL value conversion. #[derive(Debug, Clone, Copy)] @@ -121,6 +121,7 @@ pub(crate) fn mssql_cell_from_arrow_cell<'a>( mssql_datetimeoffset_value(runtime_mapping, row_index, cell)?, ))), MssqlType::NVarChar(length) => nvar_char_cell(mapping, row_index, *length, cell), + MssqlType::VarChar(length) => ascii_var_char_cell(mapping, row_index, *length, cell), MssqlType::VarBinary(length) => var_binary_cell(mapping, row_index, *length, cell), MssqlType::Binary(length) => binary_cell(mapping, row_index, *length, cell), } @@ -144,6 +145,7 @@ fn null_mssql_cell<'a>(mapping: &SchemaMapping, row_index: usize) -> Result Ok(MssqlCell::Real(None)), MssqlType::Float { .. } => Ok(MssqlCell::Float(None)), MssqlType::NVarChar(_) => Ok(MssqlCell::NVarChar(None)), + MssqlType::VarChar(_) => Ok(MssqlCell::VarChar(None)), MssqlType::VarBinary(_) => Ok(MssqlCell::VarBinary(None)), MssqlType::Binary(_) => Ok(MssqlCell::VarBinary(None)), ty => Err(unsupported_value_conversion( diff --git a/src/mssql/cell/from_arrow/variable_width.rs b/src/mssql/cell/from_arrow/variable_width.rs index 0c4f8a9..a1dbd0d 100644 --- a/src/mssql/cell/from_arrow/variable_width.rs +++ b/src/mssql/cell/from_arrow/variable_width.rs @@ -30,7 +30,7 @@ pub(super) fn nvar_char_cell<'a>( }; debug_assert_eq!(length, classified); - let value = mssql_nvarchar_value(mapping, row_index, cell)?; + let value = mssql_string_value(mapping, row_index, cell)?; let code_units = value.encode_utf16().count(); if exceeds_length(length, code_units) { @@ -47,6 +47,55 @@ pub(super) fn nvar_char_cell<'a>( Ok(MssqlCell::NVarChar(Some(value))) } +pub(super) fn ascii_var_char_cell<'a>( + mapping: &SchemaMapping, + row_index: usize, + length: MssqlTypeLength, + cell: ArrowCell<'a>, +) -> Result> { + let classified = match VariableWidthArrowToMssql::classify(mapping, row_index)? { + VariableWidthArrowToMssql::StringToAsciiVarChar { length } => length, + other => { + return Err(value_conversion_error(row_mapping_diagnostic( + mapping, + row_index, + DiagnosticCode::ValueConversionUnsupported, + format!( + "variable-width mapping {other:?} is not supported by ASCII varchar conversion" + ), + ))); + } + }; + debug_assert_eq!(length, classified); + + let value = mssql_string_value(mapping, row_index, cell)?; + if !value.is_ascii() { + return Err(value_conversion_error(row_mapping_diagnostic( + mapping, + row_index, + DiagnosticCode::ValueConversionUnsupported, + format!( + "string value contains non-ASCII characters and cannot be written as planned {}", + mapping.mssql().ty().to_sql() + ), + ))); + } + + if exceeds_length(length, value.len()) { + return Err(value_too_long_error( + mapping, + row_index, + format!( + "ASCII string value has {} byte(s), exceeding planned {}", + value.len(), + mapping.mssql().ty().to_sql() + ), + )); + } + + Ok(MssqlCell::VarChar(Some(value))) +} + pub(super) fn var_binary_cell<'a>( mapping: &SchemaMapping, row_index: usize, @@ -114,7 +163,7 @@ pub(super) fn binary_cell<'a>( Ok(MssqlCell::VarBinary(Some(value))) } -fn mssql_nvarchar_value<'a>( +fn mssql_string_value<'a>( mapping: &SchemaMapping, row_index: usize, cell: ArrowCell<'a>, @@ -269,6 +318,30 @@ mod tests { ); } + #[test] + fn accepts_bounded_ascii_varchar_and_rejects_invalid_values() { + let mappings = mappings_for_schema_with_options( + Schema::new(vec![Field::new("text", DataType::Utf8, true)]), + PlanOptions { + string_policy: StringPolicy::AsciiVarChar(2), + ..PlanOptions::default() + }, + ); + + assert_eq!( + convert_cell(&mappings[0], ArrowCell::Utf8("ab"), 0).unwrap(), + MssqlCell::VarChar(Some("ab")) + ); + + for (row_index, value, code) in [ + (1, "abc", DiagnosticCode::ValueTooLong), + (2, "e\u{301}", DiagnosticCode::ValueConversionUnsupported), + ] { + let err = convert_cell(&mappings[0], ArrowCell::Utf8(value), row_index).unwrap_err(); + assert_single_diagnostic(err, code, Some(row_index), Some((0, "text"))); + } + } + #[test] fn rejects_bounded_varbinary_by_byte_count() { let mappings = mappings_for_schema_with_options( diff --git a/src/mssql/ty.rs b/src/mssql/ty.rs index 095e1be..e4f0847 100644 --- a/src/mssql/ty.rs +++ b/src/mssql/ty.rs @@ -70,6 +70,8 @@ pub enum MssqlType { }, /// SQL Server `nvarchar(n|max)`. NVarChar(MssqlTypeLength), + /// SQL Server `varchar(n|max)` with ASCII-only Arrow string conversion. + VarChar(MssqlTypeLength), /// SQL Server `varbinary(n|max)`. VarBinary(MssqlTypeLength), /// SQL Server `binary(n)`. @@ -111,6 +113,7 @@ impl MssqlType { Self::Real => "real".to_owned(), Self::Float { precision } => format!("float({precision})"), Self::NVarChar(length) => format!("nvarchar({})", length.render()), + Self::VarChar(length) => format!("varchar({})", length.render()), Self::VarBinary(length) => format!("varbinary({})", length.render()), Self::Binary(length) => format!("binary({length})"), Self::Decimal { precision, scale } => format!("decimal({precision},{scale})"), @@ -148,6 +151,10 @@ mod tests { MssqlType::NVarChar(MssqlTypeLength::Bounded(128)).to_sql(), "nvarchar(128)" ); + assert_eq!( + MssqlType::VarChar(MssqlTypeLength::Bounded(128)).to_sql(), + "varchar(128)" + ); assert_eq!( MssqlType::VarBinary(MssqlTypeLength::Max).to_sql(), "varbinary(max)" diff --git a/src/observability/schema.rs b/src/observability/schema.rs index eb0161f..28906ca 100644 --- a/src/observability/schema.rs +++ b/src/observability/schema.rs @@ -221,6 +221,7 @@ fn mssql_type_family(ty: &MssqlType) -> &'static str { MssqlType::Real => "real", MssqlType::Float { .. } => "float", MssqlType::NVarChar(_) => "nvarchar", + MssqlType::VarChar(_) => "varchar", MssqlType::VarBinary(_) => "varbinary", MssqlType::Binary(_) => "binary", MssqlType::Decimal { .. } => "decimal", diff --git a/src/schema/type_conversion.rs b/src/schema/type_conversion.rs index 27dfb7e..0618405 100644 --- a/src/schema/type_conversion.rs +++ b/src/schema/type_conversion.rs @@ -96,6 +96,18 @@ fn plan_arrow_string_as_mssql_type( match policy { StringPolicy::NVarCharMax => Ok(MssqlType::NVarChar(MssqlTypeLength::Max)), StringPolicy::NVarChar(length) => Ok(MssqlType::NVarChar(MssqlTypeLength::Bounded(length))), + StringPolicy::AsciiVarChar(length) + if (1..=SQL_SERVER_MAX_VARCHAR_LEN).contains(&length) => + { + Ok(MssqlType::VarChar(MssqlTypeLength::Bounded(length))) + } + StringPolicy::AsciiVarChar(length) => Err(unsupported_arrow_mapping_for_arrow_to_mssql( + index, + field, + format!( + "ASCII varchar length {length} is outside SQL Server varchar(n) range 1..={SQL_SERVER_MAX_VARCHAR_LEN}" + ), + )), StringPolicy::ObservedNVarChar => Err(observed_data_required_for_arrow_to_mssql( index, field, @@ -128,7 +140,7 @@ fn plan_arrow_fixed_size_binary_as_mssql_type( field: &Field, ) -> std::result::Result { let Ok(length) = usize::try_from(length) else { - return Err(fixed_size_binary_out_of_range_for_arrow_to_mssql( + return Err(unsupported_arrow_mapping_for_arrow_to_mssql( index, field, "fixed-size binary length must be non-negative", @@ -136,7 +148,7 @@ fn plan_arrow_fixed_size_binary_as_mssql_type( }; if !(1..=SQL_SERVER_MAX_BINARY_LEN).contains(&length) { - return Err(fixed_size_binary_out_of_range_for_arrow_to_mssql( + return Err(unsupported_arrow_mapping_for_arrow_to_mssql( index, field, format!( @@ -330,7 +342,7 @@ fn decimal_out_of_range_for_arrow_to_mssql( .with_field(FieldRef::new(index, field.name())) } -fn fixed_size_binary_out_of_range_for_arrow_to_mssql( +fn unsupported_arrow_mapping_for_arrow_to_mssql( index: usize, field: &Field, message: impl Into, @@ -375,6 +387,7 @@ fn unsupported_arrow_type_family(data_type: &DataType) -> &'static str { const SQL_SERVER_MAX_DECIMAL_PRECISION: u8 = 38; const SQL_SERVER_MAX_BINARY_LEN: usize = 8000; +const SQL_SERVER_MAX_VARCHAR_LEN: usize = 8000; #[cfg(test)] mod tests { @@ -456,6 +469,17 @@ mod tests { .unwrap(), MssqlType::NVarChar(MssqlTypeLength::Bounded(128)) ); + assert_eq!( + plan_type( + DataType::Utf8, + PlanOptions { + string_policy: StringPolicy::AsciiVarChar(128), + ..PlanOptions::default() + }, + ) + .unwrap(), + MssqlType::VarChar(MssqlTypeLength::Bounded(128)) + ); assert_eq!( plan_type( DataType::Binary, @@ -480,6 +504,23 @@ mod tests { ); } + #[test] + fn rejects_ascii_varchar_lengths_outside_sql_server_range() { + for length in [0, 8001] { + let diagnostic = plan_type( + DataType::Utf8, + PlanOptions { + string_policy: StringPolicy::AsciiVarChar(length), + ..PlanOptions::default() + }, + ) + .unwrap_err(); + + assert_eq!(diagnostic.code(), DiagnosticCode::UnsupportedArrowType); + assert!(diagnostic.message().contains("range 1..=8000")); + } + } + #[test] fn maps_uint64_when_explicit_policy_is_selected() { assert_eq!( diff --git a/src/write/direct/binding.rs b/src/write/direct/binding.rs index 49dcb32..7ae0b80 100644 --- a/src/write/direct/binding.rs +++ b/src/write/direct/binding.rs @@ -30,7 +30,7 @@ use crate::{ primitive::PrimitiveArrowToMssql, temporal::TemporalArrowToMssql, variable_width::{ - VariableWidthArrowToMssql, is_binary_family_to_varbinary, is_string_family_to_nvarchar, + VariableWidthArrowToMssql, is_binary_family_to_varbinary, is_string_family_to_sql_text, }, }, write::context::RuntimeConversionContext, @@ -248,7 +248,10 @@ fn bind_direct_columns<'a>( }, DirectColumnEncoding::VariableWidth(VariableWidthArrowToMssql::StringToNVarChar { .. - }) => bind_direct_nvarchar_array( + }) + | DirectColumnEncoding::VariableWidth( + VariableWidthArrowToMssql::StringToAsciiVarChar { .. }, + ) => bind_direct_string_array( array, column, encoder.mapping_for_column_index(column_index)?, @@ -385,15 +388,15 @@ fn bind_direct_columns<'a>( Ok(columns) } -fn bind_direct_nvarchar_array<'a>( +fn bind_direct_string_array<'a>( array: &'a dyn Array, column: &'a plan::DirectColumnPlan, mapping: &SchemaMapping, ) -> Result> { - if !is_string_family_to_nvarchar(mapping) { + if !is_string_family_to_sql_text(mapping) { return Err(unsupported_planned_direct_type( column, - "nvarchar", + "SQL text", mapping.arrow().data_type(), )); } @@ -411,7 +414,7 @@ fn bind_direct_nvarchar_array<'a>( column, array: downcast_direct_array::(array, column)?, }), - other => Err(unsupported_planned_direct_type(column, "nvarchar", other)), + other => Err(unsupported_planned_direct_type(column, "SQL text", other)), } } diff --git a/src/write/direct/binding/append.rs b/src/write/direct/binding/append.rs index a206ccb..941fb2c 100644 --- a/src/write/direct/binding/append.rs +++ b/src/write/direct/binding/append.rs @@ -21,7 +21,7 @@ use super::super::types::{ append_timestamp_nanosecond_cell, append_timestamp_second_cell, }, uint64::append_uint64_decimal20_cell, - variable_width::{append_nvarchar_cell, append_varbinary_cell}, + variable_width::{append_string_cell, append_varbinary_cell}, }; use super::BoundDirectColumn; @@ -102,13 +102,13 @@ impl BoundDirectColumn<'_> { append_float64_cell(buf, array, column, row_index, measured_len) } Self::Utf8 { column, array } => { - append_nvarchar_cell(buf, *array, column, row_index, measured_len) + append_string_cell(buf, *array, column, row_index, measured_len) } Self::LargeUtf8 { column, array } => { - append_nvarchar_cell(buf, *array, column, row_index, measured_len) + append_string_cell(buf, *array, column, row_index, measured_len) } Self::Utf8View { column, array } => { - append_nvarchar_cell(buf, *array, column, row_index, measured_len) + append_string_cell(buf, *array, column, row_index, measured_len) } Self::Binary { column, array } => { append_varbinary_cell(buf, *array, column, row_index, measured_len) @@ -286,13 +286,13 @@ impl BoundDirectColumn<'_> { ) -> Result<()> { match self { Self::Utf8 { column, array } => { - append_nvarchar_cell(buf, *array, column, row_index, measured_len) + append_string_cell(buf, *array, column, row_index, measured_len) } Self::LargeUtf8 { column, array } => { - append_nvarchar_cell(buf, *array, column, row_index, measured_len) + append_string_cell(buf, *array, column, row_index, measured_len) } Self::Utf8View { column, array } => { - append_nvarchar_cell(buf, *array, column, row_index, measured_len) + append_string_cell(buf, *array, column, row_index, measured_len) } Self::Binary { column, array } => { append_varbinary_cell(buf, *array, column, row_index, measured_len) diff --git a/src/write/direct/binding/fill.rs b/src/write/direct/binding/fill.rs index 6e9af65..e42a704 100644 --- a/src/write/direct/binding/fill.rs +++ b/src/write/direct/binding/fill.rs @@ -25,7 +25,7 @@ use super::super::{ fill_timestamp_nanosecond_direct_column, fill_timestamp_second_direct_column, }, uint64::fill_uint64_decimal20_column, - variable_width::{fill_nvarchar_column, fill_varbinary_column}, + variable_width::{fill_string_column, fill_varbinary_column}, }, }; use super::BoundDirectColumn; @@ -142,13 +142,13 @@ impl BoundDirectColumn<'_> { fill_float64_column(array, column, column_index, column_count, layout, bytes) } Self::Utf8 { column, array } => { - fill_nvarchar_column(*array, column, column_index, column_count, layout, bytes) + fill_string_column(*array, column, column_index, column_count, layout, bytes) } Self::LargeUtf8 { column, array } => { - fill_nvarchar_column(*array, column, column_index, column_count, layout, bytes) + fill_string_column(*array, column, column_index, column_count, layout, bytes) } Self::Utf8View { column, array } => { - fill_nvarchar_column(*array, column, column_index, column_count, layout, bytes) + fill_string_column(*array, column, column_index, column_count, layout, bytes) } Self::Binary { column, array } => { fill_varbinary_column(*array, column, column_index, column_count, layout, bytes) diff --git a/src/write/direct/binding/measure.rs b/src/write/direct/binding/measure.rs index ea5d0ec..6b506d0 100644 --- a/src/write/direct/binding/measure.rs +++ b/src/write/direct/binding/measure.rs @@ -35,7 +35,7 @@ use super::super::{ }, uint64::measure_uint64_decimal20_cell_lengths, variable_width::{ - measure_nvarchar_column_cell_lengths, measure_varbinary_column_cell_lengths, + measure_string_column_cell_lengths, measure_varbinary_column_cell_lengths, }, }, }; @@ -189,21 +189,21 @@ impl BoundDirectColumn<'_> { column_count, cell_lengths, ), - Self::Utf8 { column, array } => measure_nvarchar_column_cell_lengths( + Self::Utf8 { column, array } => measure_string_column_cell_lengths( *array, column, column_index, column_count, cell_lengths, ), - Self::LargeUtf8 { column, array } => measure_nvarchar_column_cell_lengths( + Self::LargeUtf8 { column, array } => measure_string_column_cell_lengths( *array, column, column_index, column_count, cell_lengths, ), - Self::Utf8View { column, array } => measure_nvarchar_column_cell_lengths( + Self::Utf8View { column, array } => measure_string_column_cell_lengths( *array, column, column_index, diff --git a/src/write/direct/types/variable_width.rs b/src/write/direct/types/variable_width.rs index b064525..3b5eb83 100644 --- a/src/write/direct/types/variable_width.rs +++ b/src/write/direct/types/variable_width.rs @@ -63,33 +63,39 @@ impl RawRowsAppendTarget for Vec { } } -/// Measures one string-family-to-nvarchar column into a row-major cell length matrix. -pub(crate) fn measure_nvarchar_column_cell_lengths<'a>( +/// Measures one string-family column into a row-major cell length matrix. +pub(crate) fn measure_string_column_cell_lengths<'a>( array: impl StringArrayType<'a>, column: &DirectColumnPlan, column_index: usize, column_count: usize, cell_lengths: &mut [usize], ) -> Result<()> { - let length = match column.encoding() { + match column.encoding() { DirectColumnEncoding::VariableWidth(VariableWidthArrowToMssql::StringToNVarChar { length, - }) => length, - other => { - return Err(unsupported_batch(format!( - "direct nvarchar layout cannot measure mapping {other:?}" - ))); - } - }; - - measure_nvarchar_cell_lengths( - array, - column, - column_index, - column_count, - length, - cell_lengths, - ) + }) => measure_nvarchar_cell_lengths( + array, + column, + column_index, + column_count, + length, + cell_lengths, + ), + DirectColumnEncoding::VariableWidth(VariableWidthArrowToMssql::StringToAsciiVarChar { + length, + }) => measure_ascii_varchar_cell_lengths( + array, + column, + column_index, + column_count, + length, + cell_lengths, + ), + other => Err(unsupported_batch(format!( + "direct string layout cannot measure mapping {other:?}" + ))), + } } /// Measures one binary-family-to-varbinary column into a row-major cell length matrix. @@ -121,8 +127,8 @@ pub(crate) fn measure_varbinary_column_cell_lengths<'a>( ) } -/// Fills one string-family-to-nvarchar column into an already allocated rows payload. -pub(crate) fn fill_nvarchar_column<'a>( +/// Fills one string-family column into an already allocated rows payload. +pub(crate) fn fill_string_column<'a>( array: impl StringArrayType<'a>, column: &DirectColumnPlan, column_index: usize, @@ -130,17 +136,44 @@ pub(crate) fn fill_nvarchar_column<'a>( layout: &RowLayout, bytes: &mut [u8], ) -> Result<()> { - let length = match column.encoding() { + match column.encoding() { DirectColumnEncoding::VariableWidth(VariableWidthArrowToMssql::StringToNVarChar { length, - }) => length, - other => { - return Err(unsupported_batch(format!( - "direct nvarchar fill cannot encode mapping {other:?}" - ))); - } - }; + }) => fill_nvarchar_column( + array, + column, + column_index, + column_count, + layout, + bytes, + length, + ), + DirectColumnEncoding::VariableWidth(VariableWidthArrowToMssql::StringToAsciiVarChar { + length, + }) => fill_ascii_varchar_column( + array, + column, + column_index, + column_count, + layout, + bytes, + length, + ), + other => Err(unsupported_batch(format!( + "direct string fill cannot encode mapping {other:?}" + ))), + } +} +fn fill_nvarchar_column<'a>( + array: impl StringArrayType<'a>, + column: &DirectColumnPlan, + column_index: usize, + column_count: usize, + layout: &RowLayout, + bytes: &mut [u8], + length: MssqlTypeLength, +) -> Result<()> { for row_index in 0..array.len() { let cell = cell_position(layout, row_index, column_index, column_count)?; @@ -161,6 +194,35 @@ pub(crate) fn fill_nvarchar_column<'a>( Ok(()) } +fn fill_ascii_varchar_column<'a>( + array: impl StringArrayType<'a>, + column: &DirectColumnPlan, + column_index: usize, + column_count: usize, + layout: &RowLayout, + bytes: &mut [u8], + length: MssqlTypeLength, +) -> Result<()> { + for row_index in 0..array.len() { + let cell = cell_position(layout, row_index, column_index, column_count)?; + + if array.is_null(row_index) { + write_null_cell(bytes, cell, column, row_index, length)?; + } else { + write_ascii_varchar_cell( + bytes, + cell, + column, + row_index, + length, + array.value(row_index), + )?; + } + } + + Ok(()) +} + /// Fills one binary-family-to-varbinary column into an already allocated rows payload. pub(crate) fn fill_varbinary_column<'a>( array: impl BinaryArrayType<'a>, @@ -201,25 +263,35 @@ pub(crate) fn fill_varbinary_column<'a>( Ok(()) } -/// Appends one string-family-to-nvarchar cell to a raw bulk append buffer. -pub(crate) fn append_nvarchar_cell<'a>( +/// Appends one string-family cell to a raw bulk append buffer. +pub(crate) fn append_string_cell<'a>( buf: &mut impl RawRowsAppendTarget, array: impl StringArrayType<'a>, column: &DirectColumnPlan, row_index: usize, measured_len: usize, ) -> Result<()> { - let length = match column.encoding() { + match column.encoding() { DirectColumnEncoding::VariableWidth(VariableWidthArrowToMssql::StringToNVarChar { length, - }) => length, - other => { - return Err(unsupported_batch(format!( - "direct nvarchar append cannot encode mapping {other:?}" - ))); - } - }; + }) => append_nvarchar_cell(buf, array, column, row_index, measured_len, length), + DirectColumnEncoding::VariableWidth(VariableWidthArrowToMssql::StringToAsciiVarChar { + length, + }) => append_ascii_varchar_cell(buf, array, column, row_index, measured_len, length), + other => Err(unsupported_batch(format!( + "direct string append cannot encode mapping {other:?}" + ))), + } +} +fn append_nvarchar_cell<'a>( + buf: &mut impl RawRowsAppendTarget, + array: impl StringArrayType<'a>, + column: &DirectColumnPlan, + row_index: usize, + measured_len: usize, + length: MssqlTypeLength, +) -> Result<()> { if array.is_null(row_index) { return append_null_cell(buf, column, row_index, measured_len, length); } @@ -261,6 +333,44 @@ pub(crate) fn append_nvarchar_cell<'a>( } } +fn append_ascii_varchar_cell<'a>( + buf: &mut impl RawRowsAppendTarget, + array: impl StringArrayType<'a>, + column: &DirectColumnPlan, + row_index: usize, + measured_len: usize, + length: MssqlTypeLength, +) -> Result<()> { + if array.is_null(row_index) { + return append_null_cell(buf, column, row_index, measured_len, length); + } + + let value = array.value(row_index); + validate_ascii_value(column, row_index, value)?; + + match length { + MssqlTypeLength::Bounded(limit) => { + if value.len() > limit { + return Err(value_too_long_error( + column, + row_index, + format!( + "ASCII string value has {} byte(s), exceeding planned {}", + value.len(), + column.target_type().to_sql() + ), + )); + } + profile::record_varchar_bytes(value.len()); + append_bounded_payload_cell(buf, column, row_index, measured_len, value.as_bytes()) + } + MssqlTypeLength::Max => { + profile::record_varchar_bytes(value.len()); + append_plp_payload_cell(buf, column, row_index, measured_len, value.as_bytes()) + } + } +} + /// Appends one binary-family-to-varbinary cell to a raw bulk append buffer. pub(crate) fn append_varbinary_cell<'a>( buf: &mut impl RawRowsAppendTarget, @@ -353,6 +463,47 @@ fn measure_nvarchar_cell_lengths<'a>( Ok(()) } +fn measure_ascii_varchar_cell_lengths<'a>( + array: impl StringArrayType<'a>, + column: &DirectColumnPlan, + column_index: usize, + column_count: usize, + length: MssqlTypeLength, + cell_lengths: &mut [usize], +) -> Result<()> { + for row_index in 0..array.len() { + let cell_len = if array.is_null(row_index) { + null_cell_len(column, row_index, length)? + } else { + let value = array.value(row_index); + validate_ascii_value(column, row_index, value)?; + let encoded_bytes = value.len(); + + match length { + MssqlTypeLength::Bounded(limit) => { + if encoded_bytes > limit { + return Err(value_too_long_error( + column, + row_index, + format!( + "ASCII string value has {encoded_bytes} byte(s), exceeding planned {}", + column.target_type().to_sql() + ), + )); + } + + bounded_cell_len(encoded_bytes)? + } + MssqlTypeLength::Max => plp_cell_len(encoded_bytes)?, + } + }; + + cell_lengths[row_index * column_count + column_index] = cell_len; + } + + Ok(()) +} + fn measure_varbinary_cell_lengths<'a>( array: impl BinaryArrayType<'a>, column: &DirectColumnPlan, @@ -440,7 +591,7 @@ fn append_bounded_payload_cell( let expected_len = bounded_cell_len(value.len())?; if measured_len != expected_len { return Err(invalid_payload(format!( - "measured bounded varbinary cell at row {row_index} column {} has length {}, expected {expected_len}", + "measured bounded byte-payload cell at row {row_index} column {} has length {}, expected {expected_len}", column.source_name(), measured_len ))); @@ -474,7 +625,7 @@ fn append_plp_payload_cell( let expected_len = plp_cell_len(value.len())?; if measured_len != expected_len { return Err(invalid_payload(format!( - "measured PLP varbinary cell at row {row_index} column {} has length {}, expected {expected_len}", + "measured PLP byte-payload cell at row {row_index} column {} has length {}, expected {expected_len}", column.source_name(), measured_len ))); @@ -657,6 +808,35 @@ fn write_nvarchar_cell( } } +fn write_ascii_varchar_cell( + bytes: &mut [u8], + cell: &CellPosition, + column: &DirectColumnPlan, + row_index: usize, + length: MssqlTypeLength, + value: &str, +) -> Result<()> { + validate_ascii_value(column, row_index, value)?; + + match length { + MssqlTypeLength::Bounded(limit) => { + if value.len() > limit { + return Err(value_too_long_error( + column, + row_index, + format!( + "ASCII string value has {} byte(s), exceeding planned {}", + value.len(), + column.target_type().to_sql() + ), + )); + } + write_bounded_payload_cell(bytes, cell, value.as_bytes()) + } + MssqlTypeLength::Max => write_plp_payload_cell(bytes, cell, value.as_bytes()), + } +} + fn write_varbinary_cell( bytes: &mut [u8], cell: &CellPosition, @@ -712,7 +892,7 @@ fn write_bounded_payload_cell(bytes: &mut [u8], cell: &CellPosition, value: &[u8 let expected_len = bounded_cell_len(value.len())?; if cell.len() != expected_len { return Err(invalid_payload(format!( - "bounded varbinary cell at row {} column {} has length {}, expected {expected_len}", + "bounded byte-payload cell at row {} column {} has length {}, expected {expected_len}", cell.row_index(), cell.column_index(), cell.len() @@ -756,7 +936,7 @@ fn write_plp_payload_cell(bytes: &mut [u8], cell: &CellPosition, value: &[u8]) - let expected_len = plp_cell_len(value.len())?; if cell.len() != expected_len { return Err(invalid_payload(format!( - "PLP varbinary cell at row {} column {} has length {}, expected {expected_len}", + "PLP byte-payload cell at row {} column {} has length {}, expected {expected_len}", cell.row_index(), cell.column_index(), cell.len() @@ -817,6 +997,22 @@ fn validate_utf16_byte_len(cell: &CellPosition, encoded_bytes: usize) -> Result< ))) } +fn validate_ascii_value(column: &DirectColumnPlan, row_index: usize, value: &str) -> Result<()> { + if value.is_ascii() { + return Ok(()); + } + + Err(value_conversion_error(row_column_diagnostic( + column, + row_index, + DiagnosticCode::ValueConversionUnsupported, + format!( + "string value contains non-ASCII characters and cannot be written as planned {}", + column.target_type().to_sql() + ), + ))) +} + fn write_utf16le(dst: &mut [u8], value: &str) { for (chunk, code_unit) in dst.chunks_exact_mut(2).zip(value.encode_utf16()) { chunk.copy_from_slice(&code_unit.to_le_bytes()); @@ -955,10 +1151,10 @@ mod tests { }; use super::{ - MAX_BOUNDED_TDS_VALUE_LEN, MAX_PLP_CHUNK_LEN, append_utf16le, bounded_cell_len, - bounded_nvarchar_encoded_bytes, fill_nvarchar_column, fill_varbinary_column, - measure_nvarchar_column_cell_lengths, measure_varbinary_column_cell_lengths, plp_cell_len, - plp_nvarchar_encoded_bytes, + MAX_BOUNDED_TDS_VALUE_LEN, MAX_PLP_CHUNK_LEN, append_string_cell, append_utf16le, + bounded_cell_len, bounded_nvarchar_encoded_bytes, fill_string_column, + fill_varbinary_column, measure_string_column_cell_lengths, + measure_varbinary_column_cell_lengths, plp_cell_len, plp_nvarchar_encoded_bytes, }; #[test] @@ -972,6 +1168,38 @@ mod tests { assert_eq!(non_ascii, [0xe9, 0, 0x3d, 0xd8, 0x42, 0xde]); } + #[test] + fn encodes_ascii_varchar_as_single_bytes_and_rejects_non_ascii() { + let array = StringArray::from(vec![Some("ab"), Some(""), None]); + let plan = plan(&[mapping( + 0, + "text", + DataType::Utf8, + MssqlType::VarChar(MssqlTypeLength::Bounded(2)), + true, + )]); + let mut cell_lengths = vec![0; array.len()]; + + measure_string_column_cell_lengths(&array, &plan.columns()[0], 0, 1, &mut cell_lengths) + .unwrap(); + assert_eq!(cell_lengths, [4, 2, 2]); + + let mut encoded = Vec::new(); + append_string_cell(&mut encoded, &array, &plan.columns()[0], 0, cell_lengths[0]).unwrap(); + assert_eq!(encoded, [2, 0, b'a', b'b']); + + let non_ascii = StringArray::from(vec![Some("\u{e9}")]); + let err = + measure_string_column_cell_lengths(&non_ascii, &plan.columns()[0], 0, 1, &mut [0]) + .unwrap_err(); + assert_single_diagnostic( + err, + DiagnosticCode::ValueConversionUnsupported, + Some(0), + Some((0, "text")), + ); + } + #[test] fn measures_bounded_nvarchar_cells_by_encoded_utf16_bytes() { let array = StringArray::from(vec![Some("ab"), Some("🙂"), None]); @@ -984,7 +1212,7 @@ mod tests { )]); let mut cell_lengths = vec![0; array.len()]; - measure_nvarchar_column_cell_lengths(&array, &plan.columns()[0], 0, 1, &mut cell_lengths) + measure_string_column_cell_lengths(&array, &plan.columns()[0], 0, 1, &mut cell_lengths) .unwrap(); assert_eq!(cell_lengths, [6, 6, 2]); @@ -1002,7 +1230,7 @@ mod tests { )]); let mut cell_lengths = vec![0; array.len()]; - measure_nvarchar_column_cell_lengths(&array, &plan.columns()[0], 0, 1, &mut cell_lengths) + measure_string_column_cell_lengths(&array, &plan.columns()[0], 0, 1, &mut cell_lengths) .unwrap(); assert_eq!(cell_lengths, [18, 12, 8]); @@ -1056,14 +1284,9 @@ mod tests { )]); let mut cell_lengths = vec![0; array.len()]; - let err = measure_nvarchar_column_cell_lengths( - &array, - &plan.columns()[0], - 0, - 1, - &mut cell_lengths, - ) - .unwrap_err(); + let err = + measure_string_column_cell_lengths(&array, &plan.columns()[0], 0, 1, &mut cell_lengths) + .unwrap_err(); assert_single_diagnostic( err, @@ -1114,14 +1337,9 @@ mod tests { )]); let mut cell_lengths = vec![0; array.len()]; - let err = measure_nvarchar_column_cell_lengths( - &array, - &plan.columns()[0], - 0, - 1, - &mut cell_lengths, - ) - .unwrap_err(); + let err = + measure_string_column_cell_lengths(&array, &plan.columns()[0], 0, 1, &mut cell_lengths) + .unwrap_err(); assert_single_diagnostic( err, @@ -1184,7 +1402,7 @@ mod tests { let layout = build_fixed_width_row_layout(3, 1, &[6, 6, 2]).unwrap(); let mut bytes = allocate_rows_payload_with_tokens(&layout); - fill_nvarchar_column(&array, &plan.columns()[0], 0, 1, &layout, &mut bytes).unwrap(); + fill_string_column(&array, &plan.columns()[0], 0, 1, &layout, &mut bytes).unwrap(); assert_eq!( bytes, @@ -1207,7 +1425,7 @@ mod tests { let layout = build_fixed_width_row_layout(3, 1, &[18, 12, 8]).unwrap(); let mut bytes = allocate_rows_payload_with_tokens(&layout); - fill_nvarchar_column(&array, &plan.columns()[0], 0, 1, &layout, &mut bytes).unwrap(); + fill_string_column(&array, &plan.columns()[0], 0, 1, &layout, &mut bytes).unwrap(); assert_eq!( bytes, @@ -1277,7 +1495,7 @@ mod tests { )]); let mut cell_lengths = vec![0; array.len()]; - measure_nvarchar_column_cell_lengths(&array, &plan.columns()[0], 0, 1, &mut cell_lengths) + measure_string_column_cell_lengths(&array, &plan.columns()[0], 0, 1, &mut cell_lengths) .unwrap(); assert_eq!(cell_lengths, [6, 6, 6, 2]); @@ -1318,14 +1536,9 @@ mod tests { )]); let mut cell_lengths = vec![0; array.len()]; - let err = measure_nvarchar_column_cell_lengths( - &array, - &plan.columns()[0], - 0, - 1, - &mut cell_lengths, - ) - .unwrap_err(); + let err = + measure_string_column_cell_lengths(&array, &plan.columns()[0], 0, 1, &mut cell_lengths) + .unwrap_err(); assert_single_diagnostic( err, @@ -1377,7 +1590,7 @@ mod tests { let layout = build_fixed_width_row_layout(4, 1, &[6, 6, 2, 2]).unwrap(); let mut bytes = allocate_rows_payload_with_tokens(&layout); - fill_nvarchar_column(&array, &plan.columns()[0], 0, 1, &layout, &mut bytes).unwrap(); + fill_string_column(&array, &plan.columns()[0], 0, 1, &layout, &mut bytes).unwrap(); assert_eq!( bytes, @@ -1400,7 +1613,7 @@ mod tests { )]); let mut cell_lengths = vec![0; array.len()]; - measure_nvarchar_column_cell_lengths(&array, &plan.columns()[0], 0, 1, &mut cell_lengths) + measure_string_column_cell_lengths(&array, &plan.columns()[0], 0, 1, &mut cell_lengths) .unwrap(); assert_eq!(cell_lengths, [6, 6, 2]); @@ -1418,14 +1631,9 @@ mod tests { )]); let mut cell_lengths = vec![0; array.len()]; - let err = measure_nvarchar_column_cell_lengths( - &array, - &plan.columns()[0], - 0, - 1, - &mut cell_lengths, - ) - .unwrap_err(); + let err = + measure_string_column_cell_lengths(&array, &plan.columns()[0], 0, 1, &mut cell_lengths) + .unwrap_err(); assert_single_diagnostic( err, diff --git a/src/write/policy.rs b/src/write/policy.rs index 15fed33..21a92c7 100644 --- a/src/write/policy.rs +++ b/src/write/policy.rs @@ -33,6 +33,8 @@ pub enum StringPolicy { NVarCharMax, /// Use bounded `nvarchar(n)`. NVarChar(usize), + /// Use bounded `varchar(n)` and reject non-ASCII values. + AsciiVarChar(usize), /// Infer bounded `nvarchar(n)` from observed values. ObservedNVarChar, } @@ -202,7 +204,7 @@ mod tests { #[test] fn supports_explicit_non_default_policy_overrides() { let options = PlanOptions { - string_policy: StringPolicy::NVarChar(128), + string_policy: StringPolicy::AsciiVarChar(128), binary_policy: BinaryPolicy::VarBinary(256), timezone_policy: TimezonePolicy::DateTimeOffset, timestamp_policy: TimestampPolicy::DateTime, @@ -214,7 +216,7 @@ mod tests { date64_policy: Date64Policy::TimestampDateTime2, }; - assert_eq!(options.string_policy, StringPolicy::NVarChar(128)); + assert_eq!(options.string_policy, StringPolicy::AsciiVarChar(128)); assert_eq!(options.binary_policy, BinaryPolicy::VarBinary(256)); assert_eq!(options.timezone_policy, TimezonePolicy::DateTimeOffset); assert_eq!(options.timestamp_policy, TimestampPolicy::DateTime); diff --git a/src/write/profile.rs b/src/write/profile.rs index d262246..3a45c58 100644 --- a/src/write/profile.rs +++ b/src/write/profile.rs @@ -34,6 +34,8 @@ mod enabled { pub max_row_range_bytes: u64, /// Non-null SQL Server `nvarchar` payload bytes after UTF-16 encoding. pub nvarchar_utf16_bytes: u64, + /// Non-null SQL Server `varchar` payload bytes after ASCII validation. + pub varchar_bytes: u64, /// Non-null SQL Server `varbinary` payload bytes. pub varbinary_bytes: u64, /// Number of null cells observed by the profiled direct writer path. @@ -297,6 +299,14 @@ mod enabled { }); } + pub(crate) fn record_varchar_bytes(encoded_bytes: usize) { + with_profile(|profile| { + profile.varchar_bytes = profile + .varchar_bytes + .saturating_add(usize_to_u64_saturating(encoded_bytes)); + }); + } + pub(crate) fn record_varbinary_bytes(encoded_bytes: usize) { with_profile(|profile| { profile.varbinary_bytes = profile @@ -526,6 +536,8 @@ mod disabled { pub(crate) fn record_nvarchar_utf16_bytes(_encoded_bytes: usize) {} + pub(crate) fn record_varchar_bytes(_encoded_bytes: usize) {} + pub(crate) fn record_varbinary_bytes(_encoded_bytes: usize) {} pub(crate) fn record_null_cell() {} @@ -552,7 +564,7 @@ pub(crate) use enabled::{ direct_date_fast_path_disabled, direct_fixed_width_fast_path_disabled, record_accepted_batch, record_append_encode, record_bulk_load_stats, record_measure_batch, record_null_cell, record_nvarchar_utf16_bytes, record_row_range, record_row_range_split, record_send_total, - record_varbinary_bytes, + record_varbinary_bytes, record_varchar_bytes, }; pub(crate) fn record_elapsed(start: std::time::Instant, record: fn(Duration), value: T) -> T { @@ -574,6 +586,7 @@ mod tests { super::record_accepted_batch(7); super::record_row_range(11); super::record_nvarchar_utf16_bytes(17); + super::record_varchar_bytes(18); super::record_varbinary_bytes(19); super::record_null_cell(); super::record_bulk_load_stats(tiberius::BulkLoadStats { @@ -671,6 +684,7 @@ mod tests { assert_eq!(profile.encoded_bytes, 11); assert_eq!(profile.max_row_range_bytes, 11); assert_eq!(profile.nvarchar_utf16_bytes, 17); + assert_eq!(profile.varchar_bytes, 18); assert_eq!(profile.varbinary_bytes, 19); assert_eq!(profile.null_cells, 1); assert_eq!(profile.packet_write_calls, 23); diff --git a/src/write/token_row.rs b/src/write/token_row.rs index f0d43cc..688c1ca 100644 --- a/src/write/token_row.rs +++ b/src/write/token_row.rs @@ -49,6 +49,7 @@ pub(crate) fn mssql_cell_to_tiberius_borrowed(cell: MssqlCell<'_>) -> tiberius:: MssqlCell::Real(value) => tiberius::ColumnData::F32(value), MssqlCell::Float(value) => tiberius::ColumnData::F64(value), MssqlCell::NVarChar(value) => tiberius::ColumnData::String(value.map(Cow::Borrowed)), + MssqlCell::VarChar(value) => tiberius::ColumnData::String(value.map(Cow::Borrowed)), MssqlCell::VarBinary(value) => tiberius::ColumnData::Binary(value.map(Cow::Borrowed)), } } @@ -76,6 +77,9 @@ pub(crate) fn mssql_cell_to_tiberius_owned(cell: MssqlCell<'_>) -> tiberius::Col MssqlCell::NVarChar(value) => { tiberius::ColumnData::String(value.map(|value| Cow::Owned(value.to_owned()))) } + MssqlCell::VarChar(value) => { + tiberius::ColumnData::String(value.map(|value| Cow::Owned(value.to_owned()))) + } MssqlCell::VarBinary(value) => { tiberius::ColumnData::Binary(value.map(|value| Cow::Owned(value.to_vec()))) } diff --git a/src/write/writer.rs b/src/write/writer.rs index e80e2f8..2d973a5 100644 --- a/src/write/writer.rs +++ b/src/write/writer.rs @@ -552,6 +552,9 @@ fn expected_direct_bulk_column_type(column: &DirectColumnPlan) -> Option Some(tiberius::ColumnType::NVarchar), + DirectColumnEncoding::VariableWidth(VariableWidthArrowToMssql::StringToAsciiVarChar { + .. + }) => Some(tiberius::ColumnType::BigVarChar), DirectColumnEncoding::VariableWidth(VariableWidthArrowToMssql::BytesToVarBinary { .. }) => Some(tiberius::ColumnType::BigVarBin), @@ -1557,7 +1560,17 @@ mod tests { #[test] fn direct_bulk_target_type_validation_accepts_matching_variable_width_metadata() { - let mappings = vec![utf8_mapping_at(0, "name"), binary_mapping_at(1, "payload")]; + let mappings = vec![ + utf8_mapping_at(0, "name"), + schema_mapping_at( + 1, + "ascii_code", + DataType::Utf8, + MssqlType::VarChar(MssqlTypeLength::Bounded(16)), + false, + ), + binary_mapping_at(2, "payload"), + ]; let state = WriterState::new( WriteBackend::DirectRawBulk, SchemaCheck::Strict, @@ -1566,7 +1579,8 @@ mod tests { .unwrap(); let columns = vec![ bulk_target_column_with_type(0, "name", false, tiberius::ColumnType::NVarchar), - bulk_target_column_with_type(1, "payload", false, tiberius::ColumnType::BigVarBin), + bulk_target_column_with_type(1, "ascii_code", false, tiberius::ColumnType::BigVarChar), + bulk_target_column_with_type(2, "payload", false, tiberius::ColumnType::BigVarBin), ]; validate_direct_bulk_target_column_types( diff --git a/tests/integration_sqlserver.rs b/tests/integration_sqlserver.rs index be31b69..ae67dec 100644 --- a/tests/integration_sqlserver.rs +++ b/tests/integration_sqlserver.rs @@ -22,8 +22,8 @@ use arrow_schema::{DataType, Field, Schema, TimeUnit}; use arrow_sql_server::{ ArrowFieldRef, BulkWriter, Date64Policy, DecimalPolicy, DiagnosticCode, DiagnosticSet, Error, Identifier, MssqlColumn, MssqlProfile, MssqlType, MssqlTypeLength, NanosecondPolicy, - PlanOptions, PlanOutcome, PlannedSchema, SchemaMapping, TableName, TimestampPolicy, - TimezonePolicy, UInt64Policy, WriteBackend, WriteOptions, WritePhase, + PlanOptions, PlanOutcome, PlannedSchema, SchemaMapping, StringPolicy, TableName, + TimestampPolicy, TimezonePolicy, UInt64Policy, WriteBackend, WriteOptions, WritePhase, create_table_sql_from_mappings, }; use tokio::net::TcpStream; @@ -1166,6 +1166,92 @@ async fn writer_round_trips_float16_real_values_across_supported_backends() -> T Ok(()) } +#[tokio::test] +async fn writers_round_trip_ascii_varchar_across_supported_backends() -> TestResult<()> { + let Some((connection_string, database)) = integration_config() else { + eprintln!( + "skipping SQL Server ASCII varchar integration test: {CONNECTION_STRING_ENV} or {TEST_DATABASE_ENV} is not set" + ); + return Ok(()); + }; + + let mut client = connect(&connection_string, &database).await?; + let schema = Arc::new(Schema::new(vec![ + Field::new("row_id", DataType::Int32, false), + Field::new("code", DataType::Utf8, true), + ])); + let (mappings, _diagnostics) = plan_arrow_schema_to_mssql_mappings( + Arc::clone(&schema), + integration_mssql_profile(), + PlanOptions { + string_policy: StringPolicy::AsciiVarChar(8), + ..PlanOptions::default() + }, + )? + .into_parts(); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![1_i32, 2, 3])) as ArrayRef, + Arc::new(StringArray::from(vec![Some("ABC123"), Some(""), None])), + ], + )?; + + for backend in [ + WriteBackend::BaselineTokenRow, + WriteBackend::DirectFramedBulk, + WriteBackend::DirectRawBulk, + ] { + let table = unique_table_name()?; + execute_sql( + &mut client, + create_table_sql_from_mappings(&table, &mappings), + ) + .await?; + + let result = async { + let mut writer = BulkWriter::new( + &mut client, + table.clone(), + mappings.clone(), + WriteOptions { + backend, + ..WriteOptions::default() + }, + ) + .await?; + let stats = writer.write_batch(&batch).await?; + ensure_eq(writer.finish().await?, stats, "finish stats")?; + + let rows = client + .simple_query(format!( + "SELECT [row_id], [code], DATALENGTH([code]) FROM {} ORDER BY [row_id]", + table.quoted_sql() + )) + .await? + .into_first_result() + .await?; + + ensure_eq(rows.len(), 3, "row count")?; + ensure_eq(rows[0].get::<&str, _>(1), Some("ABC123"), "row 0 code")?; + ensure_eq(rows[0].get::(2), Some(6), "row 0 bytes")?; + ensure_eq(rows[1].get::<&str, _>(1), Some(""), "row 1 code")?; + ensure_eq(rows[1].get::(2), Some(0), "row 1 bytes")?; + ensure_eq(rows[2].get::<&str, _>(1), None, "row 2 code")?; + ensure_eq(rows[2].get::(2), None, "row 2 bytes")?; + + Ok::<(), Box>(()) + } + .await; + + let drop_result = drop_table(&mut client, &table).await; + result?; + drop_result?; + } + + Ok(()) +} + #[tokio::test] async fn direct_raw_writer_round_trips_variable_width_matrix() -> TestResult<()> { let Some((connection_string, database)) = integration_config() else {