diff --git a/crates/paimon/src/table/lumina_index_build_builder/extraction.rs b/crates/paimon/src/table/lumina_index_build_builder/extraction.rs index 1076507ac..7fb5b10fc 100644 --- a/crates/paimon/src/table/lumina_index_build_builder/extraction.rs +++ b/crates/paimon/src/table/lumina_index_build_builder/extraction.rs @@ -181,7 +181,23 @@ pub(super) fn extract_vectors_from_batches( source: None, }); } - vectors.push(values.value(value_index)); + let value = values.value(value_index); + // Java rejects these per element in `LuminaVectorGlobalIndexWriter`, and + // so does the vindex backend (`paimon_vindex_core`'s + // `validate_finite_values`). Without the check the index is built from + // undefined data and every later top-k over the shard is scored against + // it. + if !value.is_finite() { + return Err(Error::DataInvalid { + message: format!( + "Lumina vector extraction found non-finite element {value} \ + at _ROW_ID {row_id}, index {}", + value_index - start + ), + source: None, + }); + } + vectors.push(value); } } } diff --git a/crates/paimon/src/table/lumina_index_build_builder/tests.rs b/crates/paimon/src/table/lumina_index_build_builder/tests.rs index 48351c8f7..b4e598782 100644 --- a/crates/paimon/src/table/lumina_index_build_builder/tests.rs +++ b/crates/paimon/src/table/lumina_index_build_builder/tests.rs @@ -555,6 +555,25 @@ fn test_extract_vectors_rejects_non_list_float32() { assert!(matches!(err, Error::DataInvalid { message, .. } if message.contains("List"))); } +#[test] +fn test_extract_vectors_rejects_non_finite_element() { + for value in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { + let batch = vector_batch(vec![Some(vec![Some(1.0), Some(value)])], vec![Some(7)]); + + let err = extract_vectors_from_batches(&[batch], "embedding", 2, 7, 1) + .expect_err("a non-finite element should fail"); + + let Error::DataInvalid { message, .. } = err else { + panic!("expected DataInvalid for {value}, got {err:?}"); + }; + assert!(message.contains("non-finite"), "message: {message}"); + // The row id and the position inside the vector are what make the failure + // actionable, as in Java's `Vector element at rowId=%d, index=%d`. + assert!(message.contains("_ROW_ID 7"), "message: {message}"); + assert!(message.contains("index 1"), "message: {message}"); + } +} + fn fixed_size_vector_batch( rows: Vec>>, row_ids: Vec>, @@ -673,6 +692,39 @@ fn test_extract_vectors_fixed_size_list_rejects_null_element() { ); } +#[test] +fn test_extract_vectors_fixed_size_list_rejects_non_finite_element() { + // Both layouts share the element loop, so this pins that the FixedSizeList + // path reaches it too. + let element_field = Arc::new(ArrowField::new("element", ArrowDataType::Float32, true)); + let mut builder = FixedSizeListBuilder::new(Float32Builder::new(), 2).with_field(element_field); + builder.values().append_value(1.0); + builder.values().append_value(f32::NAN); + builder.append(true); + let row_ids = Arc::new(Int64Array::from(vec![Some(0)])) as ArrayRef; + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new( + "embedding", + ArrowDataType::FixedSizeList( + Arc::new(ArrowField::new("element", ArrowDataType::Float32, true)), + 2, + ), + true, + ), + ArrowField::new(ROW_ID_FIELD_NAME, ArrowDataType::Int64, true), + ])); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(builder.finish()) as ArrayRef, row_ids], + ) + .unwrap(); + + let err = extract_vectors_from_batches(&[batch], "embedding", 2, 0, 1) + .expect_err("a non-finite element should fail"); + + assert!(matches!(err, Error::DataInvalid { message, .. } if message.contains("non-finite"))); +} + #[test] fn test_checked_metadata_conversion_supports_i64_file_size() { let above_i32_max = i32::MAX as u64 + 1;