diff --git a/lib/query_processing/src/expressions/functions/str_function.rs b/lib/query_processing/src/expressions/functions/str_function.rs index dae3aa70..4aeed74e 100644 --- a/lib/query_processing/src/expressions/functions/str_function.rs +++ b/lib/query_processing/src/expressions/functions/str_function.rs @@ -1,4 +1,5 @@ use crate::expressions::cast_lang_string_to_string; +use oxrdf::vocab::xsd; use polars::datatypes::DataType; use polars::prelude::{coalesce, col, lit, Expr, LiteralValue}; use representation::cats::{maybe_decode_expr, LockedCats}; @@ -26,13 +27,15 @@ pub fn str_function(c: &str, t: &RDFNodeState, global_cats: LockedCats) -> Expr if t.is_lang_string() { cast_lang_string_to_string(c, t, s, global_cats.clone()) } else { - maybe_decode_expr( - col(c).struct_().field_by_name(&t.field_col_name()), + literal_to_string( + maybe_decode_expr( + col(c).struct_().field_by_name(&t.field_col_name()), + t, + s, + global_cats.clone(), + ), t, - s, - global_cats.clone(), ) - .cast(DataType::String) } } BaseRDFNodeType::None => lit(LiteralValue::untyped_null()).cast(DataType::String), @@ -50,10 +53,25 @@ pub fn str_function(c: &str, t: &RDFNodeState, global_cats: LockedCats) -> Expr if t.is_lang_string() { cast_lang_string_to_string(c, b, s, global_cats) } else { - maybe_decode_expr(col(c), b, s, global_cats).cast(DataType::String) + literal_to_string(maybe_decode_expr(col(c), b, s, global_cats), b) } } BaseRDFNodeType::None => lit(LiteralValue::untyped_null()).cast(DataType::String), } } } + +/// Lexical form of a non-language-tagged literal. Decimals are stored with a fixed scale, +/// so the plain string cast pads them with zeros (1.5 -> "1.5000000000000000"). Trim them +/// to the XSD canonical form, as the N-Triples writer does ("1.5", "798"). +fn literal_to_string(e: Expr, t: &BaseRDFNodeType) -> Expr { + let s = e.cast(DataType::String); + if t.is_lit_type(xsd::DECIMAL) { + s.str() + .replace_all(lit(r"(\.\d*?)0+$"), lit("${1}"), false) + .str() + .replace_all(lit(r"\.$"), lit(""), false) + } else { + s + } +} diff --git a/lib/representation/src/rdf_to_polars.rs b/lib/representation/src/rdf_to_polars.rs index 941fcb93..44add7b2 100644 --- a/lib/representation/src/rdf_to_polars.rs +++ b/lib/representation/src/rdf_to_polars.rs @@ -88,5 +88,7 @@ pub fn default_decimal_precision() -> usize { } pub fn default_decimal_scale() -> usize { - 12 + // XSD 1.1 Part 2 5.4: minimally conforming processors support decimals i / 10^k + // with i < 10^16 and k <= 16, so keep 16 fractional digits instead of truncating. + 16 } diff --git a/lib/representation/src/series_builder.rs b/lib/representation/src/series_builder.rs index 5c19dde8..1dc1d2b0 100644 --- a/lib/representation/src/series_builder.rs +++ b/lib/representation/src/series_builder.rs @@ -10,7 +10,6 @@ use oxrdf::{NamedOrBlankNode, Term}; use polars::prelude::{as_struct, col, DataType, IntoLazy, NamedFrom, PlSmallStr, Series}; use polars_core::frame::DataFrame; use polars_core::prelude::{Int128Chunked, IntoSeries, NewChunkedArray, StringChunkedBuilder}; -use rust_decimal::Decimal; use std::collections::HashMap; use std::str::FromStr; @@ -401,14 +400,48 @@ fn parse_xsd_datetime_micros(value: &str) -> Result { } } +/// Parses an xsd:decimal lexical form into the mantissa of a Decimal(precision, scale). +/// +/// This works on the digits directly instead of going through rust_decimal, whose 96-bit +/// mantissa (28 digits) cannot hold large values at the default scale and silently +/// shifted the decimal point. Fraction digits beyond the scale are truncated. fn parse_xsd_decimal(value: &str) -> Result { - match Decimal::from_str(value) { - Ok(mut d) => { - d.rescale(default_decimal_scale() as u32); - Ok(d.mantissa()) - } - Err(x) => Err(RepresentationError::LiteralParseError(x.to_string())), + let scale = default_decimal_scale(); + let precision = default_decimal_precision(); + let err = || RepresentationError::LiteralParseError(format!("Invalid xsd:decimal: {value}")); + let trimmed = value.trim(); + let (negative, unsigned) = match trimmed.as_bytes().first() { + Some(b'-') => (true, &trimmed[1..]), + Some(b'+') => (false, &trimmed[1..]), + _ => (false, trimmed), + }; + let (int_part, frac_part) = match unsigned.split_once('.') { + Some((i, f)) => (i, f), + None => (unsigned, ""), + }; + if (int_part.is_empty() && frac_part.is_empty()) + || !int_part.bytes().all(|b| b.is_ascii_digit()) + || !frac_part.bytes().all(|b| b.is_ascii_digit()) + { + return Err(err()); + } + let int_part = int_part.trim_start_matches('0'); + if int_part.len() + scale > precision { + return Err(RepresentationError::LiteralParseError(format!( + "xsd:decimal {value} has more than {} integer digits", + precision - scale + ))); + } + let mut mantissa: i128 = 0; + for b in int_part.bytes() { + mantissa = mantissa * 10 + (b - b'0') as i128; + } + let mut frac_digits = frac_part.bytes(); + for _ in 0..scale { + let d = frac_digits.next().map(|b| (b - b'0') as i128).unwrap_or(0); + mantissa = mantissa * 10 + d; } + Ok(if negative { -mantissa } else { mantissa }) } pub fn ensure_pair<'a>( diff --git a/py_maplib/tests/test_decimal_precision.py b/py_maplib/tests/test_decimal_precision.py new file mode 100644 index 00000000..f60cb158 --- /dev/null +++ b/py_maplib/tests/test_decimal_precision.py @@ -0,0 +1,67 @@ +from maplib import Model + +# XSD 1.1 Part 2, 5.4: minimally conforming processors must support decimal values +# i / 10^k with i < 10^16 and k <= 16 (sixteen total digits). + + +def _roundtrip(lexical: str) -> str: + m = Model() + m.reads( + f"""@prefix ex: . + ex:a ex:v {lexical} .""" + , format="turtle") + nt = m.writes(format="ntriples") + return nt.split('"')[1] + + +def test_decimal_with_13_fraction_digits_is_not_truncated(): + assert _roundtrip("760.3865996838672") == "760.3865996838672" + + +def test_decimal_with_16_fraction_digits_is_not_truncated(): + assert _roundtrip("0.0000000000000001") == "0.0000000000000001" + + +def test_decimal_with_large_integer_part(): + assert _roundtrip("123456789012.5") == "123456789012.5" + + +def test_decimal_query_value(): + m = Model() + m.reads( + """@prefix ex: . + ex:a ex:v 760.3865996838672 .""", format="turtle" + ) + df = m.query( + """PREFIX ex: + SELECT (STR(?v) AS ?s) WHERE { ex:a ex:v ?v }""" + ) + assert df["s"][0] == "760.3865996838672" + + +def test_decimal_str_is_canonical(): + m = Model() + m.reads( + """@prefix ex: . + ex:a ex:v 1.5 . ex:b ex:v 798.0 . ex:c ex:v 0.000 . ex:d ex:v -12.340 .""", + format="turtle", + ) + df = m.query( + """PREFIX ex: + SELECT ?x (STR(?v) AS ?s) WHERE { ?x ex:v ?v } ORDER BY ?x""" + ) + assert df["s"].to_list() == ["1.5", "798", "0", "-12.34"] + + +def test_large_decimals_keep_their_value(): + values = ["9999999999999.5", "99999999999999.5", "1234567890123456789.5", "-10000000.25"] + m = Model() + m.reads( + "@prefix ex: .\n" + + "\n".join(f"ex:a{i} ex:v {v} ." for i, v in enumerate(values)), + format="turtle", + ) + df = m.query( + """SELECT ?s (STR(?v) AS ?lex) WHERE { ?s ?p ?v } ORDER BY ?s""" + ) + assert df["lex"].to_list() == values