From 802bc7dd85cfe25f2754466fb84740567bf6f016 Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Sun, 4 Oct 2026 09:03:58 +0200 Subject: [PATCH 1/2] Keep 16 fractional digits for xsd:decimal XSD 1.1 Part 2 5.4 requires support for decimals with up to 16 fractional digits. With a scale of 12, values such as 760.3865996838672 were truncated when read. Refs DataTreehouse/maplib#70. Co-Authored-By: Claude Opus 5.5 (1M context) --- lib/representation/src/rdf_to_polars.rs | 4 ++- py_maplib/tests/test_decimal_precision.py | 39 +++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 py_maplib/tests/test_decimal_precision.py diff --git a/lib/representation/src/rdf_to_polars.rs b/lib/representation/src/rdf_to_polars.rs index 941fcb93..44add7b2 100644 --- a/lib/representation/src/rdf_to_polars.rs +++ b/lib/representation/src/rdf_to_polars.rs @@ -88,5 +88,7 @@ pub fn default_decimal_precision() -> usize { } pub fn default_decimal_scale() -> usize { - 12 + // XSD 1.1 Part 2 5.4: minimally conforming processors support decimals i / 10^k + // with i < 10^16 and k <= 16, so keep 16 fractional digits instead of truncating. + 16 } diff --git a/py_maplib/tests/test_decimal_precision.py b/py_maplib/tests/test_decimal_precision.py new file mode 100644 index 00000000..ed316e0c --- /dev/null +++ b/py_maplib/tests/test_decimal_precision.py @@ -0,0 +1,39 @@ +from maplib import Model + +# XSD 1.1 Part 2, 5.4: minimally conforming processors must support decimal values +# i / 10^k with i < 10^16 and k <= 16 (sixteen total digits). + + +def _roundtrip(lexical: str) -> str: + m = Model() + m.reads( + f"""@prefix ex: . + ex:a ex:v {lexical} .""" + , format="turtle") + nt = m.writes(format="ntriples") + return nt.split('"')[1] + + +def test_decimal_with_13_fraction_digits_is_not_truncated(): + assert _roundtrip("760.3865996838672") == "760.3865996838672" + + +def test_decimal_with_16_fraction_digits_is_not_truncated(): + assert _roundtrip("0.0000000000000001") == "0.0000000000000001" + + +def test_decimal_with_large_integer_part(): + assert _roundtrip("123456789012.5") == "123456789012.5" + + +def test_decimal_query_value(): + m = Model() + m.reads( + """@prefix ex: . + ex:a ex:v 760.3865996838672 .""", format="turtle" + ) + df = m.query( + """PREFIX ex: + SELECT (STR(?v) AS ?s) WHERE { ex:a ex:v ?v }""" + ) + assert df["s"][0] == "760.3865996838672" From 0d3bfaa5a02b38e021cc8b131b46c1743bf0e21c Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Sun, 4 Oct 2026 09:43:35 +0200 Subject: [PATCH 2/2] Parse xsd:decimal without rust_decimal and print canonical STR() for decimals rust_decimal's 96-bit mantissa cannot hold large values at the default scale, and rescale() silently shifted the decimal point (9999999999999.5 became 999999999999.95). Parse the digits into the i128 mantissa directly and reject values with more integer digits than the type allows. STR() on a decimal cast the fixed-scale value to a string with padding zeros ("1.5000000000000000"). Trim it to the canonical form, as the N-Triples writer does. Refs DataTreehouse/maplib#70. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/expressions/functions/str_function.rs | 30 +++++++++--- lib/representation/src/series_builder.rs | 47 ++++++++++++++++--- py_maplib/tests/test_decimal_precision.py | 28 +++++++++++ 3 files changed, 92 insertions(+), 13 deletions(-) diff --git a/lib/query_processing/src/expressions/functions/str_function.rs b/lib/query_processing/src/expressions/functions/str_function.rs index dae3aa70..4aeed74e 100644 --- a/lib/query_processing/src/expressions/functions/str_function.rs +++ b/lib/query_processing/src/expressions/functions/str_function.rs @@ -1,4 +1,5 @@ use crate::expressions::cast_lang_string_to_string; +use oxrdf::vocab::xsd; use polars::datatypes::DataType; use polars::prelude::{coalesce, col, lit, Expr, LiteralValue}; use representation::cats::{maybe_decode_expr, LockedCats}; @@ -26,13 +27,15 @@ pub fn str_function(c: &str, t: &RDFNodeState, global_cats: LockedCats) -> Expr if t.is_lang_string() { cast_lang_string_to_string(c, t, s, global_cats.clone()) } else { - maybe_decode_expr( - col(c).struct_().field_by_name(&t.field_col_name()), + literal_to_string( + maybe_decode_expr( + col(c).struct_().field_by_name(&t.field_col_name()), + t, + s, + global_cats.clone(), + ), t, - s, - global_cats.clone(), ) - .cast(DataType::String) } } BaseRDFNodeType::None => lit(LiteralValue::untyped_null()).cast(DataType::String), @@ -50,10 +53,25 @@ pub fn str_function(c: &str, t: &RDFNodeState, global_cats: LockedCats) -> Expr if t.is_lang_string() { cast_lang_string_to_string(c, b, s, global_cats) } else { - maybe_decode_expr(col(c), b, s, global_cats).cast(DataType::String) + literal_to_string(maybe_decode_expr(col(c), b, s, global_cats), b) } } BaseRDFNodeType::None => lit(LiteralValue::untyped_null()).cast(DataType::String), } } } + +/// Lexical form of a non-language-tagged literal. Decimals are stored with a fixed scale, +/// so the plain string cast pads them with zeros (1.5 -> "1.5000000000000000"). Trim them +/// to the XSD canonical form, as the N-Triples writer does ("1.5", "798"). +fn literal_to_string(e: Expr, t: &BaseRDFNodeType) -> Expr { + let s = e.cast(DataType::String); + if t.is_lit_type(xsd::DECIMAL) { + s.str() + .replace_all(lit(r"(\.\d*?)0+$"), lit("${1}"), false) + .str() + .replace_all(lit(r"\.$"), lit(""), false) + } else { + s + } +} diff --git a/lib/representation/src/series_builder.rs b/lib/representation/src/series_builder.rs index 5c19dde8..1dc1d2b0 100644 --- a/lib/representation/src/series_builder.rs +++ b/lib/representation/src/series_builder.rs @@ -10,7 +10,6 @@ use oxrdf::{NamedOrBlankNode, Term}; use polars::prelude::{as_struct, col, DataType, IntoLazy, NamedFrom, PlSmallStr, Series}; use polars_core::frame::DataFrame; use polars_core::prelude::{Int128Chunked, IntoSeries, NewChunkedArray, StringChunkedBuilder}; -use rust_decimal::Decimal; use std::collections::HashMap; use std::str::FromStr; @@ -401,14 +400,48 @@ fn parse_xsd_datetime_micros(value: &str) -> Result { } } +/// Parses an xsd:decimal lexical form into the mantissa of a Decimal(precision, scale). +/// +/// This works on the digits directly instead of going through rust_decimal, whose 96-bit +/// mantissa (28 digits) cannot hold large values at the default scale and silently +/// shifted the decimal point. Fraction digits beyond the scale are truncated. fn parse_xsd_decimal(value: &str) -> Result { - match Decimal::from_str(value) { - Ok(mut d) => { - d.rescale(default_decimal_scale() as u32); - Ok(d.mantissa()) - } - Err(x) => Err(RepresentationError::LiteralParseError(x.to_string())), + let scale = default_decimal_scale(); + let precision = default_decimal_precision(); + let err = || RepresentationError::LiteralParseError(format!("Invalid xsd:decimal: {value}")); + let trimmed = value.trim(); + let (negative, unsigned) = match trimmed.as_bytes().first() { + Some(b'-') => (true, &trimmed[1..]), + Some(b'+') => (false, &trimmed[1..]), + _ => (false, trimmed), + }; + let (int_part, frac_part) = match unsigned.split_once('.') { + Some((i, f)) => (i, f), + None => (unsigned, ""), + }; + if (int_part.is_empty() && frac_part.is_empty()) + || !int_part.bytes().all(|b| b.is_ascii_digit()) + || !frac_part.bytes().all(|b| b.is_ascii_digit()) + { + return Err(err()); + } + let int_part = int_part.trim_start_matches('0'); + if int_part.len() + scale > precision { + return Err(RepresentationError::LiteralParseError(format!( + "xsd:decimal {value} has more than {} integer digits", + precision - scale + ))); + } + let mut mantissa: i128 = 0; + for b in int_part.bytes() { + mantissa = mantissa * 10 + (b - b'0') as i128; + } + let mut frac_digits = frac_part.bytes(); + for _ in 0..scale { + let d = frac_digits.next().map(|b| (b - b'0') as i128).unwrap_or(0); + mantissa = mantissa * 10 + d; } + Ok(if negative { -mantissa } else { mantissa }) } pub fn ensure_pair<'a>( diff --git a/py_maplib/tests/test_decimal_precision.py b/py_maplib/tests/test_decimal_precision.py index ed316e0c..f60cb158 100644 --- a/py_maplib/tests/test_decimal_precision.py +++ b/py_maplib/tests/test_decimal_precision.py @@ -37,3 +37,31 @@ def test_decimal_query_value(): SELECT (STR(?v) AS ?s) WHERE { ex:a ex:v ?v }""" ) assert df["s"][0] == "760.3865996838672" + + +def test_decimal_str_is_canonical(): + m = Model() + m.reads( + """@prefix ex: . + ex:a ex:v 1.5 . ex:b ex:v 798.0 . ex:c ex:v 0.000 . ex:d ex:v -12.340 .""", + format="turtle", + ) + df = m.query( + """PREFIX ex: + SELECT ?x (STR(?v) AS ?s) WHERE { ?x ex:v ?v } ORDER BY ?x""" + ) + assert df["s"].to_list() == ["1.5", "798", "0", "-12.34"] + + +def test_large_decimals_keep_their_value(): + values = ["9999999999999.5", "99999999999999.5", "1234567890123456789.5", "-10000000.25"] + m = Model() + m.reads( + "@prefix ex: .\n" + + "\n".join(f"ex:a{i} ex:v {v} ." for i, v in enumerate(values)), + format="turtle", + ) + df = m.query( + """SELECT ?s (STR(?v) AS ?lex) WHERE { ?s ?p ?v } ORDER BY ?s""" + ) + assert df["lex"].to_list() == values