Keep 16 fractional digits for xsd:decimal and fix large-value parsing

lib/query_processing/src/expressions/functions/str_function.rs

@@ -1,4 +1,5 @@
 use crate::expressions::cast_lang_string_to_string;
+use oxrdf::vocab::xsd;
 use polars::datatypes::DataType;
 use polars::prelude::{coalesce, col, lit, Expr, LiteralValue};
 use representation::cats::{maybe_decode_expr, LockedCats};
@@ -26,13 +27,15 @@ pub fn str_function(c: &str, t: &RDFNodeState, global_cats: LockedCats) -> Expr
                     if t.is_lang_string() {
                         cast_lang_string_to_string(c, t, s, global_cats.clone())
                     } else {
-                        maybe_decode_expr(
-                            col(c).struct_().field_by_name(&t.field_col_name()),
+                        literal_to_string(
+                            maybe_decode_expr(
+                                col(c).struct_().field_by_name(&t.field_col_name()),
+                                t,
+                                s,
+                                global_cats.clone(),
+                            ),
                             t,
-                            s,
-                            global_cats.clone(),
                         )
-                        .cast(DataType::String)
                     }
                 }
                 BaseRDFNodeType::None => lit(LiteralValue::untyped_null()).cast(DataType::String),
@@ -50,10 +53,25 @@ pub fn str_function(c: &str, t: &RDFNodeState, global_cats: LockedCats) -> Expr
                 if t.is_lang_string() {
                     cast_lang_string_to_string(c, b, s, global_cats)
                 } else {
-                    maybe_decode_expr(col(c), b, s, global_cats).cast(DataType::String)
+                    literal_to_string(maybe_decode_expr(col(c), b, s, global_cats), b)
                 }
             }
             BaseRDFNodeType::None => lit(LiteralValue::untyped_null()).cast(DataType::String),
         }
     }
 }
+
+/// Lexical form of a non-language-tagged literal. Decimals are stored with a fixed scale,
+/// so the plain string cast pads them with zeros (1.5 -> "1.5000000000000000"). Trim them
+/// to the XSD canonical form, as the N-Triples writer does ("1.5", "798").
+fn literal_to_string(e: Expr, t: &BaseRDFNodeType) -> Expr {
+    let s = e.cast(DataType::String);
+    if t.is_lit_type(xsd::DECIMAL) {
+        s.str()
+            .replace_all(lit(r"(\.\d*?)0+$"), lit("${1}"), false)
+            .str()
+            .replace_all(lit(r"\.$"), lit(""), false)
+    } else {
+        s
+    }
+}

lib/representation/src/rdf_to_polars.rs

@@ -88,5 +88,7 @@ pub fn default_decimal_precision() -> usize {
 }
 
 pub fn default_decimal_scale() -> usize {
-    12
+    // XSD 1.1 Part 2 5.4: minimally conforming processors support decimals i / 10^k
+    // with i < 10^16 and k <= 16, so keep 16 fractional digits instead of truncating.
+    16
 }

lib/representation/src/series_builder.rs

@@ -10,7 +10,6 @@ use oxrdf::{NamedOrBlankNode, Term};
 use polars::prelude::{as_struct, col, DataType, IntoLazy, NamedFrom, PlSmallStr, Series};
 use polars_core::frame::DataFrame;
 use polars_core::prelude::{Int128Chunked, IntoSeries, NewChunkedArray, StringChunkedBuilder};
-use rust_decimal::Decimal;
 use std::collections::HashMap;
 use std::str::FromStr;
 
@@ -401,14 +400,48 @@ fn parse_xsd_datetime_micros(value: &str) -> Result<i64, RepresentationError> {
     }
 }
 
+/// Parses an xsd:decimal lexical form into the mantissa of a Decimal(precision, scale).
+///
+/// This works on the digits directly instead of going through rust_decimal, whose 96-bit
+/// mantissa (28 digits) cannot hold large values at the default scale and silently
+/// shifted the decimal point. Fraction digits beyond the scale are truncated.
 fn parse_xsd_decimal(value: &str) -> Result<i128, RepresentationError> {
-    match Decimal::from_str(value) {
-        Ok(mut d) => {
-            d.rescale(default_decimal_scale() as u32);
-            Ok(d.mantissa())
-        }
-        Err(x) => Err(RepresentationError::LiteralParseError(x.to_string())),
+    let scale = default_decimal_scale();
+    let precision = default_decimal_precision();
+    let err = || RepresentationError::LiteralParseError(format!("Invalid xsd:decimal: {value}"));
+    let trimmed = value.trim();
+    let (negative, unsigned) = match trimmed.as_bytes().first() {
+        Some(b'-') => (true, &trimmed[1..]),
+        Some(b'+') => (false, &trimmed[1..]),
+        _ => (false, trimmed),
+    };
+    let (int_part, frac_part) = match unsigned.split_once('.') {
+        Some((i, f)) => (i, f),
+        None => (unsigned, ""),
+    };
+    if (int_part.is_empty() && frac_part.is_empty())
+        || !int_part.bytes().all(|b| b.is_ascii_digit())
+        || !frac_part.bytes().all(|b| b.is_ascii_digit())
+    {
+        return Err(err());
+    }
+    let int_part = int_part.trim_start_matches('0');
+    if int_part.len() + scale > precision {
+        return Err(RepresentationError::LiteralParseError(format!(
+            "xsd:decimal {value} has more than {} integer digits",
+            precision - scale
+        )));
+    }
+    let mut mantissa: i128 = 0;
+    for b in int_part.bytes() {
+        mantissa = mantissa * 10 + (b - b'0') as i128;
+    }
+    let mut frac_digits = frac_part.bytes();
+    for _ in 0..scale {
+        let d = frac_digits.next().map(|b| (b - b'0') as i128).unwrap_or(0);
+        mantissa = mantissa * 10 + d;
     }
+    Ok(if negative { -mantissa } else { mantissa })
 }
 
 pub fn ensure_pair<'a>(

py_maplib/tests/test_decimal_precision.py

@@ -0,0 +1,67 @@
+from maplib import Model
+
+# XSD 1.1 Part 2, 5.4: minimally conforming processors must support decimal values
+# i / 10^k with i < 10^16 and k <= 16 (sixteen total digits).
+
+
+def _roundtrip(lexical: str) -> str:
+    m = Model()
+    m.reads(
+        f"""@prefix ex: <http://example.org/> .
+        ex:a ex:v {lexical} ."""
+    , format="turtle")
+    nt = m.writes(format="ntriples")
+    return nt.split('"')[1]
+
+
+def test_decimal_with_13_fraction_digits_is_not_truncated():
+    assert _roundtrip("760.3865996838672") == "760.3865996838672"
+
+
+def test_decimal_with_16_fraction_digits_is_not_truncated():
+    assert _roundtrip("0.0000000000000001") == "0.0000000000000001"
+
+
+def test_decimal_with_large_integer_part():
+    assert _roundtrip("123456789012.5") == "123456789012.5"
+
+
+def test_decimal_query_value():
+    m = Model()
+    m.reads(
+        """@prefix ex: <http://example.org/> .
+        ex:a ex:v 760.3865996838672 .""", format="turtle"
+    )
+    df = m.query(
+        """PREFIX ex: <http://example.org/>
+        SELECT (STR(?v) AS ?s) WHERE { ex:a ex:v ?v }"""
+    )
+    assert df["s"][0] == "760.3865996838672"
+
+
+def test_decimal_str_is_canonical():
+    m = Model()
+    m.reads(
+        """@prefix ex: <http://example.org/> .
+        ex:a ex:v 1.5 . ex:b ex:v 798.0 . ex:c ex:v 0.000 . ex:d ex:v -12.340 .""",
+        format="turtle",
+    )
+    df = m.query(
+        """PREFIX ex: <http://example.org/>
+        SELECT ?x (STR(?v) AS ?s) WHERE { ?x ex:v ?v } ORDER BY ?x"""
+    )
+    assert df["s"].to_list() == ["1.5", "798", "0", "-12.34"]
+
+
+def test_large_decimals_keep_their_value():
+    values = ["9999999999999.5", "99999999999999.5", "1234567890123456789.5", "-10000000.25"]
+    m = Model()
+    m.reads(
+        "@prefix ex: <http://example.org/> .\n"
+        + "\n".join(f"ex:a{i} ex:v {v} ." for i, v in enumerate(values)),
+        format="turtle",
+    )
+    df = m.query(
+        """SELECT ?s (STR(?v) AS ?lex) WHERE { ?s ?p ?v } ORDER BY ?s"""
+    )
+    assert df["lex"].to_list() == values