From e24f8edf7d4e15f5226741c8b57cf9fef976bc9a Mon Sep 17 00:00:00 2001 From: Bailey Nguyen Date: Fri, 2 Oct 2026 12:26:15 -0500 Subject: [PATCH 1/3] feat(rust): generate schema-scoped incremental item writers --- .../skills/polyxml-codegen-workflow/SKILL.md | 14 ++ crates/polyxml-core/src/codegen/rust/mod.rs | 131 ++++++++++++++++++ crates/polyxml-core/src/schema_parser/upa.rs | 18 +++ .../polyxml-core/tests/test_rust_codegen.rs | 40 ++++++ tests/fixtures/incremental/items.xsd | 10 ++ 5 files changed, 213 insertions(+) create mode 100644 tests/fixtures/incremental/items.xsd diff --git a/.agents/skills/polyxml-codegen-workflow/SKILL.md b/.agents/skills/polyxml-codegen-workflow/SKILL.md index 3c230502..cfc4c13c 100644 --- a/.agents/skills/polyxml-codegen-workflow/SKILL.md +++ b/.agents/skills/polyxml-codegen-workflow/SKILL.md @@ -1008,3 +1008,17 @@ checkout because the helper builds with that cwd. The generated-model test helper now honors this override and builds the matching CLI if needed. Keep workspace targets owned by one worktree; switching checkouts in a shared target can leave stale unchanged executables even after a core compile message. + +### Incremental Rust producer verification (#127) + +For the generated `write__items` producer, build the CLI then run +`.venv/bin/python scripts/verify_incremental_writer.py` (requires lxml and GNU +`time`). It compiles owned/borrowed generated models, checks occurrence errors +and sink failures, validates XML with libxml2, and records fresh-process RSS. +Use `--output` to retain raw samples. Ordinary `encode_xml` streams an existing +model; this producer consumes owned items from an iterator without a root Vec. +Eligibility must use `upa::Document` source QNames: flattened FieldDef namespace +metadata does not preserve every local `form`/`elementFormDefault` distinction. +Expose the API only for plain sequence roots with one repeated field and item +codecs whose element namespaces match the root. Keep unsupported schema shapes +out of the API rather than claiming full streaming schema support. diff --git a/crates/polyxml-core/src/codegen/rust/mod.rs b/crates/polyxml-core/src/codegen/rust/mod.rs index 67f6fbee..ffb04afe 100644 --- a/crates/polyxml-core/src/codegen/rust/mod.rs +++ b/crates/polyxml-core/src/codegen/rust/mod.rs @@ -1940,6 +1940,137 @@ impl RustCodegen { out.push_str(" }\n"); out.push_str("}\n"); + self.emit_incremental_roots(out, s, ir, types_with_lifetime); + } + + /// Only expose the producer API where ordinary item codecs preserve the schema. + fn emit_incremental_roots( + &self, + out: &mut String, + s: &StructDef, + ir: &SchemaIR, + lifetimes: &HashSet, + ) { + fn supported( + s: &StructDef, + ir: &SchemaIR, + namespace: &Option, + seen: &mut HashSet, + ) -> bool { + if ir + .upa_documents + .iter() + .find_map(|doc| doc.uniform_sequence(&s.qname, namespace)) + != Some(true) + { + return false; + } + if !seen.insert(s.qname.clone()) { + return false; + } + if s.base_type.is_some() + || s.is_abstract + || s.is_mixed + || ir.ordered_types.contains(&s.qname) + { + return false; + } + if ir + .content_models + .get(&s.qname) + .is_some_and(|p| p.has_choice() || p.has_reference() || p.has_repeated_sequence()) + { + return false; + } + s.fields.iter().all(|f| { + if f.nillable + || f.is_cycle_cut + || f.xml_name.contains(':') + || f.type_ref.is_list() + || f.type_ref.is_boxed() + { + return false; + } + match f.kind { + FieldKind::Element if &f.namespace != namespace => return false, + FieldKind::Attribute if f.namespace.is_some() => return false, + FieldKind::Element | FieldKind::Attribute | FieldKind::Text => {} + _ => return false, + } + match &f.type_ref { + TypeRef::Named(q) => match ir.types.get(q) { + Some(TypeDef::Struct(child)) => { + supported(child, ir, namespace, &mut seen.clone()) + } + Some(TypeDef::Enum(_)) => true, + _ => false, + }, + TypeRef::Primitive(_) => true, + _ => false, + } + }) + } + if s.fields.len() != 1 + || s.fields[0].kind != FieldKind::Element + || !s.fields[0].cardinality.is_list() + { + return; + } + let field = &s.fields[0]; + for root in ir.elements.values() { + if root.type_ref != TypeRef::Named(s.qname.clone()) + || root.nillable + || !supported(s, ir, &root.qname.namespace, &mut HashSet::new()) + { + continue; + } + let name = type_ident(&s.qname); + let item_type = self.format_rust_type_ref(&field.type_ref, lifetimes); + let impl_type = if self.options.zero_copy && lifetimes.contains(&s.qname) { + format!("impl<'a> {name}<'a>") + } else { + format!("impl {name}") + }; + let method = format!("write_{}_items", AsSnakeCase(&root.qname.local)); + writeln!(out, "\n{impl_type} {{").unwrap(); + out.push_str(" /// Produce a document incrementally, retaining only the current item.\n /// Errors may leave partial XML in the sink; publish files only after success.\n"); + writeln!(out, " pub fn {method}>(sink: &mut W, items: I) -> Result<()> {{").unwrap(); + out.push_str(" let mut writer = Writer::new(sink);\n"); + writeln!( + out, + " let {}start = BytesStart::new({:?});", + if root.qname.namespace.is_some() { + "mut " + } else { + "" + }, + root.qname.local + ) + .unwrap(); + if let Some(ns) = &root.qname.namespace { + writeln!(out, " start.push_attribute((\"xmlns\", {ns:?}));").unwrap(); + } + out.push_str(" writer.write_event(Event::Start(start))?;\n let mut count = 0usize;\n"); + let mut body = String::new(); + self.emit_element_serialize(&mut body, field, "items", ir); + body = body.replace("for item in &self.items {", "for item in items {"); + let mut checks = String::new(); + if let crate::ir::OccursLimit::Count(max) = field.cardinality.max_occurs { + writeln!(checks, " if count == {max} {{ return Err(PolyXmlError::SchemaError(\"maxOccurs exceeded\".into())); }}").unwrap(); + } + checks.push_str(" count = count.checked_add(1).ok_or_else(|| PolyXmlError::SchemaError(\"item count overflow\".into()))?;\n"); + self.emit_pattern_check(&mut checks, field, "&item.to_string()", ir); + body = body.replacen( + "for item in items {\n", + &format!("for item in items {{\n{checks}"), + 1, + ); + out.push_str(&body.replace("encode_xml(writer,", "encode_xml(&mut writer,")); + if field.cardinality.min_occurs > 0 { + writeln!(out, " if count < {} {{ return Err(PolyXmlError::SchemaError(\"minOccurs not satisfied\".into())); }}", field.cardinality.min_occurs).unwrap(); + } + writeln!(out, " writer.write_event(Event::End(BytesEnd::new({:?})))?;\n Ok(())\n }}\n}}", root.qname.local).unwrap(); + } } fn emit_integer_bounds( diff --git a/crates/polyxml-core/src/schema_parser/upa.rs b/crates/polyxml-core/src/schema_parser/upa.rs index c300891d..4554ec18 100644 --- a/crates/polyxml-core/src/schema_parser/upa.rs +++ b/crates/polyxml-core/src/schema_parser/upa.rs @@ -256,6 +256,24 @@ fn particle(node: &Node, ns: Option<&str>, qualified: bool) -> Option { Some(repeat(node, model)) } impl Document { + /// Whether a named type has plain sequence content in one element namespace. + /// Source QNames retain local `form` and `elementFormDefault` distinctions. + pub(crate) fn uniform_sequence( + &self, + name: &QName, + namespace: &Option, + ) -> Option { + fn uniform(model: &Model, namespace: &Option) -> bool { + match model { + Model::Element(q, _) => &q.namespace == namespace, + Model::Sequence(parts) => parts.iter().all(|p| uniform(p, namespace)), + Model::Repeat(part, _, _) => uniform(part, namespace), + _ => false, + } + } + self.named.get(name).map(|model| uniform(model, namespace)) + } + pub fn parse(xml: &str) -> Result { let mut reader = Reader::from_str(xml); let root = loop { diff --git a/crates/polyxml-core/tests/test_rust_codegen.rs b/crates/polyxml-core/tests/test_rust_codegen.rs index ea112a79..f41b8ebf 100644 --- a/crates/polyxml-core/tests/test_rust_codegen.rs +++ b/crates/polyxml-core/tests/test_rust_codegen.rs @@ -1505,3 +1505,43 @@ assert_eq!(parsed,value);let object:serde_json::Value=serde_json::from_str(&json ); } } + +#[test] +fn incremental_root_writer_is_schema_scoped() { + let xsd = include_str!("../../../tests/fixtures/incremental/items.xsd"); + for zero_copy in [true, false] { + let ir = XsdParser::new().parse_str(xsd).unwrap(); + let code = RustCodegen::new(RustOptions { + zero_copy, + ..Default::default() + }) + .generate_module(&ir); + assert!(code.contains("pub fn write_batch_items")); + assert!(code.contains("for item in items")); + assert!(code.contains("start.push_attribute((\"xmlns\", \"urn:items\"))")); + assert!(code.contains("if count < 1")); + assert!(code.contains("item.encode_xml(&mut writer, Some(\"Item\"))")); + let bounded = xsd.replace("maxOccurs=\"unbounded\"", "maxOccurs=\"2\""); + let ir = XsdParser::new().parse_str(&bounded).unwrap(); + let code = RustCodegen::new(RustOptions::default()).generate_module(&ir); + assert!(code.contains("if count == 2")); + for unsupported in [ + xsd.replace( + "name=\"Item\" type=\"t:ItemType\"", + "name=\"Item\" type=\"t:ItemType\" nillable=\"true\"", + ), + xsd.replace( + "elementFormDefault=\"qualified\"", + "elementFormDefault=\"unqualified\"", + ), + xsd.replace( + "name=\"Batch\" type=\"t:BatchType\"", + "name=\"Batch\" type=\"t:BatchType\" nillable=\"true\"", + ), + ] { + let ir = XsdParser::new().parse_str(&unsupported).unwrap(); + let code = RustCodegen::new(RustOptions::default()).generate_module(&ir); + assert!(!code.contains("pub fn write_batch_items"), "{unsupported}"); + } + } +} diff --git a/tests/fixtures/incremental/items.xsd b/tests/fixtures/incremental/items.xsd new file mode 100644 index 00000000..577a9626 --- /dev/null +++ b/tests/fixtures/incremental/items.xsd @@ -0,0 +1,10 @@ + + + + + + + + + + From 705379c0a4106e085fb97a850e2f25cb7fed0be9 Mon Sep 17 00:00:00 2001 From: Bailey Nguyen Date: Fri, 2 Oct 2026 12:34:01 -0500 Subject: [PATCH 2/3] test(rust): verify lazy XML production and document supported schemas --- crates/polyxml-core/src/codegen/rust/mod.rs | 39 ++- crates/polyxml-core/src/schema_parser/upa.rs | 71 ++++-- .../polyxml-core/tests/test_rust_codegen.rs | 13 +- docs/benchmarks/incremental-writer.md | 32 +++ docs/benchmarks/index.md | 3 + docs/languages/rust.md | 60 +++++ scripts/verify_incremental_writer.py | 237 ++++++++++++++++++ 7 files changed, 420 insertions(+), 35 deletions(-) create mode 100644 docs/benchmarks/incremental-writer.md create mode 100644 scripts/verify_incremental_writer.py diff --git a/crates/polyxml-core/src/codegen/rust/mod.rs b/crates/polyxml-core/src/codegen/rust/mod.rs index ffb04afe..8a67ff5a 100644 --- a/crates/polyxml-core/src/codegen/rust/mod.rs +++ b/crates/polyxml-core/src/codegen/rust/mod.rs @@ -2017,6 +2017,7 @@ impl RustCodegen { return; } let field = &s.fields[0]; + let mut methods = HashSet::new(); for root in ir.elements.values() { if root.type_ref != TypeRef::Named(s.qname.clone()) || root.nillable @@ -2031,11 +2032,17 @@ impl RustCodegen { } else { format!("impl {name}") }; - let method = format!("write_{}_items", AsSnakeCase(&root.qname.local)); + let base_method = format!("write_{}_items", AsSnakeCase(&root.qname.local)); + let mut method = base_method.clone(); + let mut suffix = 2; + while !methods.insert(method.clone()) { + method = format!("{base_method}_{suffix}"); + suffix += 1; + } writeln!(out, "\n{impl_type} {{").unwrap(); out.push_str(" /// Produce a document incrementally, retaining only the current item.\n /// Errors may leave partial XML in the sink; publish files only after success.\n"); writeln!(out, " pub fn {method}>(sink: &mut W, items: I) -> Result<()> {{").unwrap(); - out.push_str(" let mut writer = Writer::new(sink);\n"); + out.push_str(" let writer = &mut Writer::new(sink);\n"); writeln!( out, " let {}start = BytesStart::new({:?});", @@ -2051,21 +2058,13 @@ impl RustCodegen { writeln!(out, " start.push_attribute((\"xmlns\", {ns:?}));").unwrap(); } out.push_str(" writer.write_event(Event::Start(start))?;\n let mut count = 0usize;\n"); - let mut body = String::new(); - self.emit_element_serialize(&mut body, field, "items", ir); - body = body.replace("for item in &self.items {", "for item in items {"); let mut checks = String::new(); if let crate::ir::OccursLimit::Count(max) = field.cardinality.max_occurs { writeln!(checks, " if count == {max} {{ return Err(PolyXmlError::SchemaError(\"maxOccurs exceeded\".into())); }}").unwrap(); } checks.push_str(" count = count.checked_add(1).ok_or_else(|| PolyXmlError::SchemaError(\"item count overflow\".into()))?;\n"); self.emit_pattern_check(&mut checks, field, "&item.to_string()", ir); - body = body.replacen( - "for item in items {\n", - &format!("for item in items {{\n{checks}"), - 1, - ); - out.push_str(&body.replace("encode_xml(writer,", "encode_xml(&mut writer,")); + self.emit_element_serialize_from(out, field, "items", ir, Some("items"), &checks); if field.cardinality.min_occurs > 0 { writeln!(out, " if count < {} {{ return Err(PolyXmlError::SchemaError(\"minOccurs not satisfied\".into())); }}", field.cardinality.min_occurs).unwrap(); } @@ -2537,6 +2536,18 @@ impl RustCodegen { field: &FieldDef, rust_name: &str, ir: &SchemaIR, + ) { + self.emit_element_serialize_from(out, field, rust_name, ir, None, ""); + } + + fn emit_element_serialize_from( + &self, + out: &mut String, + field: &FieldDef, + rust_name: &str, + ir: &SchemaIR, + source: Option<&str>, + before_item: &str, ) { let is_list = field.cardinality.is_list() || field.type_ref.is_list(); let is_optional = field.cardinality.is_optional() || field.nillable; @@ -2546,7 +2557,11 @@ impl RustCodegen { let is_enum = self.field_is_enum(&field.type_ref, ir).is_some(); if is_list { - let _ = writeln!(out, " for item in &self.{} {{", rust_name); + let source = source + .map(str::to_owned) + .unwrap_or_else(|| format!("&self.{rust_name}")); + let _ = writeln!(out, " for item in {source} {{"); + out.push_str(before_item); if is_string { let _ = writeln!( out, diff --git a/crates/polyxml-core/src/schema_parser/upa.rs b/crates/polyxml-core/src/schema_parser/upa.rs index 4554ec18..9339f1eb 100644 --- a/crates/polyxml-core/src/schema_parser/upa.rs +++ b/crates/polyxml-core/src/schema_parser/upa.rs @@ -15,6 +15,8 @@ pub struct Document { groups: BTreeMap, named: BTreeMap, anonymous: Vec<(String, Model)>, + #[serde(default)] + unsupported_attribute_types: BTreeSet, } #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] enum Model { @@ -115,14 +117,7 @@ fn node( Event::Start(child) if matches!( strip_prefix(child.name().into_inner()), - "annotation" - | "simpleType" - | "attribute" - | "attributeGroup" - | "anyAttribute" - | "unique" - | "key" - | "keyref" + "annotation" | "simpleType" | "unique" | "key" | "keyref" ) => { reader.read_to_end(child.name())?; @@ -130,14 +125,7 @@ fn node( Event::Empty(child) if matches!( strip_prefix(child.name().into_inner()), - "annotation" - | "simpleType" - | "attribute" - | "attributeGroup" - | "anyAttribute" - | "unique" - | "key" - | "keyref" + "annotation" | "simpleType" | "unique" | "key" | "keyref" ) => {} Event::Start(child) => { result @@ -267,11 +255,15 @@ impl Document { match model { Model::Element(q, _) => &q.namespace == namespace, Model::Sequence(parts) => parts.iter().all(|p| uniform(p, namespace)), - Model::Repeat(part, _, _) => uniform(part, namespace), + Model::Repeat(part, _, _) if matches!(part.as_ref(), Model::Element(..)) => { + uniform(part, namespace) + } _ => false, } } - self.named.get(name).map(|model| uniform(model, namespace)) + self.named.get(name).map(|model| { + !self.unsupported_attribute_types.contains(name) && uniform(model, namespace) + }) } pub fn parse(xml: &str) -> Result { @@ -291,13 +283,38 @@ impl Document { groups: BTreeMap::new(), named: BTreeMap::new(), anonymous: Vec::new(), + unsupported_attribute_types: BTreeSet::new(), }; - fn collect(node: &Node, document: &mut Document, qualified: bool) { + fn qualified_attributes(node: &Node, default_qualified: bool) -> bool { + matches!(node.tag.as_str(), "attributeGroup" | "anyAttribute") + || (node.tag == "attribute" + && (node.attr("ref").is_some() + || match node.attr("form") { + Some("qualified") => true, + Some("unqualified") => false, + _ => default_qualified, + })) + || node + .children + .iter() + .any(|child| qualified_attributes(child, default_qualified)) + } + fn collect( + node: &Node, + document: &mut Document, + qualified: bool, + attributes_qualified: bool, + ) { let ns = document.namespace.as_deref(); if node.tag == "complexType" { let model = content(node, ns, qualified); if let Some(name) = node.attr("name") { document.named.insert(QName::new(ns, name), model); + if qualified_attributes(node, attributes_qualified) { + document + .unsupported_attribute_types + .insert(QName::new(ns, name)); + } } else { document.anonymous.push(( format!("anonymous complexType at byte {}", node.position), @@ -306,7 +323,7 @@ impl Document { } } for child in &node.children { - collect(child, document, qualified); + collect(child, document, qualified, attributes_qualified); } } for child in &root.children { @@ -319,7 +336,12 @@ impl Document { } } } - collect(&root, &mut document, qualified); + collect( + &root, + &mut document, + qualified, + root.attr("attributeFormDefault") == Some("qualified"), + ); Ok(document) } pub fn adopt_namespace(&mut self, namespace: &str) { @@ -362,6 +384,13 @@ impl Document { _ => {} } } + self.unsupported_attribute_types = std::mem::take(&mut self.unsupported_attribute_types) + .into_iter() + .map(|mut q| { + name(&mut q, namespace); + q + }) + .collect(); for collection in [&mut self.groups, &mut self.named] { *collection = std::mem::take(collection) .into_iter() diff --git a/crates/polyxml-core/tests/test_rust_codegen.rs b/crates/polyxml-core/tests/test_rust_codegen.rs index f41b8ebf..ac6f1dbd 100644 --- a/crates/polyxml-core/tests/test_rust_codegen.rs +++ b/crates/polyxml-core/tests/test_rust_codegen.rs @@ -1520,12 +1520,17 @@ fn incremental_root_writer_is_schema_scoped() { assert!(code.contains("for item in items")); assert!(code.contains("start.push_attribute((\"xmlns\", \"urn:items\"))")); assert!(code.contains("if count < 1")); - assert!(code.contains("item.encode_xml(&mut writer, Some(\"Item\"))")); + assert!(code.contains("item.encode_xml(writer, Some(\"Item\"))")); let bounded = xsd.replace("maxOccurs=\"unbounded\"", "maxOccurs=\"2\""); let ir = XsdParser::new().parse_str(&bounded).unwrap(); let code = RustCodegen::new(RustOptions::default()).generate_module(&ir); assert!(code.contains("if count == 2")); for unsupported in [ + xsd.replace( + "elementFormDefault=\"qualified\"", + "elementFormDefault=\"qualified\" attributeFormDefault=\"qualified\"", + ), + xsd.replace("", ""), xsd.replace( "name=\"Item\" type=\"t:ItemType\"", "name=\"Item\" type=\"t:ItemType\" nillable=\"true\"", @@ -1541,7 +1546,11 @@ fn incremental_root_writer_is_schema_scoped() { ] { let ir = XsdParser::new().parse_str(&unsupported).unwrap(); let code = RustCodegen::new(RustOptions::default()).generate_module(&ir); - assert!(!code.contains("pub fn write_batch_items"), "{unsupported}"); + assert!( + !code.contains("pub fn write_batch_items"), + "{unsupported}\n{:?}", + ir.upa_documents + ); } } } diff --git a/docs/benchmarks/incremental-writer.md b/docs/benchmarks/incremental-writer.md new file mode 100644 index 00000000..71849841 --- /dev/null +++ b/docs/benchmarks/incremental-writer.md @@ -0,0 +1,32 @@ +--- +title: Incremental Rust writer memory +description: Peak resident memory of generated Rust iterator producers with independent XSD validation. +--- + +# Incremental Rust writer memory + +The generated producer consumes items lazily and writes directly to a buffered +file. It does not construct a root model, retain a child `Vec`, or return XML +bytes. Memory depends on the current item and the caller's sink buffering. + +The [runner](https://github.com/polyxml/PolyXML/blob/main/scripts/verify_incremental_writer.py) +compiles and executes both borrowed (`Cow`) and owned (`String`) models from +[the fixture](https://github.com/polyxml/PolyXML/blob/main/tests/fixtures/incremental/items.xsd). +Each sample launches a fresh debug executable under GNU `time` (`%M`, KiB), +then validates its entire output with libxml2 through lxml's streaming XSD +validator. Validation runs in the parent process and is excluded from producer +RSS. Runtime assertions exercise minimum/maximum occurrences, empty optional +collections, sink failures, lazy iterator consumption, and unqualified output. + +Reproduce from the repository root with lxml installed in the local environment: + +```bash +cargo build -p polyxml-cli +.venv/bin/python scripts/verify_incremental_writer.py \ + --output target/incremental-rss.json +``` + +The default run measures three fresh processes per model representation at +1,000, 100,000, and 1,000,000 items. This measures peak RSS, not throughput. +A sink that retains the complete output, or an iterator that retains its +inputs, will still grow with document size. diff --git a/docs/benchmarks/index.md b/docs/benchmarks/index.md index 5420e13a..9097386a 100644 --- a/docs/benchmarks/index.md +++ b/docs/benchmarks/index.md @@ -195,3 +195,6 @@ For publishable measurements, run on an otherwise idle host, repeat the full suite, and keep raw output with the Git revision, CPU/OS, and toolchain versions. Compare read with read and write with write on the same payload; generated models, DOM parsers, and transcoding pipelines can return different values. + +- [Incremental Rust writer memory](incremental-writer.md): generated iterator + producers writing increasing item counts to files, with independent XSD validation. diff --git a/docs/languages/rust.md b/docs/languages/rust.md index d84b46af..2c68ef5b 100644 --- a/docs/languages/rust.md +++ b/docs/languages/rust.md @@ -428,3 +428,63 @@ must outlive their input, generate with `--zero-copy=false` and omit `<'_>` from example's type annotation. JSON helpers such as `from_json_str` and `to_json_vec` are conveniences over the same Serde operations. Disabling XML codecs with `--codecs=false` retains the Serde derives. + +## Incremental item production + +`model.encode_xml(&mut Writer, tag_name)` writes an existing model to a +sink, but its repeated fields are already materialized in `Vec`s. `to_xml()` +also retains the final document bytes. For a root containing one repeated +element in a plain sequence, generation additionally exposes an associated +`write__items(&mut sink, iterator)` function. It needs neither a root +instance nor a collected child list. + +For example, this schema declares a required sequence of items: + +```xml + + + + + + + + + + + +``` + +Generate Rust models using `polyxml generate items.xsd --lang rust --out src`. +The lazy iterator below creates one item at a time: + +```rust +use items::{Batch, ItemType}; +use std::io::{BufWriter, Write}; + +let mut sink = BufWriter::new(std::fs::File::create("batch.xml")?); +let items = (0..1_000_000).map(|i| ItemType { + name: format!("Item {i}").into(), +}); +Batch::write_batch_items(&mut sink, items)?; +sink.flush()?; +``` + +The producer writes the declared root name and namespace, escapes item text, +and preserves each item's field order through its generated codec. It checks +`minOccurs` at iterator exhaustion and `maxOccurs` before writing an excess +item. An excess item is consumed to detect the error. Sink errors propagate; +a failed call may leave partial XML. The caller owns buffering, flushing, and +publishing the output file. + +This API is generated for named, concrete sequence types with exactly one +repeated child, no root attributes, and element namespaces matching the root +throughout the item types. Unqualified documents and uniformly qualified +documents with local unqualified attributes are supported. Choices, repeated +or optional groups, inheritance, references, wildcards, recursion, nillable +values, named simple restrictions, and qualified attributes are excluded. +Existing model codecs remain available for those schemas. The producer checks +the root collection's occurrence bounds; it is not a complete per-item XSD +validator. Python iterable production is not exposed by this API. + +See the [memory study](../benchmarks/incremental-writer.md) for the compiled +example, independent XSD validation, and reproducible peak RSS measurements. diff --git a/scripts/verify_incremental_writer.py b/scripts/verify_incremental_writer.py new file mode 100644 index 00000000..860c3f18 --- /dev/null +++ b/scripts/verify_incremental_writer.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""Compile/run incremental Rust producers and validate output with libxml2. + +Run: uv run --with lxml python scripts/verify_incremental_writer.py +Requires the built debug CLI and GNU time. Each RSS sample is a fresh process. +An existing .venv with lxml can run this script directly. +""" + +import argparse +import json +import platform +import re +import subprocess +import tempfile +from pathlib import Path + +from lxml import etree + +REPO = Path(__file__).resolve().parents[1] +SCHEMA = REPO / "tests/fixtures/incremental/items.xsd" + + +def run(args, cwd=REPO): + result = subprocess.run( + list(map(str, args)), cwd=cwd, capture_output=True, text=True, check=False + ) + if result.returncode: + raise RuntimeError( + f"{' '.join(map(str, args))}\n{result.stdout}\n{result.stderr}" + ) + return result.stdout + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output", type=Path, default=REPO / "target/incremental-rss.json" + ) + parser.add_argument( + "--counts", type=int, nargs="+", default=[1000, 100000, 1000000] + ) + parser.add_argument("--repeats", type=int, default=3) + args = parser.parse_args() + report = { + "commit": run(["git", "rev-parse", "HEAD"]).strip(), + "working_tree": run(["git", "diff", "--stat"]), + "platform": platform.platform(), + "cpu": next( + ( + line.split(":", 1)[1].strip() + for line in Path("/proc/cpuinfo").read_text().splitlines() + if line.startswith("model name") + ), + platform.processor(), + ), + "rustc": run(["rustc", "--version"]).strip(), + "profile": "debug (RSS only; no throughput claim)", + "schema": str(SCHEMA.relative_to(REPO)), + "samples": [], + } + with tempfile.TemporaryDirectory(prefix="polyxml-incremental-") as temporary: + folder = Path(temporary) + for owned in [False, True]: + project = folder / str(owned) + source = project / "src" + source.mkdir(parents=True) + for module, xsd in [ + ( + "scalar", + '', + ), + ("items", SCHEMA.read_text()), + ( + "bounded", + SCHEMA.read_text().replace( + 'maxOccurs="unbounded"', 'maxOccurs="2"' + ), + ), + ( + "optional", + SCHEMA.read_text().replace('minOccurs="1"', 'minOccurs="0"'), + ), + ( + "plain", + SCHEMA.read_text() + .replace( + ' xmlns:t="urn:items" targetNamespace="urn:items" elementFormDefault="qualified"', + "", + ) + .replace("t:", ""), + ), + ]: + schema_file = project / f"{module}.xsd" + schema_file.write_text(xsd) + run( + [ + REPO / "target/debug/polyxml", + "generate", + schema_file, + "--lang", + "rust", + "--out", + source, + "--zero-copy", + str(not owned).lower(), + ] + ) + cargo = (REPO / "crates/polyxml-core/Cargo.toml").read_text() + version = re.search( + r'quick-xml\s*=\s*(?:\{\s*version\s*=\s*)?"([^"]+)"', cargo + ).group(1) + (project / "Cargo.toml").write_text(f'''[package] +name="incremental-producer-{str(owned).lower()}" +version="0.0.0" +edition="2021" +[dependencies] +polyxml={{path={json.dumps(str(REPO / "crates/polyxml-core"))}}} +quick-xml="{version}" +serde={{version="1",features=["derive"]}} +serde_json="1" +''') + ( + source / "main.rs" + ).write_text("""mod items; mod bounded; mod optional; mod plain; mod scalar; +use std::io::{BufWriter, Write}; +fn main() { + // Both lower and upper occurrence limits must fail at runtime. + assert!(items::Batch::write_batch_items(&mut std::io::sink(), std::iter::empty()).is_err()); + assert!(bounded::Batch::write_batch_items(&mut std::io::sink(), (0..3).map(|id| bounded::ItemType { name: "one".into(), quantity: 1, id })).is_err()); + optional::Batch::write_batch_items(&mut std::io::sink(), std::iter::empty()).unwrap(); + let mut scalar_xml = Vec::new(); + scalar::Numbers::write_numbers_items(&mut scalar_xml, -1..2).unwrap(); + std::fs::write("scalar.xml", scalar_xml).unwrap(); + let mut plain_xml = Vec::new(); + plain::Batch::write_batch_items(&mut plain_xml, [plain::ItemType { name: "plain".into(), quantity: 1, id: 0 }]).unwrap(); + std::fs::write("plain.xml", plain_xml).unwrap(); + struct Broken; + impl Write for Broken { + fn write(&mut self, _: &[u8]) -> std::io::Result { Err(std::io::Error::other("broken sink")) } + fn flush(&mut self) -> std::io::Result<()> { Ok(()) } + } + let mut consumed = 0; + assert!(items::Batch::write_batch_items(&mut Broken, (0..3).map(|id| { consumed += 1; items::ItemType { name: "one".into(), quantity: 1, id } })).is_err()); + assert_eq!(consumed, 0); // Opening write fails before asking for an item. + struct Limited { remaining: usize } + impl Write for Limited { + fn write(&mut self, data: &[u8]) -> std::io::Result { + if self.remaining == 0 { return Err(std::io::Error::other("full sink")); } + let n = self.remaining.min(data.len()); self.remaining -= n; Ok(n) + } + fn flush(&mut self) -> std::io::Result<()> { Ok(()) } + } + let mut produced_before_failure = 0; + assert!(items::Batch::write_batch_items(&mut Limited { remaining: 100 }, (0..10).map(|id| { + produced_before_failure += 1; + items::ItemType { name: "A & ".into(), quantity: 7, id } + })).is_err()); + assert!((1..10).contains(&produced_before_failure)); + let count: i32 = std::env::args().nth(1).unwrap().parse().unwrap(); + struct Observed { inner: W, bytes: std::rc::Rc> } + impl Write for Observed { + fn write(&mut self, data: &[u8]) -> std::io::Result { + let n = self.inner.write(data)?; + self.bytes.set(self.bytes.get() + n); + Ok(n) + } + fn flush(&mut self) -> std::io::Result<()> { self.inner.flush() } + } + let bytes = std::rc::Rc::new(std::cell::Cell::new(0)); + let mut file = Observed { inner: BufWriter::new(std::fs::File::create("output.xml").unwrap()), bytes: bytes.clone() }; + let mut produced = 0; + let mut previous_bytes = 0; + items::Batch::write_batch_items(&mut file, (0..count).map(|id| { + assert!(bytes.get() > previous_bytes, "previous item must be written before requesting the next"); + previous_bytes = bytes.get(); + produced += 1; + items::ItemType { name: "A & ".into(), quantity: 7, id } + })).unwrap(); + assert_eq!(produced, count); + file.flush().unwrap(); +} +""") + target = REPO / "target/incremental-writer" + run(["cargo", "build", "--offline", "--target-dir", target], project) + binary = target / "debug" / f"incremental-producer-{str(owned).lower()}" + validator = etree.XMLSchema(etree.parse(str(SCHEMA))) + scalar_validator = etree.XMLSchema(etree.parse(str(project / "scalar.xsd"))) + plain_validator = etree.XMLSchema(etree.parse(str(project / "plain.xsd"))) + for count in args.counts: + for repeat in range(args.repeats): + run( + [ + "/usr/bin/time", + "-f", + "%M", + "-o", + project / "rss.txt", + binary, + count, + ], + project, + ) + observed = 0 + for _, element in etree.iterparse( + str(project / "output.xml"), + events=("end",), + tag="{urn:items}Item", + schema=validator, + ): + assert element.findtext("{urn:items}Name") == "A & " + assert element.findtext("{urn:items}Quantity") == "7" + assert int(element.get("id")) == observed + observed += 1 + element.clear() + while element.getprevious() is not None: + del element.getparent()[0] + assert observed == count + scalar_validator.assertValid( + etree.parse(str(project / "scalar.xml")) + ) + plain_validator.assertValid(etree.parse(str(project / "plain.xml"))) + sample = { + "owned": owned, + "items": count, + "repeat": repeat, + "peak_rss_kib": int((project / "rss.txt").read_text()), + "output_bytes": (project / "output.xml").stat().st_size, + "xsd_valid": True, + } + report["samples"].append(sample) + print(json.dumps(sample), flush=True) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n") + + +if __name__ == "__main__": + main() From 40d00d898dc6d0f60d5fa9c8970b2f90c7de3cb6 Mon Sep 17 00:00:00 2001 From: Bailey Nguyen Date: Fri, 2 Oct 2026 14:17:45 -0500 Subject: [PATCH 3/3] docs(benchmarks): record bounded-memory incremental writer results --- .../data/incremental-writer-2026-10-02.json | 155 ++++++++++++++++++ docs/benchmarks/incremental-writer.md | 24 +++ 2 files changed, 179 insertions(+) create mode 100644 docs/benchmarks/data/incremental-writer-2026-10-02.json diff --git a/docs/benchmarks/data/incremental-writer-2026-10-02.json b/docs/benchmarks/data/incremental-writer-2026-10-02.json new file mode 100644 index 00000000..11fcc35d --- /dev/null +++ b/docs/benchmarks/data/incremental-writer-2026-10-02.json @@ -0,0 +1,155 @@ +{ + "commit": "b513361468ab143269495a29346a61e80edd84f0", + "working_tree": "", + "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43", + "cpu": "AMD Ryzen 5 4500 6-Core Processor", + "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)", + "profile": "debug (RSS only; no throughput claim)", + "schema": "tests/fixtures/incremental/items.xsd", + "samples": [ + { + "owned": false, + "items": 1000, + "repeat": 0, + "peak_rss_kib": 2604, + "output_bytes": 73923, + "xsd_valid": true + }, + { + "owned": false, + "items": 1000, + "repeat": 1, + "peak_rss_kib": 2628, + "output_bytes": 73923, + "xsd_valid": true + }, + { + "owned": false, + "items": 1000, + "repeat": 2, + "peak_rss_kib": 2576, + "output_bytes": 73923, + "xsd_valid": true + }, + { + "owned": false, + "items": 100000, + "repeat": 0, + "peak_rss_kib": 2528, + "output_bytes": 7588923, + "xsd_valid": true + }, + { + "owned": false, + "items": 100000, + "repeat": 1, + "peak_rss_kib": 2604, + "output_bytes": 7588923, + "xsd_valid": true + }, + { + "owned": false, + "items": 100000, + "repeat": 2, + "peak_rss_kib": 2564, + "output_bytes": 7588923, + "xsd_valid": true + }, + { + "owned": false, + "items": 1000000, + "repeat": 0, + "peak_rss_kib": 2436, + "output_bytes": 76888923, + "xsd_valid": true + }, + { + "owned": false, + "items": 1000000, + "repeat": 1, + "peak_rss_kib": 2436, + "output_bytes": 76888923, + "xsd_valid": true + }, + { + "owned": false, + "items": 1000000, + "repeat": 2, + "peak_rss_kib": 2564, + "output_bytes": 76888923, + "xsd_valid": true + }, + { + "owned": true, + "items": 1000, + "repeat": 0, + "peak_rss_kib": 2564, + "output_bytes": 73923, + "xsd_valid": true + }, + { + "owned": true, + "items": 1000, + "repeat": 1, + "peak_rss_kib": 2592, + "output_bytes": 73923, + "xsd_valid": true + }, + { + "owned": true, + "items": 1000, + "repeat": 2, + "peak_rss_kib": 2552, + "output_bytes": 73923, + "xsd_valid": true + }, + { + "owned": true, + "items": 100000, + "repeat": 0, + "peak_rss_kib": 2560, + "output_bytes": 7588923, + "xsd_valid": true + }, + { + "owned": true, + "items": 100000, + "repeat": 1, + "peak_rss_kib": 2612, + "output_bytes": 7588923, + "xsd_valid": true + }, + { + "owned": true, + "items": 100000, + "repeat": 2, + "peak_rss_kib": 2564, + "output_bytes": 7588923, + "xsd_valid": true + }, + { + "owned": true, + "items": 1000000, + "repeat": 0, + "peak_rss_kib": 2500, + "output_bytes": 76888923, + "xsd_valid": true + }, + { + "owned": true, + "items": 1000000, + "repeat": 1, + "peak_rss_kib": 2692, + "output_bytes": 76888923, + "xsd_valid": true + }, + { + "owned": true, + "items": 1000000, + "repeat": 2, + "peak_rss_kib": 2552, + "output_bytes": 76888923, + "xsd_valid": true + } + ] +} diff --git a/docs/benchmarks/incremental-writer.md b/docs/benchmarks/incremental-writer.md index 71849841..32ef19ae 100644 --- a/docs/benchmarks/incremental-writer.md +++ b/docs/benchmarks/incremental-writer.md @@ -30,3 +30,27 @@ The default run measures three fresh processes per model representation at 1,000, 100,000, and 1,000,000 items. This measures peak RSS, not throughput. A sink that retains the complete output, or an iterator that retains its inputs, will still grow with document size. + +## Recorded run: October 2, 2026 + +[Raw samples](data/incremental-writer-2026-10-02.json) record source commit, +CPU/OS, compiler version, byte lengths, and all measurements. The run used +`rustc 1.98.1 (48a229cea 2026-09-01)` on AMD Ryzen 5 4500 6-Core Processor, with no concurrent repository test/build +jobs. Three fresh processes were measured for each row; all 18 output documents +passed independent XSD validation. + +| Item strings | Items | Median peak RSS (KiB) | Range (KiB) | XML bytes | +| :--- | ---: | ---: | ---: | ---: | +| Cow | 1,000 | 2,604 | 2,576–2,628 | 73,923 | +| Cow | 100,000 | 2,564 | 2,528–2,604 | 7,588,923 | +| Cow | 1,000,000 | 2,436 | 2,436–2,564 | 76,888,923 | +| String | 1,000 | 2,564 | 2,552–2,592 | 73,923 | +| String | 100,000 | 2,564 | 2,560–2,612 | 7,588,923 | +| String | 1,000,000 | 2,552 | 2,500–2,692 | 76,888,923 | + +The output grew from 73,923 to 76,888,923 bytes while median producer RSS +remained around 2.5 MiB in both representations. This fixture supports bounded +producer memory as the item count increases; it does not establish a universal +memory ceiling or a throughput result. Per-item payloads and sink buffering +still determine the working set. The runner retains tiny fixed-size regression +outputs in memory, independently of the measured collection size.