diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5582928..ff46867 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -6,6 +6,7 @@ on: - '**/*.rs' - 'Cargo.toml' - 'Cargo.lock' + - 'crates/aster_forge_xml/fuzz/Cargo.toml' - 'templates/aster-service/**' - 'scripts/coverage-summary.mjs' - '.github/workflows/rust.yml' @@ -15,6 +16,7 @@ on: - '**/*.rs' - 'Cargo.toml' - 'Cargo.lock' + - 'crates/aster_forge_xml/fuzz/Cargo.toml' - 'templates/aster-service/**' - 'scripts/coverage-summary.mjs' - '.github/workflows/rust.yml' @@ -48,6 +50,12 @@ jobs: - name: Build run: cargo build --verbose + - name: Install nightly Rust toolchain for fuzzing + run: rustup toolchain install nightly --profile minimal + + - name: Build XML fuzz harness + run: cargo +nightly build --manifest-path crates/aster_forge_xml/fuzz/Cargo.toml --bins + - name: Format check run: cargo fmt --all -- --check diff --git a/Cargo.lock b/Cargo.lock index 7a4d853..39baa20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -801,7 +801,6 @@ dependencies = [ "httpdate", "ipnet", "md-5", - "quick-xml", "rand 0.10.2", "serde_json", "thiserror 2.0.19", @@ -819,6 +818,19 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "aster_forge_xml" +version = "0.1.0" +dependencies = [ + "aster_forge_utils", + "criterion", + "libc", + "proptest", + "quick-xml", + "roxmltree", + "xmltree", +] + [[package]] name = "astral-tokio-tar" version = "0.6.4" @@ -1013,6 +1025,21 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "2.13.1" @@ -3740,6 +3767,25 @@ dependencies = [ "thiserror 2.0.19", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "num-traits", + "rand 0.9.5", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "prost" version = "0.14.4" @@ -3807,6 +3853,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quick-xml" version = "0.41.0" @@ -3991,6 +4043,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "raw-cpuid" version = "11.6.0" @@ -4255,6 +4316,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rsa" version = "0.9.10" @@ -4403,6 +4473,18 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.23" @@ -5296,6 +5378,19 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "testcontainers" version = "0.27.3" @@ -5752,6 +5847,12 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -5903,6 +6004,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -6311,6 +6421,21 @@ dependencies = [ "rustix", ] +[[package]] +name = "xml" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" + +[[package]] +name = "xmltree" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc04313cab124e498ab1724e739720807b6dc405b9ed0edc5860164d2e4ff70" +dependencies = [ + "xml", +] + [[package]] name = "xxhash-rust" version = "0.8.17" diff --git a/Cargo.toml b/Cargo.toml index 972034e..e3fbb6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "crates/aster_forge_test", "crates/aster_forge_utils", "crates/aster_forge_validation", + "crates/aster_forge_xml", ] resolver = "3" diff --git a/README.md b/README.md index 2382e81..a498519 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ All crate names use the `aster_forge_*` prefix. The workspace targets Rust `1.94 | Web and API | [`aster_forge_api`](https://forge.astercosm.com/crates/aster_forge_api), [`aster_forge_api_docs_macros`](https://forge.astercosm.com/crates/aster_forge_api_docs_macros), [`aster_forge_actix_middleware`](https://forge.astercosm.com/crates/aster_forge_actix_middleware), [`aster_forge_actix_observability`](https://forge.astercosm.com/crates/aster_forge_actix_observability), [`aster_forge_external_auth`](https://forge.astercosm.com/crates/aster_forge_external_auth) | | Data, coordination, and background work | [`aster_forge_db`](https://forge.astercosm.com/crates/aster_forge_db), [`aster_forge_cache`](https://forge.astercosm.com/crates/aster_forge_cache), [`aster_forge_tasks`](https://forge.astercosm.com/crates/aster_forge_tasks), [`aster_forge_mail`](https://forge.astercosm.com/crates/aster_forge_mail), [`aster_forge_audit`](https://forge.astercosm.com/crates/aster_forge_audit) | | Storage and domain-neutral helpers | [`aster_forge_storage_core`](https://forge.astercosm.com/crates/aster_forge_storage_core), [`aster_forge_file_classification`](https://forge.astercosm.com/crates/aster_forge_file_classification) | -| Utilities | [`aster_forge_crypto`](https://forge.astercosm.com/crates/aster_forge_crypto), [`aster_forge_utils`](https://forge.astercosm.com/crates/aster_forge_utils), [`aster_forge_validation`](https://forge.astercosm.com/crates/aster_forge_validation) | +| Utilities | [`aster_forge_crypto`](https://forge.astercosm.com/crates/aster_forge_crypto), [`aster_forge_utils`](https://forge.astercosm.com/crates/aster_forge_utils), [`aster_forge_validation`](https://forge.astercosm.com/crates/aster_forge_validation), [`aster_forge_xml`](https://forge.astercosm.com/crates/aster_forge_xml) | ## Integration rules diff --git a/crates/aster_forge_utils/Cargo.toml b/crates/aster_forge_utils/Cargo.toml index 8054bfa..2763600 100644 --- a/crates/aster_forge_utils/Cargo.toml +++ b/crates/aster_forge_utils/Cargo.toml @@ -13,7 +13,6 @@ http.workspace = true httpdate.workspace = true ipnet.workspace = true md-5.workspace = true -quick-xml.workspace = true serde_json.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["fs", "time"] } diff --git a/crates/aster_forge_utils/src/lib.rs b/crates/aster_forge_utils/src/lib.rs index 51f53a5..d249bcc 100644 --- a/crates/aster_forge_utils/src/lib.rs +++ b/crates/aster_forge_utils/src/lib.rs @@ -30,7 +30,6 @@ pub mod paths; pub mod raii; pub mod text; pub mod url; -pub mod xml; /// Result type returned by utility helpers. pub type Result = std::result::Result; diff --git a/crates/aster_forge_utils/src/xml.rs b/crates/aster_forge_utils/src/xml.rs deleted file mode 100644 index acfa653..0000000 --- a/crates/aster_forge_utils/src/xml.rs +++ /dev/null @@ -1,270 +0,0 @@ -//! Bounded XML input validation helpers. -//! -//! The helpers intentionally validate XML as an event stream instead of constructing a DOM. -//! Callers that need to retain a document tree must validate the untrusted bytes first, then pass -//! only accepted input to their domain-specific parser. - -use quick_xml::Reader; -use quick_xml::events::Event; - -/// Default maximum allowed nesting depth for untrusted XML documents. -pub const DEFAULT_XML_MAX_DEPTH: usize = 128; - -/// Validation policy for untrusted XML input. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct XmlSafetyPolicy { - /// Maximum number of simultaneously open elements. - pub max_depth: usize, - /// Whether document type declarations must be rejected. - pub reject_doctype: bool, -} - -impl XmlSafetyPolicy { - /// A conservative policy suitable for externally supplied XML. - pub const fn untrusted() -> Self { - Self { - max_depth: DEFAULT_XML_MAX_DEPTH, - reject_doctype: true, - } - } -} - -/// Failure produced while validating XML input. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -pub enum XmlSafetyError { - /// The supplied policy cannot validate any XML element. - #[error("XML maximum depth must be positive")] - InvalidPolicy, - /// The input declares a document type or entity while declarations are prohibited. - #[error("XML external entity declarations are not allowed")] - ExternalEntity, - /// The input exceeds the configured simultaneous element depth. - #[error("XML nesting depth exceeds the configured limit")] - TooDeep, - /// The input is not a complete, single-root XML document. - #[error("malformed XML input")] - Malformed, -} - -/// Validates XML without recursively constructing a document tree. -/// -/// With `policy.reject_doctype`, a byte-level pre-scan rejects any -/// `]]>`) are -/// rejected too. That direction is fail-safe — no real declaration can slip -/// through — and products that must accept such content should use a -/// different parsing channel. -pub fn validate_xml_input( - bytes: &[u8], - policy: XmlSafetyPolicy, -) -> std::result::Result<(), XmlSafetyError> { - if policy.max_depth == 0 { - return Err(XmlSafetyError::InvalidPolicy); - } - if policy.reject_doctype && contains_external_entity_declaration(bytes) { - return Err(XmlSafetyError::ExternalEntity); - } - - let mut reader = Reader::from_reader(bytes); - let mut buffer = Vec::new(); - let mut depth = 0usize; - let mut root_count = 0usize; - - loop { - let event = reader - .read_event_into(&mut buffer) - .map_err(|_| XmlSafetyError::Malformed)?; - match event { - Event::Start(_) => { - if depth == 0 { - root_count += 1; - if root_count > 1 { - return Err(XmlSafetyError::Malformed); - } - } - depth = depth.checked_add(1).ok_or(XmlSafetyError::TooDeep)?; - if depth > policy.max_depth { - return Err(XmlSafetyError::TooDeep); - } - } - Event::Empty(_) => { - if depth == 0 { - root_count += 1; - if root_count > 1 { - return Err(XmlSafetyError::Malformed); - } - } - } - Event::End(_) => { - depth = depth.checked_sub(1).ok_or(XmlSafetyError::Malformed)?; - } - Event::DocType(_) if policy.reject_doctype => { - return Err(XmlSafetyError::ExternalEntity); - } - Event::Eof => { - if depth != 0 || root_count != 1 { - return Err(XmlSafetyError::Malformed); - } - return Ok(()); - } - _ => {} - } - buffer.clear(); - } -} - -/// Returns the local name of the document root after applying the safety policy. -pub fn xml_root_local_name( - bytes: &[u8], - policy: XmlSafetyPolicy, -) -> std::result::Result { - validate_xml_input(bytes, policy)?; - - let mut reader = Reader::from_reader(bytes); - let mut buffer = Vec::new(); - loop { - match reader - .read_event_into(&mut buffer) - .map_err(|_| XmlSafetyError::Malformed)? - { - Event::Start(element) | Event::Empty(element) => { - return std::str::from_utf8(element.local_name().as_ref()) - .map(str::to_owned) - .map_err(|_| XmlSafetyError::Malformed); - } - Event::Eof => return Err(XmlSafetyError::Malformed), - _ => buffer.clear(), - } - } -} - -fn contains_external_entity_declaration(bytes: &[u8]) -> bool { - let mut index = 0; - while let Some(offset) = bytes[index..].iter().position(|byte| *byte == b'<') { - index += offset + 1; - let Some(marker) = bytes.get(index) else { - break; - }; - let Some(after_bang) = bytes.get(index + 1..) else { - break; - }; - if *marker == b'!' - && (after_bang.len() >= b"DOCTYPE".len() - && after_bang[..b"DOCTYPE".len()].eq_ignore_ascii_case(b"DOCTYPE") - || after_bang.len() >= b"ENTITY".len() - && after_bang[..b"ENTITY".len()].eq_ignore_ascii_case(b"ENTITY")) - { - return true; - } - } - false -} - -#[cfg(test)] -mod tests { - use super::{ - DEFAULT_XML_MAX_DEPTH, XmlSafetyError, XmlSafetyPolicy, validate_xml_input, - xml_root_local_name, - }; - - #[test] - fn accepts_regular_xml_and_exact_depth_limit() { - assert!(validate_xml_input(b"", XmlSafetyPolicy::untrusted()).is_ok()); - - let mut xml = String::new(); - for _ in 0..DEFAULT_XML_MAX_DEPTH { - xml.push_str(""); - } - for _ in 0..DEFAULT_XML_MAX_DEPTH { - xml.push_str(""); - } - assert!(validate_xml_input(xml.as_bytes(), XmlSafetyPolicy::untrusted()).is_ok()); - } - - #[test] - fn rejects_doctype_and_excessive_depth() { - assert_eq!( - validate_xml_input(b"", XmlSafetyPolicy::untrusted()), - Err(XmlSafetyError::ExternalEntity) - ); - assert_eq!( - validate_xml_input( - b"", - XmlSafetyPolicy::untrusted() - ), - Err(XmlSafetyError::ExternalEntity) - ); - assert_eq!( - validate_xml_input(b"", XmlSafetyPolicy::untrusted()), - Err(XmlSafetyError::ExternalEntity) - ); - - let mut xml = String::new(); - for _ in 0..=DEFAULT_XML_MAX_DEPTH { - xml.push_str(""); - } - for _ in 0..=DEFAULT_XML_MAX_DEPTH { - xml.push_str(""); - } - assert!(validate_xml_input(xml.as_bytes(), XmlSafetyPolicy::untrusted()).is_err()); - } - - #[test] - fn rejects_empty_and_malformed_documents() { - for xml in [ - b"".as_slice(), - b"", - b"", - b"", - b"", - b"<", - ] { - assert!(validate_xml_input(xml, XmlSafetyPolicy::untrusted()).is_err()); - } - } - - #[test] - fn rejects_invalid_policy_and_can_allow_doctype_when_requested() { - assert!( - validate_xml_input( - b"", - XmlSafetyPolicy { - max_depth: 0, - reject_doctype: true, - } - ) - .is_err() - ); - - assert!( - validate_xml_input( - b"", - XmlSafetyPolicy { - max_depth: DEFAULT_XML_MAX_DEPTH, - reject_doctype: false, - } - ) - .is_ok() - ); - } - - #[test] - fn reads_prefixed_and_empty_root_local_name_after_validation() { - assert_eq!( - xml_root_local_name( - br#""#, - XmlSafetyPolicy::untrusted(), - ), - Ok("version-tree".to_string()) - ); - assert_eq!( - xml_root_local_name(b"", XmlSafetyPolicy::untrusted()), - Ok("root".to_string()) - ); - assert_eq!( - xml_root_local_name(b"", XmlSafetyPolicy::untrusted()), - Err(XmlSafetyError::Malformed) - ); - } -} diff --git a/crates/aster_forge_xml/Cargo.toml b/crates/aster_forge_xml/Cargo.toml new file mode 100644 index 0000000..fc06a9e --- /dev/null +++ b/crates/aster_forge_xml/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "aster_forge_xml" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +publish = false + +[dependencies] +aster_forge_utils = { path = "../aster_forge_utils" } +quick-xml = { workspace = true } + +[dev-dependencies] +criterion = { workspace = true } +libc = "0.2" +proptest = "1.11.0" +roxmltree = "0.21.1" +xmltree = "0.12.0" + +[[bench]] +name = "xml_bench" +harness = false + +[[bench]] +name = "xml_memory" +harness = false diff --git a/crates/aster_forge_xml/benches/support/mod.rs b/crates/aster_forge_xml/benches/support/mod.rs new file mode 100644 index 0000000..f18c213 --- /dev/null +++ b/crates/aster_forge_xml/benches/support/mod.rs @@ -0,0 +1,371 @@ +use std::io::{Read, Write}; + +use aster_forge_xml::{XmlSafetyPolicy, XmlStreamEvent, XmlStreamReader, XmlStreamWriter}; +use quick_xml::Reader; +use quick_xml::XmlVersion; +use quick_xml::escape::unescape; +use quick_xml::events::{BytesEnd, BytesStart, BytesText, Event}; +use quick_xml::name::ResolveResult; +use quick_xml::reader::NsReader; +use quick_xml::writer::Writer; + +pub(crate) fn cpu_fixtures() -> Vec<(&'static str, Vec)> { + // The 10,000-response fixture is reserved for one-shot heap/RSS probes. Generating it here + // would add about 2.5 MiB of unused setup to every multi-sample Criterion benchmark. + vec![ + ( + "propfind", + br#""# + .to_vec(), + ), + ("wopi", wopi_discovery(250).into_bytes()), + ("multistatus_1000", multistatus(1_000).into_bytes()), + ] +} + +#[allow(dead_code)] // This shared module is also compiled into the CPU-only bench target. +pub(crate) fn memory_fixtures() -> Vec<(&'static str, Vec)> { + let mut fixtures = cpu_fixtures(); + // This fixture remains below the default 10 MiB input, 100,000-element, one-million-event, + // and 64 MiB writer limits; it is retained here specifically for large-document memory data. + fixtures.push(("multistatus_10000", multistatus(10_000).into_bytes())); + fixtures +} + +pub(crate) fn wopi_discovery(actions: usize) -> String { + let mut xml = + String::from(""); + for index in 0..actions { + xml.push_str(&format!( + "" + )); + } + xml.push_str(""); + xml +} + +pub(crate) fn multistatus(responses: usize) -> String { + let mut xml = String::from(""); + for index in 0..responses { + xml.push_str(&format!( + "/files/{index}file-{index}{}"etag-{index}"HTTP/1.1 200 OK", + index * 1024 + )); + } + xml.push_str(""); + xml +} + +pub(crate) fn walk_quick_xml_events(input: &[u8]) -> usize { + let mut reader = Reader::from_reader(input); + let mut checksum = 0usize; + loop { + match reader.read_event().expect("benchmark fixture is valid XML") { + Event::Start(start) | Event::Empty(start) => { + checksum = checksum.wrapping_add(start.name().as_ref().len()); + for attribute in start.attributes() { + let attribute = attribute.expect("benchmark attributes are valid"); + checksum = checksum + .wrapping_add(attribute.key.as_ref().len()) + .wrapping_add(attribute.value.as_ref().len()); + } + } + Event::Text(text) | Event::Comment(text) => { + checksum = checksum.wrapping_add(text.as_ref().len()); + } + Event::CData(text) => checksum = checksum.wrapping_add(text.as_ref().len()), + Event::Eof => break, + _ => {} + } + } + checksum +} + +pub(crate) fn walk_quick_xml_ns_buffered(input: &[u8]) -> usize { + let mut reader = NsReader::from_reader(input.take(u64::MAX)); + let mut buffer = Vec::new(); + let mut checksum = 0usize; + loop { + buffer.clear(); + let event = reader + .read_event_into(&mut buffer) + .expect("benchmark fixture is valid XML"); + match event { + Event::Start(start) | Event::Empty(start) => { + checksum = checksum + .wrapping_add(start.name().as_ref().len()) + .wrapping_add(namespace_len( + reader.resolver().resolve_element(start.name()).0, + )); + for attribute in start.attributes() { + let attribute = attribute.expect("benchmark attributes are valid"); + checksum = checksum + .wrapping_add(attribute.key.as_ref().len()) + .wrapping_add(namespace_len( + reader.resolver().resolve_attribute(attribute.key).0, + )) + .wrapping_add( + attribute + .decoded_and_normalized_value( + XmlVersion::Explicit1_0, + reader.decoder(), + ) + .expect("benchmark value is valid") + .len(), + ); + } + } + Event::End(end) => { + checksum = checksum + .wrapping_add(end.name().as_ref().len()) + .wrapping_add(namespace_len( + reader.resolver().resolve_element(end.name()).0, + )); + } + Event::Text(text) => { + let decoded = text.decode().expect("benchmark text is valid"); + checksum = checksum.wrapping_add( + unescape(&decoded) + .expect("benchmark text entities are valid") + .len(), + ); + } + Event::GeneralRef(reference) => { + checksum = checksum.wrapping_add( + reference + .resolve_char_ref() + .expect("benchmark reference is valid") + .map_or(1, char::len_utf8), + ); + } + Event::CData(text) => { + checksum = + checksum.wrapping_add(text.decode().expect("benchmark CDATA is valid").len()); + } + Event::Comment(_) | Event::PI(_) | Event::Decl(_) | Event::DocType(_) => {} + Event::Eof => break, + } + } + checksum +} + +fn namespace_len(namespace: ResolveResult<'_>) -> usize { + match namespace { + ResolveResult::Bound(namespace) => namespace.as_ref().len(), + ResolveResult::Unbound => 0, + ResolveResult::Unknown(prefix) => prefix.len(), + } +} + +pub(crate) fn walk_forge_stream(input: &[u8]) -> usize { + let mut reader = + XmlStreamReader::new(input, stream_policy(input)).expect("benchmark policy is valid"); + let mut checksum = 0usize; + loop { + match reader.read_event().expect("benchmark fixture is valid XML") { + XmlStreamEvent::Start(start) | XmlStreamEvent::Empty(start) => { + let name = start.name().expect("benchmark name is valid"); + checksum = checksum + .wrapping_add(name.qualified().len()) + .wrapping_add(name.namespace().map_or(0, str::len)); + for attribute in start.attributes() { + let attribute = attribute.expect("benchmark attribute is valid"); + let name = attribute.name().expect("benchmark attribute name is valid"); + checksum = checksum + .wrapping_add(name.qualified().len()) + .wrapping_add(name.namespace().map_or(0, str::len)) + .wrapping_add(attribute.value().expect("benchmark value is valid").len()); + } + } + XmlStreamEvent::End(end) => { + let name = end.name().expect("benchmark name is valid"); + checksum = checksum + .wrapping_add(name.qualified().len()) + .wrapping_add(name.namespace().map_or(0, str::len)); + } + XmlStreamEvent::Text(text) => { + checksum = checksum.wrapping_add(text.value().len()); + } + XmlStreamEvent::CData(text) => { + checksum = checksum.wrapping_add(text.value().len()); + } + XmlStreamEvent::Comment(_) + | XmlStreamEvent::ProcessingInstruction(_) + | XmlStreamEvent::Declaration + | XmlStreamEvent::DocType => {} + XmlStreamEvent::Eof => break, + } + } + checksum +} + +pub(crate) fn validate_forge_stream(input: &[u8]) -> usize { + let mut reader = + XmlStreamReader::new(input, stream_policy(input)).expect("benchmark policy is valid"); + let mut events = 0usize; + loop { + let event = reader.read_event().expect("benchmark fixture is valid XML"); + if matches!(event, XmlStreamEvent::Eof) { + return events; + } + events = events.wrapping_add(1); + } +} + +fn stream_policy(input: &[u8]) -> XmlSafetyPolicy { + XmlSafetyPolicy { + max_input_bytes: input.len().max(1), + max_elements: 1_000_000, + max_text_bytes: input.len().max(1), + max_events: 10_000_000, + ..XmlSafetyPolicy::untrusted() + } +} + +pub(crate) fn write_forge_multistatus(output: W, responses: usize) -> W { + let mut writer = XmlStreamWriter::new(output).expect("benchmark writer policy is valid"); + writer + .start_element("D:multistatus", [("xmlns:D", "DAV:")]) + .expect("write multistatus root"); + for index in 0..responses { + let href = format!("/files/{index}"); + let display_name = format!("file-{index}"); + let content_length = (index * 1024).to_string(); + let etag = format!("\"etag-{index}\""); + writer.start("D:response").expect("write response"); + write_forge_text_element(&mut writer, "D:href", &href); + writer.start("D:propstat").expect("write propstat"); + writer.start("D:prop").expect("write prop"); + write_forge_text_element(&mut writer, "D:displayname", &display_name); + write_forge_text_element(&mut writer, "D:getcontentlength", &content_length); + write_forge_text_element(&mut writer, "D:getetag", &etag); + writer.end_element().expect("write prop end"); + write_forge_text_element(&mut writer, "D:status", "HTTP/1.1 200 OK"); + writer.end_element().expect("write propstat end"); + writer.end_element().expect("write response end"); + } + writer.end_element().expect("write multistatus end"); + writer.finish().expect("finish benchmark XML") +} + +fn write_forge_text_element(writer: &mut XmlStreamWriter, name: &str, value: &str) { + writer.start(name).expect("write text element"); + writer.text(value).expect("write text"); + writer.end_element().expect("write text element end"); +} + +pub(crate) fn write_quick_xml_multistatus(output: W, responses: usize) -> W { + let mut writer = Writer::new(output); + let mut root = BytesStart::new("D:multistatus"); + root.push_attribute(("xmlns:D", "DAV:")); + writer.write_event(Event::Start(root)).expect("write root"); + for index in 0..responses { + let href = format!("/files/{index}"); + let display_name = format!("file-{index}"); + let content_length = (index * 1024).to_string(); + let etag = format!("\"etag-{index}\""); + writer + .write_event(Event::Start(BytesStart::new("D:response"))) + .expect("write response"); + write_quick_text_element(&mut writer, "D:href", &href); + writer + .write_event(Event::Start(BytesStart::new("D:propstat"))) + .expect("write propstat"); + writer + .write_event(Event::Start(BytesStart::new("D:prop"))) + .expect("write prop"); + write_quick_text_element(&mut writer, "D:displayname", &display_name); + write_quick_text_element(&mut writer, "D:getcontentlength", &content_length); + write_quick_text_element(&mut writer, "D:getetag", &etag); + writer + .write_event(Event::End(BytesEnd::new("D:prop"))) + .expect("write prop end"); + write_quick_text_element(&mut writer, "D:status", "HTTP/1.1 200 OK"); + writer + .write_event(Event::End(BytesEnd::new("D:propstat"))) + .expect("write propstat end"); + writer + .write_event(Event::End(BytesEnd::new("D:response"))) + .expect("write response end"); + } + writer + .write_event(Event::End(BytesEnd::new("D:multistatus"))) + .expect("write root end"); + writer.into_inner() +} + +fn write_quick_text_element(writer: &mut Writer, name: &str, value: &str) { + writer + .write_event(Event::Start(BytesStart::new(name))) + .expect("write text element"); + writer + .write_event(Event::Text(BytesText::new(value))) + .expect("write text"); + writer + .write_event(Event::End(BytesEnd::new(name))) + .expect("write text element end"); +} + +pub(crate) fn write_xmltree_multistatus(responses: usize) -> Vec { + let mut root = dav_element("multistatus"); + let mut namespaces = xmltree::Namespace::empty(); + namespaces.put("D", "DAV:"); + root.namespaces = Some(namespaces); + for index in 0..responses { + let mut response = dav_element("response"); + response + .children + .push(xmltree::XMLNode::Element(xmltree_text_element( + "href", + format!("/files/{index}"), + ))); + let mut propstat = dav_element("propstat"); + let mut prop = dav_element("prop"); + prop.children + .push(xmltree::XMLNode::Element(xmltree_text_element( + "displayname", + format!("file-{index}"), + ))); + prop.children + .push(xmltree::XMLNode::Element(xmltree_text_element( + "getcontentlength", + (index * 1024).to_string(), + ))); + prop.children + .push(xmltree::XMLNode::Element(xmltree_text_element( + "getetag", + format!("\"etag-{index}\""), + ))); + propstat.children.push(xmltree::XMLNode::Element(prop)); + propstat + .children + .push(xmltree::XMLNode::Element(xmltree_text_element( + "status", + "HTTP/1.1 200 OK".to_owned(), + ))); + response.children.push(xmltree::XMLNode::Element(propstat)); + root.children.push(xmltree::XMLNode::Element(response)); + } + let mut options = xmltree::EmitterConfig::new(); + options.perform_indent = false; + options.write_document_declaration = false; + options.pad_self_closing = false; + options.autopad_comments = false; + let mut output = Vec::new(); + root.write_with_config(&mut output, options) + .expect("write xmltree benchmark output"); + output +} + +fn dav_element(name: &str) -> xmltree::Element { + let mut element = xmltree::Element::new(name); + element.prefix = Some("D".to_owned()); + element.namespace = Some("DAV:".to_owned()); + element +} + +fn xmltree_text_element(name: &str, value: String) -> xmltree::Element { + let mut element = dav_element(name); + element.children.push(xmltree::XMLNode::Text(value)); + element +} diff --git a/crates/aster_forge_xml/benches/xml_bench.rs b/crates/aster_forge_xml/benches/xml_bench.rs new file mode 100644 index 0000000..3ca0f8b --- /dev/null +++ b/crates/aster_forge_xml/benches/xml_bench.rs @@ -0,0 +1,159 @@ +//! CPU benchmarks for the XML implementations relevant to Aster products. + +mod support; + +use std::hint::black_box; +use std::time::Duration; + +use aster_forge_utils::numbers::usize_to_u64; +use aster_forge_xml::{BorrowedDocument, XmlSafetyPolicy, validate_xml_input}; +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use support::{ + cpu_fixtures, validate_forge_stream, walk_forge_stream, walk_quick_xml_events, + walk_quick_xml_ns_buffered, write_forge_multistatus, write_quick_xml_multistatus, + write_xmltree_multistatus, +}; + +fn bench_parse(c: &mut Criterion) { + let mut group = c.benchmark_group("xml/parse"); + group + .warm_up_time(Duration::from_secs(1)) + .measurement_time(Duration::from_secs(2)) + .sample_size(20); + for (name, input) in cpu_fixtures() { + group.throughput(Throughput::Bytes( + usize_to_u64(input.len(), "benchmark input length") + .expect("benchmark input length should fit in u64"), + )); + group.bench_with_input(BenchmarkId::new("forge_arena", name), &input, |b, input| { + b.iter(|| { + BorrowedDocument::parse(black_box(input.as_slice())).expect("fixture should parse") + }); + }); + group.bench_with_input( + BenchmarkId::new("roxmltree_borrowed", name), + &input, + |b, input| { + let input = std::str::from_utf8(input).expect("fixture should be UTF-8"); + b.iter(|| { + roxmltree::Document::parse(black_box(input)).expect("fixture should parse") + }); + }, + ); + group.bench_with_input( + BenchmarkId::new("xmltree_owned", name), + &input, + |b, input| { + b.iter(|| { + xmltree::Element::parse(black_box(input.as_slice())) + .expect("fixture should parse") + }); + }, + ); + group.bench_with_input( + BenchmarkId::new("validate_plus_xmltree", name), + &input, + |b, input| { + b.iter(|| { + validate_xml_input(black_box(input), XmlSafetyPolicy::untrusted()) + .expect("fixture should validate"); + xmltree::Element::parse(input.as_slice()).expect("fixture should parse") + }); + }, + ); + group.bench_with_input( + BenchmarkId::new("forge_stream_reader", name), + &input, + |b, input| b.iter(|| walk_forge_stream(black_box(input))), + ); + group.bench_with_input( + BenchmarkId::new("forge_stream_validation", name), + &input, + |b, input| b.iter(|| validate_forge_stream(black_box(input))), + ); + group.bench_with_input( + BenchmarkId::new("quick_xml_ns_buffered_decoded", name), + &input, + |b, input| b.iter(|| walk_quick_xml_ns_buffered(black_box(input))), + ); + group.bench_with_input( + BenchmarkId::new("quick_xml_borrowed_events", name), + &input, + |b, input| b.iter(|| walk_quick_xml_events(black_box(input))), + ); + } + group.finish(); +} + +fn bench_write(c: &mut Criterion) { + let mut group = c.benchmark_group("xml/write"); + group + .warm_up_time(Duration::from_secs(1)) + .measurement_time(Duration::from_secs(2)) + .sample_size(20); + let mut xmltree_options = xmltree::EmitterConfig::new(); + xmltree_options.perform_indent = false; + xmltree_options.write_document_declaration = false; + xmltree_options.pad_self_closing = false; + xmltree_options.autopad_comments = false; + + for (name, input) in cpu_fixtures() { + let arena = BorrowedDocument::parse(input.as_slice()).expect("fixture should parse"); + let xmltree = xmltree::Element::parse(input.as_slice()).expect("fixture should parse"); + group.throughput(Throughput::Bytes( + usize_to_u64(input.len(), "benchmark input length") + .expect("benchmark input length should fit in u64"), + )); + group.bench_function(BenchmarkId::new("arena_original", name), |b| { + b.iter(|| { + let mut output = Vec::with_capacity(input.len()); + arena + .write_original(&mut output) + .expect("fixture should write"); + black_box(output) + }); + }); + group.bench_function(BenchmarkId::new("xmltree_compact", name), |b| { + b.iter(|| { + let mut output = Vec::with_capacity(input.len()); + xmltree + .write_with_config(&mut output, xmltree_options.clone()) + .expect("fixture should write"); + black_box(output) + }); + }); + } + group.finish(); + + let responses = 1_000usize; + let expected_bytes = support::multistatus(responses).len(); + let forge_output = write_forge_multistatus(Vec::new(), responses); + let quick_output = write_quick_xml_multistatus(Vec::new(), responses); + let xmltree_output = write_xmltree_multistatus(responses); + BorrowedDocument::parse(forge_output.as_slice()).expect("Forge output should parse"); + BorrowedDocument::parse(quick_output.as_slice()).expect("quick-xml output should parse"); + BorrowedDocument::parse(xmltree_output.as_slice()).expect("xmltree output should parse"); + + let mut group = c.benchmark_group("xml/generate_multistatus"); + group + .warm_up_time(Duration::from_secs(1)) + .measurement_time(Duration::from_secs(2)) + .sample_size(20) + .throughput(Throughput::Bytes( + usize_to_u64(expected_bytes, "benchmark output length") + .expect("benchmark output length should fit in u64"), + )); + group.bench_function("forge_stream_writer/1000", |b| { + b.iter(|| black_box(write_forge_multistatus(Vec::new(), responses))); + }); + group.bench_function("quick_xml_writer/1000", |b| { + b.iter(|| black_box(write_quick_xml_multistatus(Vec::new(), responses))); + }); + group.bench_function("xmltree_build_and_write/1000", |b| { + b.iter(|| black_box(write_xmltree_multistatus(responses))); + }); + group.finish(); +} + +criterion_group!(benches, bench_parse, bench_write); +criterion_main!(benches); diff --git a/crates/aster_forge_xml/benches/xml_memory.rs b/crates/aster_forge_xml/benches/xml_memory.rs new file mode 100644 index 0000000..8fc327c --- /dev/null +++ b/crates/aster_forge_xml/benches/xml_memory.rs @@ -0,0 +1,328 @@ +//! Allocation and peak-RSS probe for XML parsing and compact writing. + +mod support; + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::hint::black_box; +use std::process::Command; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +#[cfg(any(target_os = "linux", target_os = "macos"))] +use aster_forge_utils::numbers::i64_to_u64; +use aster_forge_xml::{BorrowedDocument, ValidatedXml, XmlSafetyPolicy, validate_xml_input}; +use support::{ + memory_fixtures, validate_forge_stream, walk_forge_stream, walk_quick_xml_events, + walk_quick_xml_ns_buffered, write_forge_multistatus, write_quick_xml_multistatus, + write_xmltree_multistatus, +}; + +struct CountingAllocator; + +static ENABLED: AtomicBool = AtomicBool::new(false); +static LIVE: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); +static TOTAL: AtomicUsize = AtomicUsize::new(0); +static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0); + +#[global_allocator] +static ALLOCATOR: CountingAllocator = CountingAllocator; + +unsafe impl GlobalAlloc for CountingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // SAFETY: forwards the exact layout to the system allocator. + let pointer = unsafe { System.alloc(layout) }; + if !pointer.is_null() { + record_allocation(layout.size()); + } + pointer + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + // SAFETY: forwards the exact layout to the system allocator. + let pointer = unsafe { System.alloc_zeroed(layout) }; + if !pointer.is_null() { + record_allocation(layout.size()); + } + pointer + } + + unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { + record_deallocation(layout.size()); + // SAFETY: the pointer and layout came from this allocator. + unsafe { System.dealloc(pointer, layout) }; + } + + unsafe fn realloc(&self, pointer: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + // SAFETY: the pointer and old layout came from this allocator. + let new_pointer = unsafe { System.realloc(pointer, layout, new_size) }; + if !new_pointer.is_null() && ENABLED.load(Ordering::Relaxed) { + record_deallocation(layout.size()); + record_allocation(new_size); + } + new_pointer + } +} + +fn record_allocation(size: usize) { + if !ENABLED.load(Ordering::Relaxed) { + return; + } + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + TOTAL.fetch_add(size, Ordering::Relaxed); + let live = LIVE.fetch_add(size, Ordering::Relaxed).saturating_add(size); + PEAK.fetch_max(live, Ordering::Relaxed); +} + +fn record_deallocation(size: usize) { + if !ENABLED.load(Ordering::Relaxed) { + return; + } + let _ = LIVE.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |live| { + Some(live.saturating_sub(size)) + }); +} + +#[derive(Clone, Copy)] +struct AllocationSnapshot { + allocations: usize, + total_bytes: usize, + peak_live_bytes: usize, + retained_bytes: usize, +} + +fn begin_measurement() { + LIVE.store(0, Ordering::Relaxed); + PEAK.store(0, Ordering::Relaxed); + TOTAL.store(0, Ordering::Relaxed); + ALLOCATIONS.store(0, Ordering::Relaxed); + ENABLED.store(true, Ordering::SeqCst); +} + +fn finish_measurement() -> AllocationSnapshot { + ENABLED.store(false, Ordering::SeqCst); + AllocationSnapshot { + allocations: ALLOCATIONS.load(Ordering::Relaxed), + total_bytes: TOTAL.load(Ordering::Relaxed), + peak_live_bytes: PEAK.load(Ordering::Relaxed), + retained_bytes: LIVE.load(Ordering::Relaxed), + } +} + +fn run_child(mode: &str, fixture_name: &str) { + let input = memory_fixtures() + .into_iter() + .find_map(|(name, input)| (name == fixture_name).then_some(input)) + .unwrap_or_else(|| panic!("unknown fixture `{fixture_name}`")); + begin_measurement(); + let allocation = match mode { + "forge_arena_borrowed" => { + let document = BorrowedDocument::parse(input.as_slice()).expect("fixture should parse"); + black_box(&document); + finish_measurement() + } + "forge_validated_owned" => { + let document = ValidatedXml::new(input.clone()).expect("fixture should parse"); + black_box(&document); + finish_measurement() + } + "xmltree_owned" => { + let document = xmltree::Element::parse(input.as_slice()).expect("fixture should parse"); + black_box(&document); + finish_measurement() + } + "roxmltree_borrowed" => { + let input = std::str::from_utf8(&input).expect("fixture should be UTF-8"); + let document = roxmltree::Document::parse(input).expect("fixture should parse"); + black_box(&document); + finish_measurement() + } + "validate_plus_xmltree" => { + validate_xml_input(&input, XmlSafetyPolicy::untrusted()) + .expect("fixture should validate"); + let document = xmltree::Element::parse(input.as_slice()).expect("fixture should parse"); + black_box(&document); + finish_measurement() + } + "quick_xml_borrowed_events" => { + black_box(walk_quick_xml_events(&input)); + finish_measurement() + } + "quick_xml_ns_buffered_decoded" => { + black_box(walk_quick_xml_ns_buffered(&input)); + finish_measurement() + } + "forge_stream_reader" => { + black_box(walk_forge_stream(&input)); + finish_measurement() + } + "forge_stream_validation" => { + black_box(validate_forge_stream(&input)); + finish_measurement() + } + "forge_arena_original" => { + ENABLED.store(false, Ordering::SeqCst); + let document = BorrowedDocument::parse(input.as_slice()).expect("fixture should parse"); + begin_measurement(); + let mut output = Vec::with_capacity(input.len()); + document + .write_original(&mut output) + .expect("fixture should write"); + black_box(&output); + finish_measurement() + } + "xmltree_write" => { + ENABLED.store(false, Ordering::SeqCst); + let document = xmltree::Element::parse(input.as_slice()).expect("fixture should parse"); + let mut options = xmltree::EmitterConfig::new(); + options.perform_indent = false; + options.write_document_declaration = false; + options.pad_self_closing = false; + options.autopad_comments = false; + begin_measurement(); + let mut output = Vec::with_capacity(input.len()); + document + .write_with_config(&mut output, options) + .expect("fixture should write"); + black_box(&output); + finish_measurement() + } + "forge_stream_writer_vec" => { + let responses = multistatus_responses(fixture_name); + let output = write_forge_multistatus(Vec::new(), responses); + black_box(&output); + finish_measurement() + } + "forge_stream_writer_sink" => { + let responses = multistatus_responses(fixture_name); + black_box(write_forge_multistatus(std::io::sink(), responses)); + finish_measurement() + } + "quick_xml_writer_sink" => { + let responses = multistatus_responses(fixture_name); + black_box(write_quick_xml_multistatus(std::io::sink(), responses)); + finish_measurement() + } + "xmltree_build_and_write" => { + let responses = multistatus_responses(fixture_name); + let output = write_xmltree_multistatus(responses); + black_box(&output); + finish_measurement() + } + _ => panic!("unknown mode `{mode}`"), + }; + print_result(mode, fixture_name, input.len(), allocation); +} + +fn multistatus_responses(fixture_name: &str) -> usize { + match fixture_name { + "multistatus_1000" => 1_000, + "multistatus_10000" => 10_000, + _ => panic!("writer generation requires a multistatus fixture"), + } +} + +fn print_result( + mode: &str, + fixture_name: &str, + input_bytes: usize, + allocation: AllocationSnapshot, +) { + println!( + "{mode}\t{fixture_name}\t{input_bytes}\t{}\t{}\t{}\t{}\t{}", + allocation.allocations, + allocation.total_bytes, + allocation.peak_live_bytes, + allocation.retained_bytes, + peak_rss_bytes(), + ); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn peak_rss_bytes() -> u64 { + let mut usage = std::mem::MaybeUninit::::zeroed(); + // SAFETY: getrusage initializes the supplied rusage structure on success. + let result = unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) }; + if result != 0 { + return 0; + } + // SAFETY: getrusage returned success, so the structure is initialized. + let usage = unsafe { usage.assume_init() }; + #[cfg(target_os = "macos")] + { + i64_to_u64(usage.ru_maxrss, "peak RSS").unwrap_or(0) + } + #[cfg(target_os = "linux")] + { + i64_to_u64(usage.ru_maxrss, "peak RSS") + .unwrap_or(0) + .saturating_mul(1024) + } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn peak_rss_bytes() -> u64 { + 0 +} + +fn main() { + let arguments: Vec = std::env::args().collect(); + if arguments.iter().any(|argument| argument == "--test") { + return; + } + if let [_, child, mode, fixture_name] = arguments.as_slice() + && child == "--child" + { + run_child(mode, fixture_name); + return; + } + + println!( + "mode\tfixture\tinput_bytes\tallocations\ttotal_allocated\tpeak_live_heap\tretained_heap\tpeak_rss" + ); + let executable = std::env::current_exe().expect("current benchmark executable"); + for fixture_name in ["propfind", "wopi", "multistatus_1000", "multistatus_10000"] { + for mode in [ + "forge_arena_borrowed", + "forge_validated_owned", + "roxmltree_borrowed", + "xmltree_owned", + "validate_plus_xmltree", + "forge_stream_reader", + "forge_stream_validation", + "quick_xml_ns_buffered_decoded", + "quick_xml_borrowed_events", + "forge_arena_original", + "xmltree_write", + ] { + let output = Command::new(&executable) + .args(["--child", mode, fixture_name]) + .output() + .expect("memory benchmark child should start"); + assert!( + output.status.success(), + "memory benchmark child failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + print!("{}", String::from_utf8_lossy(&output.stdout)); + } + if fixture_name.starts_with("multistatus_") { + for mode in [ + "forge_stream_writer_vec", + "forge_stream_writer_sink", + "quick_xml_writer_sink", + "xmltree_build_and_write", + ] { + let output = Command::new(&executable) + .args(["--child", mode, fixture_name]) + .output() + .expect("memory benchmark child should start"); + assert!( + output.status.success(), + "memory benchmark child failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + print!("{}", String::from_utf8_lossy(&output.stdout)); + } + } + } +} diff --git a/crates/aster_forge_xml/fuzz/.gitignore b/crates/aster_forge_xml/fuzz/.gitignore new file mode 100644 index 0000000..9695797 --- /dev/null +++ b/crates/aster_forge_xml/fuzz/.gitignore @@ -0,0 +1,5 @@ +/artifacts/ +/corpus/ +/coverage/ +/target/ +Cargo.lock diff --git a/crates/aster_forge_xml/fuzz/Cargo.toml b/crates/aster_forge_xml/fuzz/Cargo.toml new file mode 100644 index 0000000..d763caf --- /dev/null +++ b/crates/aster_forge_xml/fuzz/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "aster_forge_xml-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +aster_forge_xml = { path = ".." } +libfuzzer-sys = "0.4.13" + +[[bin]] +name = "parse_stream" +path = "fuzz_targets/parse_stream.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "writer_roundtrip" +path = "fuzz_targets/writer_roundtrip.rs" +test = false +doc = false +bench = false + +[workspace] +members = ["."] diff --git a/crates/aster_forge_xml/fuzz/fuzz_targets/parse_stream.rs b/crates/aster_forge_xml/fuzz/fuzz_targets/parse_stream.rs new file mode 100644 index 0000000..60ec9af --- /dev/null +++ b/crates/aster_forge_xml/fuzz/fuzz_targets/parse_stream.rs @@ -0,0 +1,44 @@ +#![no_main] + +use aster_forge_xml::{ + BorrowedDocument, ParseOptions, XmlSafetyPolicy, XmlStreamEvent, XmlStreamReader, + validate_xml_input, +}; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|input: &[u8]| { + let max_input_bytes = input.len().max(1); + let policy = XmlSafetyPolicy { + max_input_bytes, + max_depth: 128, + max_elements: 65_536, + max_attributes_per_element: 1_024, + max_text_bytes: max_input_bytes, + max_events: 262_144, + reject_doctype: true, + }; + let validation = validate_xml_input(input, policy); + let options = ParseOptions::new().safety_policy(policy); + let document = BorrowedDocument::parse_with_options(input, &options); + if let Ok(document) = document { + assert!(validation.is_ok()); + let mut original = Vec::new(); + document + .write_original(&mut original) + .expect("Vec writes do not fail"); + assert_eq!(original, input); + } + + let mut reader = XmlStreamReader::new(input, policy).expect("policy is valid"); + let capture = input.first().is_some_and(|byte| byte & 1 == 1); + for _ in 0..=policy.max_events { + match reader.read_event() { + Ok(XmlStreamEvent::Start(_)) if capture => { + let capture_limit = max_input_bytes.saturating_add(4_096); + let _ = reader.capture_current(capture_limit); + } + Ok(XmlStreamEvent::Eof) | Err(_) => break, + Ok(_) => {} + } + } +}); diff --git a/crates/aster_forge_xml/fuzz/fuzz_targets/writer_roundtrip.rs b/crates/aster_forge_xml/fuzz/fuzz_targets/writer_roundtrip.rs new file mode 100644 index 0000000..a4770c7 --- /dev/null +++ b/crates/aster_forge_xml/fuzz/fuzz_targets/writer_roundtrip.rs @@ -0,0 +1,56 @@ +#![no_main] + +use aster_forge_xml::{ + BorrowedDocument, XmlSafetyPolicy, XmlStreamWriter, XmlWriteOptions, validate_xml_input, +}; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|input: &[u8]| { + exercise_writer(input); +}); + +fn exercise_writer(input: &[u8]) { + let options = XmlWriteOptions::new() + .max_output_bytes(1024 * 1024) + .max_depth(64) + .max_attributes_per_element(64); + let mut writer = XmlStreamWriter::with_options(Vec::new(), options).expect("valid options"); + writer.start("root").expect("static root is valid"); + + for (index, chunk) in input.chunks(32).take(256).enumerate() { + let value = String::from_utf8_lossy(chunk); + match index % 5 { + 0 => { + let _ = writer.text(&value); + } + 1 => { + let _ = writer.comment(&value); + } + 2 => { + let _ = writer.cdata(&value); + } + 3 => { + let _ = writer.processing_instruction(&value, Some("value")); + } + _ => { + let _ = writer.empty_element(&value, [("value", value.as_ref())]); + } + } + } + + writer + .end_element() + .expect("failed writes do not corrupt root state"); + let output = writer.finish().expect("complete root"); + let policy = XmlSafetyPolicy { + max_input_bytes: output.len().max(1), + max_text_bytes: output.len().max(1), + ..XmlSafetyPolicy::untrusted() + }; + validate_xml_input(&output, policy).expect("successful writer output validates"); + BorrowedDocument::parse_with_options( + output.as_slice(), + &aster_forge_xml::ParseOptions::new().safety_policy(policy), + ) + .expect("successful writer output reparses"); +} diff --git a/crates/aster_forge_xml/src/document.rs b/crates/aster_forge_xml/src/document.rs new file mode 100644 index 0000000..926f965 --- /dev/null +++ b/crates/aster_forge_xml/src/document.rs @@ -0,0 +1,1169 @@ +//! Source-backed, non-recursive XML document tree. + +use std::borrow::Cow; +use std::io::{Read, Write}; +use std::num::NonZeroU32; +use std::ops::Range; +use std::sync::Arc; + +use aster_forge_utils::numbers::{u32_to_usize, u64_to_usize, usize_to_u32, usize_to_u64}; +use quick_xml::Reader; +use quick_xml::XmlVersion; +use quick_xml::escape::unescape; +use quick_xml::events::{BytesStart, Event}; + +use crate::syntax::{ + XML_NAMESPACE_URI, map_quick_xml_error, split_qualified_name, utf8, validate_namespace_binding, + validate_qualified_name, +}; +use crate::{Error, ParseOptions, XmlSafetyError, XmlSafetyPolicy}; + +const OWNED_VALUE_OFFSET: u64 = u64::MAX; + +/// Stable identifier for a node in an [`XmlDocument`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct NodeId(NonZeroU32); + +impl NodeId { + fn from_index(index: usize) -> Result { + let value = usize_to_u32(index, "XML node index") + .ok() + .and_then(|index| index.checked_add(1)) + .and_then(NonZeroU32::new) + .ok_or(XmlSafetyError::TooManyElements)?; + Ok(Self(value)) + } + + fn index(self) -> usize { + stored_index(self.0.get() - 1, "XML node index") + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct ScopeId(NonZeroU32); + +impl ScopeId { + fn from_index(index: usize) -> Result { + let value = usize_to_u32(index, "XML namespace scope index") + .ok() + .and_then(|index| index.checked_add(1)) + .and_then(NonZeroU32::new) + .ok_or_else(|| Error::InvalidXml("too many namespace scopes".into()))?; + Ok(Self(value)) + } + + fn index(self) -> usize { + stored_index(self.0.get() - 1, "XML namespace scope index") + } +} + +/// A byte range in the original XML source. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SourceSpan { + pub start: u64, + pub end: u64, +} + +impl SourceSpan { + fn as_range(self, source_len: usize) -> Option> { + let start = u64_to_usize(self.start, "XML source span start").ok()?; + let end = u64_to_usize(self.end, "XML source span end").ok()?; + (start <= end && end <= source_len).then_some(start..end) + } +} + +#[derive(Debug, Clone, Copy)] +struct ValueRef { + offset: u64, + length: u32, + owned_index: u32, +} + +impl ValueRef { + fn source(offset: u64, length: u32) -> Self { + Self { + offset, + length, + owned_index: 0, + } + } + + fn owned(index: u32, length: u32) -> Self { + Self { + offset: OWNED_VALUE_OFFSET, + length, + owned_index: index, + } + } +} + +#[derive(Debug)] +struct ArenaNode { + parent: Option, + first_child: Option, + last_child: Option, + next_sibling: Option, + kind: NodeKind, +} + +#[derive(Debug)] +enum NodeKind { + Element(ElementData), + Text(ValueRef), + CData(ValueRef), + Comment(ValueRef), + ProcessingInstruction { + target: ValueRef, + content: Option, + }, +} + +#[derive(Debug)] +struct ElementData { + qualified_name: ValueRef, + attributes: Range, + namespace_scope: Option, + source: SourceSpan, +} + +#[derive(Debug)] +struct AttributeData { + qualified_name: ValueRef, + value: ValueRef, +} + +#[derive(Debug)] +struct NamespaceScope { + parent: Option, + bindings: Range, +} + +#[derive(Debug)] +struct NamespaceBinding { + prefix: ValueRef, + uri: Option, +} + +#[derive(Clone, Copy)] +struct ArenaView<'a> { + source: &'a [u8], + namespace_scopes: &'a [NamespaceScope], + namespace_bindings: &'a [NamespaceBinding], + owned_values: &'a [Box], +} + +impl<'a> ArenaView<'a> { + fn value(self, value: ValueRef) -> &'a str { + let resolved = self.checked_value(value); + debug_assert!(resolved.is_some(), "invalid internal XML value reference"); + resolved.unwrap_or("") + } + + fn checked_value(self, value: ValueRef) -> Option<&'a str> { + let length = u32_to_usize(value.length, "XML value length").ok()?; + if value.offset == OWNED_VALUE_OFFSET { + let index = u32_to_usize(value.owned_index, "owned XML value index").ok()?; + let value = self.owned_values.get(index)?.as_ref(); + return (value.len() == length).then_some(value); + } + + let start = u64_to_usize(value.offset, "XML value offset").ok()?; + let end = start.checked_add(length)?; + std::str::from_utf8(self.source.get(start..end)?).ok() + } + + fn resolve_namespace(self, scope: Option, prefix: &str) -> Option<&'a str> { + match self.checked_resolve_namespace(scope, prefix) { + Ok(namespace) => namespace, + Err(()) => { + debug_assert!(false, "invalid internal XML namespace reference"); + None + } + } + } + + fn checked_resolve_namespace( + self, + mut scope: Option, + prefix: &str, + ) -> Result, ()> { + if prefix == "xml" { + return Ok(Some(XML_NAMESPACE_URI)); + } + while let Some(scope_id) = scope { + let scope_data = self.namespace_scopes.get(scope_id.index()).ok_or(())?; + for binding_index in scope_data.bindings.clone().rev() { + let binding_index = + u32_to_usize(binding_index, "XML namespace binding index").map_err(|_| ())?; + let binding = self.namespace_bindings.get(binding_index).ok_or(())?; + if self.checked_value(binding.prefix).ok_or(())? == prefix { + return binding + .uri + .map(|uri| self.checked_value(uri).ok_or(())) + .transpose(); + } + } + scope = scope_data.parent; + } + Ok(None) + } +} + +/// An immutable XML tree whose nodes reference ranges in `source` whenever possible. +/// +/// `S` may be `&[u8]`, `Arc<[u8]>`, `Vec`, or another byte container. +#[derive(Debug)] +pub struct XmlDocument { + source: S, + nodes: Box<[ArenaNode]>, + attributes: Box<[AttributeData]>, + namespace_scopes: Box<[NamespaceScope]>, + namespace_bindings: Box<[NamespaceBinding]>, + owned_values: Box<[Box]>, + root: NodeId, +} + +/// A document borrowing its complete source buffer. +pub type BorrowedDocument<'a> = XmlDocument<&'a [u8]>; + +/// A document sharing ownership of its source buffer. +pub type OwnedDocument = XmlDocument>; + +impl> XmlDocument { + /// Parses a complete XML document with the default bounded policy. + pub fn parse(source: S) -> Result { + Self::parse_with_options(source, &ParseOptions::default()) + } + + /// Parses a complete XML document into a flat arena. + pub fn parse_with_options(source: S, options: &ParseOptions) -> Result { + options.safety.validate()?; + if source.as_ref().len() > options.safety.max_input_bytes { + return Err(XmlSafetyError::InputTooLarge.into()); + } + + let (nodes, attributes, namespace_scopes, namespace_bindings, owned_values, root) = { + let mut builder = DocumentBuilder::new(source.as_ref(), options); + builder.parse()?; + let root = builder.root.ok_or(XmlSafetyError::Malformed)?; + ( + builder.nodes.into_boxed_slice(), + builder.attributes.into_boxed_slice(), + builder.namespace_scopes.into_boxed_slice(), + builder.namespace_bindings.into_boxed_slice(), + builder.owned_values.into_boxed_slice(), + root, + ) + }; + Ok(Self { + source, + nodes, + attributes, + namespace_scopes, + namespace_bindings, + owned_values, + root, + }) + } + + pub fn source(&self) -> &[u8] { + self.source.as_ref() + } + + pub fn into_source(self) -> S { + self.source + } + + pub fn root(&self) -> ElementRef<'_, S> { + ElementRef { + document: self, + id: self.root, + } + } + + pub fn node(&self, id: NodeId) -> Option> { + let node = self.nodes.get(id.index())?; + Some(match &node.kind { + NodeKind::Element(_) => NodeRef::Element(ElementRef { document: self, id }), + NodeKind::Text(value) => NodeRef::Text(self.value(*value)), + NodeKind::CData(value) => NodeRef::CData(self.value(*value)), + NodeKind::Comment(value) => NodeRef::Comment(self.value(*value)), + NodeKind::ProcessingInstruction { target, content } => { + NodeRef::ProcessingInstruction(ProcessingInstructionRef { + target: self.value(*target), + content: content.map(|value| self.value(value)), + }) + } + }) + } + + pub fn node_count(&self) -> usize { + self.nodes.len() + } + + pub fn allocated_value_count(&self) -> usize { + self.owned_values.len() + } + + pub fn write_original(&self, mut writer: W) -> Result<(), Error> { + writer.write_all(self.source())?; + Ok(()) + } + + fn value(&self, value: ValueRef) -> &str { + self.arena_view().value(value) + } + + fn element_data(&self, id: NodeId) -> Option<&ElementData> { + match &self.nodes.get(id.index())?.kind { + NodeKind::Element(element) => Some(element), + _ => None, + } + } + + fn resolve_namespace(&self, scope: Option, prefix: &str) -> Option<&str> { + self.arena_view().resolve_namespace(scope, prefix) + } + + fn arena_view(&self) -> ArenaView<'_> { + ArenaView { + source: self.source.as_ref(), + namespace_scopes: &self.namespace_scopes, + namespace_bindings: &self.namespace_bindings, + owned_values: &self.owned_values, + } + } +} + +impl XmlDocument> { + /// Reads and parses a complete document with the default bounded policy. + pub fn from_reader(reader: R) -> Result { + Self::from_reader_with_options(reader, &ParseOptions::default()) + } + + /// Reads at most one byte beyond the configured limit before parsing an owned document. + pub fn from_reader_with_options( + reader: R, + options: &ParseOptions, + ) -> Result { + options.safety.validate()?; + let read_limit = options.safety.max_input_bytes.saturating_add(1); + let read_limit = usize_to_u64(read_limit, "XML reader byte limit").unwrap_or(u64::MAX); + let mut reader = reader.take(read_limit); + let mut source = Vec::new(); + reader.read_to_end(&mut source)?; + if source.len() > options.safety.max_input_bytes { + return Err(XmlSafetyError::InputTooLarge.into()); + } + Self::parse_with_options(Arc::from(source), options) + } +} + +/// A cheap-to-clone, validated XML document retaining the exact original bytes. +#[derive(Debug, Clone)] +pub struct ValidatedXml(Arc); + +impl ValidatedXml { + pub fn new(bytes: impl Into>) -> Result { + Self::with_policy(bytes, XmlSafetyPolicy::untrusted()) + } + + pub fn with_policy( + bytes: impl Into>, + policy: XmlSafetyPolicy, + ) -> Result { + let source = bytes.into(); + let document = + XmlDocument::parse_with_options(source, &ParseOptions::new().safety_policy(policy))?; + Ok(Self(Arc::new(document))) + } + + pub fn from_reader(reader: R) -> Result { + let document = OwnedDocument::from_reader(reader)?; + Ok(Self(Arc::new(document))) + } + + pub fn as_bytes(&self) -> &[u8] { + self.0.source() + } + + pub fn document(&self) -> &OwnedDocument { + &self.0 + } +} + +/// A borrowed view of an element node. +pub struct ElementRef<'document, S> { + document: &'document XmlDocument, + id: NodeId, +} + +impl Copy for ElementRef<'_, S> {} + +impl Clone for ElementRef<'_, S> { + fn clone(&self) -> Self { + *self + } +} + +impl<'document, S: AsRef<[u8]>> ElementRef<'document, S> { + pub fn id(self) -> NodeId { + self.id + } + + pub fn parent(self) -> Option> { + self.document.nodes[self.id.index()] + .parent + .map(|id| ElementRef { + document: self.document, + id, + }) + } + + pub fn qualified_name(self) -> &'document str { + let Some(data) = self.document.element_data(self.id) else { + return ""; + }; + self.document.value(data.qualified_name) + } + + pub fn prefix(self) -> Option<&'document str> { + split_qualified_name(self.qualified_name()).0 + } + + pub fn name(self) -> &'document str { + split_qualified_name(self.qualified_name()).1 + } + + pub fn namespace(self) -> Option<&'document str> { + let data = self.document.element_data(self.id)?; + self.document + .resolve_namespace(data.namespace_scope, self.prefix().unwrap_or("")) + } + + pub fn raw_xml(self) -> &'document [u8] { + let Some(data) = self.document.element_data(self.id) else { + return &[]; + }; + let Some(range) = data.source.as_range(self.document.source().len()) else { + return &[]; + }; + &self.document.source()[range] + } + + pub fn attributes(self) -> Attributes<'document, S> { + let range = self + .document + .element_data(self.id) + .map(|data| data.attributes.clone()) + .unwrap_or(0..0); + Attributes { + element: self, + next: range.start, + end: range.end, + } + } + + pub fn attribute(self, qualified_name: &str) -> Option<&'document str> { + self.attributes() + .find(|attribute| attribute.qualified_name() == qualified_name) + .map(AttributeRef::value) + } + + pub fn attribute_ns(self, name: &str, namespace: Option<&str>) -> Option<&'document str> { + self.attributes() + .find(|attribute| attribute.name() == name && attribute.namespace() == namespace) + .map(AttributeRef::value) + } + + pub fn children(self) -> Children<'document, S> { + Children { + document: self.document, + next: self.document.nodes[self.id.index()].first_child, + } + } + + pub fn child_elements(self) -> ChildElements<'document, S> { + ChildElements { + children: self.children(), + } + } + + pub fn get_child(self, name: &str) -> Option> { + self.child_elements().find(|element| element.name() == name) + } + + pub fn get_child_ns(self, name: &str, namespace: &str) -> Option> { + self.child_elements() + .find(|element| element.name() == name && element.namespace() == Some(namespace)) + } + + pub fn descendants(self) -> DescendantElements<'document, S> { + DescendantElements { stack: vec![self] } + } + + pub fn text(self) -> Option> { + let mut values = self.children().filter_map(|node| match node { + NodeRef::Text(text) | NodeRef::CData(text) => Some(text), + _ => None, + }); + let first = values.next()?; + match values.next() { + None => Some(Cow::Borrowed(first)), + Some(second) => { + let mut output = String::with_capacity(first.len() + second.len()); + output.push_str(first); + output.push_str(second); + values.for_each(|value| output.push_str(value)); + Some(Cow::Owned(output)) + } + } + } +} + +/// A borrowed XML node view. +pub enum NodeRef<'document, S> { + Element(ElementRef<'document, S>), + Text(&'document str), + CData(&'document str), + Comment(&'document str), + ProcessingInstruction(ProcessingInstructionRef<'document>), +} + +impl Copy for NodeRef<'_, S> {} + +impl Clone for NodeRef<'_, S> { + fn clone(&self) -> Self { + *self + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProcessingInstructionRef<'a> { + pub target: &'a str, + pub content: Option<&'a str>, +} + +pub struct Children<'document, S> { + document: &'document XmlDocument, + next: Option, +} + +impl<'document, S: AsRef<[u8]>> Iterator for Children<'document, S> { + type Item = NodeRef<'document, S>; + + fn next(&mut self) -> Option { + let id = self.next?; + self.next = self.document.nodes[id.index()].next_sibling; + self.document.node(id) + } +} + +pub struct ChildElements<'document, S> { + children: Children<'document, S>, +} + +impl<'document, S: AsRef<[u8]>> Iterator for ChildElements<'document, S> { + type Item = ElementRef<'document, S>; + + fn next(&mut self) -> Option { + self.children.find_map(|node| match node { + NodeRef::Element(element) => Some(element), + _ => None, + }) + } +} + +pub struct DescendantElements<'document, S> { + stack: Vec>, +} + +impl<'document, S: AsRef<[u8]>> Iterator for DescendantElements<'document, S> { + type Item = ElementRef<'document, S>; + + fn next(&mut self) -> Option { + let element = self.stack.pop()?; + let child_start = self.stack.len(); + self.stack.extend(element.child_elements()); + self.stack[child_start..].reverse(); + Some(element) + } +} + +pub struct Attributes<'document, S> { + element: ElementRef<'document, S>, + next: u32, + end: u32, +} + +impl<'document, S: AsRef<[u8]>> Iterator for Attributes<'document, S> { + type Item = AttributeRef<'document, S>; + + fn next(&mut self) -> Option { + if self.next >= self.end { + return None; + } + let index = self.next; + self.next += 1; + Some(AttributeRef { + element: self.element, + index, + }) + } +} + +pub struct AttributeRef<'document, S> { + element: ElementRef<'document, S>, + index: u32, +} + +impl Copy for AttributeRef<'_, S> {} + +impl Clone for AttributeRef<'_, S> { + fn clone(&self) -> Self { + *self + } +} + +impl<'document, S: AsRef<[u8]>> AttributeRef<'document, S> { + fn data(self) -> Option<&'document AttributeData> { + let data = self.checked_data(); + debug_assert!(data.is_some(), "invalid internal XML attribute index"); + data + } + + fn checked_data(self) -> Option<&'document AttributeData> { + self.document() + .attributes + .get(stored_index(self.index, "XML attribute index")) + } + + fn document(self) -> &'document XmlDocument { + self.element.document + } + + pub fn qualified_name(self) -> &'document str { + self.data() + .map(|data| self.document().value(data.qualified_name)) + .unwrap_or("") + } + + pub fn prefix(self) -> Option<&'document str> { + split_qualified_name(self.qualified_name()).0 + } + + pub fn name(self) -> &'document str { + split_qualified_name(self.qualified_name()).1 + } + + pub fn namespace(self) -> Option<&'document str> { + let prefix = self.prefix()?; + let scope = self + .document() + .element_data(self.element.id) + .and_then(|element| element.namespace_scope); + self.document().resolve_namespace(scope, prefix) + } + + pub fn value(self) -> &'document str { + self.data() + .map(|data| self.document().value(data.value)) + .unwrap_or("") + } +} + +struct DocumentBuilder<'a> { + source: &'a [u8], + options: &'a ParseOptions, + nodes: Vec, + attributes: Vec, + namespace_scopes: Vec, + namespace_bindings: Vec, + owned_values: Vec>, + open: Vec, + root: Option, + root_complete: bool, + element_count: usize, + text_bytes: usize, + event_count: usize, +} + +impl<'a> DocumentBuilder<'a> { + fn new(source: &'a [u8], options: &'a ParseOptions) -> Self { + Self { + source, + options, + nodes: Vec::new(), + attributes: Vec::new(), + namespace_scopes: Vec::new(), + namespace_bindings: Vec::new(), + owned_values: Vec::new(), + open: Vec::new(), + root: None, + root_complete: false, + element_count: 0, + text_bytes: 0, + event_count: 0, + } + } + + fn parse(&mut self) -> Result<(), Error> { + let mut reader = Reader::from_reader(self.source); + reader.config_mut().trim_text(false); + reader.config_mut().check_end_names = true; + loop { + let event_start = reader.buffer_position(); + let event = reader.read_event().map_err(map_quick_xml_error)?; + let event_end = reader.buffer_position(); + if !matches!(event, Event::Eof) { + self.count_event()?; + } + match event { + Event::Start(start) => { + self.start_element(&reader, &start, event_start, event_end)? + } + Event::Empty(start) => { + self.empty_element(&reader, &start, event_start, event_end)? + } + Event::End(_) => self.end_element(event_end)?, + Event::Text(text) => { + let raw = utf8(text.as_ref())?; + let value = + unescape(raw).map_err(|error| Error::InvalidXml(error.to_string()))?; + self.text_node(value, false)?; + } + Event::CData(text) => self.text_node(Cow::Borrowed(utf8(text.as_ref())?), true)?, + Event::Comment(comment) => { + let value = self.source_value(utf8(comment.as_ref())?)?; + self.push_content(NodeKind::Comment(value))?; + } + Event::PI(pi) => { + let target = self.source_value(utf8(pi.target())?)?; + let content = utf8(pi.content())? + .trim_start_matches(|character: char| character.is_ascii_whitespace()); + let content = (!content.is_empty()) + .then(|| self.source_value(content)) + .transpose()?; + self.push_content(NodeKind::ProcessingInstruction { target, content })?; + } + Event::GeneralRef(reference) => { + let value = if let Some(character) = reference + .resolve_char_ref() + .map_err(|error| Error::InvalidXml(error.to_string()))? + { + Cow::Owned(character.to_string()) + } else { + Cow::Owned( + match utf8(reference.as_ref())? { + "amp" => "&", + "lt" => "<", + "gt" => ">", + "apos" => "'", + "quot" => "\"", + _ => return Err(XmlSafetyError::ExternalEntity.into()), + } + .to_owned(), + ) + }; + self.text_node(value, false)?; + } + Event::Decl(_) => { + if self.root.is_some() || !self.open.is_empty() || self.root_complete { + return Err(XmlSafetyError::Malformed.into()); + } + } + Event::DocType(_) => { + if self.options.safety.reject_doctype { + return Err(XmlSafetyError::ExternalEntity.into()); + } + if self.root.is_some() || !self.open.is_empty() || self.root_complete { + return Err(XmlSafetyError::Malformed.into()); + } + } + Event::Eof => { + if !self.open.is_empty() || !self.root_complete { + return Err(XmlSafetyError::Malformed.into()); + } + return Ok(()); + } + } + } + } + + fn start_element( + &mut self, + reader: &Reader<&[u8]>, + start: &BytesStart<'a>, + source_start: u64, + source_end: u64, + ) -> Result<(), Error> { + self.check_element()?; + let id = self.build_element(reader, start, source_start, source_end)?; + self.open.push(id); + Ok(()) + } + + fn empty_element( + &mut self, + reader: &Reader<&[u8]>, + start: &BytesStart<'a>, + source_start: u64, + source_end: u64, + ) -> Result<(), Error> { + self.check_element()?; + self.build_element(reader, start, source_start, source_end)?; + if self.open.is_empty() { + self.root_complete = true; + } + Ok(()) + } + + fn end_element(&mut self, source_end: u64) -> Result<(), Error> { + let id = self.open.pop().ok_or(XmlSafetyError::Malformed)?; + let NodeKind::Element(element) = &mut self.nodes[id.index()].kind else { + return Err(XmlSafetyError::Malformed.into()); + }; + element.source.end = source_end; + if self.open.is_empty() { + self.root_complete = true; + } + Ok(()) + } + + fn build_element( + &mut self, + reader: &Reader<&[u8]>, + start: &BytesStart<'a>, + source_start: u64, + source_end: u64, + ) -> Result { + if self.open.is_empty() && self.root_complete { + return Err(XmlSafetyError::Malformed.into()); + } + let start_name = start.name(); + let qualified_name = utf8(start_name.as_ref())?; + let (prefix, _) = validate_qualified_name(qualified_name)?; + let parent_scope = self.open.last().and_then(|id| { + let NodeKind::Element(element) = &self.nodes[id.index()].kind else { + return None; + }; + element.namespace_scope + }); + let binding_start = arena_len(self.namespace_bindings.len(), "namespace bindings")?; + let mut attribute_count = 0usize; + for attribute in start.attributes() { + attribute_count = attribute_count + .checked_add(1) + .ok_or(XmlSafetyError::TooManyAttributes)?; + if attribute_count > self.options.safety.max_attributes_per_element { + return Err(XmlSafetyError::TooManyAttributes.into()); + } + let attribute = attribute.map_err(|error| Error::InvalidXml(error.to_string()))?; + let name = utf8(attribute.key.as_ref())?; + validate_qualified_name(name)?; + if name == "xmlns" || name.starts_with("xmlns:") { + let namespace_prefix = name.strip_prefix("xmlns:").unwrap_or(""); + let uri = attribute + .decoded_and_normalized_value(XmlVersion::Explicit1_0, reader.decoder()) + .map_err(|error| Error::InvalidXml(error.to_string()))?; + validate_namespace_binding(namespace_prefix, &uri)?; + let prefix_value = if namespace_prefix.is_empty() { + ValueRef::source(0, 0) + } else { + self.source_value(namespace_prefix)? + }; + let uri_value = if uri.is_empty() { + None + } else { + Some(self.cow_value(uri)?) + }; + self.namespace_bindings.push(NamespaceBinding { + prefix: prefix_value, + uri: uri_value, + }); + } + } + let binding_end = arena_len(self.namespace_bindings.len(), "namespace bindings")?; + let namespace_scope = if binding_start == binding_end { + parent_scope + } else { + let id = ScopeId::from_index(self.namespace_scopes.len())?; + self.namespace_scopes.push(NamespaceScope { + parent: parent_scope, + bindings: binding_start..binding_end, + }); + Some(id) + }; + if let Some(prefix) = prefix + && self + .arena_view() + .resolve_namespace(namespace_scope, prefix) + .is_none() + { + return Err(XmlSafetyError::Malformed.into()); + } + + let attribute_start = arena_len(self.attributes.len(), "attributes")?; + for attribute in start.attributes() { + let attribute = attribute.map_err(|error| Error::InvalidXml(error.to_string()))?; + let name = utf8(attribute.key.as_ref())?; + if name == "xmlns" || name.starts_with("xmlns:") { + continue; + } + let (prefix, _) = split_qualified_name(name); + if let Some(prefix) = prefix + && prefix != "xml" + && self + .arena_view() + .resolve_namespace(namespace_scope, prefix) + .is_none() + { + return Err(XmlSafetyError::Malformed.into()); + } + let value = attribute + .decoded_and_normalized_value(XmlVersion::Explicit1_0, reader.decoder()) + .map_err(|error| Error::InvalidXml(error.to_string()))?; + let qualified_name = self.source_value(name)?; + let value = self.cow_value(value)?; + self.attributes.push(AttributeData { + qualified_name, + value, + }); + } + let attribute_end = arena_len(self.attributes.len(), "attributes")?; + let qualified_name = self.source_value(qualified_name)?; + let id = self.push_node(NodeKind::Element(ElementData { + qualified_name, + attributes: attribute_start..attribute_end, + namespace_scope, + source: SourceSpan { + start: source_start, + end: source_end, + }, + }))?; + if self.root.is_none() { + self.root = Some(id); + } + Ok(id) + } + + fn check_element(&mut self) -> Result<(), Error> { + let depth = self + .open + .len() + .checked_add(1) + .ok_or(XmlSafetyError::TooDeep)?; + if depth > self.options.safety.max_depth { + return Err(XmlSafetyError::TooDeep.into()); + } + self.element_count = self + .element_count + .checked_add(1) + .ok_or(XmlSafetyError::TooManyElements)?; + if self.element_count > self.options.safety.max_elements { + return Err(XmlSafetyError::TooManyElements.into()); + } + Ok(()) + } + + fn count_event(&mut self) -> Result<(), Error> { + self.event_count = self + .event_count + .checked_add(1) + .ok_or(XmlSafetyError::TooManyEvents)?; + if self.event_count > self.options.safety.max_events { + return Err(XmlSafetyError::TooManyEvents.into()); + } + Ok(()) + } + + fn text_node(&mut self, value: Cow<'_, str>, cdata: bool) -> Result<(), Error> { + self.text_bytes = self + .text_bytes + .checked_add(value.len()) + .ok_or(XmlSafetyError::TextTooLarge)?; + if self.text_bytes > self.options.safety.max_text_bytes { + return Err(XmlSafetyError::TextTooLarge.into()); + } + let value = if self.options.trim_whitespace { + match value { + Cow::Borrowed(value) => Cow::Borrowed(value.trim()), + Cow::Owned(value) => Cow::Owned(value.trim().to_owned()), + } + } else { + value + }; + if value.is_empty() { + return Ok(()); + } + if self.open.is_empty() { + if value.chars().all(char::is_whitespace) { + return Ok(()); + } + return Err(XmlSafetyError::Malformed.into()); + } + let value = self.cow_value(value)?; + self.push_content(if cdata { + NodeKind::CData(value) + } else { + NodeKind::Text(value) + })?; + Ok(()) + } + + fn push_content(&mut self, kind: NodeKind) -> Result<(), Error> { + if self.open.is_empty() { + return Ok(()); + } + self.push_node(kind).map(|_| ()) + } + + fn push_node(&mut self, kind: NodeKind) -> Result { + let id = NodeId::from_index(self.nodes.len())?; + let parent = self.open.last().copied(); + self.nodes.push(ArenaNode { + parent, + first_child: None, + last_child: None, + next_sibling: None, + kind, + }); + if let Some(parent) = parent { + if let Some(previous) = self.nodes[parent.index()].last_child { + self.nodes[previous.index()].next_sibling = Some(id); + } else { + self.nodes[parent.index()].first_child = Some(id); + } + self.nodes[parent.index()].last_child = Some(id); + } + Ok(id) + } + + fn source_value(&self, value: &str) -> Result { + let source_start = self.source.as_ptr() as usize; + let value_start = value.as_ptr() as usize; + let offset = value_start + .checked_sub(source_start) + .filter(|offset| offset.saturating_add(value.len()) <= self.source.len()) + .ok_or_else(|| Error::InvalidXml("borrowed value is outside XML source".into()))?; + Ok(ValueRef::source( + usize_to_u64(offset, "XML value offset").map_err(|_| XmlSafetyError::InputTooLarge)?, + usize_to_u32(value.len(), "XML value length") + .map_err(|_| XmlSafetyError::InputTooLarge)?, + )) + } + + fn cow_value(&mut self, value: Cow<'_, str>) -> Result { + match value { + Cow::Borrowed(value) => self.source_value(value), + Cow::Owned(value) => { + let index = arena_len(self.owned_values.len(), "owned XML values")?; + let length = usize_to_u32(value.len(), "owned XML value length") + .map_err(|_| XmlSafetyError::InputTooLarge)?; + self.owned_values.push(value.into_boxed_str()); + Ok(ValueRef::owned(index, length)) + } + } + } + + fn arena_view(&self) -> ArenaView<'_> { + ArenaView { + source: self.source, + namespace_scopes: &self.namespace_scopes, + namespace_bindings: &self.namespace_bindings, + owned_values: &self.owned_values, + } + } +} + +fn arena_len(length: usize, label: &str) -> Result { + usize_to_u32(length, label).map_err(|_| Error::InvalidXml(format!("too many {label}"))) +} + +fn stored_index(value: u32, label: &str) -> usize { + // Rust's supported platforms can represent every u32 as usize. Keep the checked Forge + // conversion at the representation boundary and make malformed internal state fail indexing. + u32_to_usize(value, label).unwrap_or(usize::MAX) +} + +#[cfg(test)] +mod layout_tests { + use std::mem::size_of_val; + + use super::*; + + fn arena_payload_bytes(document: &XmlDocument) -> usize { + size_of_val(document.nodes.as_ref()) + + size_of_val(document.attributes.as_ref()) + + size_of_val(document.namespace_scopes.as_ref()) + + size_of_val(document.namespace_bindings.as_ref()) + + size_of_val(document.owned_values.as_ref()) + + document + .owned_values + .iter() + .map(|value| value.len()) + .sum::() + } + + #[test] + fn arena_view_rejects_invalid_value_ranges_and_namespace_indices() { + let source = b"value\xFF"; + let owned_values = [Box::::from("owned")]; + let scope = ScopeId::from_index(0).expect("scope id"); + let namespace_scopes = [NamespaceScope { + parent: None, + bindings: 0..1, + }]; + let view = ArenaView { + source, + namespace_scopes: &namespace_scopes, + namespace_bindings: &[], + owned_values: &owned_values, + }; + + assert_eq!(view.checked_value(ValueRef::source(0, 5)), Some("value")); + assert_eq!(view.checked_value(ValueRef::owned(0, 5)), Some("owned")); + assert_eq!(view.checked_value(ValueRef::source(5, 1)), None); + assert_eq!(view.checked_value(ValueRef::source(u64::MAX, 2)), None); + assert_eq!(view.checked_value(ValueRef::owned(1, 5)), None); + assert_eq!(view.checked_value(ValueRef::owned(0, 4)), None); + assert_eq!(view.checked_resolve_namespace(Some(scope), "p"), Err(())); + } + + #[test] + fn attribute_lookup_reports_invalid_internal_indices_without_indexing() { + let document = BorrowedDocument::parse(br#""#.as_slice()) + .expect("document should parse"); + let attribute = AttributeRef { + element: document.root(), + index: u32::MAX, + }; + + assert!(attribute.checked_data().is_none()); + } + + #[test] + fn large_owned_document_payload_stays_below_six_times_input() { + const RESPONSES: usize = 10_000; + let mut source = String::from(""); + for index in 0..RESPONSES { + source.push_str(&format!( + "/files/{index}file-{index}{}"etag-{index}"HTTP/1.1 200 OK", + index * 1024 + )); + } + source.push_str(""); + let input_bytes = source.len(); + let options = ParseOptions::new() + .max_size(input_bytes) + .max_elements(RESPONSES * 8 + 1); + let document = XmlDocument::parse_with_options(Arc::from(source.into_bytes()), &options) + .expect("large document"); + let retained_payload = input_bytes + arena_payload_bytes(&document); + + assert!( + retained_payload <= input_bytes * 6, + "retained payload {retained_payload} exceeds 6x input {input_bytes}" + ); + } +} diff --git a/crates/aster_forge_xml/src/error.rs b/crates/aster_forge_xml/src/error.rs new file mode 100644 index 0000000..728e6bd --- /dev/null +++ b/crates/aster_forge_xml/src/error.rs @@ -0,0 +1,121 @@ +//! Error types for bounded XML parsing and source I/O. + +use std::fmt; + +/// Failures produced while applying an [`XmlSafetyPolicy`](crate::XmlSafetyPolicy). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum XmlSafetyError { + /// One or more configured limits are zero. + InvalidPolicy, + /// The input is larger than the configured byte limit. + InputTooLarge, + /// Generated XML exceeds the configured byte limit. + OutputTooLarge, + /// The input declares a DTD or custom entity while declarations are prohibited. + ExternalEntity, + /// The element nesting depth exceeds the configured limit. + TooDeep, + /// The total element count exceeds the configured limit. + TooManyElements, + /// An element has more attributes than the configured limit. + TooManyAttributes, + /// The total decoded text and CDATA size exceeds the configured limit. + TextTooLarge, + /// The parser emitted more events than the configured limit. + TooManyEvents, + /// The document contains bytes that are not valid in its declared encoding. + InvalidEncoding, + /// The input is not one complete, well-formed, single-root XML document. + Malformed, +} + +impl fmt::Display for XmlSafetyError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::InvalidPolicy => "XML safety limits must be positive", + Self::InputTooLarge => "XML input exceeds the configured byte limit", + Self::OutputTooLarge => "XML output exceeds the configured byte limit", + Self::ExternalEntity => "XML DTD and custom entity declarations are not allowed", + Self::TooDeep => "XML nesting depth exceeds the configured limit", + Self::TooManyElements => "XML element count exceeds the configured limit", + Self::TooManyAttributes => "XML attribute count exceeds the configured limit", + Self::TextTooLarge => "XML text size exceeds the configured limit", + Self::TooManyEvents => "XML event count exceeds the configured limit", + Self::InvalidEncoding => "XML input contains invalid encoded text", + Self::Malformed => "malformed XML input", + }) + } +} + +impl std::error::Error for XmlSafetyError {} + +/// Errors produced while parsing or retaining an XML document. +#[derive(Debug)] +pub enum Error { + /// A configured input safety boundary was crossed. + Safety(XmlSafetyError), + /// The document is structurally invalid. The message is diagnostic only. + InvalidXml(String), + /// A writer operation would produce invalid XML or violate writer state. + InvalidData(String), + /// Reading or writing failed. + Io(std::io::Error), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Safety(error) => write!(f, "XML safety error: {error}"), + Self::InvalidXml(message) => write!(f, "invalid XML: {message}"), + Self::InvalidData(message) => write!(f, "invalid XML data: {message}"), + Self::Io(error) => write!(f, "XML I/O error: {error}"), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Safety(error) => Some(error), + Self::Io(error) => Some(error), + Self::InvalidXml(_) | Self::InvalidData(_) => None, + } + } +} + +impl From for Error { + fn from(error: XmlSafetyError) -> Self { + Self::Safety(error) + } +} + +impl From for Error { + fn from(error: std::io::Error) -> Self { + Self::Io(error) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wrapped_error_display_is_distinct_from_its_source() { + let safety = Error::Safety(XmlSafetyError::TextTooLarge); + assert_eq!( + safety.to_string(), + "XML safety error: XML text size exceeds the configured limit" + ); + assert_eq!( + std::error::Error::source(&safety).map(ToString::to_string), + Some("XML text size exceeds the configured limit".into()) + ); + + let io = Error::Io(std::io::Error::other("fixture failure")); + assert_eq!(io.to_string(), "XML I/O error: fixture failure"); + assert_eq!( + std::error::Error::source(&io).map(ToString::to_string), + Some("fixture failure".into()) + ); + } +} diff --git a/crates/aster_forge_xml/src/lib.rs b/crates/aster_forge_xml/src/lib.rs new file mode 100644 index 0000000..2dd9840 --- /dev/null +++ b/crates/aster_forge_xml/src/lib.rs @@ -0,0 +1,39 @@ +//! Bounded, source-backed XML parsing for Aster services. +//! +//! Parsed documents use a flat arena and retain source spans for names, attributes, text, and +//! subtrees. Values allocate only when XML decoding or configured normalization changes them. +#![deny(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +#![cfg_attr( + not(test), + deny( + clippy::unwrap_used, + clippy::unreachable, + clippy::expect_used, + clippy::panic, + clippy::unimplemented, + clippy::todo + ) +)] + +mod document; +mod error; +mod parser; +mod stream; +mod syntax; +mod writer; + +pub use document::{ + AttributeRef, Attributes, BorrowedDocument, ChildElements, Children, DescendantElements, + ElementRef, NodeId, NodeRef, OwnedDocument, ProcessingInstructionRef, SourceSpan, ValidatedXml, + XmlDocument, +}; +pub use error::{Error, XmlSafetyError}; +pub use parser::{ParseOptions, XmlSafetyPolicy, validate_xml_input, xml_root_local_name}; +pub use stream::{ + StreamAttribute, StreamAttributes, StreamCData, StreamComment, StreamEnd, StreamName, + StreamProcessingInstruction, StreamStart, StreamText, XmlStreamEvent, XmlStreamReader, +}; +pub use writer::{XmlStreamWriter, XmlWriteAttribute, XmlWriteOptions}; + +/// The default maximum nesting depth accepted from untrusted XML. +pub const DEFAULT_XML_MAX_DEPTH: usize = 128; diff --git a/crates/aster_forge_xml/src/parser.rs b/crates/aster_forge_xml/src/parser.rs new file mode 100644 index 0000000..c50b11d --- /dev/null +++ b/crates/aster_forge_xml/src/parser.rs @@ -0,0 +1,398 @@ +//! Event-based XML validation and shared parsing limits. + +use std::borrow::Cow; + +use quick_xml::Reader; +use quick_xml::XmlVersion; +use quick_xml::escape::unescape; +use quick_xml::events::{BytesStart, Event}; + +use crate::syntax::{ + XML_NAMESPACE_URI, map_quick_xml_error, split_qualified_name, utf8, validate_namespace_binding, + validate_qualified_name, +}; +use crate::{DEFAULT_XML_MAX_DEPTH, Error, XmlSafetyError}; + +const DEFAULT_MAX_INPUT_BYTES: usize = 10 * 1024 * 1024; +const DEFAULT_MAX_ELEMENTS: usize = 100_000; +const DEFAULT_MAX_ATTRIBUTES_PER_ELEMENT: usize = 1_024; +const DEFAULT_MAX_TEXT_BYTES: usize = 10 * 1024 * 1024; +const DEFAULT_MAX_EVENTS: usize = 1_000_000; + +/// Finite resource and declaration limits applied to untrusted XML. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct XmlSafetyPolicy { + pub max_input_bytes: usize, + pub max_depth: usize, + pub max_elements: usize, + pub max_attributes_per_element: usize, + pub max_text_bytes: usize, + pub max_events: usize, + pub reject_doctype: bool, +} + +impl XmlSafetyPolicy { + /// A conservative policy suitable for network and storage protocol input. + pub const fn untrusted() -> Self { + Self { + max_input_bytes: DEFAULT_MAX_INPUT_BYTES, + max_depth: DEFAULT_XML_MAX_DEPTH, + max_elements: DEFAULT_MAX_ELEMENTS, + max_attributes_per_element: DEFAULT_MAX_ATTRIBUTES_PER_ELEMENT, + max_text_bytes: DEFAULT_MAX_TEXT_BYTES, + max_events: DEFAULT_MAX_EVENTS, + reject_doctype: true, + } + } + + pub(crate) fn validate(self) -> Result<(), XmlSafetyError> { + if self.max_input_bytes == 0 + || self.max_depth == 0 + || self.max_elements == 0 + || self.max_attributes_per_element == 0 + || self.max_text_bytes == 0 + || self.max_events == 0 + { + Err(XmlSafetyError::InvalidPolicy) + } else { + Ok(()) + } + } +} + +impl Default for XmlSafetyPolicy { + fn default() -> Self { + Self::untrusted() + } +} + +/// Tree parsing behavior. Safety limits remain finite by default. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ParseOptions { + pub safety: XmlSafetyPolicy, + /// Drops whitespace-only text nodes and trims retained text nodes. + pub trim_whitespace: bool, +} + +impl ParseOptions { + pub fn new() -> Self { + Self::default() + } + + pub fn safety_policy(mut self, policy: XmlSafetyPolicy) -> Self { + self.safety = policy; + self + } + + pub fn max_depth(mut self, value: usize) -> Self { + self.safety.max_depth = value; + self + } + + pub fn max_elements(mut self, value: usize) -> Self { + self.safety.max_elements = value; + self + } + + pub fn max_size(mut self, value: usize) -> Self { + self.safety.max_input_bytes = value; + self + } + + pub fn max_attributes_per_element(mut self, value: usize) -> Self { + self.safety.max_attributes_per_element = value; + self + } + + pub fn max_text_bytes(mut self, value: usize) -> Self { + self.safety.max_text_bytes = value; + self + } + + pub fn max_events(mut self, value: usize) -> Self { + self.safety.max_events = value; + self + } + + pub fn allow_dtd(mut self, allow: bool) -> Self { + self.safety.reject_doctype = !allow; + self + } + + pub fn trim_whitespace(mut self, trim: bool) -> Self { + self.trim_whitespace = trim; + self + } +} + +/// Validates one complete XML document without constructing a DOM. +pub fn validate_xml_input(bytes: &[u8], policy: XmlSafetyPolicy) -> Result<(), XmlSafetyError> { + scan_xml(bytes, &ParseOptions::new().safety_policy(policy)) + .map(|_| ()) + .map_err(safety_error) +} + +/// Returns the local name of a validated document root. +pub fn xml_root_local_name( + bytes: &[u8], + policy: XmlSafetyPolicy, +) -> Result { + scan_xml(bytes, &ParseOptions::new().safety_policy(policy)) + .map_err(safety_error)? + .ok_or(XmlSafetyError::Malformed) +} + +fn safety_error(error: Error) -> XmlSafetyError { + match error { + Error::Safety(error) => error, + Error::InvalidXml(_) | Error::InvalidData(_) | Error::Io(_) => XmlSafetyError::Malformed, + } +} + +#[derive(Debug)] +struct Frame { + qualified_name: String, + binding_start: usize, +} + +#[derive(Debug)] +struct NamespaceBinding { + prefix: String, + uri: Option, +} + +#[derive(Debug, Default)] +struct ScanState { + frames: Vec, + bindings: Vec, + root_name: Option, + root_complete: bool, + elements: usize, + text_bytes: usize, + events: usize, +} + +impl ScanState { + fn count_event(&mut self, policy: XmlSafetyPolicy) -> Result<(), Error> { + self.events = self + .events + .checked_add(1) + .ok_or(XmlSafetyError::TooManyEvents)?; + if self.events > policy.max_events { + return Err(XmlSafetyError::TooManyEvents.into()); + } + Ok(()) + } + + fn count_element(&mut self, policy: XmlSafetyPolicy) -> Result<(), Error> { + let depth = self + .frames + .len() + .checked_add(1) + .ok_or(XmlSafetyError::TooDeep)?; + if depth > policy.max_depth { + return Err(XmlSafetyError::TooDeep.into()); + } + self.elements = self + .elements + .checked_add(1) + .ok_or(XmlSafetyError::TooManyElements)?; + if self.elements > policy.max_elements { + return Err(XmlSafetyError::TooManyElements.into()); + } + Ok(()) + } + + fn count_text(&mut self, text: &str, policy: XmlSafetyPolicy) -> Result<(), Error> { + self.text_bytes = self + .text_bytes + .checked_add(text.len()) + .ok_or(XmlSafetyError::TextTooLarge)?; + if self.text_bytes > policy.max_text_bytes { + return Err(XmlSafetyError::TextTooLarge.into()); + } + if self.frames.is_empty() && !text.chars().all(char::is_whitespace) { + return Err(XmlSafetyError::Malformed.into()); + } + Ok(()) + } + + fn namespace(&self, prefix: &str) -> Option<&str> { + if prefix == "xml" { + return Some(XML_NAMESPACE_URI); + } + self.bindings + .iter() + .rev() + .find(|binding| binding.prefix == prefix) + .and_then(|binding| binding.uri.as_deref()) + } +} + +fn scan_xml(bytes: &[u8], options: &ParseOptions) -> Result, Error> { + options.safety.validate()?; + if bytes.len() > options.safety.max_input_bytes { + return Err(XmlSafetyError::InputTooLarge.into()); + } + + let mut reader = Reader::from_reader(bytes); + reader.config_mut().trim_text(false); + let mut state = ScanState::default(); + + loop { + let event = reader.read_event().map_err(map_quick_xml_error)?; + if !matches!(event, Event::Eof) { + state.count_event(options.safety)?; + } + match event { + Event::Start(start) => { + state.count_element(options.safety)?; + let frame = scan_element(&reader, &mut state, &start, options.safety)?; + state.frames.push(frame); + } + Event::Empty(start) => { + state.count_element(options.safety)?; + let frame = scan_element(&reader, &mut state, &start, options.safety)?; + state.bindings.truncate(frame.binding_start); + if state.frames.is_empty() { + state.root_complete = true; + } + } + Event::End(end) => { + let end_name = end.name(); + let qualified_name = utf8(end_name.as_ref())?; + let frame = state.frames.pop().ok_or(XmlSafetyError::Malformed)?; + if frame.qualified_name != qualified_name { + return Err(XmlSafetyError::Malformed.into()); + } + state.bindings.truncate(frame.binding_start); + if state.frames.is_empty() { + state.root_complete = true; + } + } + Event::Text(text) => { + let raw = utf8(text.as_ref())?; + let value = unescape(raw).map_err(|error| Error::InvalidXml(error.to_string()))?; + state.count_text(value.as_ref(), options.safety)?; + } + Event::CData(text) => state.count_text(utf8(text.as_ref())?, options.safety)?, + Event::GeneralRef(reference) => { + let value = decode_reference(reference.as_ref(), &reference)?; + state.count_text(value.as_ref(), options.safety)?; + } + Event::Decl(_) => { + if state.root_name.is_some() || !state.frames.is_empty() || state.root_complete { + return Err(XmlSafetyError::Malformed.into()); + } + } + Event::DocType(_) => { + if options.safety.reject_doctype { + return Err(XmlSafetyError::ExternalEntity.into()); + } + if state.root_name.is_some() || !state.frames.is_empty() || state.root_complete { + return Err(XmlSafetyError::Malformed.into()); + } + } + Event::Comment(comment) => { + utf8(comment.as_ref())?; + } + Event::PI(pi) => { + utf8(pi.target())?; + utf8(pi.content())?; + } + Event::Eof => { + if !state.frames.is_empty() || !state.root_complete { + return Err(XmlSafetyError::Malformed.into()); + } + return Ok(state.root_name); + } + } + } +} + +fn scan_element( + reader: &Reader<&[u8]>, + state: &mut ScanState, + start: &BytesStart<'_>, + policy: XmlSafetyPolicy, +) -> Result { + if state.frames.is_empty() && state.root_complete { + return Err(XmlSafetyError::Malformed.into()); + } + let start_name = start.name(); + let qualified_name = utf8(start_name.as_ref())?; + let (prefix, local_name) = validate_qualified_name(qualified_name)?; + let binding_start = state.bindings.len(); + let mut attribute_count = 0usize; + + for attribute in start.attributes() { + attribute_count = attribute_count + .checked_add(1) + .ok_or(XmlSafetyError::TooManyAttributes)?; + if attribute_count > policy.max_attributes_per_element { + return Err(XmlSafetyError::TooManyAttributes.into()); + } + let attribute = attribute.map_err(|error| Error::InvalidXml(error.to_string()))?; + let name = utf8(attribute.key.as_ref())?; + validate_qualified_name(name)?; + if name == "xmlns" || name.starts_with("xmlns:") { + let namespace_prefix = name.strip_prefix("xmlns:").unwrap_or(""); + let uri = attribute + .decoded_and_normalized_value(XmlVersion::Explicit1_0, reader.decoder()) + .map_err(|error| Error::InvalidXml(error.to_string()))?; + validate_namespace_binding(namespace_prefix, &uri)?; + state.bindings.push(NamespaceBinding { + prefix: namespace_prefix.to_owned(), + uri: (!uri.is_empty()).then(|| uri.into_owned()), + }); + } + } + + if let Some(prefix) = prefix + && state.namespace(prefix).is_none() + { + return Err(XmlSafetyError::Malformed.into()); + } + for attribute in start.attributes() { + let attribute = attribute.map_err(|error| Error::InvalidXml(error.to_string()))?; + let name = utf8(attribute.key.as_ref())?; + if name == "xmlns" || name.starts_with("xmlns:") { + continue; + } + let (prefix, _) = split_qualified_name(name); + if let Some(prefix) = prefix + && prefix != "xml" + && state.namespace(prefix).is_none() + { + return Err(XmlSafetyError::Malformed.into()); + } + } + + if state.root_name.is_none() { + state.root_name = Some(local_name.to_owned()); + } + Ok(Frame { + qualified_name: qualified_name.to_owned(), + binding_start, + }) +} + +fn decode_reference<'a>( + bytes: &'a [u8], + reference: &quick_xml::events::BytesRef<'a>, +) -> Result, Error> { + if let Some(character) = reference + .resolve_char_ref() + .map_err(|error| Error::InvalidXml(error.to_string()))? + { + return Ok(Cow::Owned(character.to_string())); + } + Ok(Cow::Borrowed(match utf8(bytes)? { + "amp" => "&", + "lt" => "<", + "gt" => ">", + "apos" => "'", + "quot" => "\"", + _ => return Err(XmlSafetyError::ExternalEntity.into()), + })) +} diff --git a/crates/aster_forge_xml/src/stream.rs b/crates/aster_forge_xml/src/stream.rs new file mode 100644 index 0000000..6c7c71e --- /dev/null +++ b/crates/aster_forge_xml/src/stream.rs @@ -0,0 +1,772 @@ +//! Bounded, namespace-aware streaming XML reader. + +use std::borrow::Cow; +use std::io::{BufRead, Take, Write}; +use std::str; + +use aster_forge_utils::numbers::usize_to_u64; +use quick_xml::XmlVersion; +use quick_xml::encoding::Decoder; +use quick_xml::escape::unescape; +use quick_xml::events::attributes::{Attribute, Attributes as QuickAttributes}; +use quick_xml::events::{BytesCData, BytesEnd, BytesPI, BytesStart, BytesText, Event}; +use quick_xml::name::{NamespaceResolver, PrefixDeclaration, ResolveResult}; +use quick_xml::reader::NsReader; +use quick_xml::writer::Writer; + +use crate::{Error, ValidatedXml, XmlSafetyError, XmlSafetyPolicy}; + +/// A namespace-resolved XML name borrowed from one streaming event. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StreamName<'a> { + qualified: &'a str, + local: &'a str, + namespace: Option<&'a str>, +} + +impl<'a> StreamName<'a> { + pub fn qualified(self) -> &'a str { + self.qualified + } + + pub fn local(self) -> &'a str { + self.local + } + + pub fn namespace(self) -> Option<&'a str> { + self.namespace + } + + pub fn matches(self, local: &str, namespace: Option<&str>) -> bool { + self.local == local && self.namespace == namespace + } +} + +/// A start or empty element event. +pub struct StreamStart<'a> { + raw: BytesStart<'a>, + namespace: Option<&'a str>, + resolver: &'a NamespaceResolver, + decoder: Decoder, + cached_attribute_values: &'a [CachedAttributeValue], +} + +impl StreamStart<'_> { + pub fn name(&self) -> Result, Error> { + let qualified = utf8(self.raw.name().into_inner())?; + let local = utf8(self.raw.local_name().into_inner())?; + Ok(StreamName { + qualified, + local, + namespace: self.namespace, + }) + } + + pub fn attributes(&self) -> StreamAttributes<'_> { + StreamAttributes { + inner: self.raw.attributes(), + resolver: self.resolver, + decoder: self.decoder, + cached_values: self.cached_attribute_values, + cached_index: 0, + index: 0, + } + } + + pub fn attribute(&self, qualified_name: &str) -> Result>, Error> { + for attribute in self.attributes() { + let attribute = attribute?; + if attribute.name()?.qualified() == qualified_name { + return attribute.into_value().map(Some); + } + } + Ok(None) + } + + pub fn attribute_ns( + &self, + local: &str, + namespace: Option<&str>, + ) -> Result>, Error> { + for attribute in self.attributes() { + let attribute = attribute?; + if attribute.name()?.matches(local, namespace) { + return attribute.into_value().map(Some); + } + } + Ok(None) + } +} + +/// Iterator over attributes of a streaming start event. +pub struct StreamAttributes<'a> { + inner: QuickAttributes<'a>, + resolver: &'a NamespaceResolver, + decoder: Decoder, + cached_values: &'a [CachedAttributeValue], + cached_index: usize, + index: usize, +} + +impl<'a> Iterator for StreamAttributes<'a> { + type Item = Result, Error>; + + fn next(&mut self) -> Option { + let index = self.index; + self.index = self.index.saturating_add(1); + let cached_value = self + .cached_values + .get(self.cached_index) + .and_then(|cached| { + if cached.index == index { + self.cached_index += 1; + Some(cached.value.as_str()) + } else { + None + } + }); + self.inner.next().map(|attribute| { + attribute + .map(|raw| StreamAttribute { + raw, + resolver: self.resolver, + decoder: self.decoder, + cached_value, + }) + .map_err(|error| Error::InvalidXml(error.to_string())) + }) + } +} + +/// A namespace-resolved attribute borrowed from a streaming start event. +pub struct StreamAttribute<'a> { + raw: Attribute<'a>, + resolver: &'a NamespaceResolver, + decoder: Decoder, + cached_value: Option<&'a str>, +} + +impl<'a> StreamAttribute<'a> { + pub fn name(&self) -> Result, Error> { + let qualified = utf8(self.raw.key.into_inner())?; + let local = utf8(self.raw.key.local_name().into_inner())?; + let namespace = resolve_namespace( + self.resolver.resolve_attribute(self.raw.key).0, + "attribute namespace", + )?; + Ok(StreamName { + qualified, + local, + namespace, + }) + } + + pub fn value(&self) -> Result, Error> { + if let Some(value) = self.cached_value { + return Ok(Cow::Borrowed(value)); + } + self.raw + .decoded_and_normalized_value(XmlVersion::Explicit1_0, self.decoder) + .map_err(|error| Error::InvalidXml(error.to_string())) + } + + pub fn into_value(self) -> Result, Error> { + if let Some(value) = self.cached_value { + return Ok(Cow::Borrowed(value)); + } + self.raw + .decoded_and_normalized_value(XmlVersion::Explicit1_0, self.decoder) + .map_err(|error| Error::InvalidXml(error.to_string())) + } +} + +/// An end element event. +#[derive(Debug)] +pub struct StreamEnd<'a> { + raw: BytesEnd<'a>, + namespace: Option<&'a str>, +} + +impl StreamEnd<'_> { + pub fn name(&self) -> Result, Error> { + let qualified = utf8(self.raw.name().into_inner())?; + let local = utf8(self.raw.local_name().into_inner())?; + Ok(StreamName { + qualified, + local, + namespace: self.namespace, + }) + } +} + +/// Decoded and unescaped character data. +pub struct StreamText<'a> { + value: Cow<'a, str>, +} + +impl StreamText<'_> { + pub fn value(&self) -> &str { + &self.value + } +} + +/// Decoded CDATA content. +pub struct StreamCData<'a> { + value: Cow<'a, str>, +} + +impl StreamCData<'_> { + pub fn value(&self) -> &str { + &self.value + } +} + +/// Decoded XML comment content. +pub struct StreamComment<'a> { + value: Cow<'a, str>, +} + +impl StreamComment<'_> { + pub fn value(&self) -> &str { + &self.value + } +} + +/// A processing instruction. +pub struct StreamProcessingInstruction<'a> { + raw: BytesPI<'a>, +} + +impl StreamProcessingInstruction<'_> { + pub fn target(&self) -> Result<&str, Error> { + utf8(self.raw.target()) + } + + pub fn content(&self) -> Result, Error> { + let content = utf8(self.raw.content())? + .trim_start_matches(|character: char| character.is_ascii_whitespace()); + Ok((!content.is_empty()).then_some(content)) + } +} + +/// One bounded streaming XML event. +pub enum XmlStreamEvent<'a> { + Start(StreamStart<'a>), + Empty(StreamStart<'a>), + End(StreamEnd<'a>), + Text(StreamText<'a>), + CData(StreamCData<'a>), + Comment(StreamComment<'a>), + ProcessingInstruction(StreamProcessingInstruction<'a>), + Declaration, + DocType, + Eof, +} + +struct StreamState { + policy: XmlSafetyPolicy, + max_input_bytes_u64: u64, + depth: usize, + elements: usize, + text_bytes: usize, + events: usize, + root_seen: bool, + root_complete: bool, + current_start_available: bool, + current_start_depth: usize, + finished: bool, +} + +/// A streaming XML reader that enforces [`XmlSafetyPolicy`] without retaining a full document. +pub struct XmlStreamReader { + reader: NsReader>, + buffer: Vec, + cached_attribute_values: Vec, + state: StreamState, +} + +struct CachedAttributeValue { + index: usize, + value: String, +} + +impl XmlStreamReader { + pub fn new(reader: R, policy: XmlSafetyPolicy) -> Result { + policy.validate()?; + let read_limit = policy.max_input_bytes.saturating_add(1); + let read_limit = usize_to_u64(read_limit, "XML stream byte limit").unwrap_or(u64::MAX); + let max_input_bytes_u64 = + usize_to_u64(policy.max_input_bytes, "XML stream byte limit").unwrap_or(u64::MAX); + let mut reader = NsReader::from_reader(reader.take(read_limit)); + reader.config_mut().trim_text(false); + reader + .resolver_mut() + .set_max_declarations_per_element(policy.max_attributes_per_element); + Ok(Self { + reader, + buffer: Vec::new(), + cached_attribute_values: Vec::new(), + state: StreamState { + policy, + max_input_bytes_u64, + depth: 0, + elements: 0, + text_bytes: 0, + events: 0, + root_seen: false, + root_complete: false, + current_start_available: false, + current_start_depth: 0, + finished: false, + }, + }) + } + + pub fn read_event(&mut self) -> Result, Error> { + if self.state.finished { + return Ok(XmlStreamEvent::Eof); + } + self.state.current_start_available = false; + self.buffer.clear(); + self.cached_attribute_values.clear(); + let event = self + .reader + .read_event_into(&mut self.buffer) + .map_err(map_quick_xml_error)?; + check_stream_position(&self.reader, &self.state)?; + count_event(&mut self.state, &event)?; + + match event { + Event::Start(start) => { + let namespace = begin_element( + &mut self.state, + &self.reader, + &start, + false, + &mut self.cached_attribute_values, + )?; + Ok(XmlStreamEvent::Start(StreamStart { + raw: start, + namespace, + resolver: self.reader.resolver(), + decoder: self.reader.decoder(), + cached_attribute_values: &self.cached_attribute_values, + })) + } + Event::Empty(start) => { + let namespace = begin_element( + &mut self.state, + &self.reader, + &start, + true, + &mut self.cached_attribute_values, + )?; + Ok(XmlStreamEvent::Empty(StreamStart { + raw: start, + namespace, + resolver: self.reader.resolver(), + decoder: self.reader.decoder(), + cached_attribute_values: &self.cached_attribute_values, + })) + } + Event::End(end) => { + if self.state.depth == 0 { + return Err(XmlSafetyError::Malformed.into()); + } + let namespace = resolve_namespace( + self.reader.resolver().resolve_element(end.name()).0, + "element namespace", + )?; + utf8(end.name().into_inner())?; + self.state.depth -= 1; + if self.state.depth == 0 { + self.state.root_complete = true; + } + Ok(XmlStreamEvent::End(StreamEnd { + raw: end, + namespace, + })) + } + Event::Text(text) => { + let value = decode_text(&text)?; + count_text(&mut self.state, &value)?; + Ok(XmlStreamEvent::Text(StreamText { value })) + } + Event::CData(cdata) => { + let value = cdata + .decode() + .map_err(|_| XmlSafetyError::InvalidEncoding)?; + count_text(&mut self.state, &value)?; + Ok(XmlStreamEvent::CData(StreamCData { value })) + } + Event::Comment(comment) => { + let value = comment + .decode() + .map_err(|_| XmlSafetyError::InvalidEncoding)?; + Ok(XmlStreamEvent::Comment(StreamComment { value })) + } + Event::PI(pi) => { + utf8(pi.target())?; + utf8(pi.content())?; + Ok(XmlStreamEvent::ProcessingInstruction( + StreamProcessingInstruction { raw: pi }, + )) + } + Event::GeneralRef(reference) => { + let value = decode_reference(&reference)?; + count_text(&mut self.state, &value)?; + Ok(XmlStreamEvent::Text(StreamText { value })) + } + Event::Decl(_) => { + if self.state.root_seen || self.state.depth != 0 || self.state.root_complete { + return Err(XmlSafetyError::Malformed.into()); + } + Ok(XmlStreamEvent::Declaration) + } + Event::DocType(_) => { + if self.state.policy.reject_doctype { + return Err(XmlSafetyError::ExternalEntity.into()); + } + if self.state.root_seen || self.state.depth != 0 || self.state.root_complete { + return Err(XmlSafetyError::Malformed.into()); + } + Ok(XmlStreamEvent::DocType) + } + Event::Eof => { + if self.state.depth != 0 || !self.state.root_complete { + return Err(XmlSafetyError::Malformed.into()); + } + self.state.finished = true; + Ok(XmlStreamEvent::Eof) + } + } + } + + /// Reads direct text and CDATA until the end of the current start element. + pub fn read_text_current(&mut self) -> Result { + self.require_current_start()?; + self.state.current_start_available = false; + let mut output = String::new(); + loop { + match self.read_event()? { + XmlStreamEvent::Text(text) => output.push_str(text.value()), + XmlStreamEvent::CData(cdata) => output.push_str(cdata.value()), + XmlStreamEvent::Comment(_) | XmlStreamEvent::ProcessingInstruction(_) => {} + XmlStreamEvent::End(_) => return Ok(output), + XmlStreamEvent::Start(_) | XmlStreamEvent::Empty(_) => { + return Err(Error::InvalidXml( + "text helper encountered a nested element".into(), + )); + } + XmlStreamEvent::Declaration | XmlStreamEvent::DocType | XmlStreamEvent::Eof => { + return Err(XmlSafetyError::Malformed.into()); + } + } + } + } + + /// Skips the current start element and all descendants with constant retained memory. + pub fn skip_current(&mut self) -> Result<(), Error> { + self.require_current_start()?; + self.state.current_start_available = false; + let mut nested = 1usize; + while nested > 0 { + match self.read_event()? { + XmlStreamEvent::Start(_) => { + nested = nested.checked_add(1).ok_or(XmlSafetyError::TooDeep)?; + } + XmlStreamEvent::End(_) => nested -= 1, + XmlStreamEvent::Eof => return Err(XmlSafetyError::Malformed.into()), + _ => {} + } + } + Ok(()) + } + + /// Materializes only the current subtree as a validated owned XML value. + pub fn capture_current(&mut self, max_bytes: usize) -> Result { + self.require_current_start()?; + if max_bytes == 0 { + return Err(XmlSafetyError::InvalidPolicy.into()); + } + self.state.current_start_available = false; + let mut event_reader = quick_xml::Reader::from_reader(self.buffer.as_slice()); + let Event::Start(mut captured_start) = + event_reader.read_event().map_err(map_quick_xml_error)? + else { + return Err(Error::InvalidData( + "stream start buffer is incomplete".into(), + )); + }; + for (prefix, namespace) in self.reader.resolver().bindings() { + let already_declared = captured_start.attributes().any(|attribute| { + let Ok(attribute) = attribute else { + return false; + }; + match prefix { + PrefixDeclaration::Default => attribute.key.as_ref() == b"xmlns", + PrefixDeclaration::Named(prefix) => { + attribute.key.as_ref().strip_prefix(b"xmlns:") == Some(prefix) + } + } + }); + if already_declared { + continue; + } + let namespace = utf8(namespace.into_inner())?; + match prefix { + PrefixDeclaration::Default => captured_start.push_attribute(("xmlns", namespace)), + PrefixDeclaration::Named(prefix) => { + let prefix = utf8(prefix)?; + let name = format!("xmlns:{prefix}"); + captured_start.push_attribute((name.as_str(), namespace)); + } + } + } + let mut writer = Writer::new(LimitedVec::new(Vec::new(), max_bytes)); + write_capture_event(&mut writer, Event::Start(captured_start))?; + let mut nested = 1usize; + while nested > 0 { + let event = self.read_event()?; + match event { + XmlStreamEvent::Start(start) => { + nested = nested.checked_add(1).ok_or(XmlSafetyError::TooDeep)?; + write_capture_event(&mut writer, Event::Start(start.raw.borrow()))?; + } + XmlStreamEvent::Empty(start) => { + write_capture_event(&mut writer, Event::Empty(start.raw.borrow()))?; + } + XmlStreamEvent::End(end) => { + write_capture_event(&mut writer, Event::End(end.raw.borrow()))?; + nested -= 1; + } + XmlStreamEvent::Text(text) => { + write_capture_event(&mut writer, Event::Text(BytesText::new(text.value())))? + } + XmlStreamEvent::CData(cdata) => { + write_capture_event(&mut writer, Event::CData(BytesCData::new(cdata.value())))?; + } + XmlStreamEvent::Comment(comment) => { + write_capture_event( + &mut writer, + Event::Comment(BytesText::from_escaped(comment.value())), + )?; + } + XmlStreamEvent::ProcessingInstruction(pi) => { + write_capture_event(&mut writer, Event::PI(pi.raw.borrow()))?; + } + XmlStreamEvent::Declaration | XmlStreamEvent::DocType | XmlStreamEvent::Eof => { + return Err(XmlSafetyError::Malformed.into()); + } + } + } + let bytes = writer.into_inner().bytes; + let policy = XmlSafetyPolicy { + max_input_bytes: max_bytes, + ..self.state.policy + }; + ValidatedXml::with_policy(bytes, policy) + } + + pub fn into_inner(self) -> R { + self.reader.into_inner().into_inner() + } + + fn require_current_start(&self) -> Result<(), Error> { + if self.state.current_start_available && self.state.current_start_depth == self.state.depth + { + Ok(()) + } else { + Err(Error::InvalidData( + "operation requires the most recently read event to be Start".into(), + )) + } + } +} + +fn check_stream_position( + reader: &NsReader>, + state: &StreamState, +) -> Result<(), Error> { + if reader.buffer_position() > state.max_input_bytes_u64 { + Err(XmlSafetyError::InputTooLarge.into()) + } else { + Ok(()) + } +} + +fn count_event(state: &mut StreamState, event: &Event<'_>) -> Result<(), Error> { + if matches!(event, Event::Eof) { + return Ok(()); + } + if state.events >= state.policy.max_events { + return Err(XmlSafetyError::TooManyEvents.into()); + } + state.events += 1; + Ok(()) +} + +fn begin_element<'a, R: BufRead>( + state: &mut StreamState, + reader: &'a NsReader>, + start: &BytesStart<'_>, + empty: bool, + cached_attribute_values: &mut Vec, +) -> Result, Error> { + if state.depth == 0 && state.root_complete { + return Err(XmlSafetyError::Malformed.into()); + } + if state.depth >= state.policy.max_depth { + return Err(XmlSafetyError::TooDeep.into()); + } + let depth = state.depth + 1; + if state.elements >= state.policy.max_elements { + return Err(XmlSafetyError::TooManyElements.into()); + } + state.elements += 1; + let namespace = resolve_namespace( + reader.resolver().resolve_element(start.name()).0, + "element namespace", + )?; + utf8(start.name().into_inner())?; + + for (index, attribute) in start.attributes().enumerate() { + let attribute = attribute.map_err(|error| Error::InvalidXml(error.to_string()))?; + if index >= state.policy.max_attributes_per_element { + return Err(XmlSafetyError::TooManyAttributes.into()); + } + utf8(attribute.key.into_inner())?; + resolve_namespace( + reader.resolver().resolve_attribute(attribute.key).0, + "attribute namespace", + )?; + let value = attribute + .decoded_and_normalized_value(XmlVersion::Explicit1_0, reader.decoder()) + .map_err(|error| Error::InvalidXml(error.to_string()))?; + if let Cow::Owned(value) = value { + cached_attribute_values.push(CachedAttributeValue { index, value }); + } + } + + state.root_seen = true; + if empty { + if state.depth == 0 { + state.root_complete = true; + } + } else { + state.depth = depth; + state.current_start_depth = depth; + state.current_start_available = true; + } + Ok(namespace) +} + +fn count_text(state: &mut StreamState, value: &str) -> Result<(), Error> { + let remaining = state.policy.max_text_bytes - state.text_bytes; + if value.len() > remaining { + return Err(XmlSafetyError::TextTooLarge.into()); + } + state.text_bytes += value.len(); + if state.depth == 0 && !value.chars().all(char::is_whitespace) { + return Err(XmlSafetyError::Malformed.into()); + } + Ok(()) +} + +fn resolve_namespace<'a>(result: ResolveResult<'a>, label: &str) -> Result, Error> { + match result { + ResolveResult::Unbound => Ok(None), + ResolveResult::Bound(namespace) => utf8(namespace.into_inner()).map(Some), + ResolveResult::Unknown(prefix) => Err(Error::InvalidXml(format!( + "unknown {label} prefix `{}`", + String::from_utf8_lossy(&prefix) + ))), + } +} + +fn decode_text<'a>(text: &BytesText<'a>) -> Result, Error> { + match text.decode().map_err(|_| XmlSafetyError::InvalidEncoding)? { + Cow::Borrowed(value) => { + unescape(value).map_err(|error| Error::InvalidXml(error.to_string())) + } + Cow::Owned(value) => unescape(&value) + .map(Cow::into_owned) + .map(Cow::Owned) + .map_err(|error| Error::InvalidXml(error.to_string())), + } +} + +fn decode_reference<'a>( + reference: &quick_xml::events::BytesRef<'a>, +) -> Result, Error> { + if let Some(character) = reference + .resolve_char_ref() + .map_err(|error| Error::InvalidXml(error.to_string()))? + { + return Ok(Cow::Owned(character.to_string())); + } + Ok(Cow::Borrowed(match utf8(reference.as_ref())? { + "amp" => "&", + "lt" => "<", + "gt" => ">", + "apos" => "'", + "quot" => "\"", + _ => return Err(XmlSafetyError::ExternalEntity.into()), + })) +} + +fn utf8(bytes: &[u8]) -> Result<&str, Error> { + str::from_utf8(bytes).map_err(|_| XmlSafetyError::InvalidEncoding.into()) +} + +fn map_quick_xml_error(error: quick_xml::Error) -> Error { + match error { + quick_xml::Error::Encoding(_) => XmlSafetyError::InvalidEncoding.into(), + error => Error::InvalidXml(error.to_string()), + } +} + +fn write_capture_event(writer: &mut Writer, event: Event<'_>) -> Result<(), Error> { + if let Err(error) = writer.write_event(event) { + if writer.get_ref().exceeded { + return Err(XmlSafetyError::InputTooLarge.into()); + } + return Err(Error::Io(error)); + } + Ok(()) +} + +struct LimitedVec { + bytes: Vec, + max_bytes: usize, + exceeded: bool, +} + +impl LimitedVec { + fn new(bytes: Vec, max_bytes: usize) -> Self { + Self { + bytes, + max_bytes, + exceeded: false, + } + } +} + +impl Write for LimitedVec { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + let Some(new_len) = self.bytes.len().checked_add(buffer.len()) else { + self.exceeded = true; + return Err(std::io::Error::other("XML capture exceeds byte limit")); + }; + if new_len > self.max_bytes { + self.exceeded = true; + return Err(std::io::Error::other("XML capture exceeds byte limit")); + } + self.bytes.extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} diff --git a/crates/aster_forge_xml/src/syntax.rs b/crates/aster_forge_xml/src/syntax.rs new file mode 100644 index 0000000..30e6ef6 --- /dev/null +++ b/crates/aster_forge_xml/src/syntax.rs @@ -0,0 +1,82 @@ +//! Shared XML syntax and parser error classification. + +use crate::{Error, XmlSafetyError}; + +pub(crate) const XML_NAMESPACE_URI: &str = "http://www.w3.org/XML/1998/namespace"; +pub(crate) const XMLNS_NAMESPACE_URI: &str = "http://www.w3.org/2000/xmlns/"; + +pub(crate) fn utf8(bytes: &[u8]) -> Result<&str, Error> { + std::str::from_utf8(bytes).map_err(|_| XmlSafetyError::InvalidEncoding.into()) +} + +pub(crate) fn validate_qualified_name(name: &str) -> Result<(Option<&str>, &str), Error> { + let (prefix, local) = split_qualified_name(name); + if !valid_name(local) + || prefix.is_some_and(|prefix| !valid_name(prefix)) + || name.matches(':').count() > 1 + { + return Err(XmlSafetyError::Malformed.into()); + } + Ok((prefix, local)) +} + +pub(crate) fn split_qualified_name(name: &str) -> (Option<&str>, &str) { + match name.split_once(':') { + Some((prefix, local)) => (Some(prefix), local), + None => (None, name), + } +} + +pub(crate) fn valid_name(name: &str) -> bool { + let mut characters = name.chars(); + characters.next().is_some_and(is_name_start) && characters.all(is_name_char) +} + +fn is_name_start(character: char) -> bool { + matches!( + character, + 'A'..='Z' + | '_' + | 'a'..='z' + | '\u{00C0}'..='\u{00D6}' + | '\u{00D8}'..='\u{00F6}' + | '\u{00F8}'..='\u{02FF}' + | '\u{0370}'..='\u{037D}' + | '\u{037F}'..='\u{1FFF}' + | '\u{200C}'..='\u{200D}' + | '\u{2070}'..='\u{218F}' + | '\u{2C00}'..='\u{2FEF}' + | '\u{3001}'..='\u{D7FF}' + | '\u{F900}'..='\u{FDCF}' + | '\u{FDF0}'..='\u{FFFD}' + | '\u{10000}'..='\u{EFFFF}' + ) +} + +fn is_name_char(character: char) -> bool { + is_name_start(character) + || character.is_ascii_digit() + || matches!(character, '-' | '.' | '\u{B7}') + || ('\u{300}'..='\u{36F}').contains(&character) + || ('\u{203F}'..='\u{2040}').contains(&character) +} + +pub(crate) fn validate_namespace_binding(prefix: &str, uri: &str) -> Result<(), Error> { + if prefix == "xmlns" + || uri == XMLNS_NAMESPACE_URI + || (prefix == "xml" && uri != XML_NAMESPACE_URI) + || (prefix != "xml" && uri == XML_NAMESPACE_URI) + || (!prefix.is_empty() && uri.is_empty()) + { + Err(XmlSafetyError::Malformed.into()) + } else { + Ok(()) + } +} + +pub(crate) fn map_quick_xml_error(error: quick_xml::Error) -> Error { + match error { + quick_xml::Error::Encoding(_) => XmlSafetyError::InvalidEncoding.into(), + error => Error::InvalidXml(error.to_string()), + } +} diff --git a/crates/aster_forge_xml/src/writer.rs b/crates/aster_forge_xml/src/writer.rs new file mode 100644 index 0000000..60ba4dc --- /dev/null +++ b/crates/aster_forge_xml/src/writer.rs @@ -0,0 +1,614 @@ +//! Bounded, namespace-aware streaming XML writer. + +use std::io::{self, Write}; + +use quick_xml::events::{BytesCData, BytesDecl, BytesEnd, BytesPI, BytesStart, BytesText, Event}; +use quick_xml::writer::Writer; + +use crate::{Error, ValidatedXml, XmlSafetyError}; + +const DEFAULT_MAX_OUTPUT_BYTES: usize = 64 * 1024 * 1024; +const DEFAULT_MAX_DEPTH: usize = 128; +const DEFAULT_MAX_ATTRIBUTES_PER_ELEMENT: usize = 1_024; +const XML_NAMESPACE_URI: &str = "http://www.w3.org/XML/1998/namespace"; +const XMLNS_NAMESPACE_URI: &str = "http://www.w3.org/2000/xmlns/"; + +/// Finite output limits and document options for [`XmlStreamWriter`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct XmlWriteOptions { + pub max_output_bytes: usize, + pub max_depth: usize, + pub max_attributes_per_element: usize, + pub write_document_declaration: bool, +} + +impl XmlWriteOptions { + pub const fn new() -> Self { + Self { + max_output_bytes: DEFAULT_MAX_OUTPUT_BYTES, + max_depth: DEFAULT_MAX_DEPTH, + max_attributes_per_element: DEFAULT_MAX_ATTRIBUTES_PER_ELEMENT, + write_document_declaration: false, + } + } + + pub const fn max_output_bytes(mut self, value: usize) -> Self { + self.max_output_bytes = value; + self + } + + pub const fn max_depth(mut self, value: usize) -> Self { + self.max_depth = value; + self + } + + pub const fn max_attributes_per_element(mut self, value: usize) -> Self { + self.max_attributes_per_element = value; + self + } + + pub const fn write_document_declaration(mut self, value: bool) -> Self { + self.write_document_declaration = value; + self + } + + fn validate(self) -> Result<(), Error> { + if self.max_output_bytes == 0 || self.max_depth == 0 || self.max_attributes_per_element == 0 + { + Err(XmlSafetyError::InvalidPolicy.into()) + } else { + Ok(()) + } + } +} + +impl Default for XmlWriteOptions { + fn default() -> Self { + Self::new() + } +} + +/// One attribute passed to a streaming element write. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct XmlWriteAttribute<'a> { + pub name: &'a str, + pub value: &'a str, +} + +impl<'a> XmlWriteAttribute<'a> { + pub const fn new(name: &'a str, value: &'a str) -> Self { + Self { name, value } + } +} + +impl<'a> From<(&'a str, &'a str)> for XmlWriteAttribute<'a> { + fn from((name, value): (&'a str, &'a str)) -> Self { + Self { name, value } + } +} + +struct NamespaceBinding { + prefix: String, + uri: Option, +} + +/// Direct XML event writer with bounded output and namespace/state validation. +pub struct XmlStreamWriter { + writer: Writer>, + options: XmlWriteOptions, + depth: usize, + root_seen: bool, + root_complete: bool, + open_names: Vec, + binding_starts: Vec, + bindings: Vec, + instruction: String, +} + +impl XmlStreamWriter { + pub fn new(inner: W) -> Result { + Self::with_options(inner, XmlWriteOptions::default()) + } + + pub fn with_options(inner: W, options: XmlWriteOptions) -> Result { + options.validate()?; + let mut output = Self { + writer: Writer::new(LimitedWriter::new(inner, options.max_output_bytes)), + options, + depth: 0, + root_seen: false, + root_complete: false, + open_names: Vec::new(), + binding_starts: Vec::new(), + bindings: Vec::new(), + instruction: String::new(), + }; + if options.write_document_declaration { + output.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))?; + } + Ok(output) + } + + pub fn start_element<'a, I, A>(&mut self, name: &str, attributes: I) -> Result<(), Error> + where + I: IntoIterator, + A: Into>, + { + self.write_element(name, attributes, false) + } + + pub fn empty_element<'a, I, A>(&mut self, name: &str, attributes: I) -> Result<(), Error> + where + I: IntoIterator, + A: Into>, + { + self.write_element(name, attributes, true) + } + + pub fn start(&mut self, name: &str) -> Result<(), Error> { + self.start_element(name, std::iter::empty::>()) + } + + pub fn empty(&mut self, name: &str) -> Result<(), Error> { + self.empty_element(name, std::iter::empty::>()) + } + + pub fn end_element(&mut self) -> Result<(), Error> { + if self.depth == 0 { + return Err(Error::InvalidData( + "end_element called without an open element".into(), + )); + } + let index = self.depth - 1; + let name = self.open_names[index].as_str(); + let result = self.writer.write_event(Event::End(BytesEnd::new(name))); + if let Err(error) = result { + return self.map_io_error(error); + } + self.depth = index; + let binding_start = self.binding_starts[index]; + self.bindings.truncate(binding_start); + if self.depth == 0 { + self.root_complete = true; + } + Ok(()) + } + + pub fn text(&mut self, value: &str) -> Result<(), Error> { + validate_xml_text(value)?; + if self.depth == 0 && !value.chars().all(char::is_whitespace) { + return Err(Error::InvalidData( + "text cannot appear outside the root".into(), + )); + } + self.write_event(Event::Text(BytesText::new(value))) + } + + pub fn cdata(&mut self, value: &str) -> Result<(), Error> { + validate_xml_text(value)?; + if self.depth == 0 { + return Err(Error::InvalidData( + "CDATA cannot appear outside the root".into(), + )); + } + if value.contains("]]>") { + return Err(Error::InvalidData("CDATA contains `]]>`".into())); + } + self.write_event(Event::CData(BytesCData::new(value))) + } + + pub fn comment(&mut self, value: &str) -> Result<(), Error> { + validate_xml_text(value)?; + if value.contains("--") || value.ends_with('-') { + return Err(Error::InvalidData( + "comment contains an XML-forbidden hyphen sequence".into(), + )); + } + self.write_event(Event::Comment(BytesText::from_escaped(value))) + } + + pub fn processing_instruction( + &mut self, + target: &str, + content: Option<&str>, + ) -> Result<(), Error> { + validate_name(target, false)?; + if target.eq_ignore_ascii_case("xml") { + return Err(Error::InvalidData( + "processing-instruction target cannot be `xml`".into(), + )); + } + self.instruction.clear(); + self.instruction.push_str(target); + if let Some(content) = content { + validate_xml_text(content)?; + if content.contains("?>") { + return Err(Error::InvalidData( + "processing instruction contains `?>`".into(), + )); + } + if !content.is_empty() { + self.instruction.push(' '); + self.instruction.push_str(content); + } + } + let instruction = BytesPI::new(self.instruction.as_str()); + let result = self.writer.write_event(Event::PI(instruction)); + match result { + Ok(()) => Ok(()), + Err(error) => self.map_io_error(error), + } + } + + /// Embeds one already validated, self-contained XML root below the current element. + pub fn validated_subtree(&mut self, subtree: &ValidatedXml) -> Result<(), Error> { + if self.depth == 0 { + return Err(Error::InvalidData( + "validated subtree requires an open parent element".into(), + )); + } + let caller_has_default_namespace = self.resolve_namespace("").is_some(); + for element in subtree.document().root().descendants() { + if caller_has_default_namespace + && element.prefix().is_none() + && element.namespace().is_none() + { + return Err(Error::InvalidData( + "unprefixed validated subtree element would inherit the writer default namespace" + .into(), + )); + } + if element.attributes().count() > self.options.max_attributes_per_element { + return Err(XmlSafetyError::TooManyAttributes.into()); + } + let mut relative_depth = 1usize; + let mut parent = element.parent(); + while let Some(element) = parent { + relative_depth = relative_depth + .checked_add(1) + .ok_or(XmlSafetyError::TooDeep)?; + parent = element.parent(); + } + let combined_depth = self + .depth + .checked_add(relative_depth) + .ok_or(XmlSafetyError::TooDeep)?; + if combined_depth > self.options.max_depth { + return Err(XmlSafetyError::TooDeep.into()); + } + } + self.write_raw(subtree.document().root().raw_xml()) + } + + pub fn written_bytes(&self) -> usize { + self.writer.get_ref().written + } + + pub fn get_ref(&self) -> &W { + &self.writer.get_ref().inner + } + + pub fn finish(mut self) -> Result { + if self.depth != 0 { + return Err(Error::InvalidData( + "XML document has unclosed elements".into(), + )); + } + if !self.root_seen || !self.root_complete { + return Err(Error::InvalidData( + "XML document has no complete root".into(), + )); + } + if let Err(error) = self.writer.get_mut().flush() { + if self.writer.get_ref().exceeded { + return Err(XmlSafetyError::OutputTooLarge.into()); + } + return Err(Error::Io(error)); + } + Ok(self.writer.into_inner().inner) + } + + fn write_element<'a, I, A>( + &mut self, + name: &str, + attributes: I, + empty: bool, + ) -> Result<(), Error> + where + I: IntoIterator, + A: Into>, + { + if self.depth == 0 && self.root_complete { + return Err(Error::InvalidData( + "XML document cannot contain multiple roots".into(), + )); + } + let next_depth = self.depth.checked_add(1).ok_or(XmlSafetyError::TooDeep)?; + if next_depth > self.options.max_depth { + return Err(XmlSafetyError::TooDeep.into()); + } + validate_name(name, true)?; + + let binding_start = self.bindings.len(); + let mut start = BytesStart::new(name); + let result = (|| { + let mut attribute_count = 0usize; + for attribute in attributes { + let attribute = attribute.into(); + attribute_count = attribute_count + .checked_add(1) + .ok_or(XmlSafetyError::TooManyAttributes)?; + if attribute_count > self.options.max_attributes_per_element { + return Err(XmlSafetyError::TooManyAttributes.into()); + } + validate_name(attribute.name, true)?; + validate_xml_text(attribute.value)?; + self.record_namespace_binding(attribute)?; + start.push_attribute((attribute.name, attribute.value)); + } + for attribute in start.attributes() { + let attribute = attribute.map_err(|error| Error::InvalidData(error.to_string()))?; + let attribute_name = std::str::from_utf8(attribute.key.as_ref()) + .map_err(|_| Error::InvalidData("attribute name is not UTF-8".into()))?; + self.validate_attribute_namespace(attribute_name)?; + } + for (index, left) in start.attributes().enumerate() { + let left = left.map_err(|error| Error::InvalidData(error.to_string()))?; + let left_name = std::str::from_utf8(left.key.as_ref()) + .map_err(|_| Error::InvalidData("attribute name is not UTF-8".into()))?; + for right in start.attributes().skip(index + 1) { + let right = right.map_err(|error| Error::InvalidData(error.to_string()))?; + let right_name = std::str::from_utf8(right.key.as_ref()) + .map_err(|_| Error::InvalidData("attribute name is not UTF-8".into()))?; + if self.attributes_share_expanded_name(left_name, right_name) { + return Err(Error::InvalidData(format!( + "attributes `{left_name}` and `{right_name}` have the same expanded name" + ))); + } + } + } + self.validate_element_namespace(name) + })(); + if let Err(error) = result { + self.bindings.truncate(binding_start); + return Err(error); + } + + let event = if empty { + Event::Empty(start) + } else { + Event::Start(start) + }; + if let Err(error) = self.write_event(event) { + self.bindings.truncate(binding_start); + return Err(error); + } + self.root_seen = true; + if empty { + self.bindings.truncate(binding_start); + if self.depth == 0 { + self.root_complete = true; + } + } else { + if self.open_names.len() == self.depth { + self.open_names.push(String::new()); + self.binding_starts.push(0); + } + self.open_names[self.depth].clear(); + self.open_names[self.depth].push_str(name); + self.binding_starts[self.depth] = binding_start; + self.depth = next_depth; + } + Ok(()) + } + + fn record_namespace_binding(&mut self, attribute: XmlWriteAttribute<'_>) -> Result<(), Error> { + let Some(prefix) = namespace_declaration_prefix(attribute.name) else { + return Ok(()); + }; + validate_namespace_binding(prefix, attribute.value)?; + self.bindings.push(NamespaceBinding { + prefix: prefix.to_owned(), + uri: (!attribute.value.is_empty()).then(|| attribute.value.to_owned()), + }); + Ok(()) + } + + fn validate_element_namespace(&self, qualified_name: &str) -> Result<(), Error> { + if let Some((prefix, _)) = qualified_name.split_once(':') + && self.resolve_namespace(prefix).is_none() + { + return Err(Error::InvalidData(format!( + "element prefix `{prefix}` has no namespace binding" + ))); + } + Ok(()) + } + + fn validate_attribute_namespace(&self, qualified_name: &str) -> Result<(), Error> { + if namespace_declaration_prefix(qualified_name).is_some() { + return Ok(()); + } + if let Some((prefix, _)) = qualified_name.split_once(':') + && prefix != "xml" + && self.resolve_namespace(prefix).is_none() + { + return Err(Error::InvalidData(format!( + "attribute prefix `{prefix}` has no namespace binding" + ))); + } + Ok(()) + } + + fn attributes_share_expanded_name(&self, left: &str, right: &str) -> bool { + match ( + namespace_declaration_prefix(left), + namespace_declaration_prefix(right), + ) { + (Some(left_prefix), Some(right_prefix)) => return left_prefix == right_prefix, + (Some(_), None) | (None, Some(_)) => return false, + (None, None) => {} + } + let (left_prefix, left_local) = left.split_once(':').unwrap_or(("", left)); + let (right_prefix, right_local) = right.split_once(':').unwrap_or(("", right)); + if left_local != right_local { + return false; + } + let left_namespace = (!left_prefix.is_empty()) + .then(|| self.resolve_namespace(left_prefix)) + .flatten(); + let right_namespace = (!right_prefix.is_empty()) + .then(|| self.resolve_namespace(right_prefix)) + .flatten(); + left_namespace == right_namespace + } + + fn resolve_namespace(&self, prefix: &str) -> Option<&str> { + if prefix == "xml" { + return Some(XML_NAMESPACE_URI); + } + self.bindings + .iter() + .rev() + .find(|binding| binding.prefix == prefix) + .and_then(|binding| binding.uri.as_deref()) + } + + fn write_event(&mut self, event: Event<'_>) -> Result<(), Error> { + match self.writer.write_event(event) { + Ok(()) => Ok(()), + Err(error) => self.map_io_error(error), + } + } + + fn write_raw(&mut self, bytes: &[u8]) -> Result<(), Error> { + match self.writer.get_mut().write_all(bytes) { + Ok(()) => Ok(()), + Err(error) => self.map_io_error(error), + } + } + + fn map_io_error(&self, error: io::Error) -> Result<(), Error> { + if self.writer.get_ref().exceeded { + Err(XmlSafetyError::OutputTooLarge.into()) + } else { + Err(Error::Io(error)) + } + } +} + +fn namespace_declaration_prefix(name: &str) -> Option<&str> { + if name == "xmlns" { + Some("") + } else { + name.strip_prefix("xmlns:") + } +} + +fn validate_namespace_binding(prefix: &str, uri: &str) -> Result<(), Error> { + if prefix == "xmlns" + || uri == XMLNS_NAMESPACE_URI + || (prefix == "xml" && uri != XML_NAMESPACE_URI) + || (prefix != "xml" && uri == XML_NAMESPACE_URI) + || (!prefix.is_empty() && uri.is_empty()) + { + Err(Error::InvalidData("invalid namespace binding".into())) + } else { + Ok(()) + } +} + +fn validate_name(name: &str, qualified: bool) -> Result<(), Error> { + if name.is_empty() || (!qualified && name.contains(':')) || name.matches(':').count() > 1 { + return Err(Error::InvalidData(format!("invalid XML name `{name}`"))); + } + for part in name.split(':') { + let mut characters = part.chars(); + if !characters.next().is_some_and(is_name_start) || !characters.all(is_name_char) { + return Err(Error::InvalidData(format!("invalid XML name `{name}`"))); + } + } + Ok(()) +} + +fn is_name_start(character: char) -> bool { + matches!( + character, + 'A'..='Z' + | '_' + | 'a'..='z' + | '\u{00C0}'..='\u{00D6}' + | '\u{00D8}'..='\u{00F6}' + | '\u{00F8}'..='\u{02FF}' + | '\u{0370}'..='\u{037D}' + | '\u{037F}'..='\u{1FFF}' + | '\u{200C}'..='\u{200D}' + | '\u{2070}'..='\u{218F}' + | '\u{2C00}'..='\u{2FEF}' + | '\u{3001}'..='\u{D7FF}' + | '\u{F900}'..='\u{FDCF}' + | '\u{FDF0}'..='\u{FFFD}' + | '\u{10000}'..='\u{EFFFF}' + ) +} + +fn is_name_char(character: char) -> bool { + is_name_start(character) + || character.is_ascii_digit() + || matches!(character, '-' | '.' | '\u{B7}') + || ('\u{300}'..='\u{36F}').contains(&character) + || ('\u{203F}'..='\u{2040}').contains(&character) +} + +fn validate_xml_text(value: &str) -> Result<(), Error> { + if value.chars().all(|character| { + matches!(character, '\u{9}' | '\u{A}' | '\u{D}') + || ('\u{20}'..='\u{D7FF}').contains(&character) + || ('\u{E000}'..='\u{FFFD}').contains(&character) + || ('\u{10000}'..='\u{10FFFF}').contains(&character) + }) { + Ok(()) + } else { + Err(Error::InvalidData( + "value contains a character forbidden by XML 1.0".into(), + )) + } +} + +struct LimitedWriter { + inner: W, + max_bytes: usize, + written: usize, + exceeded: bool, +} + +impl LimitedWriter { + fn new(inner: W, max_bytes: usize) -> Self { + Self { + inner, + max_bytes, + written: 0, + exceeded: false, + } + } +} + +impl Write for LimitedWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + let Some(new_written) = self.written.checked_add(buffer.len()) else { + self.exceeded = true; + return Err(io::Error::other("XML output exceeds byte limit")); + }; + if new_written > self.max_bytes { + self.exceeded = true; + return Err(io::Error::other("XML output exceeds byte limit")); + } + self.inner.write_all(buffer)?; + self.written = new_written; + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} diff --git a/crates/aster_forge_xml/tests/document.rs b/crates/aster_forge_xml/tests/document.rs new file mode 100644 index 0000000..f295adf --- /dev/null +++ b/crates/aster_forge_xml/tests/document.rs @@ -0,0 +1,183 @@ +use std::sync::Arc; + +use aster_forge_xml::{ + BorrowedDocument, NodeRef, ParseOptions, ValidatedXml, XmlDocument, XmlSafetyError, +}; + +fn inside(source: &[u8], value: &[u8]) -> bool { + let source_start = source.as_ptr() as usize; + let source_end = source_start + source.len(); + let value_start = value.as_ptr() as usize; + let value_end = value_start + value.len(); + value_start >= source_start && value_end <= source_end +} + +#[test] +fn plain_document_values_are_source_backed() { + let source = br#"text"#; + let document = BorrowedDocument::parse(source.as_slice()).expect("document should parse"); + let root = document.root(); + let attribute = root.attributes().next().expect("plain attribute"); + let child = root.get_child_ns("child", "DAV:").expect("DAV child"); + let text = child.text().expect("child text"); + + assert_eq!(document.allocated_value_count(), 0); + assert!(inside(source, root.qualified_name().as_bytes())); + assert!(inside(source, attribute.qualified_name().as_bytes())); + assert!(inside(source, attribute.value().as_bytes())); + assert!(inside(source, text.as_bytes())); +} + +#[test] +fn decoded_entities_use_the_owned_value_pool_only_when_needed() { + let source = br#"before&after"#; + let document = BorrowedDocument::parse(source.as_slice()).expect("document should parse"); + let root = document.root(); + + assert_eq!(root.attribute("plain"), Some("value")); + assert_eq!(root.attribute("escaped"), Some("a&b")); + assert_eq!(root.text().as_deref(), Some("before&after")); + assert!(document.allocated_value_count() >= 2); +} + +#[test] +fn namespace_scopes_shadow_and_undeclare_without_per_element_maps() { + let source = br#""#; + let document = BorrowedDocument::parse(source.as_slice()).expect("document should parse"); + let root = document.root(); + let child = root + .get_child_ns("child", "urn:one") + .expect("default namespace child"); + let item = child + .get_child_ns("item", "urn:p2") + .expect("shadowed prefix"); + let plain = child.get_child("plain").expect("unqualified child"); + + assert_eq!(root.namespace(), Some("urn:one")); + assert_eq!(item.prefix(), Some("p")); + assert_eq!(plain.namespace(), None); +} + +#[test] +fn attributes_follow_xml_namespace_rules() { + let source = + br#""#; + let document = BorrowedDocument::parse(source.as_slice()).expect("document should parse"); + let root = document.root(); + + assert_eq!(root.attribute_ns("plain", None), Some("a")); + assert_eq!(root.attribute_ns("value", Some("urn:attrs")), Some("b")); + assert_eq!( + root.attribute_ns("lang", Some("http://www.w3.org/XML/1998/namespace")), + Some("en") + ); +} + +#[test] +fn subtree_raw_xml_is_an_exact_source_slice() { + let source = br#"a&b"#; + let document = BorrowedDocument::parse(source.as_slice()).expect("document should parse"); + let owner = document.root().get_child("owner").expect("owner subtree"); + + assert_eq!( + owner.raw_xml(), + br#"a&b"# + ); + assert!(inside(source, owner.raw_xml())); +} + +#[test] +fn ordered_nodes_and_parent_links_are_preserved() { + let source = br#"beforeafter"#; + let document = BorrowedDocument::parse(source.as_slice()).expect("document should parse"); + let root = document.root(); + let nodes: Vec<_> = root.children().collect(); + + assert!(matches!(nodes[0], NodeRef::Text("before"))); + assert!(matches!(nodes[1], NodeRef::Element(element) if element.name() == "child")); + assert!(matches!(nodes[2], NodeRef::CData("raw"))); + assert!(matches!(nodes[3], NodeRef::Comment("note"))); + assert!(matches!( + nodes[4], + NodeRef::ProcessingInstruction(pi) if pi.target == "work" && pi.content == Some("now") + )); + assert!(matches!(nodes[5], NodeRef::Text("after"))); + let child = root.get_child("child").expect("child"); + assert_eq!(child.parent().map(|parent| parent.id()), Some(root.id())); +} + +#[test] +fn descendants_preserve_depth_first_document_order() { + let source = b""; + let document = BorrowedDocument::parse(source.as_slice()).expect("document should parse"); + let names: Vec<_> = document + .root() + .descendants() + .map(|element| element.name()) + .collect(); + + assert_eq!(names, ["root", "a", "b", "c", "d"]); +} + +#[test] +fn validated_xml_owns_exact_bytes_and_clones_cheaply() { + let source = br#"mailto:a@example.test"#.to_vec(); + let validated = ValidatedXml::new(source.clone()).expect("validated XML"); + let cloned = validated.clone(); + + assert_eq!(validated.as_bytes(), source); + assert_eq!(cloned.as_bytes(), source); + assert_eq!(validated.document().root().namespace(), Some("DAV:")); +} + +#[test] +fn original_write_preserves_the_complete_document() { + let source = br#""#; + let document = BorrowedDocument::parse(source.as_slice()).expect("document should parse"); + let mut output = Vec::new(); + + document + .write_original(&mut output) + .expect("document should write"); + + assert_eq!(output, source); + assert_eq!(document.root().raw_xml(), b""); +} + +#[test] +fn generic_owned_source_does_not_require_self_references() { + let source: Arc<[u8]> = Arc::from(br#""#.as_slice()); + let document = XmlDocument::parse(Arc::clone(&source)).expect("owned document"); + drop(source); + + assert_eq!( + document.root().get_child("child").map(|child| child.name()), + Some("child") + ); +} + +#[test] +fn deep_arena_document_drops_without_tree_recursion() { + const DEPTH: usize = 20_000; + let mut source = "".repeat(DEPTH); + source.push_str(&"".repeat(DEPTH)); + let options = ParseOptions::new() + .max_size(source.len()) + .max_depth(DEPTH) + .max_elements(DEPTH) + .max_events(DEPTH * 2 + 1); + let document = BorrowedDocument::parse_with_options(source.as_bytes(), &options) + .expect("deep arena document"); + + assert_eq!(document.node_count(), DEPTH); + drop(document); +} + +#[test] +fn arena_parser_keeps_safety_error_classification() { + let error = BorrowedDocument::parse(b"".as_slice()).expect_err("second root"); + assert!(matches!( + error, + aster_forge_xml::Error::Safety(XmlSafetyError::Malformed) + )); +} diff --git a/crates/aster_forge_xml/tests/property.proptest-regressions b/crates/aster_forge_xml/tests/property.proptest-regressions new file mode 100644 index 0000000..36932c9 --- /dev/null +++ b/crates/aster_forge_xml/tests/property.proptest-regressions @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc fc032be8a565eb159cbbf3a4f18511794c7264567a2257a1fa92535043092863 # shrinks to model = ElementModel { name: "a", attributes: {"a": "\n"}, text: "", children: [] } diff --git a/crates/aster_forge_xml/tests/property.rs b/crates/aster_forge_xml/tests/property.rs new file mode 100644 index 0000000..8be8207 --- /dev/null +++ b/crates/aster_forge_xml/tests/property.rs @@ -0,0 +1,176 @@ +use std::collections::BTreeMap; + +use aster_forge_xml::{ + BorrowedDocument, ElementRef, ParseOptions, XmlSafetyPolicy, XmlStreamEvent, XmlStreamReader, + XmlStreamWriter, validate_xml_input, +}; +use proptest::collection::{btree_map, vec}; +use proptest::prelude::*; +use proptest::string::string_regex; + +#[derive(Clone, Debug)] +struct ElementModel { + name: String, + attributes: BTreeMap, + text: String, + children: Vec, +} + +fn xml_name() -> impl Strategy { + string_regex("[a-z][a-z0-9]{0,7}").expect("static name regex") +} + +fn xml_text() -> impl Strategy { + vec(0u8..=16, 0..32).prop_map(|values| { + values + .into_iter() + .map(|value| match value { + 0 => '&', + 1 => '<', + 2 => '>', + 3 => '\'', + 4 => '"', + 5 => '\n', + 6 => '\t', + 7 => '中', + _ => char::from(b'a' + (value % 10)), + }) + .collect() + }) +} + +fn element_model() -> impl Strategy { + ( + xml_name(), + btree_map(xml_name(), xml_text(), 0..4), + xml_text(), + ) + .prop_map(|(name, attributes, text)| ElementModel { + name, + attributes, + text, + children: Vec::new(), + }) + .prop_recursive(4, 96, 4, |children| { + ( + xml_name(), + btree_map(xml_name(), xml_text(), 0..4), + xml_text(), + vec(children, 0..4), + ) + .prop_map(|(name, attributes, text, children)| ElementModel { + name, + attributes, + text, + children, + }) + }) +} + +fn write_model(writer: &mut XmlStreamWriter>, model: &ElementModel) { + writer + .start_element( + &model.name, + model + .attributes + .iter() + .map(|(name, value)| (name.as_str(), value.as_str())), + ) + .expect("generated element is valid"); + writer.text(&model.text).expect("generated text is valid"); + for child in &model.children { + write_model(writer, child); + } + writer.end_element().expect("generated element end"); +} + +fn assert_forge_model(element: ElementRef<'_, &[u8]>, model: &ElementModel) { + assert_eq!(element.name(), model.name); + assert_eq!(element.text().unwrap_or_default(), model.text); + assert_eq!(element.attributes().count(), model.attributes.len()); + for (name, value) in &model.attributes { + let normalized = normalize_attribute_value(value); + assert_eq!(element.attribute(name), Some(normalized.as_str())); + } + let actual_children: Vec<_> = element.child_elements().collect(); + assert_eq!(actual_children.len(), model.children.len()); + for (actual, expected) in actual_children.into_iter().zip(&model.children) { + assert_forge_model(actual, expected); + } +} + +fn assert_roxmltree_model(node: roxmltree::Node<'_, '_>, model: &ElementModel) { + assert_eq!(node.tag_name().name(), model.name); + let direct_text = node + .children() + .filter(roxmltree::Node::is_text) + .filter_map(|child| child.text()) + .collect::(); + assert_eq!(direct_text, model.text); + assert_eq!(node.attributes().len(), model.attributes.len()); + for (name, value) in &model.attributes { + let normalized = normalize_attribute_value(value); + assert_eq!(node.attribute(name.as_str()), Some(normalized.as_str())); + } + let actual_children: Vec<_> = node + .children() + .filter(roxmltree::Node::is_element) + .collect(); + assert_eq!(actual_children.len(), model.children.len()); + for (actual, expected) in actual_children.into_iter().zip(&model.children) { + assert_roxmltree_model(actual, expected); + } +} + +fn normalize_attribute_value(value: &str) -> String { + value + .chars() + .map(|character| match character { + '\n' | '\r' | '\t' => ' ', + character => character, + }) + .collect() +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(256))] + + #[test] + fn streaming_writer_round_trips_generated_trees(model in element_model()) { + let mut writer = XmlStreamWriter::new(Vec::new()).expect("writer"); + write_model(&mut writer, &model); + let output = writer.finish().expect("complete generated document"); + + validate_xml_input(&output, XmlSafetyPolicy::untrusted()).expect("generated document validates"); + let forge = BorrowedDocument::parse(output.as_slice()).expect("Forge reparses generated document"); + assert_forge_model(forge.root(), &model); + let text = std::str::from_utf8(&output).expect("writer emits UTF-8"); + let roxmltree = roxmltree::Document::parse(text).expect("roxmltree reparses generated document"); + assert_roxmltree_model(roxmltree.root_element(), &model); + } + + #[test] + fn arbitrary_byte_inputs_remain_bounded_and_never_panic(input in vec(any::(), 0..4096)) { + let max_input_bytes = input.len().max(1); + let policy = XmlSafetyPolicy { + max_input_bytes, + max_depth: 64, + max_elements: 4096, + max_attributes_per_element: 128, + max_text_bytes: max_input_bytes, + max_events: 16_384, + reject_doctype: true, + }; + let _ = validate_xml_input(&input, policy); + let options = ParseOptions::new().safety_policy(policy); + let _ = BorrowedDocument::parse_with_options(input.as_slice(), &options); + + let mut reader = XmlStreamReader::new(input.as_slice(), policy).expect("valid policy"); + for _ in 0..=policy.max_events { + match reader.read_event() { + Ok(XmlStreamEvent::Eof) | Err(_) => break, + Ok(_) => {} + } + } + } +} diff --git a/crates/aster_forge_xml/tests/roxmltree_compat.rs b/crates/aster_forge_xml/tests/roxmltree_compat.rs new file mode 100644 index 0000000..c128389 --- /dev/null +++ b/crates/aster_forge_xml/tests/roxmltree_compat.rs @@ -0,0 +1,121 @@ +use std::collections::BTreeMap; + +use aster_forge_xml::{BorrowedDocument, ElementRef}; + +#[derive(Debug, PartialEq, Eq)] +struct ElementSnapshot { + local_name: String, + namespace: Option, + attributes: BTreeMap<(Option, String), String>, + direct_text: String, + children: Vec, +} + +fn forge_snapshot(element: ElementRef<'_, &[u8]>) -> ElementSnapshot { + ElementSnapshot { + local_name: element.name().to_owned(), + namespace: element.namespace().map(str::to_owned), + attributes: element + .attributes() + .map(|attribute| { + ( + ( + attribute.namespace().map(str::to_owned), + attribute.name().to_owned(), + ), + attribute.value().to_owned(), + ) + }) + .collect(), + direct_text: element.text().unwrap_or_default().into_owned(), + children: element.child_elements().map(forge_snapshot).collect(), + } +} + +fn roxmltree_snapshot(node: roxmltree::Node<'_, '_>) -> ElementSnapshot { + ElementSnapshot { + local_name: node.tag_name().name().to_owned(), + namespace: node + .tag_name() + .namespace() + .filter(|namespace| !namespace.is_empty()) + .map(str::to_owned), + attributes: node + .attributes() + .map(|attribute| { + ( + ( + attribute + .namespace() + .filter(|namespace| !namespace.is_empty()) + .map(str::to_owned), + attribute.name().to_owned(), + ), + attribute.value().to_owned(), + ) + }) + .collect(), + direct_text: node + .children() + .filter(roxmltree::Node::is_text) + .filter_map(|child| child.text()) + .collect::(), + children: node + .children() + .filter(roxmltree::Node::is_element) + .map(roxmltree_snapshot) + .collect(), + } +} + +#[test] +fn forge_matches_roxmltree_for_source_backed_tree_queries() { + for source in [ + br#""#.as_slice(), + br#"beforeafter"#, + br#"a&b]]>"#, + br#""#, + ] { + let forge = BorrowedDocument::parse(source).expect("Forge fixture"); + let text = std::str::from_utf8(source).expect("UTF-8 fixture"); + let roxmltree = roxmltree::Document::parse(text).expect("roxmltree fixture"); + + assert_eq!( + forge_snapshot(forge.root()), + roxmltree_snapshot(roxmltree.root_element()) + ); + } +} + +#[test] +fn forge_and_roxmltree_expose_the_same_exact_root_range() { + let source = br#"text"#; + let forge = BorrowedDocument::parse(source.as_slice()).expect("Forge fixture"); + let text = std::str::from_utf8(source).expect("UTF-8 fixture"); + let roxmltree = roxmltree::Document::parse(text).expect("roxmltree fixture"); + let range = roxmltree.root_element().range(); + + assert_eq!(forge.root().raw_xml(), &source[range]); +} + +#[test] +fn roxmltree_comparison_keeps_intentional_node_model_differences_explicit() { + let source = br#"text"#; + let forge = BorrowedDocument::parse(source.as_slice()).expect("Forge fixture"); + let text = std::str::from_utf8(source).expect("UTF-8 fixture"); + let roxmltree = roxmltree::Document::parse(text).expect("roxmltree fixture"); + + assert_eq!(forge.root().text().as_deref(), Some("textcdata")); + assert_eq!(roxmltree.root_element().text(), Some("textcdata")); + assert_eq!( + roxmltree + .root_element() + .children() + .filter(roxmltree::Node::is_text) + .filter_map(|node| node.text()) + .collect::(), + "textcdata" + ); + assert_eq!(forge.root().children().count(), 4); + assert_eq!(roxmltree.root_element().children().count(), 3); +} diff --git a/crates/aster_forge_xml/tests/stream.rs b/crates/aster_forge_xml/tests/stream.rs new file mode 100644 index 0000000..09f282c --- /dev/null +++ b/crates/aster_forge_xml/tests/stream.rs @@ -0,0 +1,285 @@ +use std::fmt::Write as _; +use std::io::{BufReader, Cursor}; + +use aster_forge_xml::{Error, XmlSafetyError, XmlSafetyPolicy, XmlStreamEvent, XmlStreamReader}; + +fn reader(input: &[u8]) -> XmlStreamReader> { + XmlStreamReader::new(BufReader::new(input), XmlSafetyPolicy::untrusted()).expect("valid policy") +} + +#[test] +fn streams_namespace_resolved_names_attributes_and_text() { + let input = + br#"a&b"#; + let mut reader = reader(input); + + let XmlStreamEvent::Start(root) = reader.read_event().expect("root") else { + panic!("expected root start"); + }; + assert!(root.name().expect("name").matches("root", Some("DAV:"))); + assert_eq!( + root.attribute_ns("id", Some("urn:attr")) + .expect("attribute") + .as_deref(), + Some("7") + ); + + let XmlStreamEvent::Start(name) = reader.read_event().expect("name") else { + panic!("expected name start"); + }; + assert!(name.name().expect("name").matches("name", Some("DAV:"))); + let text = reader.read_text_current().expect("direct text"); + assert_eq!(text, "a&b"); + + assert!(matches!(reader.read_event(), Ok(XmlStreamEvent::End(_)))); + assert!(matches!(reader.read_event(), Ok(XmlStreamEvent::Eof))); +} + +#[test] +fn reuses_validated_values_and_captures_equivalent_mixed_content() { + let input = br#"A&"#; + let mut reader = reader(input); + let XmlStreamEvent::Start(root) = reader.read_event().expect("root") else { + panic!("expected root start"); + }; + let escaped = root + .attributes() + .find_map(|attribute| { + let attribute = attribute.expect("valid attribute"); + (attribute.name().expect("name").qualified() == "escaped").then_some(attribute) + }) + .expect("escaped attribute"); + assert_eq!(escaped.value().expect("first value"), "a&b"); + assert_eq!(escaped.value().expect("cached value"), "a&b"); + + assert!(matches!(reader.read_event(), Ok(XmlStreamEvent::Start(_)))); + let captured = reader.capture_current(1024).expect("captured target"); + assert_eq!(captured.document().root().text().as_deref(), Some("x")); + assert!(captured.contains("")); + assert!(captured.contains("A&")); +} + +#[test] +fn skips_unselected_subtrees_without_a_token_stack() { + let input = + b"ignoredyes"; + let mut reader = reader(input); + assert!(matches!(reader.read_event(), Ok(XmlStreamEvent::Start(_)))); + assert!(matches!(reader.read_event(), Ok(XmlStreamEvent::Start(_)))); + reader.skip_current().expect("skip subtree"); + + let XmlStreamEvent::Start(keep) = reader.read_event().expect("keep") else { + panic!("expected keep start"); + }; + assert_eq!(keep.name().expect("name").local(), "keep"); + assert_eq!(reader.read_text_current().expect("keep text"), "yes"); +} + +#[test] +fn captures_only_selected_subtree_and_injects_in_scope_namespaces() { + let input = br#"a&b"#; + let mut reader = reader(input); + assert!(matches!(reader.read_event(), Ok(XmlStreamEvent::Start(_)))); + assert!(matches!(reader.read_event(), Ok(XmlStreamEvent::Start(_)))); + let XmlStreamEvent::Start(color) = reader.read_event().expect("color") else { + panic!("expected color start"); + }; + assert!( + color + .name() + .expect("name") + .matches("color", Some("urn:property")) + ); + + let captured = reader.capture_current(1024).expect("captured subtree"); + let root = captured.document().root(); + assert_eq!(root.name(), "color"); + assert_eq!(root.namespace(), Some("urn:property")); + assert_eq!(root.attribute("shade"), Some("dark")); + assert_eq!(root.text().as_deref(), Some("a&b")); + assert!(captured.as_bytes().starts_with(b""); + assert!(matches!(reader.read_event(), Ok(XmlStreamEvent::Empty(_)))); + assert!(matches!(reader.skip_current(), Err(Error::InvalidData(_)))); + assert!(matches!( + reader.capture_current(128), + Err(Error::InvalidData(_)) + )); +} + +#[test] +fn stream_enforces_input_depth_attribute_text_and_event_limits() { + let base = XmlSafetyPolicy::untrusted(); + for (input, policy, expected) in [ + ( + b"".as_slice(), + XmlSafetyPolicy { + max_input_bytes: 6, + ..base + }, + XmlSafetyError::InputTooLarge, + ), + ( + b"".as_slice(), + XmlSafetyPolicy { + max_depth: 1, + ..base + }, + XmlSafetyError::TooDeep, + ), + ( + b"".as_slice(), + XmlSafetyPolicy { + max_attributes_per_element: 1, + ..base + }, + XmlSafetyError::TooManyAttributes, + ), + ( + b"text".as_slice(), + XmlSafetyPolicy { + max_text_bytes: 3, + ..base + }, + XmlSafetyError::TextTooLarge, + ), + ( + b"".as_slice(), + XmlSafetyPolicy { + max_events: 2, + ..base + }, + XmlSafetyError::TooManyEvents, + ), + ] { + let mut reader = XmlStreamReader::new(BufReader::new(input), policy).expect("policy"); + let error = loop { + match reader.read_event() { + Ok(XmlStreamEvent::Eof) => panic!("fixture should cross a limit"), + Ok(_) => {} + Err(error) => break error, + } + }; + assert!(matches!(error, Error::Safety(actual) if actual == expected)); + } +} + +#[test] +fn stream_rejects_doctype_unknown_prefix_and_multiple_roots() { + for input in [ + b"".as_slice(), + b"", + b"", + ] { + let mut reader = reader(input); + let error = loop { + match reader.read_event() { + Ok(XmlStreamEvent::Eof) => panic!("fixture should fail"), + Ok(_) => {} + Err(error) => break error, + } + }; + assert!(matches!(error, Error::Safety(_) | Error::InvalidXml(_))); + } +} + +#[test] +fn selective_capture_enforces_its_own_byte_limit() { + let mut reader = reader(b"0123456789"); + assert!(matches!(reader.read_event(), Ok(XmlStreamEvent::Start(_)))); + assert!(matches!(reader.read_event(), Ok(XmlStreamEvent::Start(_)))); + assert!(matches!( + reader.capture_current(16), + Err(Error::Safety(XmlSafetyError::InputTooLarge)) + )); +} + +#[test] +fn streams_and_selectively_captures_a_hundred_thousand_element_document() { + let items = 100_000usize; + let mut input = String::from(""); + for index in 0..items { + write!( + input, + "/files/{index}" + ) + .expect("write string fixture"); + } + input.push_str(""); + let policy = XmlSafetyPolicy { + max_input_bytes: input.len(), + max_elements: items * 3, + max_text_bytes: input.len(), + max_events: items * 8, + ..XmlSafetyPolicy::untrusted() + }; + let mut reader = XmlStreamReader::new(BufReader::new(Cursor::new(input.as_bytes())), policy) + .expect("reader"); + let mut responses = 0usize; + let mut captured = None; + + loop { + match reader.read_event().expect("stream event") { + XmlStreamEvent::Start(start) + if start + .name() + .expect("name") + .matches("response", Some("DAV:")) => + { + responses += 1; + if responses == items / 2 { + captured = Some(reader.capture_current(256).expect("capture one response")); + } + } + XmlStreamEvent::Eof => break, + _ => {} + } + } + + assert_eq!(responses, items); + let captured = captured.expect("selected response"); + assert_eq!( + captured.document().root().attribute("id"), + Some((items / 2 - 1).to_string().as_str()) + ); + assert_eq!( + captured + .document() + .root() + .get_child_ns("href", "DAV:") + .expect("captured href") + .text() + .as_deref(), + Some(format!("/files/{}", items / 2 - 1).as_str()) + ); +} + +#[test] +fn input_limit_consumes_only_one_probe_byte_past_the_boundary() { + let input = vec![b'x'; 128 * 1024]; + let policy = XmlSafetyPolicy { + max_input_bytes: 32, + ..XmlSafetyPolicy::untrusted() + }; + let mut reader = XmlStreamReader::new(Cursor::new(input), policy).expect("reader"); + let error = loop { + match reader.read_event() { + Ok(XmlStreamEvent::Eof) => panic!("fixture should cross the input limit"), + Ok(_) => {} + Err(error) => break error, + } + }; + + assert!(matches!( + error, + Error::Safety(XmlSafetyError::InputTooLarge) + )); + assert_eq!(reader.into_inner().position(), 33); +} diff --git a/crates/aster_forge_xml/tests/writer.rs b/crates/aster_forge_xml/tests/writer.rs new file mode 100644 index 0000000..f2938c3 --- /dev/null +++ b/crates/aster_forge_xml/tests/writer.rs @@ -0,0 +1,348 @@ +use std::io::{self, Write}; + +use aster_forge_xml::{ + BorrowedDocument, Error, ValidatedXml, XmlSafetyError, XmlStreamWriter, XmlWriteAttribute, + XmlWriteOptions, +}; + +fn finish(writer: XmlStreamWriter>) -> Vec { + writer.finish().expect("writer should finish") +} + +fn assert_invalid_data(result: Result<(), Error>) { + assert!(matches!(result, Err(Error::InvalidData(_)))); +} + +#[test] +fn writes_namespace_aware_webdav_multistatus_and_reparses() { + let mut writer = XmlStreamWriter::new(Vec::new()).expect("writer"); + writer + .start_element("D:multistatus", [("xmlns:D", "DAV:")]) + .expect("root"); + writer.start("D:response").expect("response"); + writer.start("D:href").expect("href"); + writer.text("/files/a&b").expect("text"); + writer.end_element().expect("href end"); + writer + .empty_element( + "x:color", + [ + XmlWriteAttribute::new("xmlns:x", "urn:property"), + XmlWriteAttribute::new("x:shade", "dark"), + ], + ) + .expect("extension property"); + writer.end_element().expect("response end"); + writer.end_element().expect("root end"); + + let output = finish(writer); + assert_eq!( + output, + br#"/files/a&b"# + ); + let document = BorrowedDocument::parse(output.as_slice()).expect("Forge reparses output"); + assert_eq!(document.root().namespace(), Some("DAV:")); + xmltree::Element::parse(output.as_slice()).expect("xmltree reparses output"); +} + +#[test] +fn supports_inherited_prefix_default_namespace_and_undeclaration() { + let mut writer = XmlStreamWriter::new(Vec::new()).expect("writer"); + writer + .start_element("root", [("xmlns", "urn:default"), ("xmlns:p", "urn:p")]) + .expect("root"); + writer + .start_element("p:child", [("p:id", "7")]) + .expect("prefixed child"); + writer + .empty_element("plain", [("xmlns", "")]) + .expect("default namespace undeclaration"); + writer.end_element().expect("child end"); + writer.end_element().expect("root end"); + + let output = finish(writer); + let document = BorrowedDocument::parse(output.as_slice()).expect("parse output"); + assert_eq!(document.root().namespace(), Some("urn:default")); + let child = document + .root() + .get_child_ns("child", "urn:p") + .expect("child"); + assert_eq!(child.attribute_ns("id", Some("urn:p")), Some("7")); + let plain = child.get_child("plain").expect("plain"); + assert_eq!(plain.namespace(), None); +} + +#[test] +fn escapes_attribute_and_text_values_once() { + let mut writer = XmlStreamWriter::new(Vec::new()).expect("writer"); + writer + .start_element("root", [("value", "&<>\"'")]) + .expect("root"); + writer.text("&<>\"'").expect("text"); + writer.end_element().expect("root end"); + let output = finish(writer); + + assert_eq!( + output, + br#"&<>"'"# + ); + let document = BorrowedDocument::parse(output.as_slice()).expect("parse output"); + assert_eq!(document.root().attribute("value"), Some("&<>\"'")); + assert_eq!(document.root().text().as_deref(), Some("&<>\"'")); +} + +#[test] +fn writes_declaration_cdata_comment_processing_instruction_and_empty_root() { + let options = XmlWriteOptions::new().write_document_declaration(true); + let mut writer = XmlStreamWriter::with_options(Vec::new(), options).expect("writer"); + writer.comment("before").expect("comment"); + writer + .processing_instruction("build", Some("id=7")) + .expect("processing instruction"); + writer.start("root").expect("root"); + writer.cdata("a < b & c").expect("CDATA"); + writer.comment("inside").expect("comment"); + writer.end_element().expect("root end"); + writer.comment("after").expect("comment"); + + let output = finish(writer); + assert_eq!( + output, + br#""# + ); + BorrowedDocument::parse(output.as_slice()).expect("parse output"); + + let mut empty = XmlStreamWriter::new(Vec::new()).expect("writer"); + empty.empty("root").expect("empty root"); + assert_eq!(finish(empty), b""); +} + +#[test] +fn rejects_invalid_document_state_namespaces_names_and_values() { + let mut writer = XmlStreamWriter::new(Vec::new()).expect("writer"); + assert_invalid_data(writer.text("outside")); + assert_invalid_data(writer.cdata("outside")); + assert_invalid_data(writer.end_element()); + assert_invalid_data(writer.start("p:root")); + assert_invalid_data(writer.start("1root")); + assert_invalid_data(writer.start("a:b:c")); + assert_invalid_data(writer.start_element("root", [("p:id", "7")])); + assert_invalid_data(writer.start_element("root", [("xmlns:xml", "urn:wrong")])); + assert_invalid_data(writer.start_element("root", [("xmlns:xmlns", "urn:x")])); + assert_invalid_data(writer.start_element("root", [("xmlns:p", "")])); + assert_invalid_data(writer.start_element( + "root", + [("xmlns:p", "http://www.w3.org/XML/1998/namespace")], + )); + assert_invalid_data(writer.start_element("root", [("xmlns", "http://www.w3.org/2000/xmlns/")])); + assert_invalid_data(writer.start_element("root", [("x", "\u{0}")])); + + writer.start("root").expect("valid root after failures"); + assert_invalid_data(writer.comment("a--b")); + assert_invalid_data(writer.comment("trailing-")); + assert_invalid_data(writer.cdata("a]]>b")); + assert_invalid_data(writer.processing_instruction("xml", None)); + assert_invalid_data(writer.processing_instruction("p:target", None)); + assert_invalid_data(writer.processing_instruction("target", Some("a?>b"))); + assert_invalid_data(writer.text("\u{1}")); + writer.end_element().expect("root end"); + assert_invalid_data(writer.empty("second")); +} + +#[test] +fn rejects_duplicate_attributes_and_mismatched_writer_lifecycle() { + let mut writer = XmlStreamWriter::new(Vec::new()).expect("writer"); + assert_invalid_data(writer.start_element("root", [("id", "1"), ("id", "2")])); + assert_invalid_data( + writer.start_element("root", [("xmlns:a", "urn:one"), ("xmlns:a", "urn:one")]), + ); + assert_invalid_data(writer.start_element("root", [("xmlns", "urn:one"), ("xmlns", "urn:two")])); + assert_invalid_data(writer.start_element( + "root", + [ + ("xmlns:a", "urn:same"), + ("xmlns:b", "urn:same"), + ("a:id", "1"), + ("b:id", "2"), + ], + )); + writer.start("root").expect("root"); + assert!(matches!(writer.finish(), Err(Error::InvalidData(_)))); + + let writer = XmlStreamWriter::new(Vec::new()).expect("writer"); + assert!(matches!(writer.finish(), Err(Error::InvalidData(_)))); +} + +#[test] +fn validated_subtree_does_not_inherit_the_writer_default_namespace() { + let unqualified = ValidatedXml::new(b"".to_vec()).expect("subtree"); + let prefixed = + ValidatedXml::new(br#""#.to_vec()) + .expect("prefixed subtree"); + + let mut writer = XmlStreamWriter::new(Vec::new()).expect("writer"); + writer + .start_element("root", [("xmlns", "urn:caller")]) + .expect("root"); + assert_invalid_data(writer.validated_subtree(&unqualified)); + writer + .validated_subtree(&prefixed) + .expect("prefixed subtree remains self-contained"); + writer.end_element().expect("root end"); + + let output = finish(writer); + let document = BorrowedDocument::parse(output.as_slice()).expect("parse output"); + assert!(document.root().get_child_ns("child", "urn:child").is_some()); + + let mut writer = XmlStreamWriter::new(Vec::new()).expect("writer"); + writer.start("root").expect("root"); + writer + .validated_subtree(&unqualified) + .expect("no caller default namespace"); + writer.end_element().expect("root end"); + let output = finish(writer); + let document = BorrowedDocument::parse(output.as_slice()).expect("parse output"); + assert!(document.root().get_child("child").is_some()); +} + +#[test] +fn enforces_exact_depth_attribute_and_output_boundaries() { + let options = XmlWriteOptions::new().max_depth(2); + let mut writer = XmlStreamWriter::with_options(Vec::new(), options).expect("writer"); + writer.start("root").expect("depth one"); + writer.start("child").expect("depth two"); + assert!(matches!( + writer.start("too-deep"), + Err(Error::Safety(XmlSafetyError::TooDeep)) + )); + writer.end_element().expect("child end"); + writer.end_element().expect("root end"); + finish(writer); + + let options = XmlWriteOptions::new().max_attributes_per_element(2); + let mut writer = XmlStreamWriter::with_options(Vec::new(), options).expect("writer"); + writer + .empty_element("root", [("a", "1"), ("b", "2")]) + .expect("exact attribute limit"); + finish(writer); + let mut writer = XmlStreamWriter::with_options(Vec::new(), options).expect("writer"); + assert!(matches!( + writer.empty_element("root", [("a", "1"), ("b", "2"), ("c", "3")]), + Err(Error::Safety(XmlSafetyError::TooManyAttributes)) + )); + + let exact = b"123"; + let options = XmlWriteOptions::new().max_output_bytes(exact.len()); + let mut writer = XmlStreamWriter::with_options(Vec::new(), options).expect("writer"); + writer.start("root").expect("root"); + writer.text("123").expect("text"); + writer.end_element().expect("root end"); + assert_eq!(finish(writer), exact); + + let options = XmlWriteOptions::new().max_output_bytes(exact.len() - 1); + let mut writer = XmlStreamWriter::with_options(Vec::new(), options).expect("writer"); + writer.start("root").expect("root"); + writer.text("123").expect("text"); + assert!(matches!( + writer.end_element(), + Err(Error::Safety(XmlSafetyError::OutputTooLarge)) + )); +} + +#[test] +fn embeds_validated_subtree_without_building_an_output_dom() { + let subtree = ValidatedXml::new( + br#"blue"#.to_vec(), + ) + .expect("subtree"); + let mut writer = XmlStreamWriter::new(Vec::new()).expect("writer"); + assert_invalid_data(writer.validated_subtree(&subtree)); + writer.start("root").expect("root"); + writer.validated_subtree(&subtree).expect("subtree"); + writer.end_element().expect("root end"); + + let output = finish(writer); + let document = BorrowedDocument::parse(output.as_slice()).expect("parse output"); + let color = document + .root() + .get_child_ns("color", "urn:property") + .expect("embedded subtree"); + assert_eq!(color.text().as_deref(), Some("blue")); +} + +#[test] +fn embedded_subtree_obeys_writer_depth_and_attribute_limits() { + let deep = ValidatedXml::new(b"".to_vec()).expect("subtree"); + let options = XmlWriteOptions::new().max_depth(2); + let mut writer = XmlStreamWriter::with_options(Vec::new(), options).expect("writer"); + writer.start("root").expect("root"); + assert!(matches!( + writer.validated_subtree(&deep), + Err(Error::Safety(XmlSafetyError::TooDeep)) + )); + + let attributed = ValidatedXml::new(b"".to_vec()).expect("subtree"); + let options = XmlWriteOptions::new().max_attributes_per_element(1); + let mut writer = XmlStreamWriter::with_options(Vec::new(), options).expect("writer"); + writer.start("root").expect("root"); + assert!(matches!( + writer.validated_subtree(&attributed), + Err(Error::Safety(XmlSafetyError::TooManyAttributes)) + )); +} + +struct AlwaysFails; + +impl Write for AlwaysFails { + fn write(&mut self, _buffer: &[u8]) -> io::Result { + Err(io::Error::other("fixture failure")) + } + + fn flush(&mut self) -> io::Result<()> { + Err(io::Error::other("fixture failure")) + } +} + +#[test] +fn preserves_underlying_io_failures() { + let mut writer = XmlStreamWriter::new(AlwaysFails).expect("writer construction"); + assert!(matches!(writer.empty("root"), Err(Error::Io(_)))); + + struct FlushFails(Vec); + impl Write for FlushFails { + fn write(&mut self, buffer: &[u8]) -> io::Result { + self.0.extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Err(io::Error::other("flush failure")) + } + } + + let mut writer = XmlStreamWriter::new(FlushFails(Vec::new())).expect("writer"); + writer.empty("root").expect("root"); + assert!(matches!(writer.finish(), Err(Error::Io(_)))); +} + +#[test] +fn writes_large_response_directly() { + let responses = 25_000usize; + let mut writer = XmlStreamWriter::new(Vec::new()).expect("writer"); + writer + .start_element("D:multistatus", [("xmlns:D", "DAV:")]) + .expect("root"); + for index in 0..responses { + let href = format!("/files/{index}"); + writer.start("D:response").expect("response"); + writer.start("D:href").expect("href"); + writer.text(&href).expect("href text"); + writer.end_element().expect("href end"); + writer.end_element().expect("response end"); + } + writer.end_element().expect("root end"); + let output = finish(writer); + + let document = BorrowedDocument::parse(output.as_slice()).expect("parse large output"); + assert_eq!(document.root().child_elements().count(), responses); +} diff --git a/crates/aster_forge_xml/tests/xml.rs b/crates/aster_forge_xml/tests/xml.rs new file mode 100644 index 0000000..0e739e8 --- /dev/null +++ b/crates/aster_forge_xml/tests/xml.rs @@ -0,0 +1,381 @@ +use std::io::{self, Write}; + +use aster_forge_xml::{ + BorrowedDocument, Error, NodeRef, OwnedDocument, ParseOptions, XmlSafetyError, XmlSafetyPolicy, + validate_xml_input, xml_root_local_name, +}; + +#[test] +fn preserves_every_node_type_and_mixed_content_in_order() { + let input = br#"

before]]>after

"#; + let document = BorrowedDocument::parse(input.as_slice()).expect("mixed document should parse"); + let nodes: Vec<_> = document.root().children().collect(); + + assert!(matches!(nodes[0], NodeRef::Text("before"))); + assert!(matches!(nodes[1], NodeRef::Element(element) if element.name() == "b")); + assert!(matches!(nodes[2], NodeRef::CData(""))); + assert!(matches!(nodes[3], NodeRef::Comment("note"))); + assert!(matches!( + nodes[4], + NodeRef::ProcessingInstruction(pi) if pi.target == "work" && pi.content == Some("now") + )); + assert!(matches!(nodes[5], NodeRef::Text("after"))); + assert_eq!( + document + .root() + .get_child("b") + .and_then(|b| b.attribute("key")), + Some("a&b") + ); +} + +#[test] +fn resolves_default_prefix_shadowing_and_attribute_namespaces() { + let input = br#""#; + let document = BorrowedDocument::parse(input.as_slice()).expect("namespaces should parse"); + let root = document.root(); + + assert_eq!(root.namespace(), Some("urn:one")); + assert_eq!(root.attribute_ns("id", Some("urn:attr")), Some("7")); + assert_eq!(root.attribute_ns("id", None), None); + assert!(root.get_child_ns("child", "urn:one").is_some()); + let item = root + .get_child_ns("item", "urn:two") + .expect("prefixed namespace"); + assert_eq!(item.prefix(), Some("p")); + assert!(item.get_child_ns("leaf", "urn:three").is_some()); +} + +#[test] +fn preserves_default_namespace_undeclaration() { + let input = br#""#; + let document = BorrowedDocument::parse(input.as_slice()).expect("document should parse"); + let child = document.root().get_child("child").expect("child"); + + assert_eq!(child.namespace(), None); + assert_eq!( + child.get_child("leaf").and_then(|leaf| leaf.namespace()), + None + ); + assert_eq!(child.raw_xml(), b""); +} + +#[test] +fn preserves_webdav_dead_property_and_lock_owner_subtrees_exactly() { + let input = br#"blue!mailto:a@example.test"#; + let document = BorrowedDocument::parse(input.as_slice()).expect("WebDAV XML should parse"); + let root = document.root(); + let color = root + .get_child_ns("color", "urn:example") + .expect("dead property"); + let owner = root.get_child_ns("lockowner", "DAV:").expect("lock owner"); + + assert_eq!(color.attribute("shade"), Some("dark")); + assert_eq!(color.text().as_deref(), Some("blue")); + assert_eq!( + owner.raw_xml(), + br#"mailto:a@example.test"# + ); +} + +#[test] +fn validates_complete_single_root_and_declarations() { + let policy = XmlSafetyPolicy::untrusted(); + assert_eq!( + xml_root_local_name(br#""#, policy), + Ok("prop".into()) + ); + for input in [ + b"".as_slice(), + b"garbage", + b"", + b"", + ] { + assert_eq!( + validate_xml_input(input, policy), + Err(XmlSafetyError::Malformed) + ); + } + assert_eq!( + validate_xml_input(b"]>&x;", policy), + Err(XmlSafetyError::ExternalEntity) + ); + assert!(validate_xml_input(b"]]>", policy).is_ok()); +} + +#[test] +fn applies_depth_and_element_limits_to_start_and_empty_elements() { + let base = XmlSafetyPolicy::untrusted(); + let depth_policy = XmlSafetyPolicy { + max_depth: 2, + ..base + }; + assert!(validate_xml_input(b"", depth_policy).is_ok()); + assert_eq!( + validate_xml_input(b"", depth_policy), + Err(XmlSafetyError::TooDeep) + ); + + let element_policy = XmlSafetyPolicy { + max_elements: 3, + ..base + }; + assert!(validate_xml_input(b"", element_policy).is_ok()); + assert_eq!( + validate_xml_input(b"", element_policy), + Err(XmlSafetyError::TooManyElements) + ); +} + +#[test] +fn applies_input_attribute_text_and_event_limits() { + let base = XmlSafetyPolicy::untrusted(); + assert!( + validate_xml_input( + b"", + XmlSafetyPolicy { + max_input_bytes: 7, + ..base + } + ) + .is_ok() + ); + assert_eq!( + validate_xml_input( + b"", + XmlSafetyPolicy { + max_input_bytes: 6, + ..base + } + ), + Err(XmlSafetyError::InputTooLarge) + ); + assert!( + validate_xml_input( + b"", + XmlSafetyPolicy { + max_attributes_per_element: 2, + ..base + } + ) + .is_ok() + ); + assert_eq!( + validate_xml_input( + b"", + XmlSafetyPolicy { + max_attributes_per_element: 1, + ..base + } + ), + Err(XmlSafetyError::TooManyAttributes) + ); + assert!( + validate_xml_input( + b"four", + XmlSafetyPolicy { + max_text_bytes: 4, + ..base + } + ) + .is_ok() + ); + assert_eq!( + validate_xml_input( + b"four", + XmlSafetyPolicy { + max_text_bytes: 3, + ..base + } + ), + Err(XmlSafetyError::TextTooLarge) + ); + assert!( + validate_xml_input( + b"", + XmlSafetyPolicy { + max_events: 3, + ..base + } + ) + .is_ok() + ); + assert_eq!( + validate_xml_input( + b"", + XmlSafetyPolicy { + max_events: 2, + ..base + } + ), + Err(XmlSafetyError::TooManyEvents) + ); +} + +#[test] +fn rejects_zero_limits_and_invalid_utf8_in_all_textual_events() { + let base = XmlSafetyPolicy::untrusted(); + for policy in [ + XmlSafetyPolicy { + max_input_bytes: 0, + ..base + }, + XmlSafetyPolicy { + max_depth: 0, + ..base + }, + XmlSafetyPolicy { + max_elements: 0, + ..base + }, + XmlSafetyPolicy { + max_attributes_per_element: 0, + ..base + }, + XmlSafetyPolicy { + max_text_bytes: 0, + ..base + }, + XmlSafetyPolicy { + max_events: 0, + ..base + }, + ] { + assert_eq!( + validate_xml_input(b"", policy), + Err(XmlSafetyError::InvalidPolicy) + ); + } + for input in [ + b"\xFF".as_slice(), + b"", + b"", + ] { + assert_eq!( + validate_xml_input(input, base), + Err(XmlSafetyError::InvalidEncoding) + ); + } +} + +#[test] +fn owned_reader_enforces_the_exact_input_size_boundary() { + let accepted = ParseOptions::new().max_size(7); + assert!(OwnedDocument::from_reader_with_options(b"".as_slice(), &accepted).is_ok()); + + let rejected = ParseOptions::new().max_size(6); + assert!(matches!( + OwnedDocument::from_reader_with_options(b"".as_slice(), &rejected), + Err(Error::Safety(XmlSafetyError::InputTooLarge)) + )); +} + +#[test] +fn rejects_invalid_or_unbound_namespace_prefixes() { + let policy = XmlSafetyPolicy::untrusted(); + for input in [ + br#""#.as_slice(), + br#""#, + br#""#, + br#""#, + br#""#, + br#""#, + ] { + assert_eq!( + validate_xml_input(input, policy), + Err(XmlSafetyError::Malformed) + ); + } + assert!(matches!( + BorrowedDocument::parse(br#""#.as_slice()), + Err(Error::Safety(XmlSafetyError::Malformed)) + )); +} + +#[test] +fn arena_and_validator_classify_invalid_encoding_consistently() { + let input = b"\xFF"; + + assert_eq!( + validate_xml_input(input, XmlSafetyPolicy::untrusted()), + Err(XmlSafetyError::InvalidEncoding) + ); + assert!(matches!( + BorrowedDocument::parse(input.as_slice()), + Err(Error::Safety(XmlSafetyError::InvalidEncoding)) + )); +} + +#[test] +fn parses_and_drops_deep_trees_without_recursive_walkers() { + const DEPTH: usize = 20_000; + let mut input = "".repeat(DEPTH); + input.push_str(&"".repeat(DEPTH)); + let options = ParseOptions::new() + .max_size(input.len()) + .max_depth(DEPTH) + .max_elements(DEPTH) + .max_events(DEPTH * 2 + 1); + let document = BorrowedDocument::parse_with_options(input.as_bytes(), &options) + .expect("deep tree should parse"); + + assert_eq!(document.node_count(), DEPTH); + drop(document); +} + +#[test] +fn trim_whitespace_is_explicit_and_source_backed_when_possible() { + let input = b" before after "; + let document = BorrowedDocument::parse_with_options( + input.as_slice(), + &ParseOptions::new().trim_whitespace(true), + ) + .expect("trimmed document"); + let nodes: Vec<_> = document.root().children().collect(); + + assert!(matches!(nodes[0], NodeRef::Text("before"))); + assert!(matches!(nodes[2], NodeRef::Text("after"))); + assert_eq!(document.allocated_value_count(), 0); +} + +#[test] +fn trim_whitespace_counts_original_decoded_text_against_the_limit() { + let input = b" x "; + let policy = XmlSafetyPolicy { + max_text_bytes: 4, + ..XmlSafetyPolicy::untrusted() + }; + let options = ParseOptions::new() + .safety_policy(policy) + .trim_whitespace(true); + + assert_eq!( + validate_xml_input(input, policy), + Err(XmlSafetyError::TextTooLarge) + ); + assert!(matches!( + BorrowedDocument::parse_with_options(input.as_slice(), &options), + Err(Error::Safety(XmlSafetyError::TextTooLarge)) + )); +} + +#[test] +fn original_writer_propagates_io_errors() { + struct FailingWriter; + + impl Write for FailingWriter { + fn write(&mut self, _buffer: &[u8]) -> io::Result { + Err(io::Error::other("fixture failure")) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + let document = BorrowedDocument::parse(b"".as_slice()).expect("document"); + assert!(matches!( + document.write_original(FailingWriter), + Err(Error::Io(_)) + )); +} diff --git a/crates/aster_forge_xml/tests/xmltree_compat.rs b/crates/aster_forge_xml/tests/xmltree_compat.rs new file mode 100644 index 0000000..2799b79 --- /dev/null +++ b/crates/aster_forge_xml/tests/xmltree_compat.rs @@ -0,0 +1,235 @@ +use aster_forge_xml::{BorrowedDocument, ElementRef, NodeRef}; +use quick_xml::Reader; +use quick_xml::events::Event; + +#[derive(Debug, PartialEq, Eq)] +enum NodeSnapshot { + Element(ElementSnapshot), + Text(String), + CData(String), + Comment(String), + ProcessingInstruction(String, Option), +} + +#[derive(Debug, PartialEq, Eq)] +struct ElementSnapshot { + prefix: Option, + namespace: Option, + name: String, + attributes: std::collections::BTreeMap, + children: Vec, +} + +fn forge_snapshot>(element: ElementRef<'_, S>) -> ElementSnapshot { + let mut children = Vec::new(); + for node in element.children() { + let snapshot = match node { + NodeRef::Element(element) => NodeSnapshot::Element(forge_snapshot(element)), + NodeRef::Text(text) => NodeSnapshot::Text(text.to_owned()), + NodeRef::CData(text) => NodeSnapshot::CData(text.to_owned()), + NodeRef::Comment(text) => NodeSnapshot::Comment(text.to_owned()), + NodeRef::ProcessingInstruction(pi) => NodeSnapshot::ProcessingInstruction( + pi.target.to_owned(), + pi.content.map(str::to_owned), + ), + }; + if let (Some(NodeSnapshot::Text(previous)), NodeSnapshot::Text(text)) = + (children.last_mut(), &snapshot) + { + previous.push_str(text); + } else { + children.push(snapshot); + } + } + ElementSnapshot { + prefix: element.prefix().map(str::to_owned), + namespace: element.namespace().map(str::to_owned), + name: element.name().to_owned(), + attributes: element + .attributes() + .map(|attribute| { + ( + attribute.qualified_name().to_owned(), + attribute.value().to_owned(), + ) + }) + .collect(), + children, + } +} + +fn xmltree_snapshot(element: &xmltree::Element) -> ElementSnapshot { + ElementSnapshot { + prefix: element.prefix.clone(), + namespace: element.namespace.clone(), + name: element.name.clone(), + attributes: element + .attributes + .iter() + .map(|(name, value)| (name.clone(), value.clone())) + .collect(), + children: element + .children + .iter() + .map(|node| match node { + xmltree::XMLNode::Element(element) => { + NodeSnapshot::Element(xmltree_snapshot(element)) + } + xmltree::XMLNode::Text(text) => NodeSnapshot::Text(text.clone()), + xmltree::XMLNode::CData(text) => NodeSnapshot::CData(text.clone()), + xmltree::XMLNode::Comment(text) => NodeSnapshot::Comment(text.clone()), + xmltree::XMLNode::ProcessingInstruction(target, content) => { + NodeSnapshot::ProcessingInstruction(target.clone(), content.clone()) + } + }) + .collect(), + } +} + +fn inside(source: &[u8], slice: &[u8]) -> bool { + let source_start = source.as_ptr() as usize; + let source_end = source_start + source.len(); + let slice_start = slice.as_ptr() as usize; + let slice_end = slice_start + slice.len(); + slice_start >= source_start && slice_end <= source_end +} + +#[test] +fn quick_xml_slice_events_borrow_plain_names_attributes_and_text() { + let source = br#"text"#; + let mut reader = Reader::from_reader(source.as_slice()); + + let Event::Start(start) = reader.read_event().expect("start event") else { + panic!("expected start event"); + }; + assert!(inside(source, start.name().as_ref())); + let attribute = start + .attributes() + .next() + .expect("attribute") + .expect("valid attribute"); + assert!(inside(source, attribute.key.as_ref())); + assert!(inside(source, attribute.value.as_ref())); + + let Event::Text(text) = reader.read_event().expect("text event") else { + panic!("expected text event"); + }; + assert!(inside(source, text.as_ref())); +} + +#[test] +fn forge_arena_borrows_plain_values_from_the_source() { + let source = br#"text"#; + let document = BorrowedDocument::parse(source.as_slice()).expect("fixture should parse"); + let root = document.root(); + + assert!(inside(source, root.qualified_name().as_bytes())); + assert!(inside( + source, + root.attribute("plain").expect("attribute").as_bytes() + )); + assert!(inside(source, root.text().expect("text").as_bytes())); + assert_eq!(document.allocated_value_count(), 0); +} + +#[test] +fn forge_and_xmltree_preserve_the_same_ordered_node_kinds() { + let source = br#"beforeafter"#; + let forge = BorrowedDocument::parse(source.as_slice()).expect("Forge fixture"); + let xmltree = xmltree::Element::parse(source.as_slice()).expect("xmltree fixture"); + + let forge_kinds: Vec<&str> = forge + .root() + .children() + .map(|node| match node { + NodeRef::Element(_) => "element", + NodeRef::Text(_) => "text", + NodeRef::CData(_) => "cdata", + NodeRef::Comment(_) => "comment", + NodeRef::ProcessingInstruction(_) => "pi", + }) + .collect(); + let xmltree_kinds: Vec<&str> = xmltree + .children + .iter() + .map(|node| match node { + xmltree::XMLNode::Element(_) => "element", + xmltree::XMLNode::Text(_) => "text", + xmltree::XMLNode::CData(_) => "cdata", + xmltree::XMLNode::Comment(_) => "comment", + xmltree::XMLNode::ProcessingInstruction(_, _) => "pi", + }) + .collect(); + assert_eq!(forge_kinds, xmltree_kinds); +} + +#[test] +fn forge_and_xmltree_resolve_element_namespaces_consistently() { + let source = br#""#; + let forge = BorrowedDocument::parse(source.as_slice()).expect("Forge fixture"); + let xmltree = xmltree::Element::parse(source.as_slice()).expect("xmltree fixture"); + + assert_eq!(forge.root().namespace(), xmltree.namespace.as_deref()); + assert!(forge.root().get_child_ns("item", "urn:example").is_some()); + assert!(xmltree.get_child(("item", "urn:example")).is_some()); + assert!(forge.root().get_child_ns("item", "DAV:").is_some()); + assert!(xmltree.get_child(("item", "DAV:")).is_some()); +} + +#[test] +fn forge_matches_xmltree_for_the_supported_parse_contract() { + for source in [ + br#""#.as_slice(), + br#"plain & escaped"#, + br#"beforeafter"#, + br#""#, + br#"]]>"#, + "中文 日本語 한국어".as_bytes(), + ] { + let forge = BorrowedDocument::parse(source).expect("Forge fixture"); + let xmltree = xmltree::Element::parse(source).expect("xmltree fixture"); + assert_eq!(forge_snapshot(forge.root()), xmltree_snapshot(&xmltree)); + } +} + +#[test] +fn forge_intentionally_preserves_element_only_whitespace_that_xmltree_drops() { + let source = b"\n \n"; + let forge = BorrowedDocument::parse(source.as_slice()).expect("Forge fixture"); + let xmltree = xmltree::Element::parse(source.as_slice()).expect("xmltree fixture"); + let nodes: Vec<_> = forge.root().children().collect(); + + assert!(matches!(nodes[0], NodeRef::Text("\n "))); + assert_eq!(xmltree.children.len(), 1); +} + +#[test] +fn exact_forge_output_and_xmltree_output_round_trip_through_each_other() { + let source = br#"beforeafter"#; + let forge = BorrowedDocument::parse(source.as_slice()).expect("Forge fixture"); + let expected = forge_snapshot(forge.root()); + let mut forge_output = Vec::new(); + forge + .write_original(&mut forge_output) + .expect("Forge exact output"); + let xmltree_from_forge = + xmltree::Element::parse(forge_output.as_slice()).expect("xmltree parses Forge output"); + assert_eq!(expected, xmltree_snapshot(&xmltree_from_forge)); + + let xmltree = xmltree::Element::parse(source.as_slice()).expect("xmltree fixture"); + let mut options = xmltree::EmitterConfig::new(); + options.perform_indent = false; + options.write_document_declaration = false; + options.pad_self_closing = false; + options.autopad_comments = false; + let mut xmltree_output = Vec::new(); + xmltree + .write_with_config(&mut xmltree_output, options) + .expect("xmltree output"); + let forge_from_xmltree = + BorrowedDocument::parse(xmltree_output.as_slice()).expect("Forge parses xmltree output"); + assert_eq!( + xmltree_snapshot(&xmltree), + forge_snapshot(forge_from_xmltree.root()) + ); +} diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index ce02580..1d8e0da 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -46,6 +46,7 @@ const cratePages = [ ["Tasks", "/crates/aster_forge_tasks"], ["Utils", "/crates/aster_forge_utils"], ["Validation", "/crates/aster_forge_validation"], + ["XML", "/crates/aster_forge_xml"], ] as const function crateSidebar() { diff --git a/docs/crates/aster_forge_utils.md b/docs/crates/aster_forge_utils.md index 8c7318a..183ea6f 100644 --- a/docs/crates/aster_forge_utils.md +++ b/docs/crates/aster_forge_utils.md @@ -196,58 +196,6 @@ Unicode segmentation 策略。 适合 external auth callback、CORS origin、公开 base URL 等配置规范化。`public_site_*` helper 只处理产品无关的 origin 解析、去重、请求 origin 匹配和 URL 拼接；产品侧仍然保留具体 config key、runtime snapshot、日志上下文和错误映射。 -### xml - -主要 API： - -- `DEFAULT_XML_MAX_DEPTH` -- `XmlSafetyPolicy` -- `XmlSafetyPolicy::untrusted()` -- `XmlSafetyError` -- `validate_xml_input(bytes, policy)` -- `xml_root_local_name(bytes, policy)` - -`validate_xml_input` 使用 `quick-xml` 事件读取器校验输入，不递归构造 DOM。默认的不可信 -输入策略拒绝 `DOCTYPE` / `ENTITY` 声明，并把最大同时打开元素数限制为 128。校验同时要求 -输入是完整、格式正确、仅有一个根元素的 XML 文档。 - -`reject_doctype` 策略会在解析前做字节级预扫描：预扫描不理解 CDATA 和注释，文本内容里合法出现 -`]]>`）也会被拒。误报方向是"拒绝"， -属于 fail-safe；产品必须接受这类内容时应走其他解析通道。 - -```rust -use aster_forge_utils::xml::{XmlSafetyPolicy, validate_xml_input}; - -validate_xml_input(body, XmlSafetyPolicy::untrusted())?; -let document = product_xml_parser(body)?; -``` - -只需要分派根元素、无需保留 DOM 时，直接使用 `xml_root_local_name`。它会先执行同一套安全 -校验，再返回去除命名空间前缀后的根元素本地名： - -```rust -use aster_forge_utils::xml::{XmlSafetyPolicy, xml_root_local_name}; - -let report_type = xml_root_local_name(body, XmlSafetyPolicy::untrusted())?; -``` - -该 helper 只负责 XML 结构安全，不负责： - -- HTTP response body 的累计字节上限；调用方必须在聚合响应前单独限制大小。 -- XML 业务 schema、元素名、命名空间和字段语义。 -- 构建 DOM 或把 XML 映射为产品 DTO。 -- 把错误映射为 REST、WebDAV、WOPI 或对象存储协议响应。 - -需要保留完整 XML 子树时，应先调用 `validate_xml_input`，再把通过校验的字节交给 -`xmltree` 或产品自己的解析器。不要把外部输入直接交给递归 DOM parser。 - -`XmlSafetyError` 的稳定分类为： - -- `InvalidPolicy`：最大深度为零等调用方配置错误。 -- `ExternalEntity`：输入含被策略禁止的 DTD 或 ENTITY 声明。 -- `TooDeep`：嵌套深度超过策略上限。 -- `Malformed`：空文档、多个根元素、标签错配、未闭合或其他格式错误。 - ## 错误边界 `UtilsError` 分为： @@ -257,20 +205,14 @@ let report_type = xml_root_local_name(body, XmlSafetyPolicy::untrusted())?; 产品侧应在配置加载、API handler 或 service 边界映射成具体错误。不要把 `UtilsError` 直接作为产品 API error 类型。 -`xml` 模块使用独立的强类型 `XmlSafetyError`，方便调用方保留 `ExternalEntity`、`TooDeep` -和 `Malformed` 的协议差异；它不经过通用 `UtilsError` 文案匹配。 - ## 测试要求 - 每个产品接入点覆盖非法输入。 - 数值转换测试要包含负数、超上限和边界值。 - URL/origin 测试要覆盖 loopback HTTP、HTTPS、wildcard。 - trusted proxy 测试要覆盖多代理链。 -- XML 测试要覆盖正常输入、精确深度上限、超限一层、空文档、多个根元素、标签错配、 - DTD/ENTITY 大小写变体、尾部不完整 markup，以及允许 DTD 的显式策略分支。 ## 参考项目 -- AsterDrive：URL、proxy、upload chunk、token、临时路径、数值转换，以及 WebDAV/WOPI/ - 对象存储 XML 输入安全场景丰富。 +- AsterDrive：URL、proxy、upload chunk、token、临时路径和数值转换场景丰富。 - AsterYggdrasil：适合看轻量项目如何直接调用 Forge utils，避免保留无意义 facade。 diff --git a/docs/crates/aster_forge_xml.md b/docs/crates/aster_forge_xml.md new file mode 100644 index 0000000..f691d1b --- /dev/null +++ b/docs/crates/aster_forge_xml.md @@ -0,0 +1,229 @@ +# aster_forge_xml + +`aster_forge_xml` 是 Aster 产品共用的有界 XML 内核，提供 source-backed arena DOM、namespace-aware streaming reader、选择性 subtree capture 和 direct streaming writer。它用 `quick-xml` 读取和写出事件，不会把每个 name、attribute、text 和 subtree 都复制成独立 `String`，也不会构造递归拥有子树的 DOM。 + +## 用途边界 + +Forge XML 负责： + +- 用平坦 arena 保存 node，并用 `NodeId` 连接 parent、child 和 sibling。 +- 普通 UTF-8 name、attribute、text 和 subtree 直接引用原始输入区间。 +- entity 解码或显式 whitespace normalization 改变值时，才进入 owned value pool。 +- 保留 mixed content、CDATA、comment、processing instruction 的顺序。 +- 正确处理 default namespace、prefix shadowing、undeclaration 和 attribute namespace。 +- 统一限制输入字节、深度、元素、属性、文本和事件数量。 +- 默认拒绝 DTD、自定义 entity、多个 root、尾部垃圾和非法编码。 +- 对任意 `BufRead` 做有界流式读取，并允许直接跳过或只保留选中的 subtree。 +- 对任意 `Write` 直接生成 XML，并检查 document state、namespace binding、名称、字符、深度、属性数和输出字节上限。 + +产品侧仍负责 XML schema、协议字段、HTTP body 聚合上限、错误映射、权限和存储语义。已知 schema 的 WOPI/COS/WebDAV request 应优先使用 `XmlStreamReader` 做 typed/event parsing；需要随机查询时使用 `BorrowedDocument` 或 `OwnedDocument`；只需保留未知扩展节点时使用 `capture_current`；生成中大型 response 时使用 `XmlStreamWriter`。 + +## Cargo 接入 + +```toml +[dependencies] +aster_forge_xml = { git = "https://github.com/AsterCommunity/AsterForge" } +``` + +## Borrowed document + +```rust +use aster_forge_xml::BorrowedDocument; + +let body = br#"a&b"#; +let document = BorrowedDocument::parse(body.as_slice())?; +let display_name = document + .root() + .get_child_ns("displayname", "DAV:") + .expect("fixture contains DAV:displayname"); + +assert_eq!(display_name.text().as_deref(), Some("a&b")); +assert_eq!(display_name.raw_xml(), br#"a&b"#); +``` + +`BorrowedDocument<'a>` 不复制完整输入；document 的生命周期受输入 byte slice 约束。name、普通 attribute/text 和 raw subtree payload 直接引用 source span，只有 entity decoding 或 normalization 改变值时才进入 owned value pool；arena node、link、namespace scope 等索引元数据仍会分配，因此准确说法是 source-backed、payload zero-copy DOM，而不是“整个 DOM 零分配”。`raw_xml()` 返回 element 在原始输入中的精确 byte slice，适合 WebDAV dead property、LOCK owner 和未知扩展 subtree。 + +## Owned document 与 validated value + +需要跨 task、cache 或 store 保留文档时使用 owned 形态： + +```rust +use std::sync::Arc; +use aster_forge_xml::{OwnedDocument, ValidatedXml}; + +let source: Arc<[u8]> = Arc::from(br#"mailto:a@example.test"#.as_slice()); +let document = OwnedDocument::parse(Arc::clone(&source))?; + +let validated = ValidatedXml::new(source)?; +let cheap_clone = validated.clone(); +assert_eq!(cheap_clone.as_bytes(), validated.as_bytes()); +``` + +- `OwnedDocument = XmlDocument>`：arena 和 source 一起拥有。 +- `OwnedDocument::from_reader`：只读取到配置上限再解析。 +- `ValidatedXml`：`Arc` 包装，clone 不复制 source 或 arena。 +- `write_original`：写出完整原始文档，包括 declaration、root 外 comment/PI 和空白。 + +`write_original` 是 exact byte copy，不是结构化 serializer。生成 WebDAV/WOPI response 时应使用后面的 `XmlStreamWriter`，不要先构造完整 DOM。 + +## Streaming reader 与选择性 capture + +```rust +use std::io::BufReader; +use aster_forge_xml::{XmlSafetyPolicy, XmlStreamEvent, XmlStreamReader}; + +let body = br#"/a"#; +let mut reader = XmlStreamReader::new(BufReader::new(body.as_slice()), XmlSafetyPolicy::untrusted())?; + +loop { + match reader.read_event()? { + XmlStreamEvent::Start(start) if start.name()?.matches("href", Some("DAV:")) => { + assert_eq!(reader.read_text_current()?, "/a"); + } + XmlStreamEvent::Start(start) if start.name()?.matches("unknown", Some("urn:extension")) => { + let retained = reader.capture_current(64 * 1024)?; + assert_eq!(retained.document().root().namespace(), Some("urn:extension")); + } + XmlStreamEvent::Start(start) if start.name()?.matches("ignored", None) => reader.skip_current()?, + XmlStreamEvent::Eof => break, + _ => {} + } +} +``` + +`XmlStreamReader` 复用 event buffer，只保留当前事件、namespace resolver 和与深度成正比的 namespace state，不保留完整 token 列表或 DOM。`capture_current(max_bytes)` 只 materialize 当前 subtree，并自动补齐祖先作用域内、该 subtree 独立解析所需的 namespace declaration；`skip_current()` 继续执行完整安全和 well-formedness 检查，但不会保留被跳过节点。 + +## Streaming writer + +```rust +use aster_forge_xml::XmlStreamWriter; + +let mut writer = XmlStreamWriter::new(Vec::new())?; +writer.start_element("D:multistatus", [("xmlns:D", "DAV:")])?; +writer.start("D:response")?; +writer.start("D:href")?; +writer.text("/files/a&b")?; +writer.end_element()?; +writer.end_element()?; +writer.end_element()?; +let output = writer.finish()?; + +assert_eq!(output, br#"/files/a&b"#); +``` + +`XmlStreamWriter` 直接把事件写到 socket、file、buffer 或 compression stream。`XmlWriteOptions` 控制 XML declaration、最大输出字节、最大深度和单元素最大属性数；writer 会检查 root lifecycle、prefix binding、namespace-expanded duplicate attribute、XML name/control character、CDATA/comment/PI 约束和 I/O failure。`validated_subtree` 可以嵌入一个自包含的 `ValidatedXml` root，并继续执行 writer depth、attribute 和 byte limit。 + +## 查询 API + +- `root` / `node`：取得 root 或按稳定 `NodeId` 查询。 +- `children` / `child_elements`：按文档顺序遍历直接子节点。 +- `get_child` / `get_child_ns`：按 local name 和 namespace 查询直接子元素。 +- `attributes` / `attribute` / `attribute_ns`：按 qualified name 或 namespace 查询属性。 +- `parent`：访问 parent element。 +- `descendants`：非递归深度优先遍历，包含当前元素。 +- `text`：拼接直接 Text 和 CDATA；单节点返回 borrowed `Cow`。 +- `raw_xml`：返回精确原始 subtree bytes。 + +tree 是 immutable view。需要修改业务字段时，应解析成 typed DTO；需要输出新文档时，应走 event writer，而不是复制 arena 生成另一棵可变递归树。 + +## 安全策略 + +```rust +use aster_forge_xml::{BorrowedDocument, ParseOptions, XmlSafetyPolicy, validate_xml_input}; + +let policy = XmlSafetyPolicy { + max_input_bytes: 1024 * 1024, + max_depth: 64, + max_elements: 10_000, + max_attributes_per_element: 128, + max_text_bytes: 512 * 1024, + max_events: 100_000, + reject_doctype: true, +}; + +validate_xml_input(body, policy)?; // event-only，不构建 DOM +let document = BorrowedDocument::parse_with_options( + body, + &ParseOptions::new().safety_policy(policy), +)?; +``` + +如果只需要分派 root： + +```rust +use aster_forge_xml::{XmlSafetyPolicy, xml_root_local_name}; + +let method = xml_root_local_name(body, XmlSafetyPolicy::untrusted())?; +``` + +`XmlSafetyError` 保留这些可映射分类：`InvalidPolicy`、`InputTooLarge`、`OutputTooLarge`、`ExternalEntity`、`TooDeep`、`TooManyElements`、`TooManyAttributes`、`TextTooLarge`、`TooManyEvents`、`InvalidEncoding`、`Malformed`。 + +## 错误边界 + +- `Error::Safety`：输入越过 `XmlSafetyPolicy`。 +- `Error::InvalidXml`：底层 XML 结构错误。 +- `Error::InvalidData`：writer state、name、namespace 或 XML value 不合法。 +- `Error::Io`：reader/writer 的底层 I/O 失败。 + +产品 API 层负责映射成 WebDAV/WOPI/对象存储协议响应和面向用户的文案。 + +## 测试要求 + +- plain value 必须落在 source buffer 内，且 owned value pool 为空。 +- entity/normalization 只为发生变化的值分配。 +- mixed content、所有 node kind、parent/sibling link 和 text join。 +- default namespace、shadowing、undeclaration、namespaced attribute。 +- DTD/entity、多个 root、尾部垃圾、非法 UTF-8 和每一种 limit 边界。 +- WebDAV dead property / LOCK owner subtree exact bytes。 +- 20,000 层 parse、traversal 和 drop 不依赖递归 destructor。 +- 100,000 response streaming walk 和 selective capture 不构造整棵 DOM。 +- streaming writer 覆盖 namespace、escaping、所有 node kind、document lifecycle、limit 精确边界、I/O failure、subtree embedding 和 25,000-response direct generation。 +- 与 `xmltree` 对照 supported parse contract 和 round-trip。 +- 与 `roxmltree` 对照 source-backed tree、namespace、attribute、text、child query 和 exact source range，并明确 CDATA/default namespace empty-URI 的 node model 差异。 +- `proptest` 每次生成 256 组有界随机树，执行 writer → validator → Forge DOM → `roxmltree` round-trip；随机 byte input 同时喂给 validator、arena parser 和 stream reader，检查资源边界内不 panic。 +- borrowed arena、owned validated document、event walk 和 writer 的 allocation/heap/RSS probe。 + +长期 fuzz harness 位于 `crates/aster_forge_xml/fuzz`，将 parser/stream/capture 与 writer rejection/round-trip 分成两个 target： + +```bash +cargo fuzz run parse_stream +cargo fuzz run writer_roundtrip +``` + +## Benchmark + +`xmltree`、`roxmltree` 和 `proptest` 只在本 crate 的 dev-dependencies 中，用于行为对照、property test 和 benchmark，不进入正式依赖图；`libfuzzer-sys` 只存在于独立的 `fuzz` workspace。 + +```bash +cargo bench -p aster_forge_xml --bench xml_bench +cargo bench -p aster_forge_xml --bench xml_memory +``` + +CPU benchmark 固定 `PROPFIND`、WOPI discovery 和 1,000-response `multistatus` workload，比较： + +- `forge_arena` +- `roxmltree_borrowed` +- `forge_stream_reader` +- `forge_stream_validation`（执行完整 stream safety/namespace/value validation，但不读取 event field） +- `xmltree_owned` +- `validate_plus_xmltree` +- `quick_xml_ns_buffered_decoded`（buffered `NsReader`、namespace resolution、attribute normalization、text decode/unescape 和 end-name processing） +- `quick_xml_borrowed_events` lower bound +- `arena_original` 与 `xmltree_compact`（两者语义不同，禁止用这组结果宣称 serializer 倍率） +- `forge_stream_writer`、`quick_xml_writer` lower bound 与 `xmltree_build_and_write`（同一个 1,000-response 业务生成 workload） + +内存 probe 让每个 implementation/fixture 在独立子进程运行，并输出 allocation count、累计申请字节、peak live heap、retained heap 和 peak RSS；最大 fixture 是 10,000-response `multistatus`。reader 输入在 measurement 前创建；writer 同时测量保留完整 output `Vec` 和写入 `sink` 两种形态，后者用于确认 writer 自身 retained heap 与文档大小无关。 + +2026-07-25 在 `review/pr-3` working tree（base `305ef17343fc`）、`rustc 1.97.1`、`aarch64-apple-darwin` 的 release benchmark 样本中，arena parse 相对 `xmltree_owned` 的 Criterion central estimate 分别是 `2.9×`（PROPFIND）、`2.9×`（WOPI）和 `7.8×`（multistatus 1000），因此“约 3–8×”在这组固定 workload 上有实测支持；`forge_stream_writer` 生成 multistatus 1000 为 `1.034 ms`，`xmltree_build_and_write` 为 `3.326 ms`，即 `3.22×`，落在“2–5×”范围内。这些是 workload-specific evidence，不是跨机器常量，也不应用 `arena_original` exact copy 与 serializer 的差值替代。 + +同一轮 2,625,862-byte multistatus 10000 memory probe 中，`forge_arena_borrowed` retained heap 为 11,140,124 bytes，`forge_validated_owned` 为 13,766,124 bytes，`xmltree_owned` 为 111,004,222 bytes；优化后的 `forge_stream_reader` peak live heap 为 404 bytes、retained heap 为 0。direct writer 写入 `sink` 时 peak live heap 为 606 bytes、retained heap 为 0；写入 output `Vec` 时 retained capacity 为 3,801,088 bytes；`xmltree_build_and_write` peak live heap 为 65,039,004 bytes。RSS 包含进程、allocator 和预先创建的 input，只能比较同一轮独立子进程结果，不能当作 retained DOM bytes。 + +加入真正的 source-backed DOM 对照后，`roxmltree_borrowed` 在三个 parse workload 上分别比 Forge arena 快约 `1.5×`、`1.5×` 和 `1.7×`；multistatus 10000 retained heap 为 9,680,148 bytes，Forge arena 为 11,140,124 bytes。Forge 为 namespace persistent scope、所有 node kind 的独立保留、owned value pool、统一 safety classification 和 exact document/subtree API 付出了约 15% retained heap 与 1.5–1.7× CPU 成本；大文档只做顺序处理时仍应选择 `XmlStreamReader`，不应为了 API 统一强行构建 arena。 + +`quick_xml_borrowed_events` 只是 slice-specialized tokenizer 下界，没有执行 Forge 的 namespace、完整 value decode、single-root 和资源限制，不能直接用来判断 wrapper 开销。加入工作量更接近的 `quick_xml_ns_buffered_decoded` 后，当前样本中 `forge_stream_validation` 约为其 `1.0–1.4×`，读取 name/attribute/text 的完整 `forge_stream_reader` 约为其 `1.5–1.9×`；剩余差值主要来自安全计数、root/depth state，以及 attribute name/namespace 的验证后再次访问。stream hot path 已去掉每个 Start event 的备用 raw copy，复用已验证的 owned attribute value，并让 Text/CDATA/Comment event 直接携带首次 decode 结果；固定 2,000 次 multistatus 1000 walk 的交替 A/B 样本中，优化前后中位数为 `4.968 s` 与 `4.160 s`，约快 `16%`。WOPI allocation probe 从 1,016 次、59,921 bytes 降至 767 次、46,549 bytes；multistatus stream allocation 从 15 次、738 bytes 降至 14 次、710 bytes。这些同样是本机 workload-specific evidence。 + +## 参考项目 + +- AsterDrive：WOPI discovery、WebDAV properties/LOCK owner、Tencent COS CORS/media metadata。 +- `xmltree-rs`：行为与 benchmark 对照，不进入生产依赖。 +- `roxmltree`：source-backed read-only tree 的性能与内存基线，不进入生产依赖。 diff --git a/docs/guide/new-project-integration.md b/docs/guide/new-project-integration.md index fac5a03..fff36ac 100644 --- a/docs/guide/new-project-integration.md +++ b/docs/guide/new-project-integration.md @@ -83,6 +83,7 @@ aster_forge_runtime = { git = "https://github.com/AsterCommunity/AsterForge", pa aster_forge_tasks = { git = "https://github.com/AsterCommunity/AsterForge", package = "aster_forge_tasks", features = ["runtime-component"] } aster_forge_utils = { git = "https://github.com/AsterCommunity/AsterForge", package = "aster_forge_utils" } aster_forge_validation = { git = "https://github.com/AsterCommunity/AsterForge", package = "aster_forge_validation" } +aster_forge_xml = { git = "https://github.com/AsterCommunity/AsterForge", package = "aster_forge_xml" } ``` 按需开启 feature： diff --git a/docs/index.md b/docs/index.md index 3b99076..2ba66c8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -64,6 +64,7 @@ aster_forge_runtime::AsterRuntime::builder() - [`aster_forge_tasks`](./crates/aster_forge_tasks.md) - [`aster_forge_utils`](./crates/aster_forge_utils.md) - [`aster_forge_validation`](./crates/aster_forge_validation.md) +- [`aster_forge_xml`](./crates/aster_forge_xml.md) ## 参考项目