From 80cdbb9657ea476c86a067bec3d03ca0047bc440 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 13:04:53 +0000 Subject: [PATCH 1/2] feat(spiders): XML and CSV feed spiders + transparent gzip bodies (upstream v0.4.13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the one applicable feature pair from upstream v0.4.13 (v0.4.14 is a Python-packaging fix, n/a): - XmlFeedSpider: iterate feed nodes (RSS by default, any CSS selector via iter_tag, e.g. "entry" for Atom). Without a callback, each node becomes an object of its child elements' text — RSS-to-items with zero parsing code. HTML-void feed tags (notably , which carries every RSS item's URL and whose text html5ever would swallow) are rewritten to xmlfeed-* for parsing and translated back in item keys; custom callbacks address them by the rewritten name (documented). - CsvFeedSpider: rows as header-keyed objects via a hand-rolled RFC 4180 parser (quoted fields with "" escapes, embedded delimiters and newlines, LF/CRLF, no phantom trailing row), configurable delimiter, header override (first row becomes data), short rows padded, extra cells dropped. - Transparent gzip decompression in the body decoder (magic-byte detection, flate2): raw .xml.gz / .csv.gz FILES served without Content-Encoding now arrive as text — this also closes the documented SitemapSpider .xml.gz gap. Decompression is capped at the fetcher's max_body_bytes so a decompression bomb cannot bypass the body-size cap; oversized or corrupt gzip falls back to the raw bytes (previous behavior). Tests: 4 gzip decoder unit tests (roundtrip, bomb fallback, corrupt fallback, non-gzip untouched), 3 CSV parser unit tests, 4 feed spider parse tests (RSS default incl. the case, Atom iter_tag + custom callback, CSV header/override/delimiter), and a live-server end-to-end test proving a .gz file body arrives decompressed. README gains a feed spider section. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KDFsMaKk764vogjUW3nqpk --- Cargo.lock | 1 + Cargo.toml | 3 + README.md | 18 +- src/fetchers/client.rs | 6 +- src/fetchers/encoding.rs | 77 +++++++- src/lib.rs | 4 +- src/spiders/templates/csv_feed.rs | 294 ++++++++++++++++++++++++++++++ src/spiders/templates/mod.rs | 4 + src/spiders/templates/sitemap.rs | 6 +- src/spiders/templates/xml_feed.rs | 248 +++++++++++++++++++++++++ tests/fetchers_client.rs | 42 +++++ tests/spiders_templates.rs | 99 ++++++++++ 12 files changed, 795 insertions(+), 7 deletions(-) create mode 100644 src/spiders/templates/csv_feed.rs create mode 100644 src/spiders/templates/xml_feed.rs diff --git a/Cargo.lock b/Cargo.lock index 6007701..8eb7efd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1710,6 +1710,7 @@ dependencies = [ "ego-tree", "encoding_rs", "env_logger", + "flate2", "indexmap", "log", "percent-encoding", diff --git a/Cargo.toml b/Cargo.toml index 41366c0..53d273d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,9 @@ serde_json = { version = "1", features = ["preserve_order"] } rusqlite = { version = "0.40", features = ["bundled"] } regex = "1" encoding_rs = "0.8" +# Transparent gzip decompression of .gz feed/sitemap bodies (pure-Rust +# miniz_oxide backend by default). +flate2 = "1" sha2 = "0.11" url = "2" percent-encoding = "2" diff --git a/README.md b/README.md index f1ed6eb..c0e90ec 100644 --- a/README.md +++ b/README.md @@ -545,6 +545,22 @@ impl Spider for MySpider { > resume without re-visiting pages. For very large or open-ended crawls, set > `allowed_domains()` to bound scope. +### Feed spiders (RSS/Atom/CSV) + +Iterate structured feeds without writing parsing code — each RSS `` +(or Atom `` via `iter_tag("entry")`) becomes an item object of its +child elements, and each CSV row becomes a header-keyed object. Gzipped +feed files (`.xml.gz`, `.csv.gz`) are decompressed transparently. + +```rust +use rust_scrapling::XmlFeedSpider; + +let spider = XmlFeedSpider::builder("news") + .feed_url("https://example.com/rss.xml") + .build(); +// items: [{"title": "...", "link": "...", "description": "..."}, ...] +``` + ### CrawlSpider, SitemapSpider & LinkExtractor For the common cases you don't need to implement `Spider` yourself: @@ -649,7 +665,7 @@ rust_scrapling/ |-- spider.rs # Spider trait (user-facing API) |-- engine.rs # CrawlerEngine: async orchestrator |-- links.rs # LinkExtractor: URL discovery primitive - |-- templates/ # CrawlSpider + CrawlRule, SitemapSpider + |-- templates/ # CrawlSpider, SitemapSpider, feed spiders, Shopify |-- request.rs # SpiderRequest: fingerprinting + priority |-- response.rs # SpiderResponse: parser integration |-- result.rs # CrawlResult, CrawlStats, ItemList diff --git a/src/fetchers/client.rs b/src/fetchers/client.rs index b38a96c..d2deef9 100644 --- a/src/fetchers/client.rs +++ b/src/fetchers/client.rs @@ -336,7 +336,11 @@ impl Fetcher { } } } - let body_text = crate::fetchers::encoding::decode_body(&bytes, &content_type); + let body_text = crate::fetchers::encoding::decode_body_capped( + &bytes, + &content_type, + max_body, + ); let mut response = Response::new( status_code, diff --git a/src/fetchers/encoding.rs b/src/fetchers/encoding.rs index 19bf19a..42eb422 100644 --- a/src/fetchers/encoding.rs +++ b/src/fetchers/encoding.rs @@ -44,13 +44,47 @@ pub fn charset_from_content_type(content_type: &str) -> Option<&str> { /// scraper, falling back to lossy UTF-8 preserves far more of the content. #[must_use] pub fn decode_body(bytes: &[u8], content_type: &str) -> String { + decode_body_capped(bytes, content_type, usize::MAX) +} + +/// [`decode_body`] with a decompression cap: gzip-compressed bodies +/// (detected by their `1f 8b` magic bytes — e.g. raw `.xml.gz` feed and +/// sitemap files served without `Content-Encoding`) are transparently +/// decompressed before charset decoding, matching upstream Scrapling's +/// feed handling. `max_bytes` bounds the DECOMPRESSED size so a +/// decompression bomb cannot bypass the fetcher's body-size cap; oversized +/// or corrupt gzip data falls back to decoding the raw bytes as before. +pub fn decode_body_capped(bytes: &[u8], content_type: &str, max_bytes: usize) -> String { + let effective: std::borrow::Cow<'_, [u8]> = match gunzip_capped(bytes, max_bytes) { + Some(decompressed) => std::borrow::Cow::Owned(decompressed), + None => std::borrow::Cow::Borrowed(bytes), + }; let encoding = charset_from_content_type(content_type) .and_then(|label| encoding_rs::Encoding::for_label_no_replacement(label.as_bytes())) .unwrap_or(encoding_rs::UTF_8); - let (text, _, _) = encoding.decode(bytes); + let (text, _, _) = encoding.decode(&effective); text.into_owned() } +/// Decompress `bytes` when they carry the gzip magic; `None` when they are +/// not gzip, are corrupt, or would decompress beyond `max_bytes`. +fn gunzip_capped(bytes: &[u8], max_bytes: usize) -> Option> { + use std::io::Read; + + if bytes.len() < 2 || bytes[0] != 0x1f || bytes[1] != 0x8b { + return None; + } + let mut out = Vec::new(); + let limit = max_bytes as u64; + let mut reader = flate2::read::GzDecoder::new(bytes).take(limit.saturating_add(1)); + match reader.read_to_end(&mut out) { + Ok(_) if out.len() as u64 <= limit => Some(out), + // Oversized (bomb) or corrupt: keep the raw bytes, exactly the + // pre-gzip-support behavior. + _ => None, + } +} + #[cfg(test)] mod tests { use super::*; @@ -191,4 +225,45 @@ mod tests { bytes.extend_from_slice("café".as_bytes()); assert_eq!(decode_body(&bytes, "text/html; charset=ISO-8859-1"), "café"); } + #[test] + fn gzip_bodies_are_transparently_decompressed() { + use std::io::Write; + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all("hi".as_bytes()) + .unwrap(); + let gz = enc.finish().unwrap(); + assert_eq!( + decode_body_capped(&gz, "application/gzip", 1024 * 1024), + "hi" + ); + } + + #[test] + fn gzip_bomb_falls_back_to_raw_bytes() { + use std::io::Write; + // 1 MiB of zeros compresses to ~1 KiB; cap the decompressed size + // below 1 MiB so the bomb guard trips and the raw (compressed) + // bytes are decoded instead — the pre-gzip behavior. + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(&vec![0u8; 1024 * 1024]).unwrap(); + let gz = enc.finish().unwrap(); + let out = decode_body_capped(&gz, "application/gzip", 64 * 1024); + assert!( + out.len() < 1024 * 1024, + "bomb must not be fully decompressed" + ); + } + + #[test] + fn corrupt_gzip_falls_back_to_raw_bytes() { + // Valid magic, garbage stream. + let fake = [0x1f, 0x8b, b'n', b'o', b't', b'g', b'z']; + let out = decode_body_capped(&fake, "text/plain", 1024); + assert!(!out.is_empty(), "raw-bytes fallback must decode something"); + } + + #[test] + fn non_gzip_bodies_are_untouched() { + assert_eq!(decode_body_capped(b"hello", "text/plain", 10), "hello"); + } } diff --git a/src/lib.rs b/src/lib.rs index 5e1170a..e72ba4d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -158,4 +158,6 @@ pub use spiders::request::SpiderRequest; pub use spiders::result::{CrawlResult, CrawlStats, ItemList}; pub use spiders::session::{SessionError, SessionManager}; pub use spiders::spider::Spider; -pub use spiders::templates::{CrawlRule, CrawlSpider, ShopifySpider, SitemapSpider}; +pub use spiders::templates::{ + CrawlRule, CrawlSpider, CsvFeedSpider, ShopifySpider, SitemapSpider, XmlFeedSpider, +}; diff --git a/src/spiders/templates/csv_feed.rs b/src/spiders/templates/csv_feed.rs new file mode 100644 index 0000000..9441951 --- /dev/null +++ b/src/spiders/templates/csv_feed.rs @@ -0,0 +1,294 @@ +//! `CsvFeedSpider`, ported from upstream Scrapling's `CSVFeedSpider` +//! (v0.4.13): iterate over the rows of CSV feeds as keyed objects. +//! +//! Gzip-compressed feed files (`.csv.gz`) are decompressed transparently +//! by the fetcher (magic-byte detection in the body decoder), in addition +//! to transport-level `Content-Encoding: gzip`. + +use crate::spiders::request::SpiderRequest; +use crate::spiders::response::SpiderResponse; +use crate::spiders::spider::Spider; +use async_trait::async_trait; +use std::collections::HashSet; +use std::sync::Arc; + +/// Callback turning one CSV row (header → cell map, in column order) into +/// items. +pub type ParseRowFn = Arc< + dyn Fn(&SpiderResponse, &serde_json::Map) -> Vec + + Send + + Sync, +>; + +/// A [`Spider`] that iterates over CSV feed rows. +/// +/// Each response body is parsed as RFC 4180 CSV (configurable delimiter, +/// `"`-quoted fields with `""` escapes, LF or CRLF row endings). The first +/// row provides the column names unless `headers` overrides them (in which +/// case the first row is data). Every row becomes a header → cell object +/// handed to `parse_row`; without a callback the row object itself is the +/// item. Rows shorter than the header get empty strings for the missing +/// columns; extra cells beyond the header are dropped. Feeds are terminal: +/// no follow-up requests are generated. +pub struct CsvFeedSpider { + name: String, + feed_urls: Vec, + delimiter: char, + headers: Option>, + parse_row: Option, + allowed_domains: HashSet, + concurrent_requests: u32, + development_mode: bool, + robots_txt_obey: bool, +} + +impl CsvFeedSpider { + /// Start building a `CsvFeedSpider` with the given spider name. + pub fn builder(name: &str) -> CsvFeedSpiderBuilder { + CsvFeedSpiderBuilder::new(name) + } +} + +/// Parse an RFC 4180 CSV document into rows of fields. Handles quoted +/// fields (with `""` escapes and embedded delimiters/newlines), LF and +/// CRLF row endings, and a trailing newline without emitting a phantom +/// empty row. +fn parse_csv(text: &str, delimiter: char) -> Vec> { + let mut rows: Vec> = Vec::new(); + let mut row: Vec = Vec::new(); + let mut field = String::new(); + let mut in_quotes = false; + let mut chars = text.chars().peekable(); + // Tracks whether the current row has any content at all, so a trailing + // newline doesn't produce a phantom ["" ] row while a genuinely empty + // line mid-file still does. + let mut row_started = false; + + while let Some(c) = chars.next() { + if in_quotes { + match c { + '"' => { + if chars.peek() == Some(&'"') { + chars.next(); + field.push('"'); + } else { + in_quotes = false; + } + } + c => field.push(c), + } + continue; + } + match c { + '"' if field.is_empty() => { + in_quotes = true; + row_started = true; + } + c if c == delimiter => { + row.push(std::mem::take(&mut field)); + row_started = true; + } + '\r' => { + // Consumed as part of CRLF; a bare \r inside an unquoted + // field is kept verbatim. + if chars.peek() == Some(&'\n') { + continue; + } + field.push('\r'); + } + '\n' => { + if row_started || !field.is_empty() { + row.push(std::mem::take(&mut field)); + rows.push(std::mem::take(&mut row)); + } + row_started = false; + } + c => { + field.push(c); + row_started = true; + } + } + } + if row_started || !field.is_empty() { + row.push(field); + rows.push(row); + } + rows +} + +#[async_trait] +impl Spider for CsvFeedSpider { + fn name(&self) -> &str { + &self.name + } + fn start_urls(&self) -> Vec { + self.feed_urls.clone() + } + fn allowed_domains(&self) -> HashSet { + self.allowed_domains.clone() + } + fn concurrent_requests(&self) -> u32 { + self.concurrent_requests + } + fn development_mode(&self) -> bool { + self.development_mode + } + fn robots_txt_obey(&self) -> bool { + self.robots_txt_obey + } + + async fn parse( + &self, + response: SpiderResponse, + ) -> (Vec, Vec) { + let rows = parse_csv(response.text(), self.delimiter); + let mut rows_iter = rows.into_iter(); + + let headers: Vec = match &self.headers { + Some(h) => h.clone(), + None => match rows_iter.next() { + Some(first) => first, + None => return (vec![], vec![]), + }, + }; + + let mut items = Vec::new(); + for row in rows_iter { + let mut obj = serde_json::Map::new(); + for (i, header) in headers.iter().enumerate() { + let cell = row.get(i).cloned().unwrap_or_default(); + obj.insert(header.clone(), serde_json::Value::String(cell)); + } + match &self.parse_row { + Some(f) => items.extend(f(&response, &obj)), + None => items.push(serde_json::Value::Object(obj)), + } + } + (items, vec![]) + } +} + +/// Builder for [`CsvFeedSpider`]. +#[must_use = "a builder does nothing until `.build()` is called"] +pub struct CsvFeedSpiderBuilder { + spider: CsvFeedSpider, +} + +impl CsvFeedSpiderBuilder { + fn new(name: &str) -> Self { + Self { + spider: CsvFeedSpider { + name: name.to_string(), + feed_urls: Vec::new(), + delimiter: ',', + headers: None, + parse_row: None, + allowed_domains: HashSet::new(), + concurrent_requests: 4, + development_mode: false, + robots_txt_obey: false, + }, + } + } + + /// Add a feed URL to fetch. + pub fn feed_url(mut self, url: &str) -> Self { + self.spider.feed_urls.push(url.to_string()); + self + } + + /// Add several feed URLs. + pub fn feed_urls(mut self, urls: impl IntoIterator) -> Self { + self.spider.feed_urls.extend(urls); + self + } + + /// Field delimiter (default `,`; use `;` or `\t` for such feeds). + pub fn delimiter(mut self, delimiter: char) -> Self { + self.spider.delimiter = delimiter; + self + } + + /// Override the column names. When set, the first CSV row is treated + /// as data instead of a header row. + pub fn headers(mut self, headers: impl IntoIterator) -> Self { + self.spider.headers = Some(headers.into_iter().collect()); + self + } + + /// Callback turning one row object into items; without it, each row + /// object itself becomes an item. + pub fn parse_row(mut self, f: ParseRowFn) -> Self { + self.spider.parse_row = Some(f); + self + } + + /// Restrict the crawl to these domains. + pub fn allowed_domains(mut self, domains: impl IntoIterator) -> Self { + self.spider.allowed_domains.extend(domains); + self + } + + /// Global concurrency limit (default 4). + pub fn concurrent_requests(mut self, n: u32) -> Self { + self.spider.concurrent_requests = n; + self + } + + /// Cache responses to disk for development iteration. + pub fn development_mode(mut self, on: bool) -> Self { + self.spider.development_mode = on; + self + } + + /// Respect robots.txt. + pub fn robots_txt_obey(mut self, on: bool) -> Self { + self.spider.robots_txt_obey = on; + self + } + + /// Finish building the spider. + pub fn build(self) -> CsvFeedSpider { + self.spider + } +} + +#[cfg(test)] +mod tests { + use super::parse_csv; + + #[test] + fn parses_quoted_fields_delimiters_and_crlf() { + let text = "a,b,c\r\n\"x,1\",\"say \"\"hi\"\"\",plain\r\nlast,\"multi\nline\",end\r\n"; + let rows = parse_csv(text, ','); + assert_eq!( + rows, + vec![ + vec!["a", "b", "c"], + vec!["x,1", "say \"hi\"", "plain"], + vec!["last", "multi\nline", "end"], + ] + ); + } + + #[test] + fn trailing_newline_produces_no_phantom_row() { + assert_eq!( + parse_csv("a,b\n1,2\n", ','), + vec![vec!["a", "b"], vec!["1", "2"]] + ); + // No trailing newline works too. + assert_eq!( + parse_csv("a,b\n1,2", ','), + vec![vec!["a", "b"], vec!["1", "2"]] + ); + } + + #[test] + fn custom_delimiter_and_empty_fields() { + assert_eq!( + parse_csv("a;;c\n;;\n", ';'), + vec![vec!["a", "", "c"], vec!["", "", ""]] + ); + } +} diff --git a/src/spiders/templates/mod.rs b/src/spiders/templates/mod.rs index 33333a0..875f4b2 100644 --- a/src/spiders/templates/mod.rs +++ b/src/spiders/templates/mod.rs @@ -1,9 +1,13 @@ //! Generic spider templates that build on the [`Spider`](crate::spiders::spider::Spider) trait. pub mod crawler; +pub mod csv_feed; pub mod shopify; pub mod sitemap; +pub mod xml_feed; pub use crawler::{CrawlRule, CrawlSpider}; +pub use csv_feed::CsvFeedSpider; pub use shopify::ShopifySpider; pub use sitemap::SitemapSpider; +pub use xml_feed::XmlFeedSpider; diff --git a/src/spiders/templates/sitemap.rs b/src/spiders/templates/sitemap.rs index a2bb22d..e67e11f 100644 --- a/src/spiders/templates/sitemap.rs +++ b/src/spiders/templates/sitemap.rs @@ -8,9 +8,9 @@ //! other response is treated as a content page. //! //! Gzipped sitemaps: responses compressed at the transport level -//! (`Content-Encoding: gzip`) are decompressed transparently by the HTTP -//! client. Raw `.xml.gz` *files* are not supported because response bodies -//! are stored as text. +//! (`Content-Encoding: gzip`) are decompressed by the HTTP client, and raw +//! `.xml.gz` *files* are decompressed transparently by the body decoder +//! (gzip magic-byte detection), so both forms work. use crate::parser::Selector; use crate::spiders::links::LinkExtractor; diff --git a/src/spiders/templates/xml_feed.rs b/src/spiders/templates/xml_feed.rs new file mode 100644 index 0000000..1220a75 --- /dev/null +++ b/src/spiders/templates/xml_feed.rs @@ -0,0 +1,248 @@ +//! `XmlFeedSpider`, ported from upstream Scrapling's `XMLFeedSpider` +//! (v0.4.13): iterate over the nodes of an XML feed (RSS, Atom, product +//! feeds) and turn each into items. +//! +//! Gzip-compressed feed files (`.xml.gz`) are decompressed transparently +//! by the fetcher (magic-byte detection in the body decoder), in addition +//! to transport-level `Content-Encoding: gzip`. + +use crate::parser::Selector; +use crate::spiders::request::SpiderRequest; +use crate::spiders::response::SpiderResponse; +use crate::spiders::spider::Spider; +use async_trait::async_trait; +use std::collections::HashSet; +use std::sync::Arc; + +/// Callback turning one feed node (e.g. one RSS ``) into items. +/// Receives the full response plus the node's [`Selector`]. +pub type ParseNodeFn = + Arc Vec + Send + Sync>; + +/// A [`Spider`] that iterates over the nodes of XML feeds. +/// +/// Every response is parsed and each element matching `iter_tag` (default +/// `"item"`, the RSS entry tag — use `"entry"` for Atom) is handed to the +/// `parse_node` callback. Without a callback, each node is converted to an +/// object mapping its **child element names to their text content** +/// (`X` → `"title": "X"`); when a child tag repeats, the +/// last occurrence wins. Feeds are terminal: no follow-up requests are +/// generated. +pub struct XmlFeedSpider { + name: String, + feed_urls: Vec, + iter_tag: String, + parse_node: Option, + allowed_domains: HashSet, + concurrent_requests: u32, + development_mode: bool, + robots_txt_obey: bool, +} + +/// XML tags that the HTML parser treats as void elements, swallowing their +/// text content — `` is THE critical one (every RSS item's URL). +/// They are rewritten to `xmlfeed-*` before parsing and translated back +/// when item keys are emitted. +const VOID_TAG_REWRITES: &[(&str, &str)] = &[("link", "xmlfeed-link"), ("meta", "xmlfeed-meta")]; + +impl XmlFeedSpider { + /// Start building an `XmlFeedSpider` with the given spider name. + pub fn builder(name: &str) -> XmlFeedSpiderBuilder { + XmlFeedSpiderBuilder::new(name) + } + + /// Rewrite XML tags that HTML parsing would mangle (HTML void elements + /// like `` cannot have children, so their text would be lost). + /// Case-insensitive, whole-tag-name matches only. + fn rewrite_void_tags(body: &str) -> String { + let mut out = String::with_capacity(body.len()); + let bytes = body.as_bytes(); + let mut i = 0; + 'outer: while i < bytes.len() { + if bytes[i] == b'<' { + let (start, closing) = if bytes.get(i + 1) == Some(&b'/') { + (i + 2, true) + } else { + (i + 1, false) + }; + for (from, to) in VOID_TAG_REWRITES { + let end = start + from.len(); + let next = bytes.get(end); + let name_matches = body + .get(start..end) + .is_some_and(|s| s.eq_ignore_ascii_case(from)); + let boundary_ok = + matches!(next, Some(b'>') | Some(b' ') | Some(b'\t') | Some(b'/')) + || (closing && next.is_none()); + if name_matches && boundary_ok { + out.push('<'); + if closing { + out.push('/'); + } + out.push_str(to); + i = end; + continue 'outer; + } + } + } + // Advance by whole characters so multibyte input stays intact. + let ch_len = body[i..].chars().next().map_or(1, char::len_utf8); + out.push_str(&body[i..i + ch_len]); + i += ch_len; + } + out + } + + /// Translate a possibly-rewritten tag name back to its original. + fn original_tag(tag: &str) -> &str { + VOID_TAG_REWRITES + .iter() + .find(|(_, to)| *to == tag) + .map_or(tag, |(from, _)| from) + } + + /// Default node conversion: child element names → recursive text. + fn node_to_item(node: &Selector) -> serde_json::Value { + let mut obj = serde_json::Map::new(); + for child in &node.children() { + let tag = child.tag().to_string(); + if tag.starts_with('#') { + continue; // text/comment placeholder tags + } + let text = child.get_all_text("", false, &[], None); + obj.insert( + Self::original_tag(&tag).to_string(), + serde_json::Value::String(text.as_str().trim().to_string()), + ); + } + serde_json::Value::Object(obj) + } +} + +#[async_trait] +impl Spider for XmlFeedSpider { + fn name(&self) -> &str { + &self.name + } + fn start_urls(&self) -> Vec { + self.feed_urls.clone() + } + fn allowed_domains(&self) -> HashSet { + self.allowed_domains.clone() + } + fn concurrent_requests(&self) -> u32 { + self.concurrent_requests + } + fn development_mode(&self) -> bool { + self.development_mode + } + fn robots_txt_obey(&self) -> bool { + self.robots_txt_obey + } + + async fn parse( + &self, + response: SpiderResponse, + ) -> (Vec, Vec) { + // Parse a rewritten copy of the body so HTML-void feed tags + // (notably ) keep their text; translate the user's iter_tag + // too in case it names one of them. Custom parse_node callbacks see + // the rewritten tree: address rewritten tags as `xmlfeed-link` / + // `xmlfeed-meta` (documented on the builder). + let rewritten = Self::rewrite_void_tags(response.text()); + let selector = Selector::from_html_with_url(&rewritten, response.url()); + let iter_tag = VOID_TAG_REWRITES + .iter() + .find(|(from, _)| from.eq_ignore_ascii_case(&self.iter_tag)) + .map_or(self.iter_tag.clone(), |(_, to)| (*to).to_string()); + let mut items = Vec::new(); + for node in &selector.css(&iter_tag) { + match &self.parse_node { + Some(f) => items.extend(f(&response, node)), + None => items.push(Self::node_to_item(node)), + } + } + (items, vec![]) + } +} + +/// Builder for [`XmlFeedSpider`]. +#[must_use = "a builder does nothing until `.build()` is called"] +pub struct XmlFeedSpiderBuilder { + spider: XmlFeedSpider, +} + +impl XmlFeedSpiderBuilder { + fn new(name: &str) -> Self { + Self { + spider: XmlFeedSpider { + name: name.to_string(), + feed_urls: Vec::new(), + iter_tag: "item".to_string(), + parse_node: None, + allowed_domains: HashSet::new(), + concurrent_requests: 4, + development_mode: false, + robots_txt_obey: false, + }, + } + } + + /// Add a feed URL to fetch. + pub fn feed_url(mut self, url: &str) -> Self { + self.spider.feed_urls.push(url.to_string()); + self + } + + /// Add several feed URLs. + pub fn feed_urls(mut self, urls: impl IntoIterator) -> Self { + self.spider.feed_urls.extend(urls); + self + } + + /// The element to iterate over (default `"item"`; use `"entry"` for + /// Atom feeds). Any CSS selector works. + pub fn iter_tag(mut self, tag: &str) -> Self { + self.spider.iter_tag = tag.to_string(); + self + } + + /// Callback turning one feed node into items; overrides the default + /// child-elements-to-object conversion. Note: the parsed tree has + /// HTML-void feed tags rewritten (`` → ``, + /// `` → ``) so their text survives HTML parsing — + /// address them by the rewritten name in CSS queries. + pub fn parse_node(mut self, f: ParseNodeFn) -> Self { + self.spider.parse_node = Some(f); + self + } + + /// Restrict the crawl to these domains. + pub fn allowed_domains(mut self, domains: impl IntoIterator) -> Self { + self.spider.allowed_domains.extend(domains); + self + } + + /// Global concurrency limit (default 4). + pub fn concurrent_requests(mut self, n: u32) -> Self { + self.spider.concurrent_requests = n; + self + } + + /// Cache responses to disk for development iteration. + pub fn development_mode(mut self, on: bool) -> Self { + self.spider.development_mode = on; + self + } + + /// Respect robots.txt. + pub fn robots_txt_obey(mut self, on: bool) -> Self { + self.spider.robots_txt_obey = on; + self + } + + /// Finish building the spider. + pub fn build(self) -> XmlFeedSpider { + self.spider + } +} diff --git a/tests/fetchers_client.rs b/tests/fetchers_client.rs index e5619c8..7ef0237 100644 --- a/tests/fetchers_client.rs +++ b/tests/fetchers_client.rs @@ -329,3 +329,45 @@ async fn live_responses_carry_fetch_latency() { ); assert_eq!(synthetic.latency(), std::time::Duration::ZERO); } + +#[tokio::test] +async fn gzip_file_bodies_are_transparently_decompressed() { + // A raw .xml.gz FILE served without Content-Encoding (so reqwest's + // transport decompression does not apply) must still arrive as text: + // the body decoder detects the gzip magic bytes. + use std::io::Write as _; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(b"gz") + .unwrap(); + let gz = enc.finish().unwrap(); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut buf = [0u8; 4096]; + let _ = socket.read(&mut buf).await; + let header = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + gz.len() + ); + let _ = socket.write_all(header.as_bytes()).await; + let _ = socket.write_all(&gz).await; + let _ = socket.shutdown().await; + }); + + let fetcher = Fetcher::new(FetcherConfig::default()).unwrap(); + let response = fetcher + .get(&format!("http://{}/feed.xml.gz", addr)) + .await + .unwrap(); + server.await.unwrap(); + assert_eq!( + response.text(), + "gz", + "gzip file bodies must be transparently decompressed" + ); +} diff --git a/tests/spiders_templates.rs b/tests/spiders_templates.rs index 40c5e8f..3d4b672 100644 --- a/tests/spiders_templates.rs +++ b/tests/spiders_templates.rs @@ -382,3 +382,102 @@ async fn content_page_mentioning_urlset_is_not_misclassified() { assert!(items.is_empty()); assert_eq!(requests.len(), 1); } + +// ── Feed spiders (upstream v0.4.13) ── + +use rust_scrapling::spiders::spider::Spider as _; +use rust_scrapling::spiders::templates::{CsvFeedSpider, XmlFeedSpider}; + +fn feed_response(content_type: &str, body: &str) -> SpiderResponse { + SpiderResponse::new(Response::new( + 200, + content_type.to_string(), + body.to_string(), + "https://feeds.example/feed".to_string(), + HashMap::new(), + )) +} + +#[tokio::test] +async fn xml_feed_default_converts_rss_items_to_objects() { + let rss = r#" + + Chan + Firsthttps://a.example/1one & only + Secondhttps://a.example/2two +"#; + let spider = XmlFeedSpider::builder("rss") + .feed_url("https://feeds.example/feed") + .build(); + let (items, follow) = spider + .parse(feed_response("application/rss+xml", rss)) + .await; + + assert!(follow.is_empty(), "feeds are terminal"); + assert_eq!(items.len(), 2); + assert_eq!(items[0]["title"], "First"); + assert_eq!(items[0]["link"], "https://a.example/1"); + assert_eq!(items[0]["description"], "one & only"); + assert_eq!(items[1]["title"], "Second"); +} + +#[tokio::test] +async fn xml_feed_iter_tag_selects_atom_entries_and_callback_overrides() { + let atom = r#" + + A1 + A2 +"#; + let spider = XmlFeedSpider::builder("atom") + .feed_url("https://feeds.example/atom") + .iter_tag("entry") + .parse_node(Arc::new(|_resp, node| { + vec![serde_json::json!({ + "custom": node.css_get("title::text").map(|t| t.as_str().to_string()), + })] + })) + .build(); + let (items, _) = spider + .parse(feed_response("application/atom+xml", atom)) + .await; + + assert_eq!(items.len(), 2); + assert_eq!(items[0]["custom"], "A1"); + assert_eq!(items[1]["custom"], "A2"); +} + +#[tokio::test] +async fn csv_feed_first_row_is_header_and_rows_become_items() { + let csv = "name,price\r\nWidget,9.99\r\n\"Gadget, Large\",19.99\r\n"; + let spider = CsvFeedSpider::builder("csv") + .feed_url("https://feeds.example/f.csv") + .build(); + let (items, follow) = spider.parse(feed_response("text/csv", csv)).await; + + assert!(follow.is_empty()); + assert_eq!(items.len(), 2); + assert_eq!(items[0]["name"], "Widget"); + assert_eq!(items[0]["price"], "9.99"); + assert_eq!(items[1]["name"], "Gadget, Large"); +} + +#[tokio::test] +async fn csv_feed_header_override_short_rows_and_custom_delimiter() { + let csv = "1;only-one-cell\n2;b;EXTRA\n"; + let spider = CsvFeedSpider::builder("csv2") + .feed_url("https://feeds.example/f2.csv") + .delimiter(';') + .headers(["id".to_string(), "val".to_string()]) + .parse_row(Arc::new(|_resp, row| { + vec![serde_json::json!({ "id": row["id"], "val": row["val"] })] + })) + .build(); + let (items, _) = spider.parse(feed_response("text/csv", csv)).await; + + // Header override: the first row is DATA. Extra cells dropped. + assert_eq!(items.len(), 2); + assert_eq!(items[0]["id"], "1"); + assert_eq!(items[0]["val"], "only-one-cell"); + assert_eq!(items[1]["id"], "2"); + assert_eq!(items[1]["val"], "b"); +} From e9f4117f7b50c47f4d248ca67f5178ff94d7c20f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 13:17:10 +0000 Subject: [PATCH 2/2] fix(spiders): review fixes for feed spiders and gzip decoding - rewrite_void_tags: expand self-closing to an explicit end tag (html5ever ignores the self-closing flag on unknown elements, which left the rewritten tag open and swallowed all following Atom entry siblings); accept CR/LF as tag-name boundaries per XML S; track quoted attribute values when locating the tag-closing '>' - gunzip_capped: use MultiGzDecoder so concatenated gzip members (pigz/bgzip output) are fully decompressed instead of truncated - parse_csv: correct the blank-line comment to match behavior (skipped, like Python's csv module) and pin it with a test - docs: XmlFeedSpider limitations (namespaces, CDATA); decode_body's uncapped transparent gzip Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KDFsMaKk764vogjUW3nqpk --- src/fetchers/encoding.rs | 27 ++++++- src/spiders/templates/csv_feed.rs | 14 +++- src/spiders/templates/xml_feed.rs | 130 ++++++++++++++++++++++++++++-- 3 files changed, 160 insertions(+), 11 deletions(-) diff --git a/src/fetchers/encoding.rs b/src/fetchers/encoding.rs index 42eb422..74ecf93 100644 --- a/src/fetchers/encoding.rs +++ b/src/fetchers/encoding.rs @@ -38,6 +38,11 @@ pub fn charset_from_content_type(content_type: &str) -> Option<&str> { /// byte-order mark in the body takes precedence over the header, matching /// the WHATWG encoding standard behaviour. /// +/// Gzip-compressed bodies (detected by their magic bytes) are transparently +/// decompressed first, with **no size cap** — call [`decode_body_capped`] +/// to bound the decompressed size (the fetcher does, using its configured +/// body-size limit). +/// /// `for_label_no_replacement` is used because the WHATWG spec maps a few /// legacy labels (`hz-gb-2312`, `iso-2022-kr`, …) to the *replacement* /// encoding, which decodes the entire body to a single U+FFFD — for a @@ -76,7 +81,10 @@ fn gunzip_capped(bytes: &[u8], max_bytes: usize) -> Option> { } let mut out = Vec::new(); let limit = max_bytes as u64; - let mut reader = flate2::read::GzDecoder::new(bytes).take(limit.saturating_add(1)); + // MultiGzDecoder reads ALL members of a concatenated gzip stream + // (pigz/bgzip/log-rotation output), matching gzip(1) and Python's + // GzipFile; the single-member GzDecoder would silently truncate. + let mut reader = flate2::read::MultiGzDecoder::new(bytes).take(limit.saturating_add(1)); match reader.read_to_end(&mut out) { Ok(_) if out.len() as u64 <= limit => Some(out), // Oversized (bomb) or corrupt: keep the raw bytes, exactly the @@ -238,6 +246,23 @@ mod tests { ); } + #[test] + fn multi_member_gzip_is_fully_decompressed() { + use std::io::Write; + // Concatenated gzip members (as produced by pigz, bgzip, or + // `cat a.gz b.gz`) must all be read, not just the first. + let mut gz = Vec::new(); + for part in ["first ", "second"] { + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(part.as_bytes()).unwrap(); + gz.extend(enc.finish().unwrap()); + } + assert_eq!( + decode_body_capped(&gz, "application/gzip", 1024), + "first second" + ); + } + #[test] fn gzip_bomb_falls_back_to_raw_bytes() { use std::io::Write; diff --git a/src/spiders/templates/csv_feed.rs b/src/spiders/templates/csv_feed.rs index 9441951..b0f3afb 100644 --- a/src/spiders/templates/csv_feed.rs +++ b/src/spiders/templates/csv_feed.rs @@ -59,9 +59,9 @@ fn parse_csv(text: &str, delimiter: char) -> Vec> { let mut field = String::new(); let mut in_quotes = false; let mut chars = text.chars().peekable(); - // Tracks whether the current row has any content at all, so a trailing - // newline doesn't produce a phantom ["" ] row while a genuinely empty - // line mid-file still does. + // Tracks whether the current row has any content at all, so neither a + // trailing newline nor a blank line mid-file produces a phantom [""] + // row (Python's csv module likewise yields nothing for blank lines). let mut row_started = false; while let Some(c) = chars.next() { @@ -284,6 +284,14 @@ mod tests { ); } + #[test] + fn blank_lines_are_skipped() { + assert_eq!( + parse_csv("a,b\n\n1,2\n\r\n3,4\n", ','), + vec![vec!["a", "b"], vec!["1", "2"], vec!["3", "4"]] + ); + } + #[test] fn custom_delimiter_and_empty_fields() { assert_eq!( diff --git a/src/spiders/templates/xml_feed.rs b/src/spiders/templates/xml_feed.rs index 1220a75..a9c6f0f 100644 --- a/src/spiders/templates/xml_feed.rs +++ b/src/spiders/templates/xml_feed.rs @@ -28,6 +28,18 @@ pub type ParseNodeFn = /// (`X` → `"title": "X"`); when a child tag repeats, the /// last occurrence wins. Feeds are terminal: no follow-up requests are /// generated. +/// +/// # Limitations +/// +/// Feeds are parsed with the HTML parser, not a real XML parser: +/// +/// - **XML namespaces** are not understood. Namespaced child tags show up +/// literally as keys (`` → `"g:price"`), but a namespaced +/// `iter_tag` such as `"media:content"` cannot be used — the `:` is +/// parsed as a CSS pseudo-class and the selector matches nothing. +/// - **CDATA sections** (``) are not supported and their +/// content may be truncated or mangled; feeds that wrap descriptions in +/// CDATA will lose markup-heavy text. pub struct XmlFeedSpider { name: String, feed_urls: Vec, @@ -53,7 +65,10 @@ impl XmlFeedSpider { /// Rewrite XML tags that HTML parsing would mangle (HTML void elements /// like `` cannot have children, so their text would be lost). - /// Case-insensitive, whole-tag-name matches only. + /// Case-insensitive, whole-tag-name matches only. Self-closing forms + /// (``) are expanded to an explicit end tag because html5ever + /// ignores the self-closing flag on unknown elements — the rewritten tag + /// would stay open and swallow every following sibling. fn rewrite_void_tags(body: &str) -> String { let mut out = String::with_capacity(body.len()); let bytes = body.as_bytes(); @@ -71,16 +86,60 @@ impl XmlFeedSpider { let name_matches = body .get(start..end) .is_some_and(|s| s.eq_ignore_ascii_case(from)); - let boundary_ok = - matches!(next, Some(b'>') | Some(b' ') | Some(b'\t') | Some(b'/')) - || (closing && next.is_none()); + // XML `S` after a tag name: space, tab, CR, LF. + let boundary_ok = matches!( + next, + Some(b'>') + | Some(b' ') + | Some(b'\t') + | Some(b'\r') + | Some(b'\n') + | Some(b'/') + ) || (closing && next.is_none()); if name_matches && boundary_ok { - out.push('<'); if closing { - out.push('/'); + out.push_str("` ending this tag; XML + // attribute values may legally contain `>`. + let mut j = end; + let mut quote: Option = None; + let gt = loop { + match bytes.get(j) { + None => break None, + Some(&b) => match quote { + Some(q) if b == q => quote = None, + Some(_) => {} + None => match b { + b'"' | b'\'' => quote = Some(b), + b'>' => break Some(j), + _ => {} + }, + }, + } + j += 1; + }; + out.push('<'); out.push_str(to); - i = end; + match gt { + Some(gt) if bytes[gt - 1] == b'/' => { + // Self-closing: emit an explicit end tag. + out.push_str(&body[end..gt - 1]); + out.push_str(">'); + i = gt + 1; + } + Some(gt) => { + out.push_str(&body[end..=gt]); + i = gt + 1; + } + // Truncated tag at EOF: copy the rest verbatim. + None => i = end, + } continue 'outer; } } @@ -246,3 +305,60 @@ impl XmlFeedSpiderBuilder { self.spider } } + +#[cfg(test)] +mod tests { + use super::XmlFeedSpider; + use crate::parser::Selector; + + #[test] + fn self_closing_link_does_not_swallow_siblings() { + // html5ever ignores the self-closing flag on unknown elements; the + // rewriter must emit an explicit end tag or `id`/`updated` would + // become children of `xmlfeed-link` and vanish from the item. + let body = r#"A1urn:12020"#; + let rewritten = XmlFeedSpider::rewrite_void_tags(body); + assert!(rewritten.contains(r#""#)); + let selector = Selector::from_html(&rewritten); + let entries = selector.css("entry"); + let item = XmlFeedSpider::node_to_item(&entries[0]); + let obj = item.as_object().unwrap(); + assert_eq!(obj.get("title").unwrap(), "A1"); + assert_eq!(obj.get("id").unwrap(), "urn:1"); + assert_eq!(obj.get("updated").unwrap(), "2020"); + assert_eq!(obj.get("link").unwrap(), ""); + } + + #[test] + fn newline_after_tag_name_is_a_boundary() { + // Pretty-printers wrap attributes: `` is valid XML `S`. + let rewritten = XmlFeedSpider::rewrite_void_tags("https://a/1"); + assert_eq!(rewritten, "https://a/1"); + let rewritten = XmlFeedSpider::rewrite_void_tags("t"); + assert_eq!(rewritten, "t"); + } + + #[test] + fn gt_inside_quoted_attribute_does_not_end_the_tag() { + // `>` is legal inside XML attribute values. + let rewritten = XmlFeedSpider::rewrite_void_tags(r#"1"#); + assert_eq!( + rewritten, + r#"1"# + ); + } + + #[test] + fn non_matching_and_truncated_tags_pass_through() { + assert_eq!( + XmlFeedSpider::rewrite_void_tags("x"), + "x" + ); + // Truncated tag at EOF: copied verbatim after the rewritten name. + assert_eq!( + XmlFeedSpider::rewrite_void_tags("