diff --git a/Cargo.lock b/Cargo.lock index e7beb95..f5816da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -85,6 +91,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "async-trait" version = "0.1.91" @@ -251,6 +266,7 @@ dependencies = [ "async-trait", "bullpen-llm", "bullpen-sandbox", + "flate2", "futures", "globset", "ignore", @@ -260,9 +276,11 @@ dependencies = [ "serde", "serde_json", "sha2", + "tar", "tempfile", "thiserror", "tokio", + "zip", ] [[package]] @@ -403,6 +421,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.7" @@ -497,6 +524,17 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -558,12 +596,32 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "foldhash" version = "0.1.5" @@ -759,6 +817,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "hashlink" version = "0.9.1" @@ -997,6 +1061,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + [[package]] name = "indoc" version = "2.0.7" @@ -1143,6 +1217,16 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -1684,6 +1768,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "slab" version = "0.4.12" @@ -1794,6 +1884,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -2386,6 +2487,16 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + [[package]] name = "yoke" version = "0.8.3" @@ -2489,8 +2600,37 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index a3a6ddd..b1b4cbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,9 @@ globset = "0.4" ignore = "0.4" uuid = { version = "1", features = ["v4", "v5"] } sha2 = "0.10" +zip = { version = "2", default-features = false, features = ["deflate"] } +tar = "0.4" +flate2 = "1" base64 = "0.22" getrandom = "0.3" diff --git a/README.md b/README.md index 7aad639..441a3ef 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ crash-recoverable and inspectable from the CLI the whole way. | Tool | What it does | |---|---| | `bash` | Shell in the workspace; sandboxed with its children under Seatbelt on macOS | -| `read_file` | One path for files (hashline `line#hash` anchors), directories (sorted listings), SQLite databases (schema view or read-only `query`), and http(s) URLs (streamed cap; refused when the sandbox denies network) | +| `read_file` | One path for files (hashline `line#hash` anchors), directories (sorted listings), SQLite databases (schema view or read-only `query`), zip/tar/tar.gz archives (listing or one `entry`; plain gzip decompresses as text), and http(s) URLs (streamed cap; refused when the sandbox denies network) | | `write_file` / `edit_file` | Writes under sandbox confinement; edits by exact string or by anchored hashline patch with stale-anchor recovery | | `grep` / `glob` | Regex content search and path patterns, `.gitignore`-aware | | `ast_grep` / `ast_edit` | Structural search and rewrite over the syntax tree via [ast-grep](https://ast-grep.github.io) (when installed); rewrites preview by default and write only on `apply: true` | diff --git a/crates/tools/Cargo.toml b/crates/tools/Cargo.toml index 60cb2d9..45422dc 100644 --- a/crates/tools/Cargo.toml +++ b/crates/tools/Cargo.toml @@ -12,6 +12,9 @@ futures.workspace = true globset.workspace = true reqwest.workspace = true rusqlite.workspace = true +zip.workspace = true +tar.workspace = true +flate2.workspace = true ignore.workspace = true regex.workspace = true serde.workspace = true diff --git a/crates/tools/src/archive.rs b/crates/tools/src/archive.rs new file mode 100644 index 0000000..596937b --- /dev/null +++ b/crates/tools/src/archive.rs @@ -0,0 +1,542 @@ +//! Archives behind the one read path. +//! +//! Zip, tar, and gzipped tar are recognized by their magic bytes — never +//! their extensions — and render as an entry listing like a directory +//! read. Passing `entry` extracts one member instead, which `read_file` +//! renders as hashline text like any file. Everything is bounded: the +//! listing caps its entry count, an extracted member caps its bytes while +//! streaming — an archive has no business unpacking further than the +//! transcript can carry. + +use std::io::Read; +use std::path::Path; + +use crate::{ToolError, truncate_middle}; + +/// Listing cap; the true total is always reported. +const MAX_ENTRIES: usize = 1_000; +/// Extraction cap, enforced while streaming (zip bombs stay in the bottle). +pub(crate) const MAX_ENTRY_BYTES: usize = 262_144; // matches file reads +/// Rendered-listing cap, matching the other read views. +const MAX_OUTPUT_BYTES: usize = 100_000; +/// How much of a tar stream a scan may consume, measured after +/// decompression. Listing or searching a tar walks the whole stream, and +/// without this a small gzip bomb buys unbounded CPU; hitting the bound +/// caps the listing (reported) or fails the entry search (explained). +const MAX_SCAN_BYTES: u64 = 64 * 1024 * 1024; +/// Distinctive marker carried by the cap's io error, so the tar iterator's +/// wrapped failure is recognizable as "capped", not "corrupt". +const SCAN_CAP: &str = "bullpen-archive-scan-cap"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Kind { + Zip, + Tar, + TarGz, + /// A gzip stream that is not a tar — plain compressed content, read as + /// text rather than rejected as a malformed archive. + Gz, +} + +/// Recognize an archive by content: zip's `PK` local/empty headers, gzip's +/// two magic bytes, or tar's `ustar` at offset 257. A gzip stream is only +/// a tar.gz if the *decompressed* head carries the tar magic too. +pub(crate) fn detect(path: &Path) -> Option { + let mut file = std::fs::File::open(path).ok()?; + let mut head = [0u8; 265]; + let n = file.read(&mut head).ok()?; + let head = &head[..n]; + if head.starts_with(b"PK\x03\x04") || head.starts_with(b"PK\x05\x06") { + return Some(Kind::Zip); + } + if head.starts_with(&[0x1f, 0x8b]) { + let file = std::fs::File::open(path).ok()?; + let mut inner = [0u8; 262]; + let mut decoder = flate2::read::GzDecoder::new(file); + let mut got = 0; + while got < inner.len() { + match decoder.read(&mut inner[got..]) { + Ok(0) => break, + Ok(n) => got += n, + Err(_) => break, + } + } + return Some(if got >= 262 && &inner[257..262] == b"ustar" { + Kind::TarGz + } else { + Kind::Gz + }); + } + if n >= 262 && &head[257..262] == b"ustar" { + return Some(Kind::Tar); + } + None +} + +/// A reader that refuses to hand out more than `remaining` bytes; the +/// refusal is an io error carrying [`SCAN_CAP`]. +struct LimitedReader { + inner: R, + remaining: u64, +} + +impl Read for LimitedReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if self.remaining == 0 { + return Err(std::io::Error::other(SCAN_CAP)); + } + let want = buf + .len() + .min(self.remaining.min(usize::MAX as u64) as usize); + let n = self.inner.read(&mut buf[..want])?; + self.remaining -= n as u64; + Ok(n) + } +} + +/// Whether a tar iteration error is the scan cap biting (as opposed to a +/// genuinely corrupt archive). +fn is_scan_cap(e: &std::io::Error) -> bool { + e.to_string().contains(SCAN_CAP) +} + +fn arch_err(path: &Path, e: impl std::fmt::Display) -> ToolError { + ToolError::Failed(format!("archive {}: {e}", path.display())) +} + +/// One listed member: directories render with a trailing slash. +struct Entry { + name: String, + size: u64, + is_dir: bool, +} + +/// `total` counts every member scanned; `entries` holds at most +/// [`MAX_ENTRIES`] of them — the cap is applied while scanning, so a +/// million-member archive costs a count, not a vector. `capped` means the +/// scan budget ran out first: the total is a floor, not a count. +fn render(path: &Path, kind: Kind, entries: Vec, total: usize, capped: bool) -> String { + let label = match kind { + Kind::Zip => "zip", + Kind::Tar => "tar", + Kind::TarGz => "tar.gz", + Kind::Gz => "gzip", + }; + if entries.is_empty() && !capped { + return format!("(empty {label} archive {})", path.display()); + } + let mut out = format!( + "{label} archive {} ({total}{} entries) — pass `entry` to read one:\n", + path.display(), + if capped { "+" } else { "" } + ); + for entry in &entries { + if entry.is_dir { + out.push_str(&format!(" {}\n", entry.name)); + } else { + out.push_str(&format!(" {} {} bytes\n", entry.name, entry.size)); + } + } + if capped { + out.push_str(&format!( + "[scan capped at {} MiB — listing incomplete]\n", + MAX_SCAN_BYTES / (1024 * 1024) + )); + } else if total > entries.len() { + out.push_str(&format!("[listed the first {MAX_ENTRIES}]\n")); + } + truncate_middle(out, MAX_OUTPUT_BYTES) +} + +/// Read `reader` into a bounded buffer; `true` when the cap cut it short. +fn bounded_read(mut reader: impl Read) -> std::io::Result<(Vec, bool)> { + let mut buf = Vec::new(); + let clipped = (&mut reader) + .take(MAX_ENTRY_BYTES as u64 + 1) + .read_to_end(&mut buf) + .map(|_| buf.len() > MAX_ENTRY_BYTES)?; + buf.truncate(MAX_ENTRY_BYTES); + Ok((buf, clipped)) +} + +fn zip_archive(path: &Path) -> Result, ToolError> { + let file = std::fs::File::open(path).map_err(|e| arch_err(path, e))?; + zip::ZipArchive::new(file).map_err(|e| arch_err(path, e)) +} + +fn tar_archive(path: &Path, kind: Kind) -> Result>, ToolError> { + let file = std::fs::File::open(path).map_err(|e| arch_err(path, e))?; + let raw: Box = match kind { + Kind::TarGz => Box::new(flate2::read::GzDecoder::new(file)), + _ => Box::new(file), + }; + // The whole scan — headers plus the payloads the iterator skips over — + // draws from one decompressed-byte budget, so a small gzip bomb stops + // at the budget instead of costing unbounded CPU. + Ok(tar::Archive::new(Box::new(LimitedReader { + inner: raw, + remaining: MAX_SCAN_BYTES, + }))) +} + +/// The listing view. +pub(crate) fn list(path: &Path, kind: Kind) -> Result { + let (entries, total, capped) = match kind { + Kind::Gz => { + return Err(ToolError::InvalidInput(format!( + "{} is a plain gzip stream, not an archive — read it without \ + `entry` to see its decompressed content", + path.display() + ))); + } + Kind::Zip => { + // The central directory answers without touching member data. + let mut archive = zip_archive(path)?; + let total = archive.len(); + let mut entries = Vec::new(); + for i in 0..total.min(MAX_ENTRIES) { + let member = archive.by_index(i).map_err(|e| arch_err(path, e))?; + entries.push(Entry { + name: member.name().to_string(), + size: member.size(), + is_dir: member.is_dir(), + }); + } + (entries, total, false) + } + Kind::Tar | Kind::TarGz => { + let mut archive = tar_archive(path, kind)?; + let mut entries = Vec::new(); + let mut total = 0; + let mut capped = false; + for member in archive.entries().map_err(|e| arch_err(path, e))? { + let member = match member { + Ok(member) => member, + Err(e) if is_scan_cap(&e) => { + capped = true; + break; + } + Err(e) => return Err(arch_err(path, e)), + }; + total += 1; + if entries.len() < MAX_ENTRIES { + entries.push(Entry { + name: member + .path() + .map_err(|e| arch_err(path, e))? + .display() + .to_string(), + size: member.size(), + is_dir: member.header().entry_type().is_dir(), + }); + } + } + (entries, total, capped) + } + }; + Ok(render(path, kind, entries, total, capped)) +} + +/// Extract one member's bytes (bounded). The name must match a listed +/// entry exactly; a miss says so rather than guessing. +pub(crate) fn read_entry( + path: &Path, + kind: Kind, + entry: &str, +) -> Result<(Vec, bool), ToolError> { + if entry.is_empty() { + return Err(ToolError::InvalidInput( + "`entry` must name an archive member — read the archive without \ + `entry` to list them" + .into(), + )); + } + let missing = || { + ToolError::InvalidInput(format!( + "no entry `{entry}` in {} — read the archive without `entry` to list them", + path.display() + )) + }; + match kind { + Kind::Gz => Err(ToolError::InvalidInput(format!( + "{} is a plain gzip stream, not an archive — read it without \ + `entry` to see its decompressed content", + path.display() + ))), + Kind::Zip => { + let mut archive = zip_archive(path)?; + let member = match archive.by_name(entry) { + Ok(member) => member, + Err(zip::result::ZipError::FileNotFound) => return Err(missing()), + Err(e) => return Err(arch_err(path, e)), + }; + bounded_read(member).map_err(|e| arch_err(path, e)) + } + Kind::Tar | Kind::TarGz => { + let scan_out = || { + ToolError::Failed(format!( + "the {} MiB scan budget ran out before `{entry}` in {} — \ + extract oversized archives with bash instead", + MAX_SCAN_BYTES / (1024 * 1024), + path.display() + )) + }; + let mut archive = tar_archive(path, kind)?; + for member in archive.entries().map_err(|e| arch_err(path, e))? { + let member = match member { + Ok(member) => member, + Err(e) if is_scan_cap(&e) => return Err(scan_out()), + Err(e) => return Err(arch_err(path, e)), + }; + if member + .path() + .map_err(|e| arch_err(path, e))? + .display() + .to_string() + == entry + { + return bounded_read(member).map_err(|e| { + if is_scan_cap(&e) { + scan_out() + } else { + arch_err(path, e) + } + }); + } + } + Err(missing()) + } + } +} + +/// A plain gzip stream: its decompressed content, bounded like any entry. +pub(crate) fn read_gz(path: &Path) -> Result<(Vec, bool), ToolError> { + let file = std::fs::File::open(path).map_err(|e| arch_err(path, e))?; + bounded_read(flate2::read::GzDecoder::new(file)).map_err(|e| arch_err(path, e)) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn make_zip(dir: &tempfile::TempDir) -> std::path::PathBuf { + let path = dir.path().join("a.zip"); + let file = std::fs::File::create(&path).unwrap(); + let mut writer = zip::ZipWriter::new(file); + let options = zip::write::SimpleFileOptions::default(); + writer.add_directory("docs/", options).unwrap(); + writer.start_file("docs/readme.md", options).unwrap(); + writer.write_all(b"# hello\nworld\n").unwrap(); + writer.start_file("main.rs", options).unwrap(); + writer.write_all(b"fn main() {}\n").unwrap(); + writer.finish().unwrap(); + path + } + + fn make_tar_gz(dir: &tempfile::TempDir) -> std::path::PathBuf { + let path = dir.path().join("a.tgz"); + let file = std::fs::File::create(&path).unwrap(); + let gz = flate2::write::GzEncoder::new(file, flate2::Compression::default()); + let mut builder = tar::Builder::new(gz); + let data = b"key = value\n"; + let mut header = tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "conf/app.toml", &data[..]) + .unwrap(); + builder.into_inner().unwrap().finish().unwrap(); + path + } + + #[test] + fn detects_by_magic_not_extension() { + let dir = tempfile::tempdir().unwrap(); + let zip = make_zip(&dir); + let tgz = make_tar_gz(&dir); + // Deliberately misleading names. + let disguised = dir.path().join("archive.txt"); + std::fs::copy(&zip, &disguised).unwrap(); + + assert_eq!(detect(&zip), Some(Kind::Zip)); + assert_eq!(detect(&tgz), Some(Kind::TarGz)); + assert_eq!(detect(&disguised), Some(Kind::Zip)); + let plain = dir.path().join("notes.zip"); + std::fs::write(&plain, "not an archive").unwrap(); + assert_eq!(detect(&plain), None); + } + + #[test] + fn plain_tar_is_detected_by_ustar() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("a.tar"); + let mut builder = tar::Builder::new(std::fs::File::create(&path).unwrap()); + let data = b"x"; + let mut header = tar::Header::new_gnu(); + header.set_size(1); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "one.txt", &data[..]) + .unwrap(); + builder.finish().unwrap(); + + assert_eq!(detect(&path), Some(Kind::Tar)); + let out = list(&path, Kind::Tar).unwrap(); + assert!(out.contains("one.txt 1 bytes"), "{out}"); + } + + #[test] + fn listings_show_entries_sizes_and_the_read_hint() { + let dir = tempfile::tempdir().unwrap(); + let out = list(&make_zip(&dir), Kind::Zip).unwrap(); + assert!(out.contains("zip archive"), "{out}"); + assert!(out.contains("3 entries"), "{out}"); + assert!(out.contains("docs/\n"), "{out}"); + assert!(out.contains("docs/readme.md 14 bytes"), "{out}"); + assert!(out.contains("pass `entry`"), "{out}"); + + let out = list(&make_tar_gz(&dir), Kind::TarGz).unwrap(); + assert!(out.contains("tar.gz archive"), "{out}"); + assert!(out.contains("conf/app.toml 12 bytes"), "{out}"); + } + + #[test] + fn entries_extract_bounded_and_misses_name_the_listing() { + let dir = tempfile::tempdir().unwrap(); + let zip = make_zip(&dir); + let (bytes, clipped) = read_entry(&zip, Kind::Zip, "docs/readme.md").unwrap(); + assert_eq!(bytes, b"# hello\nworld\n"); + assert!(!clipped); + + let (bytes, _) = read_entry(&make_tar_gz(&dir), Kind::TarGz, "conf/app.toml").unwrap(); + assert_eq!(bytes, b"key = value\n"); + + let err = read_entry(&zip, Kind::Zip, "nope.txt").unwrap_err(); + assert!(matches!(err, ToolError::InvalidInput(_)), "{err}"); + assert!(err.to_string().contains("without `entry`"), "{err}"); + } + + #[test] + fn listings_cap_what_they_keep_but_count_everything() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("many.tar"); + let mut builder = tar::Builder::new(std::fs::File::create(&path).unwrap()); + // Long names, so the rendered listing would blow past the output + // cap if it were unbounded. + let name = |i: usize| format!("{}{i:04}.txt", "n".repeat(150)); + for i in 0..(MAX_ENTRIES + 5) { + let mut header = tar::Header::new_gnu(); + header.set_size(1); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, name(i), &b"x"[..]) + .unwrap(); + } + builder.finish().unwrap(); + + let out = list(&path, Kind::Tar).unwrap(); + assert!( + out.contains(&format!("({} entries)", MAX_ENTRIES + 5)), + "{out}" + ); + assert!( + out.contains(&format!("[listed the first {MAX_ENTRIES}]")), + "{out}" + ); + assert!(!out.contains(&name(MAX_ENTRIES + 2)), "{out}"); + // The rendered output itself is bounded, not just the entry count. + assert!(out.len() <= MAX_OUTPUT_BYTES + 200, "{}", out.len()); + } + + #[test] + fn plain_gzip_text_is_gz_not_a_broken_tar() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("notes.gz"); + let mut gz = flate2::write::GzEncoder::new( + std::fs::File::create(&path).unwrap(), + flate2::Compression::default(), + ); + gz.write_all(b"hello\nworld\n").unwrap(); + gz.finish().unwrap(); + + assert_eq!(detect(&path), Some(Kind::Gz)); + let (bytes, clipped) = read_gz(&path).unwrap(); + assert_eq!(bytes, b"hello\nworld\n"); + assert!(!clipped); + // The archive views refuse it with a pointer to the text path. + assert!(matches!( + list(&path, Kind::Gz), + Err(ToolError::InvalidInput(_)) + )); + assert!(matches!( + read_entry(&path, Kind::Gz, "x"), + Err(ToolError::InvalidInput(_)) + )); + } + + #[test] + fn tar_scans_stop_at_the_decompressed_byte_budget() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("bomb.tgz"); + let gz = flate2::write::GzEncoder::new( + std::fs::File::create(&path).unwrap(), + flate2::Compression::fast(), + ); + let mut builder = tar::Builder::new(gz); + // First member decompresses past the scan budget; a marker hides + // behind it. Compressed, the whole thing stays tiny. + let big = MAX_SCAN_BYTES + 8 * 1024 * 1024; + let mut header = tar::Header::new_gnu(); + header.set_size(big); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "large.bin", std::io::repeat(0).take(big)) + .unwrap(); + let mut header = tar::Header::new_gnu(); + header.set_size(1); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "marker.txt", &b"x"[..]) + .unwrap(); + builder.into_inner().unwrap().finish().unwrap(); + + let out = list(&path, Kind::TarGz).unwrap(); + assert!(out.contains("1+ entries"), "{out}"); + assert!(out.contains("large.bin"), "{out}"); + assert!(out.contains("scan capped"), "{out}"); + assert!(!out.contains("marker.txt"), "{out}"); + + let err = read_entry(&path, Kind::TarGz, "marker.txt").unwrap_err(); + assert!(err.to_string().contains("scan budget"), "{err}"); + } + + #[test] + fn an_empty_entry_name_is_invalid_input() { + let dir = tempfile::tempdir().unwrap(); + let err = read_entry(&make_zip(&dir), Kind::Zip, "").unwrap_err(); + assert!(matches!(err, ToolError::InvalidInput(_)), "{err}"); + } + + #[test] + fn extraction_is_capped_while_streaming() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("big.zip"); + let mut writer = zip::ZipWriter::new(std::fs::File::create(&path).unwrap()); + writer + .start_file("big.bin", zip::write::SimpleFileOptions::default()) + .unwrap(); + // Highly compressible: a small archive that expands well past the cap. + writer.write_all(&vec![b'a'; MAX_ENTRY_BYTES * 4]).unwrap(); + writer.finish().unwrap(); + + let (bytes, clipped) = read_entry(&path, Kind::Zip, "big.bin").unwrap(); + assert_eq!(bytes.len(), MAX_ENTRY_BYTES); + assert!(clipped); + } +} diff --git a/crates/tools/src/fs.rs b/crates/tools/src/fs.rs index 917c160..f82b612 100644 --- a/crates/tools/src/fs.rs +++ b/crates/tools/src/fs.rs @@ -36,6 +36,31 @@ fn hashlines(lines: &[&str], first: usize, limit: usize) -> String { .collect() } +/// What the content probes made of a path, decided in blocking work. +enum Probed { + Sqlite, + Archive(crate::archive::Kind), + Plain, +} + +/// Decompressed or extracted bytes rendered as hashline text, with the +/// clip note when a bound cut them short. +fn entry_text(bytes: &[u8], clipped: bool, empty: &str) -> String { + let text = String::from_utf8_lossy(bytes); + let lines: Vec<&str> = text.lines().collect(); + let mut out = hashlines(&lines, 1, usize::MAX); + if out.is_empty() { + out = empty.to_string(); + } + if clipped { + out.push_str(&format!( + "[content clipped at {} bytes]\n", + crate::archive::MAX_ENTRY_BYTES + )); + } + truncate_middle(out, MAX_READ_BYTES) +} + pub struct ReadFile; #[async_trait::async_trait] @@ -48,24 +73,27 @@ impl Tool for ReadFile { ToolSpec { name: "read_file".into(), description: "Read through one path: a file, a directory, a SQLite \ - database, or an http(s) URL. Files render as 1-indexed \ - `line#hashcontent`, capped at 256 KiB — the \ - `line#hash` token is an anchor that `edit_file` \ - patches accept — with an optional line window. \ - Directories render a sorted listing. A SQLite file \ - (detected by content) renders its schema and row \ + database, an archive, or an http(s) URL. Files render \ + as 1-indexed `line#hashcontent`, capped at 256 \ + KiB — the `line#hash` token is an anchor that \ + `edit_file` patches accept — with an optional line \ + window. Directories render a sorted listing. A SQLite \ + file (detected by content) renders its schema and row \ counts, or runs a read-only `query` (writes are \ - rejected by the engine). URLs are fetched with GET \ + rejected by the engine). A zip/tar/tar.gz archive \ + lists its entries, or renders one via `entry`; plain \ + gzip decompresses as text. URLs are fetched with GET \ (capped, 30s timeout); a sandbox that denies network \ refuses them." .into(), input_schema: json!({ "type": "object", "properties": { - "path": {"type": "string", "description": "File, directory, or SQLite path, or an http(s):// URL"}, + "path": {"type": "string", "description": "File, directory, SQLite, or archive path, or an http(s):// URL"}, "offset": {"type": "integer", "description": "1-indexed first line (files only)"}, "limit": {"type": "integer", "description": "Max lines to return (files only)"}, - "query": {"type": "string", "description": "Read-only SQL to run (SQLite files only)"} + "query": {"type": "string", "description": "Read-only SQL to run (SQLite files only)"}, + "entry": {"type": "string", "description": "Archive member to read (archives only)"} }, "required": ["path"] }), @@ -89,17 +117,60 @@ impl Tool for ReadFile { if tokio::fs::metadata(&path).await.is_ok_and(|m| m.is_dir()) { return list_dir(&path).await; } - if crate::sqlite::is_sqlite(&path) { - let query = input - .get("query") - .and_then(Value::as_str) - .map(str::to_string); - let db = path.clone(); - return tokio::task::spawn_blocking(move || { - crate::sqlite::read_sqlite(&db, query.as_deref()) - }) - .await - .map_err(|e| ToolError::Failed(format!("sqlite task failed: {e}")))?; + // Content probes read file heads synchronously — off the async + // worker, like every other blocking touch of the file. + let (path, probed) = tokio::task::spawn_blocking(move || { + let probed = if crate::sqlite::is_sqlite(&path) { + Probed::Sqlite + } else if let Some(kind) = crate::archive::detect(&path) { + Probed::Archive(kind) + } else { + Probed::Plain + }; + (path, probed) + }) + .await + .map_err(|e| ToolError::Failed(format!("probe task failed: {e}")))?; + + match probed { + Probed::Plain => {} + Probed::Sqlite => { + let query = input + .get("query") + .and_then(Value::as_str) + .map(str::to_string); + return tokio::task::spawn_blocking(move || { + crate::sqlite::read_sqlite(&path, query.as_deref()) + }) + .await + .map_err(|e| ToolError::Failed(format!("sqlite task failed: {e}")))?; + } + Probed::Archive(kind) => { + let entry = input + .get("entry") + .and_then(Value::as_str) + .map(str::to_string); + return tokio::task::spawn_blocking(move || { + use crate::archive::Kind; + match (kind, entry) { + (Kind::Gz, None) => { + let (bytes, clipped) = crate::archive::read_gz(&path)?; + Ok(entry_text(&bytes, clipped, "(empty gzip stream)")) + } + (_, None) => crate::archive::list(&path, kind), + (kind, Some(entry)) => { + let (bytes, clipped) = crate::archive::read_entry(&path, kind, &entry)?; + Ok(entry_text( + &bytes, + clipped, + &format!("(empty entry {entry})"), + )) + } + } + }) + .await + .map_err(|e| ToolError::Failed(format!("archive task failed: {e}")))?; + } } let raw = tokio::fs::read(&path) .await @@ -868,6 +939,71 @@ mod tests { assert!(out.contains("W12x26"), "{out}"); } + #[tokio::test] + async fn plain_gzip_reads_as_decompressed_text() { + let dir = tempfile::tempdir().unwrap(); + let c = ctx(&dir); + let mut gz = flate2::write::GzEncoder::new( + std::fs::File::create(dir.path().join("notes.gz")).unwrap(), + flate2::Compression::default(), + ); + std::io::Write::write_all(&mut gz, b"hello\nworld\n").unwrap(); + gz.finish().unwrap(); + + let out = ReadFile + .run(&c, "t", json!({"path": "notes.gz"})) + .await + .unwrap(); + assert_eq!( + out, + format!( + "1#{}\thello\n2#{}\tworld\n", + line_hash("hello"), + line_hash("world") + ) + ); + + let err = ReadFile + .run(&c, "t", json!({"path": "notes.gz", "entry": "x"})) + .await + .unwrap_err(); + assert!(matches!(err, ToolError::InvalidInput(_)), "{err}"); + } + + #[tokio::test] + async fn an_archive_reads_as_a_listing_then_an_entry() { + let dir = tempfile::tempdir().unwrap(); + let c = ctx(&dir); + let mut writer = + zip::ZipWriter::new(std::fs::File::create(dir.path().join("bundle.zip")).unwrap()); + writer + .start_file("src/lib.rs", zip::write::SimpleFileOptions::default()) + .unwrap(); + std::io::Write::write_all(&mut writer, b"pub fn one() {}\n").unwrap(); + writer.finish().unwrap(); + + let out = ReadFile + .run(&c, "t", json!({"path": "bundle.zip"})) + .await + .unwrap(); + assert!(out.contains("zip archive"), "{out}"); + assert!(out.contains("src/lib.rs 16 bytes"), "{out}"); + + let out = ReadFile + .run( + &c, + "t", + json!({"path": "bundle.zip", "entry": "src/lib.rs"}), + ) + .await + .unwrap(); + // The entry renders as ordinary hashline text. + assert_eq!( + out, + format!("1#{}\tpub fn one() {{}}\n", line_hash("pub fn one() {}")) + ); + } + /// One canned HTTP exchange on a local port; returns the URL to hit. fn serve_once(response: &'static str) -> String { let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/crates/tools/src/lib.rs b/crates/tools/src/lib.rs index 7f6ef52..1146dab 100644 --- a/crates/tools/src/lib.rs +++ b/crates/tools/src/lib.rs @@ -4,6 +4,7 @@ //! application, never by the agent loop. The runtime — not the model — owns //! the parallel-safety decision via [`Tool::parallel_safe`]. +mod archive; mod ask; mod ast; mod bash; diff --git a/docs/TOOLS.md b/docs/TOOLS.md index ce9b453..967bd1c 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -14,7 +14,7 @@ what bullpen ships today, and in what order the rest should land. | Tool | Catalog role | Notes | |---|---|---| | `bash` | runtime shell | Serial, sandboxable (Seatbelt on macOS), timeout-bounded. | -| `read_file` | read | One path for files, directories, SQLite databases, and http(s) URLs. Files are hashline output — every line carries a `line#hash` anchor — capped head+tail; directories render sorted listings; SQLite files (detected by magic, not extension) render schema + row counts or run a read-only `query` (writes rejected by the engine via `mode=ro` + `query_only`, with an `immutable=1` fallback for live-WAL stores); URLs fetch with a streamed cap and honor the sandbox's network policy. | +| `read_file` | read | One path for files, directories, SQLite databases, archives, and http(s) URLs. Files are hashline output — every line carries a `line#hash` anchor — capped head+tail; directories render sorted listings; SQLite files (detected by magic, not extension) render schema + row counts or run a read-only `query` (writes rejected by the engine via `mode=ro` + `query_only`, with an `immutable=1` fallback for live-WAL stores); zip/tar/tar.gz archives (also magic-detected) list entries or render one via `entry`, with extraction capped while streaming and tar scans bounded by a decompressed-byte budget; plain gzip decompresses as text; URLs fetch with a streamed cap and honor the sandbox's network policy. | | `write_file` / `edit_file` | write / edit | Sandbox write-confinement applies. `edit_file` takes exact-string replacements or hashline patches — hunks addressed by anchors, spans via `to`, with stale-anchor recovery: a moved line is followed while its content hash is unique, a changed line fails with fresh context instead of misapplying. | | `grep` | content search | Regex over the tree, `.gitignore`-aware. | | `glob` | path find | Pattern lookup; reach for `grep` when you need content. | @@ -39,9 +39,9 @@ names: `find` is `glob`, `search` is `grep`, and `task` is the pen's ## Then: files & search, deepened -- **richer `read`** — directories, URLs, and SQLite are in; archives and - PDFs remain, each an incremental, independently testable decoder behind - the existing tool. +- **richer `read`** — directories, URLs, SQLite, and archives are in; + PDFs remain, an incremental, independently testable decoder behind the + existing tool. ## Then: reaching outside the workspace