diff --git a/.gitignore b/.gitignore index e6c4b16..6d63025 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ .serena/ .whetstone/ STACK-SETUP.md +spindle-debug.log diff --git a/Cargo.lock b/Cargo.lock index 9ff77f8..1d77093 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -131,6 +131,9 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] [[package]] name = "arg_enum_proc_macro" @@ -297,7 +300,7 @@ dependencies = [ "arrayvec", "cc", "cfg-if", - "constant_time_eq", + "constant_time_eq 0.4.2", "cpufeatures 0.3.0", "rayon-core", ] @@ -372,6 +375,25 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cassowary" version = "0.3.0" @@ -527,6 +549,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -567,6 +595,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -694,6 +737,29 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +[[package]] +name = "deflate64" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "difflib" version = "0.4.0" @@ -708,6 +774,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -1136,6 +1203,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "1.4.2" @@ -1716,6 +1792,27 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "matchers" version = "0.2.0" @@ -1874,6 +1971,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-derive" version = "0.4.2" @@ -1977,6 +2080,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "pdf-extract" version = "0.10.0" @@ -2006,6 +2119,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "png" version = "0.18.1" @@ -2040,6 +2159,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2691,6 +2816,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" @@ -2828,6 +2964,7 @@ dependencies = [ "tracing-subscriber", "walkdir", "wiremock", + "zip", ] [[package]] @@ -3027,6 +3164,25 @@ dependencies = [ "zune-jpeg", ] +[[package]] +name = "time" +version = "0.3.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + [[package]] name = "tinystr" version = "0.8.3" @@ -4061,6 +4217,15 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "y4m" version = "0.8.0" @@ -4136,6 +4301,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "zerotrie" @@ -4170,12 +4349,82 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq 0.3.1", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "getrandom 0.3.4", + "hmac", + "indexmap", + "lzma-rs", + "memchr", + "pbkdf2", + "sha1", + "thiserror 2.0.18", + "time", + "xz2", + "zeroize", + "zopfli", + "zstd", +] + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "zune-core" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 4c711a5..6f8564b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,6 +81,8 @@ hex = "0.4" atty = "0.2" dotenv = "0.15.0" pdf-extract = "0.10.0" +zip = "2" +tempfile = "3" [dev-dependencies] tempfile = "3" diff --git a/src/ai/claude.rs b/src/ai/claude.rs index 5479b7b..59e15a3 100644 --- a/src/ai/claude.rs +++ b/src/ai/claude.rs @@ -709,6 +709,81 @@ impl AiProvider for ClaudeProvider { Ok(groups) } + + async fn propose_groups_with_organized_context( + &self, + files: &[FileSummary], + existing_labels: &[String], + organized_context: &[( + String, + Vec, + )], + ) -> Result> { + if organized_context.is_empty() { + return self + .propose_groups_with_context(files, existing_labels) + .await; + } + + let user_prompt = format!( + "{}{}{}", + super::group_user_prompt(files), + super::group_organized_context(organized_context), + super::group_existing_groups_note(existing_labels), + ); + let request = cached_api_request( + self.model.clone(), + 32_768, + Some(vec![cached_system_block(super::group_system_prompt())]), + vec![Message { + role: "user", + content: vec![ContentBlock::Text { + text: user_prompt, + cache_control: None, + }], + }], + ); + + let text = self.send_request(request).await?; + + #[derive(Deserialize)] + struct GroupResponse { + groups: Vec, + } + + let response: GroupResponse = serde_json::from_str(&text) + .with_context(|| { + let preview = if text.len() > 500 { + format!( + "{}...(truncated, {} bytes total)", + &text[..500], + text.len() + ) + } else { + text.clone() + }; + format!( + "Failed to parse groups JSON from Claude. Raw response:\n{}", + preview + ) + })?; + + let groups = response + .groups + .into_iter() + .map(|mut g| { + if !g.member_destinations.is_empty() + && g.member_indices.is_empty() + { + g.member_indices = + g.member_destinations.iter().map(|m| m.index).collect(); + } + g + }) + .collect(); + + Ok(groups) + } } #[cfg(test)] diff --git a/src/ai/mod.rs b/src/ai/mod.rs index 1d79e20..59c1761 100644 --- a/src/ai/mod.rs +++ b/src/ai/mod.rs @@ -67,6 +67,18 @@ pub trait AiProvider: Send + Sync { self.propose_groups(files) } + /// Group files with both label names and rich content descriptions + /// from the organized pool. Falls back to label-only context. + fn propose_groups_with_organized_context( + &self, + files: &[FileSummary], + existing_labels: &[String], + _organized_context: &[(String, Vec)], + ) -> impl std::future::Future>> + Send + { + self.propose_groups_with_context(files, existing_labels) + } + /// Describe many files in one operation. The default falls back to /// sequential individual calls; providers with a native batch API /// (50% discount) should override this. diff --git a/src/ai/prompts.rs b/src/ai/prompts.rs index 82fd0a0..bafbf30 100644 --- a/src/ai/prompts.rs +++ b/src/ai/prompts.rs @@ -71,6 +71,42 @@ pub fn group_existing_groups_note( note } +/// Rich context about existing groups: includes sample file +/// summaries/tags from each group so the model can match new files +/// against the actual content of the organized pool. +pub fn group_organized_context( + groups: &[(String, Vec)], +) -> String { + if groups.is_empty() { + return String::new(); + } + let mut note = String::from( + "\nThese folders already exist with the following contents. \ + Route new files into an existing group when the content clearly \ + fits — reuse its exact label:\n", + ); + for (label, descriptions) in groups { + note.push_str(&format!("\n## {label}\n")); + for desc in descriptions.iter().take(5) { + let tags = desc.tags.join(", "); + let _ = std::fmt::Write::write_fmt( + &mut note, + format_args!("- {} (tags: {})\n", desc.summary, tags), + ); + } + if descriptions.len() > 5 { + let _ = std::fmt::Write::write_fmt( + &mut note, + format_args!( + " ... and {} more files\n", + descriptions.len() - 5 + ), + ); + } + } + note +} + pub fn describe_text_user_prompt( context: &DescribeContext, excerpt: &str, @@ -377,4 +413,58 @@ mod tests { assert!(prompt.contains("SPECIFIC groups")); assert!(!prompt.contains("Prefer fewer, larger groups")); } + + #[test] + fn organized_context_is_empty_without_groups() { + assert!(group_organized_context(&[]).is_empty()); + } + + #[test] + fn organized_context_includes_summaries_and_tags() { + let groups = vec![( + "Beach Photos".to_string(), + vec![ + ContentDescription { + summary: "Sandy beach at sunset".to_string(), + tags: vec!["beach".to_string(), "sunset".to_string()], + suggested_category: "travel".to_string(), + confidence: 0.9, + }, + ContentDescription { + summary: "Ocean waves crashing".to_string(), + tags: vec!["ocean".to_string(), "waves".to_string()], + suggested_category: "nature".to_string(), + confidence: 0.85, + }, + ], + )]; + + let context = group_organized_context(&groups); + + assert!(context.contains("Beach Photos")); + assert!(context.contains("Sandy beach at sunset")); + assert!(context.contains("Ocean waves crashing")); + assert!(context.contains("beach, sunset")); + assert!(context.to_lowercase().contains("reuse")); + } + + #[test] + fn organized_context_truncates_beyond_five() { + let descriptions: Vec = (0..8) + .map(|i| ContentDescription { + summary: format!("File {i}"), + tags: vec![], + suggested_category: "other".to_string(), + confidence: 0.5, + }) + .collect(); + let groups = vec![("Big Group".to_string(), descriptions)]; + + let context = group_organized_context(&groups); + + assert!(context.contains("File 0")); + assert!(context.contains("File 4")); + assert!(!context.contains("File 5")); + assert!(context.contains("3 more")); + } } diff --git a/src/analyze/mod.rs b/src/analyze/mod.rs index d3293bd..fc2f31f 100644 --- a/src/analyze/mod.rs +++ b/src/analyze/mod.rs @@ -17,6 +17,9 @@ pub struct AnalyzeOptions { /// Use the Batch API (50% cheaper, async) instead of concurrent /// individual requests. pub use_batch_api: bool, + pub introspect_archives: bool, + pub max_archive_files: usize, + pub max_archive_file_size_mb: u64, } impl Default for AnalyzeOptions { @@ -25,6 +28,9 @@ impl Default for AnalyzeOptions { cache_dir: default_cache_dir(), max_concurrent: 5, use_batch_api: false, + introspect_archives: true, + max_archive_files: 20, + max_archive_file_size_mb: 50, } } } @@ -245,6 +251,12 @@ pub async fn analyze_file( crate::model::FileType::Document(_) ) { analyze_document(provider, file, &filename).await? + } else if matches!( + file.scanned.file_type, + crate::model::FileType::Archive(_) + ) && options.introspect_archives + { + analyze_archive(provider, file, &filename, options).await? } else { describe_by_filename(file, &filename) }; @@ -282,6 +294,223 @@ fn describe_by_filename( } } +const BYTES_PER_MB: u64 = 1_000_000; + +/// Analyze an archive by extracting its contents to a temp dir, +/// running each inner file through the normal analysis pipeline, and +/// synthesizing a single description. +async fn analyze_archive( + provider: &impl AiProvider, + file: &FingerprintedFile, + filename: &str, + options: &AnalyzeOptions, +) -> Result { + let inner_descriptions = + extract_and_analyze_archive(provider, file, options).await; + + if inner_descriptions.is_empty() { + return Ok(describe_by_filename(file, filename)); + } + + let mut all_tags = Vec::new(); + let mut summaries = Vec::new(); + let mut categories: HashMap = HashMap::new(); + + for (inner_name, desc) in &inner_descriptions { + summaries.push(format!("{inner_name}: {}", desc.summary)); + all_tags.extend(desc.tags.clone()); + *categories + .entry(desc.suggested_category.clone()) + .or_default() += 1; + } + + all_tags.sort(); + all_tags.dedup(); + + let top_category = categories + .into_iter() + .max_by_key(|(_, count)| *count) + .map(|(cat, _)| cat) + .unwrap_or_else(|| "other".to_string()); + + let items_summary = if summaries.len() <= 5 { + summaries.join("; ") + } else { + let first_five = summaries[..5].join("; "); + format!("{}; ... and {} more", first_five, summaries.len() - 5) + }; + + Ok(ContentDescription { + summary: format!( + "Archive ({} files): {}", + inner_descriptions.len(), + items_summary, + ), + tags: all_tags, + suggested_category: top_category, + confidence: 0.8, + }) +} + +async fn extract_and_analyze_archive( + provider: &impl AiProvider, + file: &FingerprintedFile, + options: &AnalyzeOptions, +) -> Vec<(String, ContentDescription)> { + use crate::model::ArchiveFormat; + + let format = match file.scanned.file_type { + crate::model::FileType::Archive(f) => f, + _ => return Vec::new(), + }; + + let tmp = match tempfile::TempDir::new() { + Ok(t) => t, + Err(e) => { + tracing::warn!(error = %e, "Failed to create temp dir for archive introspection"); + return Vec::new(); + } + }; + + let extracted = match format { + ArchiveFormat::Zip => { + extract_zip(&file.scanned.path, tmp.path(), options) + } + _ => { + tracing::debug!( + format = ?format, + "Archive format not yet supported for introspection" + ); + return Vec::new(); + } + }; + + let extracted = match extracted { + Ok(files) => files, + Err(e) => { + tracing::warn!( + path = %file.scanned.path.display(), + error = %e, + "Failed to extract archive" + ); + return Vec::new(); + } + }; + + let inner_opts = AnalyzeOptions { + cache_dir: options.cache_dir.clone(), + max_concurrent: options.max_concurrent, + use_batch_api: false, + introspect_archives: false, + max_archive_files: 0, + max_archive_file_size_mb: 0, + }; + + let mut results = Vec::new(); + for inner_file in &extracted { + let inner_name = inner_file + .scanned + .path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .to_string(); + match Box::pin(analyze_file(provider, inner_file, &inner_opts)) + .await + { + Ok(desc) => results.push((inner_name, desc)), + Err(e) => { + tracing::debug!( + file = %inner_name, + error = %e, + "Failed to analyze inner archive file" + ); + } + } + } + results +} + +fn extract_zip( + archive_path: &std::path::Path, + dest: &std::path::Path, + options: &AnalyzeOptions, +) -> Result> { + use crate::scanner::scan_directory; + + let file = std::fs::File::open(archive_path) + .with_context(|| format!("Opening {}", archive_path.display()))?; + let mut archive = + zip::ZipArchive::new(file).with_context(|| { + format!("Reading ZIP {}", archive_path.display()) + })?; + + let max_size = options.max_archive_file_size_mb * BYTES_PER_MB; + let mut extracted_count = 0; + + for i in 0..archive.len() { + if extracted_count >= options.max_archive_files { + break; + } + + let mut entry = match archive.by_index(i) { + Ok(e) => e, + Err(_) => continue, + }; + + if entry.is_dir() { + continue; + } + + if entry.size() > max_size { + continue; + } + + let entry_name = match entry.enclosed_name() { + Some(name) => name.to_path_buf(), + None => continue, + }; + + if entry_name + .components() + .any(|c| c.as_os_str().to_string_lossy().starts_with('.')) + { + continue; + } + + let inner_ext = entry_name + .extension() + .and_then(|e| e.to_str()) + .unwrap_or(""); + let inner_type = + crate::model::FileType::from_extension(inner_ext); + if matches!(inner_type, crate::model::FileType::Archive(_)) + || matches!(inner_type, crate::model::FileType::Other) + { + continue; + } + + let dest_path = dest + .join(entry_name.file_name().unwrap_or(entry_name.as_os_str())); + + let mut out = match std::fs::File::create(&dest_path) { + Ok(f) => f, + Err(_) => continue, + }; + if std::io::copy(&mut entry, &mut out).is_err() { + continue; + } + + extracted_count += 1; + } + + let scanned = scan_directory(dest).unwrap_or_default(); + Ok( + crate::fingerprint::fingerprint_files(scanned) + .unwrap_or_default(), + ) +} + /// Max bytes of extracted text sent to the API per document /// (~2k tokens). const MAX_TEXT_EXCERPT_BYTES: usize = 8 * 1024; @@ -906,6 +1135,7 @@ mod tests { cache_dir: dir.path().to_path_buf(), max_concurrent: 1, use_batch_api: false, + ..Default::default() }; struct PanicProvider; @@ -943,6 +1173,7 @@ mod tests { cache_dir: cache_dir.path().to_path_buf(), max_concurrent: 1, use_batch_api: false, + ..Default::default() }; struct FakeProvider; @@ -985,6 +1216,7 @@ mod tests { cache_dir: cache_dir.path().to_path_buf(), max_concurrent: 1, use_batch_api: false, + ..Default::default() }; struct FakeProvider; @@ -1046,6 +1278,7 @@ mod tests { cache_dir: cache_dir.path().to_path_buf(), max_concurrent: 1, use_batch_api: false, + ..Default::default() }; struct UnusedProvider; @@ -1087,6 +1320,7 @@ mod tests { cache_dir: cache_dir.path().to_path_buf(), max_concurrent: 1, use_batch_api: false, + ..Default::default() }; struct StubProvider; @@ -1182,6 +1416,7 @@ mod tests { cache_dir: cache_dir.path().to_path_buf(), max_concurrent: 1, use_batch_api: false, + ..Default::default() }; let provider = TextCapturingProvider { @@ -1213,6 +1448,7 @@ mod tests { cache_dir: cache_dir.path().to_path_buf(), max_concurrent: 1, use_batch_api: false, + ..Default::default() }; let provider = TextCapturingProvider { @@ -1241,6 +1477,7 @@ mod tests { cache_dir: cache_dir.path().to_path_buf(), max_concurrent: 1, use_batch_api: false, + ..Default::default() }; let provider = TextCapturingProvider { @@ -1352,6 +1589,7 @@ mod tests { cache_dir: cache_dir.path().to_path_buf(), max_concurrent: 1, use_batch_api: true, + ..Default::default() }; let provider = BatchOnlyProvider::new(); @@ -1397,6 +1635,7 @@ mod tests { cache_dir: cache_dir.path().to_path_buf(), max_concurrent: 1, use_batch_api: true, + ..Default::default() }; let provider = BatchOnlyProvider::new(); @@ -1430,6 +1669,7 @@ mod tests { cache_dir: cache_dir.path().to_path_buf(), max_concurrent: 1, use_batch_api: true, + ..Default::default() }; let provider = BatchOnlyProvider::new(); @@ -1442,4 +1682,89 @@ mod tests { // Nothing was submitted to the API assert!(provider.batch_sizes.lock().unwrap().is_empty()); } + + #[test] + fn extract_zip_extracts_supported_files() { + let dir = TempDir::new().unwrap(); + let zip_path = dir.path().join("test.zip"); + + let file = std::fs::File::create(&zip_path).unwrap(); + let mut writer = zip::ZipWriter::new(file); + let options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored); + writer.start_file("hello.txt", options).unwrap(); + std::io::Write::write_all(&mut writer, b"Hello world").unwrap(); + writer.start_file("photo.jpg", options).unwrap(); + std::io::Write::write_all(&mut writer, b"\xFF\xD8\xFF\xE0fake") + .unwrap(); + writer.finish().unwrap(); + + let dest = dir.path().join("extracted"); + std::fs::create_dir(&dest).unwrap(); + + let opts = AnalyzeOptions { + max_archive_files: 10, + max_archive_file_size_mb: 50, + ..Default::default() + }; + let files = extract_zip(&zip_path, &dest, &opts).unwrap(); + + assert_eq!(files.len(), 2); + } + + #[test] + fn extract_zip_skips_hidden_and_nested_archives() { + let dir = TempDir::new().unwrap(); + let zip_path = dir.path().join("test.zip"); + + let file = std::fs::File::create(&zip_path).unwrap(); + let mut writer = zip::ZipWriter::new(file); + let options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored); + writer.start_file("visible.txt", options).unwrap(); + std::io::Write::write_all(&mut writer, b"ok").unwrap(); + writer.start_file(".hidden/secret.txt", options).unwrap(); + std::io::Write::write_all(&mut writer, b"nope").unwrap(); + writer.start_file("inner.zip", options).unwrap(); + std::io::Write::write_all(&mut writer, b"nested").unwrap(); + writer.finish().unwrap(); + + let dest = dir.path().join("extracted"); + std::fs::create_dir(&dest).unwrap(); + + let opts = AnalyzeOptions::default(); + let files = extract_zip(&zip_path, &dest, &opts).unwrap(); + + assert_eq!(files.len(), 1); + let name = + files[0].scanned.path.file_name().unwrap().to_string_lossy(); + assert_eq!(name, "visible.txt"); + } + + #[test] + fn extract_zip_respects_max_files_limit() { + let dir = TempDir::new().unwrap(); + let zip_path = dir.path().join("test.zip"); + + let file = std::fs::File::create(&zip_path).unwrap(); + let mut writer = zip::ZipWriter::new(file); + let options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored); + for i in 0..10 { + writer.start_file(format!("file{i}.txt"), options).unwrap(); + std::io::Write::write_all(&mut writer, b"data").unwrap(); + } + writer.finish().unwrap(); + + let dest = dir.path().join("extracted"); + std::fs::create_dir(&dest).unwrap(); + + let opts = AnalyzeOptions { + max_archive_files: 3, + ..Default::default() + }; + let files = extract_zip(&zip_path, &dest, &opts).unwrap(); + + assert_eq!(files.len(), 3); + } } diff --git a/src/config.rs b/src/config.rs index 6c02461..f7b93b3 100644 --- a/src/config.rs +++ b/src/config.rs @@ -86,6 +86,14 @@ pub struct CliArgs { /// Path to the organized ledger (defaults to the global data dir). #[arg(long)] pub ledger: Option, + + /// Disable using organized-folder contents as context for grouping. + #[arg(long)] + pub no_organized_context: bool, + + /// Disable introspecting archive contents (zip, tar, gz). + #[arg(long)] + pub no_introspect_archives: bool, } /// Resolve the ledger path for this run: `None` when disabled, otherwise the @@ -114,6 +122,8 @@ pub struct Config { pub ai: AiConfig, #[serde(default)] pub duplicates: DuplicateConfig, + #[serde(default)] + pub matching: MatchingConfig, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -165,6 +175,37 @@ pub struct DuplicateConfig { pub near_duplicate_threshold: u32, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MatchingConfig { + #[serde(default = "default_true")] + pub use_organized_context: bool, + #[serde(default = "default_true")] + pub introspect_archives: bool, + #[serde(default = "default_max_archive_files")] + pub max_archive_files: usize, + #[serde(default = "default_max_archive_file_size_mb")] + pub max_archive_file_size_mb: u64, +} + +impl Default for MatchingConfig { + fn default() -> Self { + Self { + use_organized_context: true, + introspect_archives: true, + max_archive_files: 20, + max_archive_file_size_mb: 50, + } + } +} + +fn default_max_archive_files() -> usize { + 20 +} + +fn default_max_archive_file_size_mb() -> u64 { + 50 +} + impl Default for GeneralConfig { fn default() -> Self { Self { @@ -261,6 +302,7 @@ impl Config { general: GeneralConfig::default(), ai: AiConfig::default(), duplicates: DuplicateConfig::default(), + matching: MatchingConfig::default(), } }; @@ -279,6 +321,12 @@ impl Config { if let Some(ref url) = cli.api_base_url { config.ai.base_url = Some(url.clone()); } + if cli.no_organized_context { + config.matching.use_organized_context = false; + } + if cli.no_introspect_archives { + config.matching.introspect_archives = false; + } Ok(config) } @@ -317,6 +365,8 @@ mod tests { file_types: vec![], no_ledger: false, ledger: None, + no_organized_context: false, + no_introspect_archives: false, } } @@ -432,6 +482,7 @@ near_duplicate_threshold = 12 general: GeneralConfig::default(), ai: AiConfig::default(), duplicates: DuplicateConfig::default(), + matching: MatchingConfig::default(), }; let result = config.api_key(); @@ -446,6 +497,7 @@ near_duplicate_threshold = 12 general: GeneralConfig::default(), ai: AiConfig::default(), duplicates: DuplicateConfig::default(), + matching: MatchingConfig::default(), }; config.ai.api_key = Some("sk-my-key".to_string()); @@ -627,4 +679,94 @@ base_url = "https://from-file.example.com" CliArgs::parse_from(["spindle", "--batch", "some-dir"]); assert!(batch_args.batch); } + + #[test] + fn matching_config_defaults_are_sane() { + let matching = MatchingConfig::default(); + + assert!(matching.use_organized_context); + assert!(matching.introspect_archives); + assert_eq!(matching.max_archive_files, 20); + assert_eq!(matching.max_archive_file_size_mb, 50); + } + + #[test] + fn matching_config_deserializes_with_defaults() { + let config: Config = + toml::from_str("[ai]\nmodel = \"test\"").unwrap(); + + assert!(config.matching.use_organized_context); + assert!(config.matching.introspect_archives); + } + + #[test] + fn matching_config_overrides_from_toml() { + let dir = TempDir::new().unwrap(); + let toml_path = dir.path().join("config.toml"); + fs::write( + &toml_path, + r#" +[matching] +use_organized_context = false +introspect_archives = false +max_archive_files = 5 +max_archive_file_size_mb = 10 +"#, + ) + .unwrap(); + + let cli = CliArgs { + config: Some(toml_path), + ..empty_cli() + }; + + let config = Config::load(&cli).unwrap(); + + assert!(!config.matching.use_organized_context); + assert!(!config.matching.introspect_archives); + assert_eq!(config.matching.max_archive_files, 5); + assert_eq!(config.matching.max_archive_file_size_mb, 10); + } + + #[test] + fn cli_no_organized_context_overrides_config() { + let dir = TempDir::new().unwrap(); + let cli = CliArgs { + config: Some(dir.path().join("nonexistent.toml")), + no_organized_context: true, + ..empty_cli() + }; + + let config = Config::load(&cli).unwrap(); + + assert!(!config.matching.use_organized_context); + } + + #[test] + fn cli_no_introspect_archives_overrides_config() { + let dir = TempDir::new().unwrap(); + let cli = CliArgs { + config: Some(dir.path().join("nonexistent.toml")), + no_introspect_archives: true, + ..empty_cli() + }; + + let config = Config::load(&cli).unwrap(); + + assert!(!config.matching.introspect_archives); + } + + #[test] + fn cli_matching_flags_parse() { + use clap::Parser; + + let args = CliArgs::parse_from([ + "spindle", + "--no-organized-context", + "--no-introspect-archives", + "some-dir", + ]); + assert!(args.no_organized_context); + assert!(args.no_introspect_archives); + } } diff --git a/src/ledger/mod.rs b/src/ledger/mod.rs index 2b6eaf3..2435e25 100644 --- a/src/ledger/mod.rs +++ b/src/ledger/mod.rs @@ -144,6 +144,29 @@ impl Ledger { self.entries.iter().find(|e| e.blake3_hex == blake3_hex) } + /// Content hashes of files in each existing group, keyed by label. + /// Returns at most `max_per_group` hashes per group so callers can + /// load cached descriptions without pulling the whole ledger into memory. + pub fn group_content_hashes( + &self, + output_dir: &Path, + max_per_group: usize, + ) -> Vec<(String, Vec)> { + let mut groups: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for entry in &self.entries { + if !entry.dest_path.starts_with(output_dir) { + continue; + } + let hashes = + groups.entry(entry.group_label.clone()).or_default(); + if hashes.len() < max_per_group { + hashes.push(entry.blake3_hex.clone()); + } + } + groups.into_iter().collect() + } + /// Distinct group folders previously created under `output_dir`, so new /// files can be routed into them rather than into fresh near-duplicates. pub fn existing_groups_under( @@ -363,4 +386,64 @@ mod tests { bytes[1] = 0xad; assert!(hash_hex(&bytes).starts_with("dead")); } + + #[test] + fn group_content_hashes_returns_hashes_per_group() { + let mut ledger = Ledger::default(); + ledger.record(entry( + "/src/a.jpg", + "/out/beach/a.jpg", + "h1", + "Beach", + )); + ledger.record(entry( + "/src/b.jpg", + "/out/beach/b.jpg", + "h2", + "Beach", + )); + ledger.record(entry( + "/src/c.jpg", + "/out/beach/c.jpg", + "h3", + "Beach", + )); + ledger.record(entry( + "/src/d.jpg", + "/out/dogs/d.jpg", + "h4", + "Dogs", + )); + + let groups = ledger.group_content_hashes(Path::new("/out"), 2); + + assert_eq!(groups.len(), 2); + let beach = groups.iter().find(|(l, _)| l == "Beach").unwrap(); + assert_eq!(beach.1.len(), 2); + let dogs = groups.iter().find(|(l, _)| l == "Dogs").unwrap(); + assert_eq!(dogs.1.len(), 1); + assert_eq!(dogs.1[0], "h4"); + } + + #[test] + fn group_content_hashes_filters_by_output_dir() { + let mut ledger = Ledger::default(); + ledger.record(entry( + "/src/a.jpg", + "/out/beach/a.jpg", + "h1", + "Beach", + )); + ledger.record(entry( + "/src/b.jpg", + "/elsewhere/cats/b.jpg", + "h2", + "Cats", + )); + + let groups = ledger.group_content_hashes(Path::new("/out"), 5); + + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].0, "Beach"); + } } diff --git a/src/main.rs b/src/main.rs index fc6ed6f..7183b2c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -70,6 +70,12 @@ async fn main() -> Result<()> { include_trash: cli.include_trash, type_filter: cli.file_types.clone(), use_batch_api: cli.batch, + introspect_archives: config.matching.introspect_archives, + max_archive_files: config.matching.max_archive_files, + max_archive_file_size_mb: config + .matching + .max_archive_file_size_mb, + use_organized_context: config.matching.use_organized_context, ledger_path: ledger_path.clone(), }; diff --git a/src/pipeline.rs b/src/pipeline.rs index 6a248f2..d7bda7f 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -19,8 +19,8 @@ use crate::group::build_groups; use crate::ledger::{Ledger, OrganizedDuplicate}; use crate::model::FileCategory; use crate::model::{ - DuplicateSet, FileSummary, FingerprintedFile, ProposedGroup, - ReorgPlan, + ContentDescription, DuplicateSet, FileSummary, FingerprintedFile, + ProposedGroup, ReorgPlan, }; use crate::plan::propose_plan; use crate::scanner::{scan_directories_filtered, ScanOptions}; @@ -81,6 +81,10 @@ pub struct PipelineConfig { pub include_trash: bool, pub type_filter: Vec, pub use_batch_api: bool, + pub introspect_archives: bool, + pub max_archive_files: usize, + pub max_archive_file_size_mb: u64, + pub use_organized_context: bool, /// Path to the persistent "already organized" ledger. `None` disables /// both candidate exclusion and recording. pub ledger_path: Option, @@ -215,10 +219,21 @@ pub async fn run( .collect() }) .unwrap_or_default(); + let organized_context = if config.use_organized_context { + load_organized_context( + ledger.as_ref(), + &config.output_dir, + &config.cache_dir, + ) + .await + } else { + Vec::new() + }; run_ai_pipeline( provider, &fingerprinted, &existing_labels, + &organized_context, config, &tx, ) @@ -285,10 +300,38 @@ fn apply_ledger_exclusion( duplicates } +async fn load_organized_context( + ledger: Option<&Ledger>, + output_dir: &std::path::Path, + cache_dir: &std::path::Path, +) -> Vec<(String, Vec)> { + let Some(ledger) = ledger else { + return Vec::new(); + }; + let group_hashes = ledger.group_content_hashes(output_dir, 5); + let mut context = Vec::with_capacity(group_hashes.len()); + for (label, hashes) in group_hashes { + let mut descriptions = Vec::new(); + for hex in &hashes { + let mut hash_bytes = [0u8; 32]; + if hex::decode_to_slice(hex, &mut hash_bytes).is_ok() { + if let Some(desc) = read_cache(cache_dir, &hash_bytes).await { + descriptions.push(desc); + } + } + } + if !descriptions.is_empty() { + context.push((label, descriptions)); + } + } + context +} + async fn run_ai_pipeline( provider: &P, fingerprinted: &[FingerprintedFile], existing_labels: &[String], + organized_context: &[(String, Vec)], config: &PipelineConfig, tx: &mpsc::Sender, ) -> Result> { @@ -347,6 +390,9 @@ async fn run_ai_pipeline( cache_dir: config.cache_dir.clone(), max_concurrent: config.max_concurrent, use_batch_api: config.use_batch_api, + introspect_archives: config.introspect_archives, + max_archive_files: config.max_archive_files, + max_archive_file_size_mb: config.max_archive_file_size_mb, }; let subset: Vec<_> = @@ -434,10 +480,19 @@ async fn run_ai_pipeline( return Ok(groups); } - match provider - .propose_groups_with_context(&summaries, existing_labels) - .await - { + match if organized_context.is_empty() { + provider + .propose_groups_with_context(&summaries, existing_labels) + .await + } else { + provider + .propose_groups_with_organized_context( + &summaries, + existing_labels, + organized_context, + ) + .await + } { Ok(groups) => { let _ = write_cached_grouping( &config.cache_dir, @@ -557,6 +612,10 @@ mod tests { include_trash: false, type_filter: vec![], use_batch_api: false, + introspect_archives: false, + max_archive_files: 20, + max_archive_file_size_mb: 50, + use_organized_context: false, ledger_path: None, }; @@ -601,6 +660,10 @@ mod tests { include_trash: false, type_filter: vec![], use_batch_api: false, + introspect_archives: false, + max_archive_files: 20, + max_archive_file_size_mb: 50, + use_organized_context: false, ledger_path, } } @@ -793,6 +856,10 @@ mod tests { include_trash: false, type_filter: vec![], use_batch_api: false, + introspect_archives: false, + max_archive_files: 20, + max_archive_file_size_mb: 50, + use_organized_context: false, ledger_path: None, }; @@ -844,6 +911,10 @@ mod tests { include_trash: false, type_filter: vec![], use_batch_api: false, + introspect_archives: false, + max_archive_files: 20, + max_archive_file_size_mb: 50, + use_organized_context: false, ledger_path: None, }; @@ -886,6 +957,10 @@ mod tests { include_trash: false, type_filter: vec![], use_batch_api: false, + introspect_archives: false, + max_archive_files: 20, + max_archive_file_size_mb: 50, + use_organized_context: false, ledger_path: None, }; @@ -926,6 +1001,10 @@ mod tests { include_trash: false, type_filter: vec![], use_batch_api: false, + introspect_archives: false, + max_archive_files: 20, + max_archive_file_size_mb: 50, + use_organized_context: false, ledger_path: None, }; @@ -962,6 +1041,10 @@ mod tests { include_trash: false, type_filter: vec![], use_batch_api: false, + introspect_archives: false, + max_archive_files: 20, + max_archive_file_size_mb: 50, + use_organized_context: false, ledger_path: None, }; diff --git a/src/tui/review.rs b/src/tui/review.rs index 0ffc9cd..cf8afa6 100644 --- a/src/tui/review.rs +++ b/src/tui/review.rs @@ -26,15 +26,17 @@ fn decode_image(path: &Path) -> Option { { Ok(reader) => match reader.decode() { Ok(img) => return Some(img), - Err(e) => tracing::debug!(?path, %e, "image crate failed, trying magick"), + Err(e) => { + tracing::debug!(?path, %e, "image crate failed, trying magick") + } }, - Err(e) => tracing::debug!(?path, %e, "image crate failed, trying magick"), + Err(e) => { + tracing::debug!(?path, %e, "image crate failed, trying magick") + } } - let tmp = std::env::temp_dir().join(format!( - "spindle-preview-{}.png", - std::process::id() - )); + let tmp = std::env::temp_dir() + .join(format!("spindle-preview-{}.png", std::process::id())); let ok = std::process::Command::new("magick") .arg("convert") .arg(path) @@ -267,8 +269,7 @@ impl ReviewState { let other_approved = vec![true; og.len()]; let other_file_keep = Self::init_file_keep(&other_group_moves, omode); - let other_file_marked = - vec![HashSet::new(); og.len()]; + let other_file_marked = vec![HashSet::new(); og.len()]; ModeData { groups: og, group_moves: other_group_moves, @@ -446,7 +447,11 @@ impl ReviewState { deletions } - pub fn is_file_marked(&self, group_idx: usize, file_idx: usize) -> bool { + pub fn is_file_marked( + &self, + group_idx: usize, + file_idx: usize, + ) -> bool { self .file_marked .get(group_idx) @@ -658,8 +663,10 @@ impl ReviewState { self.diff_state = Some(ds); } else if is_text { let text = std::fs::read_to_string(&path).unwrap_or_default(); - let lines: Vec = - text.lines().map(|l| DiffLine::Same(l.to_string())).collect(); + let lines: Vec = text + .lines() + .map(|l| DiffLine::Same(l.to_string())) + .collect(); self.diff_state = Some(DiffState { primary_preview: PreviewState::None, secondary_preview: PreviewState::None, @@ -1119,9 +1126,7 @@ impl ReviewState { ds.scroll = ds.scroll.saturating_add(3); } } - KeyCode::Esc - | KeyCode::Char(' ') - | KeyCode::Char('q') => { + KeyCode::Esc | KeyCode::Char(' ') | KeyCode::Char('q') => { self.mode = Mode::Normal; self.exit_diff_view(); } @@ -1198,8 +1203,7 @@ impl ReviewState { if i >= self.group_moves[self.selected].len() { continue; } - let mut file_move = - self.group_moves[self.selected].remove(i); + let mut file_move = self.group_moves[self.selected].remove(i); let was_kept = self.file_keep[self.selected].remove(i); let filename = file_move @@ -1268,8 +1272,7 @@ impl ReviewState { if i >= self.group_moves[self.selected].len() { continue; } - let mut file_move = - self.group_moves[self.selected].remove(i); + let mut file_move = self.group_moves[self.selected].remove(i); let was_kept = self.file_keep[self.selected].remove(i); let filename = file_move @@ -1481,10 +1484,7 @@ fn render_middle_panel( } } -fn render_preview_modal( - frame: &mut Frame, - state: &mut ReviewState, -) { +fn render_preview_modal(frame: &mut Frame, state: &mut ReviewState) { let area = frame.area(); let modal_w = (area.width * 75 / 100).max(40).min(area.width - 2); let modal_h = (area.height * 75 / 100).max(10).min(area.height - 2); diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 5c5799e..bc081fa 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -277,6 +277,7 @@ async fn analyze_caches_and_reuses_results() { cache_dir: cache_dir.path().to_path_buf(), max_concurrent: 2, use_batch_api: false, + ..Default::default() }; let result1 = @@ -323,6 +324,7 @@ async fn analyze_batch_processes_multiple_files() { cache_dir: cache_dir.path().to_path_buf(), max_concurrent: 2, use_batch_api: false, + ..Default::default() }; let results = analyze_batch(&FakeAiProvider, &files, &opts).await; @@ -363,6 +365,7 @@ async fn full_pipeline_end_to_end() { cache_dir: cache_dir.path().to_path_buf(), max_concurrent: 2, use_batch_api: false, + ..Default::default() }; let descriptions = analyze_batch(&FakeAiProvider, &fingerprinted, &opts).await;