Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 5 additions & 26 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 1 addition & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,7 @@ infer = "0.16"
directories = "5"
base64 = "0.22"
hex = "0.4"
atty = "0.2"
dotenv = "0.15.0"
dotenvy = "0.15"
pdf-extract = "0.10.0"
zip = "2"
tempfile = "3"
Expand All @@ -89,7 +88,6 @@ tempfile = "3"
wiremock = "0.6"
tokio-test = "0.4"
assert_fs = "1"
dotenv = "0.15.0"


[features]
Expand Down
6 changes: 1 addition & 5 deletions src/ai/claude.rs
Original file line number Diff line number Diff line change
Expand Up @@ -245,10 +245,7 @@ impl ClaudeProvider {

/// Use a different (usually cheaper) model for per-file
/// descriptions than for grouping.
pub fn with_describe_model(
self,
model: impl Into<String>,
) -> Self {
pub fn with_describe_model(self, model: impl Into<String>) -> Self {
let mut this = self;
this.describe_model = model.into();
this
Expand Down Expand Up @@ -878,7 +875,6 @@ mod tests {
cache_control: None,
}],
}],

None,
);

Expand Down
8 changes: 2 additions & 6 deletions src/analyze/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -586,9 +586,7 @@ async fn extract_document_text(
}

/// docx is a zip: the body text lives in `word/document.xml`.
async fn extract_docx_text(
path: &std::path::Path,
) -> Option<String> {
async fn extract_docx_text(path: &std::path::Path) -> Option<String> {
let path = path.to_path_buf();
tokio::task::spawn_blocking(move || {
let file = std::fs::File::open(&path).ok()?;
Expand Down Expand Up @@ -638,9 +636,7 @@ fn strip_docx_xml(xml: &str) -> String {
}

/// RTF: drop control words and groups, keep plain text.
async fn extract_rtf_text(
path: &std::path::Path,
) -> Option<String> {
async fn extract_rtf_text(path: &std::path::Path) -> Option<String> {
let raw = read_text_excerpt(path).await?;
let text = strip_rtf(&raw);
(!text.trim().is_empty()).then_some(text)
Expand Down
12 changes: 8 additions & 4 deletions src/cost.rs
Original file line number Diff line number Diff line change
Expand Up @@ -116,10 +116,13 @@ mod tests {

#[test]
fn pricing_varies_by_model_family() {
let haiku = estimate_cost(100, "claude-haiku-4-5", "claude-haiku-4-5");
let sonnet = estimate_cost(100, "claude-sonnet-5", "claude-sonnet-5");
let haiku =
estimate_cost(100, "claude-haiku-4-5", "claude-haiku-4-5");
let sonnet =
estimate_cost(100, "claude-sonnet-5", "claude-sonnet-5");
let opus = estimate_cost(100, OPUS, OPUS);
let fable = estimate_cost(100, "claude-fable-5", "claude-fable-5");
let fable =
estimate_cost(100, "claude-fable-5", "claude-fable-5");

assert!(haiku.estimated_cost_usd < sonnet.estimated_cost_usd);
assert!(sonnet.estimated_cost_usd < opus.estimated_cost_usd);
Expand All @@ -128,7 +131,8 @@ mod tests {

#[test]
fn unknown_model_uses_opus_pricing() {
let unknown = estimate_cost(10, "some-future-model", "some-future-model");
let unknown =
estimate_cost(10, "some-future-model", "some-future-model");
let opus = estimate_cost(10, OPUS, OPUS);

assert_eq!(
Expand Down
5 changes: 2 additions & 3 deletions src/executor/journal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,8 @@ pub fn load(journal_dir: &Path, run_id: &str) -> Result<RunJournal> {
std::fs::read_to_string(&path).with_context(|| {
format!("No journal for run {run_id} at {}", path.display())
})?;
serde_json::from_str(&content).with_context(|| {
format!("Corrupt journal: {}", path.display())
})
serde_json::from_str(&content)
.with_context(|| format!("Corrupt journal: {}", path.display()))
}

/// Run ids of journals that have not been undone, newest first.
Expand Down
3 changes: 1 addition & 2 deletions src/executor/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,7 @@ pub fn execute_plan(
let mut deletions_staged = Vec::new();
let mut bytes_staged = 0u64;
for path in &plan.deletions {
let size =
std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
let dest = trash::trash_path_for(&paths.trash_dir, &run_id, path);
match move_file(path, &dest) {
Ok(()) => {
Expand Down
3 changes: 2 additions & 1 deletion src/executor/trash.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ pub fn trash_path_for(
Component::Normal(part) => dest.push(part),
Component::Prefix(prefix) => {
// Windows drive prefix — keep it as a plain directory name.
dest.push(prefix.as_os_str().to_string_lossy().replace(':', ""))
dest
.push(prefix.as_os_str().to_string_lossy().replace(':', ""))
}
_ => {}
}
Expand Down
3 changes: 1 addition & 2 deletions src/fingerprint/text.rs
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,7 @@ mod tests {

fn detect(dir: &TempDir) -> Vec<DuplicateSet> {
let files =
fingerprint_files(scan_directory(dir.path()).unwrap())
.unwrap();
fingerprint_files(scan_directory(dir.path()).unwrap()).unwrap();
find_similar_text(&files)
}

Expand Down
52 changes: 35 additions & 17 deletions src/main.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
use std::collections::HashMap;
use std::path::Path;

use std::io::IsTerminal;

use anyhow::Result;
use clap::Parser;
use dotenv::dotenv;
use dotenvy::dotenv;
use tracing_subscriber::EnvFilter;

use spindle::ai::ClaudeProvider;
Expand Down Expand Up @@ -103,17 +105,37 @@ async fn main() -> Result<()> {

let (tx, mut rx) = tokio::sync::mpsc::channel::<PipelineEvent>(64);

let event_handle = tokio::spawn(async move {
let mut progress = PipelineProgress::new();
while let Some(event) = rx.recv().await {
progress.handle_event(&event);
}
});
// Full TUI progress in a real terminal; plain line output when
// piped or redirected.
let use_tui = std::io::stdout().is_terminal();
let event_handle: tokio::task::JoinHandle<Result<()>> = if use_tui {
tokio::task::spawn_blocking(move || {
tui::run_pipeline_progress(rx)
})
} else {
tokio::spawn(async move {
let mut progress = PipelineProgress::new();
while let Some(event) = rx.recv().await {
progress.handle_event(&event);
}
Ok(())
})
};

let result = pipeline::run(&provider, &pipeline_config, tx).await?;
if let Err(e) = event_handle.await {
tracing::error!(error = %e, "Event handler task panicked");
// Join the progress task before propagating pipeline errors so the
// terminal is restored first.
let pipeline_result =
pipeline::run(&provider, &pipeline_config, tx).await;
match event_handle.await {
Ok(Ok(())) => {}
Ok(Err(e)) => {
tracing::warn!(error = %e, "Progress display failed")
}
Err(e) => {
tracing::error!(error = %e, "Event handler task panicked")
}
}
let result = pipeline_result?;

let plan = &result.plan;

Expand Down Expand Up @@ -234,8 +256,7 @@ fn run_list_undo() -> Result<()> {
for run_id in runs {
match journal::load(&paths.journal_dir, &run_id) {
Ok(j) => {
let staged: u64 =
j.deletions.iter().map(|d| d.size).sum();
let staged: u64 = j.deletions.iter().map(|d| d.size).sum();
println!(
" {} {} moves, {} deletions ({} in trash)",
run_id,
Expand Down Expand Up @@ -599,11 +620,8 @@ fn execute_review(
deletions,
skipped_files: vec![],
};
let report = execute_plan(
&plan,
&exec_paths,
&config.general.output_dir,
);
let report =
execute_plan(&plan, &exec_paths, &config.general.output_dir);

println!(
"\nDone! {} duplicates staged to trash ({} reclaimable \
Expand Down
12 changes: 9 additions & 3 deletions src/model/group.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,22 @@ use serde::{Deserialize, Serialize};
)]
pub enum DuplicateType {
Exact,
NearDuplicate { distance: u32 },
NearDuplicate {
distance: u32,
},
/// An archive whose entries are all present, extracted, in a
/// directory — the archive is redundant.
ArchiveMatch,
/// Text files whose normalized content is near-identical
/// (simhash hamming distance).
SimilarText { distance: u32 },
SimilarText {
distance: u32,
},
/// Audio files with matching acoustic fingerprints
/// (percent bit-similarity of chromaprint streams).
SimilarAudio { score: u32 },
SimilarAudio {
score: u32,
},
}

#[derive(Debug, Clone, Serialize, Deserialize)]
Expand Down
23 changes: 8 additions & 15 deletions src/pipeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -152,9 +152,7 @@ pub async fn run<P: AiProvider>(
config.near_duplicate_threshold,
);
near_dupes.extend(
crate::fingerprint::archive::find_archive_matches(
&fingerprinted,
),
crate::fingerprint::archive::find_archive_matches(&fingerprinted),
);
near_dupes.extend(crate::fingerprint::text::find_similar_text(
&fingerprinted,
Expand Down Expand Up @@ -814,19 +812,14 @@ mod tests {
}
}

let summaries: Vec<FileSummary> =
(0..MAX_GROUPING_BATCH * 2 + 5)
.map(|i| summary(i, 0.9))
.collect();
let summaries: Vec<FileSummary> = (0..MAX_GROUPING_BATCH * 2 + 5)
.map(|i| summary(i, 0.9))
.collect();

let groups = propose_groups_batched(
&OneLabelProvider,
&summaries,
&[],
&[],
)
.await
.unwrap();
let groups =
propose_groups_batched(&OneLabelProvider, &summaries, &[], &[])
.await
.unwrap();

assert_eq!(groups.len(), 1);
assert_eq!(groups[0].member_indices.len(), summaries.len());
Expand Down
Loading