diff --git a/toolproof/src/main.rs b/toolproof/src/main.rs index 99f3a36..5a5c9b2 100644 --- a/toolproof/src/main.rs +++ b/toolproof/src/main.rs @@ -975,7 +975,7 @@ async fn main_inner() -> Result<(), ()> { _ => None, }) .collect::>(); - let mut resolved_errors = 0; + let mut resolved: Vec = vec![]; println!("\n{}\n", "Finished running tests".bold()); @@ -1008,7 +1008,7 @@ async fn main_inner() -> Result<(), ()> { HoldingError::TestFailure => {} HoldingError::SnapFailure { out } => { if confirm_snapshot(&term, &file, &out).is_ok_and(|v| v) { - resolved_errors += 1; + resolved.push(file.file_path.clone()); if let Err(e) = tokio::fs::write(&file.file_path, out).await { eprintln!("Unable to write updates snapshot to disk.\n{e}"); @@ -1033,12 +1033,31 @@ async fn main_inner() -> Result<(), ()> { ) }; - let failing = results.iter().filter(|r| r.is_err()).count() - resolved_errors; + let hard_failures = results + .iter() + .filter_map(|r| match r { + Err((file, HoldingError::TestFailure)) => Some(file), + _ => None, + }) + .collect::>(); + let changed_snapshots = results + .iter() + .filter_map(|r| match r { + Err((file, HoldingError::SnapFailure { .. })) + if !resolved.contains(&file.file_path) => + { + Some(file) + } + _ => None, + }) + .collect::>(); + + let failing = hard_failures.len() + changed_snapshots.len(); let passing = results .iter() .filter(|r| matches!(r, Ok(ToolproofTestSuccess::Passed { .. }))) .count() - + resolved_errors; + + resolved.len(); let skipped = results .iter() .filter(|r| matches!(r, Ok(ToolproofTestSuccess::Skipped))) @@ -1054,13 +1073,42 @@ async fn main_inner() -> Result<(), ()> { }; println!( - "{}\n{}\n{}\n{}", + "{}\n{}\n{}\n{}\n{}", style(&format!("Total passing tests: {}", passing)).cyan(), style(&format!("Passed after retry: {}", retried_passed)).cyan(), - style(&format!("Failing tests: {}", failing)).cyan(), + style(&format!("Failing tests: {}", hard_failures.len())).cyan(), + style(&format!("Changed snapshots: {}", changed_snapshots.len())).cyan(), style(&format!("Skipped tests: {}", skipped)).cyan(), ); + let shell_quote = |s: &str| format!("'{}'", s.replace('\'', "'\\''")); + + if !hard_failures.is_empty() { + println!("\n{}", "Failing tests:".red().bold()); + for file in &hard_failures { + println!(" {} {}", "✘".red().bold(), file.name.red()); + } + println!("\n{}", "Rerun the failing tests with:".bold()); + for file in &hard_failures { + println!(" toolproof --name {}", shell_quote(&file.name)); + } + } + + if !changed_snapshots.is_empty() { + println!("\n{}", "Changed snapshots:".yellow().bold()); + for file in &changed_snapshots { + println!(" {} {}", "⚠".yellow().bold(), file.name.yellow()); + } + println!( + "\n{}", + "Review the changed snapshots. If a change is correct, accept it with --update (-u):" + .bold() + ); + for file in &changed_snapshots { + println!(" toolproof --name {} --update", shell_quote(&file.name)); + } + } + if failing > 0 { println!( "{}", diff --git a/toolproof/test_suite/base/default_placeholders.toolproof.yml b/toolproof/test_suite/base/default_placeholders.toolproof.yml index caffed7..5efb5ff 100644 --- a/toolproof/test_suite/base/default_placeholders.toolproof.yml +++ b/toolproof/test_suite/base/default_placeholders.toolproof.yml @@ -23,6 +23,7 @@ steps: ╎Total passing tests: 1 ╎Passed after retry: 0 ╎Failing tests: 0 + ╎Changed snapshots: 0 ╎Skipped tests: 0 ╎ ╎All tests passed diff --git a/toolproof/test_suite/base/extract.toolproof.yml b/toolproof/test_suite/base/extract.toolproof.yml index 7a04633..6283307 100644 --- a/toolproof/test_suite/base/extract.toolproof.yml +++ b/toolproof/test_suite/base/extract.toolproof.yml @@ -25,6 +25,7 @@ steps: ╎Total passing tests: 1 ╎Passed after retry: 0 ╎Failing tests: 0 + ╎Changed snapshots: 0 ╎Skipped tests: 0 ╎ ╎All tests passed diff --git a/toolproof/test_suite/base/snapshot.toolproof.yml b/toolproof/test_suite/base/snapshot.toolproof.yml index 435be0e..92b939e 100644 --- a/toolproof/test_suite/base/snapshot.toolproof.yml +++ b/toolproof/test_suite/base/snapshot.toolproof.yml @@ -23,6 +23,7 @@ steps: ╎Total passing tests: 1 ╎Passed after retry: 0 ╎Failing tests: 0 + ╎Changed snapshots: 0 ╎Skipped tests: 0 ╎ ╎All tests passed @@ -63,7 +64,14 @@ steps: ╎ ╎Total passing tests: 0 ╎Passed after retry: 0 - ╎Failing tests: 1 + ╎Failing tests: 0 + ╎Changed snapshots: 1 ╎Skipped tests: 0 ╎ + ╎Changed snapshots: + ╎ ⚠ Inner failing snapshot test + ╎ + ╎Review the changed snapshots. If a change is correct, accept it with --update (-u): + ╎ toolproof --name 'Inner failing snapshot test' --update + ╎ ╎Some tests failed diff --git a/toolproof/test_suite/base/snapshot_file.toolproof.yml b/toolproof/test_suite/base/snapshot_file.toolproof.yml index c5de9d1..27e24ee 100644 --- a/toolproof/test_suite/base/snapshot_file.toolproof.yml +++ b/toolproof/test_suite/base/snapshot_file.toolproof.yml @@ -31,6 +31,7 @@ steps: ╎Total passing tests: 1 ╎Passed after retry: 0 ╎Failing tests: 0 + ╎Changed snapshots: 0 ╎Skipped tests: 0 ╎ ╎All tests passed diff --git a/toolproof/test_suite/base/snapshot_update.toolproof.yml b/toolproof/test_suite/base/snapshot_update.toolproof.yml index 617c561..c9cc199 100644 --- a/toolproof/test_suite/base/snapshot_update.toolproof.yml +++ b/toolproof/test_suite/base/snapshot_update.toolproof.yml @@ -23,6 +23,7 @@ steps: ╎Total passing tests: 1 ╎Passed after retry: 0 ╎Failing tests: 0 + ╎Changed snapshots: 0 ╎Skipped tests: 0 ╎ ╎All tests passed diff --git a/toolproof/test_suite/base/step_platforms.toolproof.yml b/toolproof/test_suite/base/step_platforms.toolproof.yml index b0b2da6..85d7aa5 100644 --- a/toolproof/test_suite/base/step_platforms.toolproof.yml +++ b/toolproof/test_suite/base/step_platforms.toolproof.yml @@ -26,6 +26,7 @@ steps: ╎Total passing tests: 1 ╎Passed after retry: 0 ╎Failing tests: 0 + ╎Changed snapshots: 0 ╎Skipped tests: 0 ╎ ╎All tests passed diff --git a/toolproof/test_suite/base/summary_mixed_failures.toolproof.yml b/toolproof/test_suite/base/summary_mixed_failures.toolproof.yml new file mode 100644 index 0000000..a5957ca --- /dev/null +++ b/toolproof/test_suite/base/summary_mixed_failures.toolproof.yml @@ -0,0 +1,80 @@ +name: Summary separates snapshot changes from failures + +steps: + - step: I have a "normal.toolproof.yml" file with the content {yaml} + yaml: |- + name: Normal failing test + + steps: + - I run 'echo "hello"' + - stdout should contain "goodbye" + - step: I have a "snap.toolproof.yml" file with the content {yaml} + yaml: |- + name: Snapshot changing test + + steps: + - I run 'echo "actual output"' + - snapshot: stdout + snapshot_content: |- + ╎stale output + - I run "%toolproof_path% --porcelain -c 1" and expect it to fail + - snapshot: stdout + snapshot_content: |- + ╎ + ╎Running tests + ╎ + ╎✘ Normal failing test + ╎--- STEPS --- + ╎✓ I run 'echo "hello"' + ╎✘ stdout should contain "goodbye" + ╎--- ERROR --- + ╎Error in step "stdout should contain "goodbye"": + ╎-- + ╎Failed assertion: The value + ╎--- + ╎"hello\n" + ╎--- + ╎does not contain + ╎--- + ╎"goodbye" + ╎--- + ╎⚠ Snapshot changing test + ╎--- SNAPSHOT CHANGED --- + ╎ + ╎ 0 0 │ name: Snapshot changing test + ╎ 1 1 │ + ╎ 2 2 │ steps: + ╎ 3 3 │ - I run 'echo "actual output"' + ╎ 4 4 │ - snapshot: stdout + ╎ 5 5 │ snapshot_content: |- + ╎ 6 │- ╎stale output + ╎ 6 │+ ╎actual output + ╎ + ╎ + ╎--- END SNAPSHOT CHANGE --- + ╎ + ╎Run in interactive mode (-i) or with --update (-u) to accept new snapshots + ╎ + ╎ + ╎Finished running tests + ╎ + ╎Total passing tests: 0 + ╎Passed after retry: 0 + ╎Failing tests: 1 + ╎Changed snapshots: 1 + ╎Skipped tests: 0 + ╎ + ╎Failing tests: + ╎ ✘ Normal failing test + ╎ + ╎Rerun the failing tests with: + ╎ toolproof --name 'Normal failing test' + ╎ + ╎Changed snapshots: + ╎ ⚠ Snapshot changing test + ╎ + ╎Review the changed snapshots. If a change is correct, accept it with --update (-u): + ╎ toolproof --name 'Snapshot changing test' --update + ╎ + ╎Some tests failed + - stderr should be empty diff --git a/toolproof/test_suite/base/summary_multiple_failures.toolproof.yml b/toolproof/test_suite/base/summary_multiple_failures.toolproof.yml new file mode 100644 index 0000000..51b90c7 --- /dev/null +++ b/toolproof/test_suite/base/summary_multiple_failures.toolproof.yml @@ -0,0 +1,72 @@ +name: Summary lists every failing test + +steps: + - step: I have a "alpha.toolproof.yml" file with the content {yaml} + yaml: |- + name: Alpha failing test + + steps: + - I run 'echo "hello"' + - stdout should contain "goodbye" + - step: I have a "beta.toolproof.yml" file with the content {yaml} + yaml: |- + name: Beta failing test + + steps: + - I run 'echo "hello"' + - stdout should contain "farewell" + - I run "%toolproof_path% --porcelain -c 1" and expect it to fail + - snapshot: stdout + snapshot_content: |- + ╎ + ╎Running tests + ╎ + ╎✘ Alpha failing test + ╎--- STEPS --- + ╎✓ I run 'echo "hello"' + ╎✘ stdout should contain "goodbye" + ╎--- ERROR --- + ╎Error in step "stdout should contain "goodbye"": + ╎-- + ╎Failed assertion: The value + ╎--- + ╎"hello\n" + ╎--- + ╎does not contain + ╎--- + ╎"goodbye" + ╎--- + ╎✘ Beta failing test + ╎--- STEPS --- + ╎✓ I run 'echo "hello"' + ╎✘ stdout should contain "farewell" + ╎--- ERROR --- + ╎Error in step "stdout should contain "farewell"": + ╎-- + ╎Failed assertion: The value + ╎--- + ╎"hello\n" + ╎--- + ╎does not contain + ╎--- + ╎"farewell" + ╎--- + ╎ + ╎Finished running tests + ╎ + ╎Total passing tests: 0 + ╎Passed after retry: 0 + ╎Failing tests: 2 + ╎Changed snapshots: 0 + ╎Skipped tests: 0 + ╎ + ╎Failing tests: + ╎ ✘ Alpha failing test + ╎ ✘ Beta failing test + ╎ + ╎Rerun the failing tests with: + ╎ toolproof --name 'Alpha failing test' + ╎ toolproof --name 'Beta failing test' + ╎ + ╎Some tests failed + - stderr should be empty diff --git a/toolproof/test_suite/base/test_platforms.toolproof.yml b/toolproof/test_suite/base/test_platforms.toolproof.yml index c1176eb..63eb0b7 100644 --- a/toolproof/test_suite/base/test_platforms.toolproof.yml +++ b/toolproof/test_suite/base/test_platforms.toolproof.yml @@ -46,6 +46,7 @@ steps: ╎Total passing tests: 1 ╎Passed after retry: 0 ╎Failing tests: 0 + ╎Changed snapshots: 0 ╎Skipped tests: 2 ╎ ╎All tests passed @@ -64,6 +65,7 @@ steps: ╎Total passing tests: 1 ╎Passed after retry: 0 ╎Failing tests: 0 + ╎Changed snapshots: 0 ╎Skipped tests: 2 ╎ ╎All tests passed @@ -82,6 +84,7 @@ steps: ╎Total passing tests: 1 ╎Passed after retry: 0 ╎Failing tests: 0 + ╎Changed snapshots: 0 ╎Skipped tests: 2 ╎ ╎All tests passed diff --git a/toolproof/test_suite/browser/selectors.toolproof.yml b/toolproof/test_suite/browser/selectors.toolproof.yml index 70cbe17..8f0d550 100644 --- a/toolproof/test_suite/browser/selectors.toolproof.yml +++ b/toolproof/test_suite/browser/selectors.toolproof.yml @@ -100,6 +100,13 @@ steps: ╎Total passing tests: 0 ╎Passed after retry: 0 ╎Failing tests: 1 + ╎Changed snapshots: 0 ╎Skipped tests: 0 ╎ + ╎Failing tests: + ╎ ✘ Inner failing test + ╎ + ╎Rerun the failing tests with: + ╎ toolproof --name 'Inner failing test' + ╎ ╎Some tests failed diff --git a/toolproof/test_suite/browser/snapshot.toolproof.yml b/toolproof/test_suite/browser/snapshot.toolproof.yml index f7005a8..bc8e207 100644 --- a/toolproof/test_suite/browser/snapshot.toolproof.yml +++ b/toolproof/test_suite/browser/snapshot.toolproof.yml @@ -46,6 +46,6 @@ steps: ╎Hello World - I run "%toolproof_path% --timeout 60" and expect it to fail - step: "stdout should contain 'Total passing tests: 0'" - - step: "stdout should contain 'Failing tests: 1'" + - step: "stdout should contain 'Changed snapshots: 1'" - step: "stdout should contain 'Some tests failed'" - stderr should be empty diff --git a/toolproof/test_suite/browser/wait_for.toolproof.yml b/toolproof/test_suite/browser/wait_for.toolproof.yml index 2ea2516..dc7a472 100644 --- a/toolproof/test_suite/browser/wait_for.toolproof.yml +++ b/toolproof/test_suite/browser/wait_for.toolproof.yml @@ -76,6 +76,13 @@ steps: ╎Total passing tests: 0 ╎Passed after retry: 0 ╎Failing tests: 1 + ╎Changed snapshots: 0 ╎Skipped tests: 0 ╎ + ╎Failing tests: + ╎ ✘ Inner failing test + ╎ + ╎Rerun the failing tests with: + ╎ toolproof --name 'Inner failing test' + ╎ ╎Some tests failed diff --git a/toolproof/test_suite/hooks/before_all.toolproof.yml b/toolproof/test_suite/hooks/before_all.toolproof.yml index 9ecd0ab..308a041 100644 --- a/toolproof/test_suite/hooks/before_all.toolproof.yml +++ b/toolproof/test_suite/hooks/before_all.toolproof.yml @@ -27,6 +27,7 @@ steps: ╎Total passing tests: 1 ╎Passed after retry: 0 ╎Failing tests: 0 + ╎Changed snapshots: 0 ╎Skipped tests: 0 ╎ ╎All tests passed diff --git a/toolproof/test_suite/hooks/macros.toolproof.yml b/toolproof/test_suite/hooks/macros.toolproof.yml index 256964c..3054d44 100644 --- a/toolproof/test_suite/hooks/macros.toolproof.yml +++ b/toolproof/test_suite/hooks/macros.toolproof.yml @@ -33,6 +33,7 @@ steps: ╎Total passing tests: 1 ╎Passed after retry: 0 ╎Failing tests: 0 + ╎Changed snapshots: 0 ╎Skipped tests: 0 ╎ ╎All tests passed diff --git a/toolproof/test_suite/refs/ref-file.toolproof.yml b/toolproof/test_suite/refs/ref-file.toolproof.yml index 404a0d9..2310082 100644 --- a/toolproof/test_suite/refs/ref-file.toolproof.yml +++ b/toolproof/test_suite/refs/ref-file.toolproof.yml @@ -78,7 +78,14 @@ steps: ╎Total passing tests: 0 ╎Passed after retry: 0 ╎Failing tests: 1 + ╎Changed snapshots: 0 ╎Skipped tests: 0 ╎ + ╎Failing tests: + ╎ ✘ Inner failing test + ╎ + ╎Rerun the failing tests with: + ╎ toolproof --name 'Inner failing test' + ╎ ╎Some tests failed - stderr should be empty