diff --git a/crates/forge-cli/src/main.rs b/crates/forge-cli/src/main.rs index ea47641..3dab154 100644 --- a/crates/forge-cli/src/main.rs +++ b/crates/forge-cli/src/main.rs @@ -3,8 +3,8 @@ use forge_core::{ capture_example, from_collection, from_openapi, is_openapi, merge_inherited, parse, }; use forge_engine::{ - execute_with_scripts, inherited_vars, run_file, settings_chain, to_curl, ExecOptions, - ExecutedResponse, RequestRun, Settings, VarContext, + execute_with_scripts, inherited_vars, json_report, junit_xml, run_file, settings_chain, + to_curl, ExecOptions, ExecutedResponse, ReportSuite, RequestRun, Settings, VarContext, }; use forge_history::History; use std::collections::HashMap; @@ -78,6 +78,12 @@ enum Command { /// Accept invalid TLS certificates (use with care). #[arg(long)] insecure: bool, + /// Write a JUnit XML report to this path (for CI). + #[arg(long = "reporter-junit", value_name = "PATH")] + reporter_junit: Option, + /// Write a JSON report to this path (for CI). + #[arg(long = "reporter-json", value_name = "PATH")] + reporter_json: Option, }, /// Print a request as an equivalent curl command (variables resolved). Curl { @@ -156,7 +162,20 @@ async fn main() { env_file, timeout, insecure, - } => cmd_run_all(&path, &vars, env_file.as_deref(), timeout, insecure).await, + reporter_junit, + reporter_json, + } => { + cmd_run_all(RunAllArgs { + path, + vars, + env_file, + timeout, + insecure, + reporter_junit, + reporter_json, + }) + .await + } Command::Curl { file, request, @@ -454,28 +473,32 @@ fn cmd_import(file: &Path, out: &Path) { // ─── run-all (collection runner) ─────────────────────────────────────────────── -async fn cmd_run_all( - path: &Path, - vars: &[String], - env_file: Option<&Path>, +struct RunAllArgs { + path: PathBuf, + vars: Vec, + env_file: Option, timeout: u64, insecure: bool, -) { + reporter_junit: Option, + reporter_json: Option, +} + +async fn cmd_run_all(a: RunAllArgs) { let mut files: Vec = Vec::new(); - if path.is_dir() { - collect_http(path, &mut files); + if a.path.is_dir() { + collect_http(&a.path, &mut files); files.sort(); } else { - files.push(path.to_path_buf()); + files.push(a.path.clone()); } if files.is_empty() { - eprintln!("error: no .http files found at {}", path.display()); + eprintln!("error: no .http files found at {}", a.path.display()); exit(1); } let opts = ExecOptions { - timeout_secs: timeout, - insecure, + timeout_secs: a.timeout, + insecure: a.insecure, proxy: None, ca_cert_path: None, client_cert_path: None, @@ -485,6 +508,8 @@ async fn cmd_run_all( }; let mut passed = 0usize; let mut failed = 0usize; + // (suite name, runs) groups retained for the CI reporters. + let mut suites: Vec<(String, Vec)> = Vec::new(); for file in &files { let mut parsed = parse(&read_file(file), &stem(file)); @@ -495,7 +520,7 @@ async fn cmd_run_all( for r in &mut parsed.requests { *r = merge_inherited(r, &chain); } - let ctx = match build_context(&parsed, &chain, env_file, &HashMap::new(), vars) { + let ctx = match build_context(&parsed, &chain, a.env_file.as_deref(), &HashMap::new(), &a.vars) { Ok(c) => c, Err(e) => { eprintln!("error: {e}"); @@ -505,22 +530,58 @@ async fn cmd_run_all( if files.len() > 1 { println!("\n{}", stem(file)); } - for run in run_file(&parsed, ctx, &opts).await { - print_run(&run); + let runs = run_file(&parsed, ctx, &opts).await; + for run in &runs { + print_run(run); if run.ok { passed += 1; } else { failed += 1; } } + suites.push((stem(file), runs)); } + write_reports(&suites, a.reporter_junit.as_deref(), a.reporter_json.as_deref()); + println!("\n{passed} passed, {failed} failed"); if failed > 0 { exit(1); } } +/// Write the requested CI reports from the accumulated run suites. +fn write_reports( + suites: &[(String, Vec)], + junit: Option<&Path>, + json: Option<&Path>, +) { + if junit.is_none() && json.is_none() { + return; + } + let report_suites: Vec = suites + .iter() + .map(|(name, runs)| ReportSuite { + name, + runs, + }) + .collect(); + if let Some(path) = junit { + if let Err(e) = std::fs::write(path, junit_xml(&report_suites)) { + eprintln!("warning: could not write JUnit report: {e}"); + } else { + eprintln!("✎ wrote JUnit report → {}", path.display()); + } + } + if let Some(path) = json { + if let Err(e) = std::fs::write(path, json_report(&report_suites)) { + eprintln!("warning: could not write JSON report: {e}"); + } else { + eprintln!("✎ wrote JSON report → {}", path.display()); + } + } +} + fn collect_http(dir: &Path, out: &mut Vec) { let Ok(entries) = std::fs::read_dir(dir) else { return; diff --git a/crates/forge-cli/tests/cli.rs b/crates/forge-cli/tests/cli.rs index d788ec5..1e19690 100644 --- a/crates/forge-cli/tests/cli.rs +++ b/crates/forge-cli/tests/cli.rs @@ -472,3 +472,64 @@ fn run_with_preexisting_session_and_env_file() { assert!(out.status.success()); std::fs::remove_dir_all(&dir).ok(); } + +#[test] +fn run_all_writes_junit_and_json_reports() { + let dir = unique_dir("reporters"); + let server = TestServer::start(); + let pass = dir.join("pass.http"); + write( + &pass, + &format!("### ok\nGET {}\n# @assert status == 200\n", server.url("/ok")), + ); + let fail = dir.join("fail.http"); + write( + &fail, + &format!("### bad\nGET {}\n# @assert status == 200\n", server.url("/fail")), + ); + + let junit = dir.join("report.xml"); + let json = dir.join("report.json"); + let out = run( + &[ + "run-all", + dir.to_str().unwrap(), + "--reporter-junit", + junit.to_str().unwrap(), + "--reporter-json", + json.to_str().unwrap(), + ], + &dir, + ); + // One assertion fails → exit 1. + assert_eq!(out.status.code(), Some(1)); + + let xml = std::fs::read_to_string(&junit).unwrap(); + assert!(xml.starts_with(" { + pub name: &'a str, + pub runs: &'a [RequestRun], +} + +/// True when a run should count as a failure (transport error, a failed +/// assertion, or a failed post-response `test()`). +fn run_failed(run: &RequestRun) -> bool { + run.error.is_some() + || !run.ok + || run.assertions.iter().any(|a| !a.passed) + || run.tests.iter().any(|t| !t.passed) +} + +/// Human-readable description of why a run failed (empty if it passed). +fn failure_detail(run: &RequestRun) -> String { + let mut lines = Vec::new(); + for a in run.assertions.iter().filter(|a| !a.passed) { + let v = &a.assertion; + let val = if v.value.is_empty() { + String::new() + } else { + format!(" {}", v.value) + }; + lines.push(format!("assert {} {}{} (got {})", v.subject, v.op, val, a.actual)); + } + for t in run.tests.iter().filter(|t| !t.passed) { + let why = t.error.as_deref().unwrap_or("test failed"); + lines.push(format!("test \"{}\": {}", t.name, why)); + } + lines.join("\n") +} + +// ─── JUnit XML ────────────────────────────────────────────────────────────── + +fn xml_escape(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + +/// Render the runs as a JUnit `` document. Each suite becomes a +/// `` and each request a ``; transport errors map to +/// `` and assertion/test failures to ``. +pub fn junit_xml(suites: &[ReportSuite]) -> String { + let total: usize = suites.iter().map(|s| s.runs.len()).sum(); + let failures: usize = suites + .iter() + .flat_map(|s| s.runs.iter()) + .filter(|r| r.error.is_none() && run_failed(r)) + .count(); + let errors: usize = suites + .iter() + .flat_map(|s| s.runs.iter()) + .filter(|r| r.error.is_some()) + .count(); + let total_time: f64 = suites + .iter() + .flat_map(|s| s.runs.iter()) + .map(|r| r.timing_ms as f64 / 1000.0) + .sum(); + + let mut out = String::new(); + out.push_str("\n"); + let _ = writeln!( + out, + "" + ); + + for suite in suites { + let s_fail = suite + .runs + .iter() + .filter(|r| r.error.is_none() && run_failed(r)) + .count(); + let s_err = suite.runs.iter().filter(|r| r.error.is_some()).count(); + let s_time: f64 = suite.runs.iter().map(|r| r.timing_ms as f64 / 1000.0).sum(); + let _ = writeln!( + out, + " ", + xml_escape(suite.name), + suite.runs.len() + ); + + for run in suite.runs { + let time = run.timing_ms as f64 / 1000.0; + let name = xml_escape(&format!("[{}] {}", run.seq, run.name)); + let classname = xml_escape(suite.name); + if let Some(err) = &run.error { + let _ = writeln!( + out, + " " + ); + let _ = writeln!( + out, + " {}", + xml_escape(err), + xml_escape(err) + ); + out.push_str(" \n"); + } else if run_failed(run) { + let detail = failure_detail(run); + let _ = writeln!( + out, + " " + ); + let _ = writeln!( + out, + " {}", + xml_escape("assertions failed"), + xml_escape(&detail) + ); + out.push_str(" \n"); + } else { + let _ = writeln!( + out, + " " + ); + } + } + out.push_str(" \n"); + } + out.push_str("\n"); + out +} + +// ─── JSON ─────────────────────────────────────────────────────────────────── + +#[derive(Serialize)] +struct JsonReport<'a> { + summary: Summary, + suites: Vec>, +} + +#[derive(Serialize)] +struct Summary { + total: usize, + passed: usize, + failed: usize, +} + +#[derive(Serialize)] +struct JsonSuite<'a> { + name: &'a str, + runs: &'a [RequestRun], +} + +/// Render the runs as a pretty-printed JSON report (a summary plus the full, +/// already-`serde`-serializable run details). +pub fn json_report(suites: &[ReportSuite]) -> String { + let total: usize = suites.iter().map(|s| s.runs.len()).sum(); + let failed: usize = suites + .iter() + .flat_map(|s| s.runs.iter()) + .filter(|r| run_failed(r)) + .count(); + let report = JsonReport { + summary: Summary { + total, + passed: total - failed, + failed, + }, + suites: suites + .iter() + .map(|s| JsonSuite { + name: s.name, + runs: s.runs, + }) + .collect(), + }; + serde_json::to_string_pretty(&report).unwrap_or_else(|_| "{}".into()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::assert::AssertionResult; + use crate::script::ScriptTest; + use forge_core::Assertion; + + fn passing_run() -> RequestRun { + RequestRun { + seq: 1, + name: "ok".into(), + ok: true, + status: 200, + timing_ms: 12, + error: None, + assertions: vec![AssertionResult { + assertion: Assertion { + subject: "status".into(), + op: "==".into(), + value: "200".into(), + }, + passed: true, + actual: "200".into(), + }], + captured: vec![], + tests: vec![], + } + } + + fn failing_assertion_run() -> RequestRun { + RequestRun { + seq: 2, + name: "bad".into(), + ok: false, + status: 500, + timing_ms: 8, + error: None, + assertions: vec![AssertionResult { + assertion: Assertion { + subject: "status".into(), + op: "==".into(), + value: "200".into(), + }, + passed: false, + actual: "500".into(), + }], + captured: vec![], + tests: vec![ScriptTest { + name: "smoke".into(), + passed: false, + error: Some("boom".into()), + }], + } + } + + fn errored_run() -> RequestRun { + RequestRun { + seq: 3, + name: "dead & ".into(), + ok: false, + status: 0, + timing_ms: 0, + error: Some("connection refused".into()), + assertions: vec![], + captured: vec![], + tests: vec![], + } + } + + #[test] + fn junit_counts_and_shapes() { + let runs = vec![passing_run(), failing_assertion_run(), errored_run()]; + let suites = [ReportSuite { + name: "suite-a", + runs: &runs, + }]; + let xml = junit_xml(&suites); + + assert!(xml.starts_with("")); + // Failing assertion → with detail. + assert!(xml.contains(". + assert!(xml.contains("")); + // XML escaping of names. + assert!(xml.contains("[3] dead & <gone>")); + assert!(xml.trim_end().ends_with("")); + } + + #[test] + fn json_summary_and_details() { + let runs = vec![passing_run(), failing_assertion_run(), errored_run()]; + let suites = [ReportSuite { + name: "suite-a", + runs: &runs, + }]; + let json = json_report(&suites); + let v: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(v["summary"]["total"], 3); + assert_eq!(v["summary"]["passed"], 1); + assert_eq!(v["summary"]["failed"], 2); + assert_eq!(v["suites"][0]["name"], "suite-a"); + // RequestRun serializes camelCase via its derive. + assert_eq!(v["suites"][0]["runs"][0]["seq"], 1); + assert_eq!(v["suites"][0]["runs"][2]["error"], "connection refused"); + } + + #[test] + fn empty_input_is_valid() { + let xml = junit_xml(&[]); + assert!(xml.contains("tests=\"0\"")); + let json = json_report(&[]); + let v: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(v["summary"]["total"], 0); + } +}