From 99a5cc458980bffc04c64973fd7899d095de999c Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Sun, 23 Aug 2026 03:13:09 +0200 Subject: [PATCH 1/3] fix(diff): make per-file analysis errors advisory Emit structured analysis diagnostics without failing the whole diff by default, preserve valid metrics and thresholds from other files, and add opt-in strict gating for the CLI and GitHub Action. --- action.yml | 8 + crates/mehen-cli/tests/cli_smoke.rs | 65 +++- crates/mehen-cli/tests/config_thresholds.rs | 24 +- crates/mehen-core/src/diagnostic.rs | 8 +- crates/mehen-engine/src/diff.rs | 316 ++++++++++++++------ docs/commands/diff.mdx | 18 +- docs/concepts/output-formats.mdx | 3 + docs/configuration.mdx | 13 +- docs/guides/github-action.mdx | 13 + docs/guides/pr-comment-design.mdx | 17 ++ scripts/github-action.mjs | 120 +++++++- scripts/github-action.test.mjs | 107 ++++++- 12 files changed, 593 insertions(+), 119 deletions(-) diff --git a/action.yml b/action.yml index 4778b6ab1..96852b000 100644 --- a/action.yml +++ b/action.yml @@ -70,6 +70,10 @@ inputs: description: "Fail the action when any configured threshold is exceeded." required: false default: "true" + fail-on-analysis-error: + description: "Fail the action after publishing the report when any file has an error- or fatal-severity analysis diagnostic." + required: false + default: "false" comment-title: description: "Markdown heading used for the sticky pull request comment." required: false @@ -113,6 +117,9 @@ outputs: violations: description: "Number of threshold violations (delta thresholds from the `thresholds` input plus repository `mehen.toml` breaches)." value: ${{ steps.run.outputs.violations }} + analysis_errors: + description: "Number of unique error- or fatal-severity analysis diagnostics." + value: ${{ steps.run.outputs.analysis_errors }} report_json: description: "Path to the JSON diff report produced by mehen." value: ${{ steps.run.outputs.report_json }} @@ -206,6 +213,7 @@ runs: GHA_MEHEN_GITHUB_TOKEN: ${{ inputs.github-token || github.token }} GHA_MEHEN_THRESHOLDS: ${{ inputs.thresholds }} GHA_MEHEN_FAIL_ON_THRESHOLD: ${{ inputs.fail-on-threshold }} + GHA_MEHEN_FAIL_ON_ANALYSIS_ERROR: ${{ inputs.fail-on-analysis-error }} GHA_MEHEN_COMMENT_TITLE: ${{ inputs.comment-title }} GHA_MEHEN_COVERAGE_FILES: ${{ inputs.coverage-files }} GHA_MEHEN_COVERAGE_BASE_SOURCE: ${{ inputs.coverage-base-source }} diff --git a/crates/mehen-cli/tests/cli_smoke.rs b/crates/mehen-cli/tests/cli_smoke.rs index 9a4cdfb0a..7b27d961c 100644 --- a/crates/mehen-cli/tests/cli_smoke.rs +++ b/crates/mehen-cli/tests/cli_smoke.rs @@ -1962,7 +1962,7 @@ fn diff_reports_history_only_for_unparsable_files() { // A malformed head file has no trustworthy static metrics: the // row must show zeros for statics (not partial values blended // into history composites) while history reads real values. The - // run still exits non-zero for the blocking diagnostics. + // diagnostic is advisory unless strict analysis gating is requested. let dir = tempfile::tempdir().expect("tempdir"); init_git_repo(dir.path()); @@ -1994,8 +1994,9 @@ fn diff_reports_history_only_for_unparsable_files() { .output() .expect("failed to run mehen diff"); assert!( - !output.status.success(), - "blocking diagnostics must fail the run" + output.status.success(), + "per-file diagnostics are advisory by default: stderr={}", + String::from_utf8_lossy(&output.stderr) ); let value: serde_json::Value = @@ -2028,6 +2029,64 @@ fn diff_reports_history_only_for_unparsable_files() { metric("history.churn.relative")["current"].as_f64(), Some(0.0) ); + assert_eq!( + metric("cognitive")["current_unavailable"].as_bool(), + Some(true) + ); + let analysis_errors = value["analysis_errors"] + .as_array() + .expect("analysis_errors must be an array"); + assert_eq!(analysis_errors.len(), 1, "{analysis_errors:?}"); + assert_eq!(analysis_errors[0]["path"].as_str(), Some("broken.py")); + assert_eq!(analysis_errors[0]["side"].as_str(), Some("head")); + assert!( + analysis_errors[0]["diagnostics"] + .as_array() + .expect("diagnostics array") + .iter() + .any(|diagnostic| { + matches!(diagnostic["severity"].as_str(), Some("error" | "fatal")) + && diagnostic["code"] + .as_str() + .is_some_and(|code| code.contains("syntax") || code.contains("parse")) + }), + "{analysis_errors:?}" + ); + + let strict = Command::new(env!("CARGO_BIN_EXE_mehen")) + .current_dir(dir.path()) + .args([ + "diff", + "--from", + "broken-cli-base", + "--to", + "broken-cli-head", + "--metrics", + "cognitive", + "--output-format", + "json", + "--fail-on-analysis-error", + ]) + .env_remove("GITHUB_ACTIONS") + .env_remove("GITHUB_EVENT_NAME") + .env_remove("GITHUB_BASE_REF") + .env_remove("GITHUB_SHA") + .env_remove("GITHUB_REPOSITORY") + .output() + .expect("failed to run strict mehen diff"); + assert_eq!( + strict.status.code(), + Some(1), + "strict analysis gating must fail after emitting the report" + ); + let strict_value: serde_json::Value = + serde_json::from_slice(&strict.stdout).expect("strict diff output must be JSON"); + assert!( + strict_value["analysis_errors"] + .as_array() + .is_some_and(|errors| !errors.is_empty()), + "{strict_value}" + ); } #[test] diff --git a/crates/mehen-cli/tests/config_thresholds.rs b/crates/mehen-cli/tests/config_thresholds.rs index 1d257c23b..ca385a9c8 100644 --- a/crates/mehen-cli/tests/config_thresholds.rs +++ b/crates/mehen-cli/tests/config_thresholds.rs @@ -488,11 +488,10 @@ fn diff_passes_when_head_within_thresholds() { } #[test] -fn diff_analysis_failure_outranks_the_threshold_gate() { +fn diff_analysis_errors_do_not_suppress_the_threshold_gate() { // One file crosses the threshold, another has a hard syntax error: - // the run must fail as an analysis failure — JSON without the - // machine-readable gate signal — so CI consumers do not publish a - // partial report as an ordinary gate failure. + // the parseable file's gate remains authoritative, while the broken + // file is disclosed separately and contributes no partial statics. let dir = tempfile::tempdir().expect("tempdir"); git_ok(dir.path(), &["init", "-q", "-b", "main"]); git_ok(dir.path(), &["config", "commit.gpgsign", "false"]); @@ -525,10 +524,16 @@ fn diff_analysis_failure_outranks_the_threshold_gate() { assert_eq!(output.status.code(), Some(1)); let value: serde_json::Value = serde_json::from_slice(&output.stdout).expect("JSON still emitted"); - assert!( - value.get("threshold_violations").is_none(), - "an analysis failure must withhold the gate signal: {value}" - ); + let violations = value["threshold_violations"] + .as_array() + .expect("parseable threshold breaches must remain visible"); + assert_eq!(violations.len(), 1, "{value}"); + assert_eq!(violations[0]["path"].as_str(), Some("sample.py")); + let errors = value["analysis_errors"] + .as_array() + .expect("analysis_errors must be present"); + assert_eq!(errors.len(), 1, "{value}"); + assert_eq!(errors[0]["path"].as_str(), Some("broken.py")); } #[test] @@ -565,8 +570,7 @@ fn diff_json_output_still_emitted_before_threshold_failure() { .expect("machine output must stay parseable when the gate fails"); assert!(value["source_code"].is_array()); // The explicit gate signal machine consumers (e.g. the GitHub - // Action) use to distinguish a quality-gate exit from an analysis - // failure, which also exits 1 but without this key. + // Action) use to identify a configured quality-gate exit. let violations = value["threshold_violations"] .as_array() .expect("gate failures must carry threshold_violations"); diff --git a/crates/mehen-core/src/diagnostic.rs b/crates/mehen-core/src/diagnostic.rs index 56b1327c0..f9b0314b9 100644 --- a/crates/mehen-core/src/diagnostic.rs +++ b/crates/mehen-core/src/diagnostic.rs @@ -9,9 +9,11 @@ use crate::span::SourceSpan; /// /// Per the rewrite plan §9.3: /// - `Warning`: recoverable, exit 0 unless thresholds fail. -/// - `Error`: analysis incomplete; `mehen metrics` exits 1, `mehen diff` -/// records under `analysis_errors`. -/// - `Fatal`: IO/toolchain/invariant failure; exit 1. +/// - `Error`: analysis incomplete; `mehen metrics` exits 1, while +/// `mehen diff` records the affected side under `analysis_errors` +/// and only exits 1 when strict analysis gating is requested. +/// - `Fatal`: toolchain/invariant failure; follows the same command-level +/// policy as `Error`. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum DiagnosticSeverity { diff --git a/crates/mehen-engine/src/diff.rs b/crates/mehen-engine/src/diff.rs index 278074875..36c31adca 100644 --- a/crates/mehen-engine/src/diff.rs +++ b/crates/mehen-engine/src/diff.rs @@ -620,11 +620,12 @@ fn collect_diagnostics( /// /// Per the diagnostic contract (rewrite plan §9.3), `Warning` is /// informational, while `Error` or `Fatal` signals that the analysis is -/// incomplete — diff orchestrators must surface those (CLI exit 1, JSON -/// `analysis_errors`). Returns `true` iff any diagnostic in `diagnostics` -/// reaches the blocking threshold. Lives in the post-1.0 `diff` module -/// so it survives the legacy-engine teardown; the legacy diff path -/// re-uses it via `pub(crate)`. +/// incomplete — diff orchestrators must reject the partial static tree +/// and surface the diagnostics. Process failure is a caller policy. +/// Returns `true` iff any diagnostic in `diagnostics` reaches the +/// blocking threshold. Lives in the post-1.0 `diff` module so it +/// survives the legacy-engine teardown; the legacy diff path re-uses +/// it via `pub(crate)`. pub(crate) fn has_blocking_diagnostic(diagnostics: &[ParseDiagnostic]) -> bool { diagnostics.iter().any(|d| { matches!( @@ -780,6 +781,11 @@ pub struct DiffOpts { value_parser = parse_fail_on_flag, )] fail_on: Vec, + /// Exit with status 1 after rendering when any analyzed side has + /// an error- or fatal-severity diagnostic. By default those sides + /// are reported as unavailable while the rest of the diff succeeds. + #[clap(long)] + fail_on_analysis_error: bool, /// Head-side coverage: the shared `--coverage` flag /// (`PATH|auto|off`, repeatable, bare means `auto`). Also loads /// lazily when a `coverage.*` column or configured threshold asks, @@ -1140,13 +1146,13 @@ fn run_diff_inner( // longer used; we drive `LanguageAnalyzer::analyze` and read // selector values out of the root `MetricSpace`'s `MetricSet`. // - // Recoverable parser errors are surfaced as - // `DiagnosticSeverity::Error` / `Fatal` by the per-language - // analyzers (plan §9.3). Track whether any analyzed side reported - // an error/fatal so the diff exits non-zero at the end — partial - // metrics from a broken parse must not pass CI silently. + // Recoverable parser errors are surfaced as structured + // diagnostics. A blocked side contributes no partial static + // metrics, but it does not invalidate measurements from other + // files; callers can opt back into strict gating with + // `--fail-on-analysis-error`. let mut diffs = Vec::new(); - let mut analysis_failed = false; + let mut analysis_errors: Vec = Vec::new(); // Configured metric thresholds (`mehen.toml`): evaluated per file // against the *head* side of the metrics this diff reports. let threshold_policy = config @@ -1183,6 +1189,25 @@ fn run_diff_inner( cf.path.display(), language.canonical() ); + for side in [ + (!is_new).then_some(DiffSide::Base), + (!is_deleted).then_some(DiffSide::Head), + ] + .into_iter() + .flatten() + { + analysis_errors.push(AnalysisErrorRecord { + path: utf8_path.clone(), + side, + diagnostics: vec![ParseDiagnostic::warning( + "engine.analyzer_unavailable", + format!( + "no analyzer registered for `{}` in this build; static metrics are unavailable", + language.canonical() + ), + )], + }); + } } // Selectors for *this* file: the explicit list, or this language's @@ -1206,52 +1231,96 @@ fn run_diff_inner( langs }; - let mut analyze = |bytes: Vec, side: &str| -> Option { + let mut analyze = |bytes: Vec, side: DiffSide| -> Option { let analyzer = analyzer.as_deref()?; - let text = String::from_utf8(bytes).ok()?; + let side_label = match side { + DiffSide::Base => "baseline", + DiffSide::Head => "current", + }; + let text = match String::from_utf8(bytes) { + Ok(text) => text, + Err(_) => { + let diagnostic = ParseDiagnostic::warning( + "engine.undecodable", + "not valid UTF-8; static analysis unavailable for this side", + ); + log::warn!( + "{} ({side_label}): {}: {}", + cf.path.display(), + diagnostic.code, + diagnostic.message + ); + analysis_errors.push(AnalysisErrorRecord { + path: utf8_path.clone(), + side, + diagnostics: vec![diagnostic], + }); + return None; + } + }; let source = SourceFile::new(utf8_path.clone(), *language, text); let analysis = match analyzer.analyze(&source, &analysis_config) { Ok(a) => a, Err(err) => { - log::error!("{} ({side}): analyzer failed: {err}", cf.path.display()); - analysis_failed = true; + let diagnostic = ParseDiagnostic::error("analysis.error", err.to_string()); + log::error!( + "{} ({side_label}): {}: {}", + cf.path.display(), + diagnostic.code, + diagnostic.message + ); + analysis_errors.push(AnalysisErrorRecord { + path: utf8_path.clone(), + side, + diagnostics: vec![diagnostic], + }); return None; } }; for diag in &analysis.diagnostics { match diag.severity { DiagnosticSeverity::Warning => log::warn!( - "{} ({side}): {}: {}", + "{} ({side_label}): {}: {}", cf.path.display(), diag.code, diag.message ), DiagnosticSeverity::Error | DiagnosticSeverity::Fatal => log::error!( - "{} ({side}): {}: {}", + "{} ({side_label}): {}: {}", cf.path.display(), diag.code, diag.message ), } } + if !analysis.diagnostics.is_empty() { + analysis_errors.push(AnalysisErrorRecord { + path: utf8_path.clone(), + side, + diagnostics: analysis.diagnostics.clone(), + }); + } if has_blocking_diagnostic(&analysis.diagnostics) { - analysis_failed = true; // A partial tree behind an `Error`/`Fatal` diagnostic // is not a measurement (§9.3): emitting its truncated // statics — or blending them into the history - // composites — would mislead even though the run - // already exits non-zero. The side falls back to the - // history-only synthetic space below. + // composites — would mislead. The side falls back to + // the history-only synthetic space below. return None; } Some(analysis.root) }; + let mut baseline_static_available = false; let mut baseline_space: Option = if is_new { None } else { match mehen_git::read_blob(&repo, &from_ref, base_path) { - Ok(Some(bytes)) => analyze(bytes, "baseline"), + Ok(Some(bytes)) => { + let space = analyze(bytes, DiffSide::Base); + baseline_static_available = space.is_some(); + space + } Ok(None) => None, Err(e) => { log::warn!("Skipping baseline for {}: {e}", cf.path.display()); @@ -1260,11 +1329,16 @@ fn run_diff_inner( } }; + let mut current_static_available = false; let mut current_space: Option = if is_deleted { None } else { match mehen_git::read_blob(&repo, &to_ref, &cf.path) { - Ok(Some(bytes)) => analyze(bytes, "current"), + Ok(Some(bytes)) => { + let space = analyze(bytes, DiffSide::Head); + current_static_available = space.is_some(); + space + } Ok(None) => None, Err(e) => { log::warn!("Skipping current for {}: {e}", cf.path.display()); @@ -1558,26 +1632,29 @@ fn run_diff_inner( baseline_space.is_some() && !base_measured, current_space.is_some() && !head_measured, ) - } else { + } else if sel.name.starts_with("history.") { let baseline_history_missing = - matches!(histories.as_ref(), Some((None, _))) - && !is_new_row - && sel.name.starts_with("history."); + matches!(histories.as_ref(), Some((None, _))) && !is_new; ( baseline_history_missing - || (baseline_space.is_some() + || (!is_new && !history_metrics::selector_available( sel.name, baseline_composites, baseline_history_available, )), - current_space.is_some() + !is_deleted && !history_metrics::selector_available( sel.name, current_composites, current_history_available, ), ) + } else { + ( + !is_new && !baseline_static_available, + !is_deleted && !current_static_available, + ) }; let baseline = baseline_space .as_ref() @@ -1714,7 +1791,14 @@ fn run_diff_inner( let format = opts.output_format.unwrap_or(DiffFormat::Markdown); match format { DiffFormat::Markdown => { - print_markdown(&diffs, &display_selectors, &from_label, &from_ref, &to_ref); + print_markdown( + &diffs, + &display_selectors, + &analysis_errors, + &from_label, + &from_ref, + &to_ref, + ); if !doc_files.is_empty() { let mut ctx = DocRenderCtx::new(&from_label); let repo_url = ci_ctx @@ -1736,17 +1820,7 @@ fn run_diff_inner( } else { Some(&doc_files) }; - // A failed analysis means the measurements behind the - // breaches are partial: withhold the machine-readable gate - // signal so consumers (the GitHub Action) fail fast on the - // analysis error instead of publishing an incomplete - // report as an ordinary gate failure. - let publishable_breaches: &[crate::config_file::ThresholdBreach] = if analysis_failed { - &[] - } else { - &threshold_breaches - }; - if let Err(e) = print_json(&diffs, doc_ref, publishable_breaches) { + if let Err(e) = print_json(&diffs, doc_ref, &analysis_errors, &threshold_breaches) { // Surface the error loudly — exit code 2 mirrors the // --fail-on gate and is distinct from the generic exit 1 // that covers setup/IO errors in run_diff_inner. @@ -1776,15 +1850,21 @@ fn run_diff_inner( std::process::exit(2); } - // Per the diagnostic contract (rewrite plan §9.3), recoverable - // parser errors must surface as a non-zero exit so CI cannot pass - // partial metrics computed from a known-broken parse. Checked - // before the threshold gate: a broken analysis outranks a quality - // gate evaluated on the parseable remainder. Exit 1 lines up with - // the generic setup/IO bucket and is distinct from exit 2 (doc - // gate). Diagnostics are already logged above; this gate only - // flips the exit code. - if analysis_failed { + // Advisory by default: each blocked side is represented by + // `analysis_errors`, and its static cells render as unavailable. + // Strict callers can restore the historical non-zero behavior + // without losing the complete report. + if opts.fail_on_analysis_error + && analysis_errors + .iter() + .flat_map(|record| &record.diagnostics) + .any(|diagnostic| { + matches!( + diagnostic.severity, + DiagnosticSeverity::Error | DiagnosticSeverity::Fatal + ) + }) + { std::process::exit(1); } @@ -2081,6 +2161,7 @@ fn legacy_path_is_selected(path: &Path, paths: &[PathBuf]) -> bool { fn print_markdown( diffs: &[FileDiff], selectors: &[MetricSelector], + analysis_errors: &[AnalysisErrorRecord], from_label: &str, from: &str, to: &str, @@ -2095,42 +2176,101 @@ fn print_markdown( if diffs.is_empty() { out.push_str("No metric changes detected.\n"); - write!(std::io::stdout().lock(), "{out}").unwrap(); - return; - } + } else { + // Header + out.push_str("| File |"); + for sel in selectors { + out.push_str(&format!(" {} |", sel.label)); + } + out.push('\n'); - // Header - out.push_str("| File |"); - for sel in selectors { - out.push_str(&format!(" {} |", sel.label)); + // Separator + out.push_str("|---|"); + for _ in selectors { + out.push_str("---:|"); + } + out.push('\n'); + + // Rows. Each cell is looked up by selector *name* against the file's + // metrics, so a file that doesn't publish a given column (e.g. a SQL file + // under the `cyclomatic` column of a mixed PR) renders an em dash rather + // than a misaligned value. + for diff in diffs { + out.push_str(&format!("| {} |", diff.path.display())); + for sel in selectors { + out.push(' '); + match diff.metrics.iter().find(|m| m.name == sel.name) { + Some(md) => out.push_str(&format_metric_cell(md, from_label)), + None => out.push('\u{2013}'), // – (column not applicable to this file) + } + out.push_str(" |"); + } + out.push('\n'); + } } - out.push('\n'); - // Separator - out.push_str("|---|"); - for _ in selectors { - out.push_str("---:|"); + write_analysis_diagnostics(&mut out, analysis_errors); + write!(std::io::stdout().lock(), "{out}").unwrap(); +} + +fn write_analysis_diagnostics(out: &mut String, records: &[AnalysisErrorRecord]) { + if records.is_empty() { + return; } - out.push('\n'); - // Rows. Each cell is looked up by selector *name* against the file's - // metrics, so a file that doesn't publish a given column (e.g. a SQL file - // under the `cyclomatic` column of a mixed PR) renders an em dash rather - // than a misaligned value. - for diff in diffs { - out.push_str(&format!("| {} |", diff.path.display())); - for sel in selectors { - out.push(' '); - match diff.metrics.iter().find(|m| m.name == sel.name) { - Some(md) => out.push_str(&format_metric_cell(md, from_label)), - None => out.push('\u{2013}'), // – (column not applicable to this file) + out.push_str("\n## Analysis diagnostics\n\n"); + out.push_str( + "Static metrics are unavailable for sides with error or fatal diagnostics; other measurements remain valid.\n\n", + ); + out.push_str("| File | Side | Severity | Diagnostic |\n"); + out.push_str("|---|---|---|---|\n"); + + let mut seen = std::collections::BTreeSet::new(); + for record in records { + let side = match record.side { + DiffSide::Base => "base", + DiffSide::Head => "head", + }; + for diagnostic in &record.diagnostics { + let severity = match diagnostic.severity { + DiagnosticSeverity::Warning => "warning", + DiagnosticSeverity::Error => "error", + DiagnosticSeverity::Fatal => "fatal", + }; + let span_key = diagnostic + .span + .as_ref() + .map(|span| { + format!( + "{}:{}:{}:{}", + span.start_byte, span.end_byte, span.start_line, span.end_line + ) + }) + .unwrap_or_default(); + let key = format!( + "{}\0{side}\0{severity}\0{}\0{}\0{span_key}", + record.path, diagnostic.code, diagnostic.message, + ); + if !seen.insert(key) { + continue; } - out.push_str(" |"); + let path = escape_markdown_cell(record.path.as_str()); + let code = escape_markdown_cell(&diagnostic.code); + let message = escape_markdown_cell(&diagnostic.message); + let line = diagnostic + .span + .as_ref() + .map(|span| format!(" at line {}", span.start_line)) + .unwrap_or_default(); + out.push_str(&format!( + "| {path} | {side} | {severity} | `{code}`{line}: {message} |\n" + )); } - out.push('\n'); } +} - write!(std::io::stdout().lock(), "{out}").unwrap(); +fn escape_markdown_cell(value: &str) -> String { + value.replace('|', "\\|").replace(['\r', '\n'], " ") } fn format_metric_cell(md: &MetricDiff, from: &str) -> String { @@ -2215,29 +2355,32 @@ fn format_f64(v: f64) -> String { // ── JSON output ──────────────────────────────────────────────────────── -/// Emit a single JSON document with a `source_code` key and an optional -/// `markdown` key. Downstream consumers (`jq`, `serde_json`) see one top-level -/// object, not two concatenated arrays. +/// Emit a single JSON document with `source_code` and `analysis_errors` +/// keys plus optional `markdown` and `threshold_violations` sections. +/// Downstream consumers (`jq`, `serde_json`) see one top-level object, +/// not concatenated arrays. /// /// Serialization errors bubble up as `Err` so `run_diff_inner` exits /// non-zero instead of silently writing an empty `""` to stdout. fn print_json( diffs: &[FileDiff], docs: Option<&[DocDiffFile]>, + analysis_errors: &[AnalysisErrorRecord], threshold_breaches: &[crate::config_file::ThresholdBreach], ) -> Result<(), Box> { let mut payload = serde_json::Map::new(); payload.insert("source_code".to_string(), serde_json::to_value(diffs)?); + payload.insert( + "analysis_errors".to_string(), + serde_json::to_value(analysis_errors)?, + ); if let Some(docs) = docs { payload.insert( "markdown".to_string(), serde_json::Value::Array(doc_json_payload(docs)), ); } - // Present only when a configured `mehen.toml` gate fired — the - // explicit signal machine consumers (e.g. the GitHub Action) use - // to distinguish a quality-gate exit (1, with this key) from an - // analysis failure (also exit 1, but without it). + // Present only when a configured `mehen.toml` gate fired. if !threshold_breaches.is_empty() { payload.insert( "threshold_violations".to_string(), @@ -3630,6 +3773,7 @@ binary.md binary show_unchanged: false, ignore_git_attributes: true, fail_on: vec![], + fail_on_analysis_error: false, coverage: Default::default(), base_coverage: vec![], } @@ -3962,7 +4106,7 @@ src/archive.txt binary is_deleted: false, functions: 0, }]; - let res = print_json(&diffs, None, &[]); + let res = print_json(&diffs, None, &[], &[]); assert!(res.is_ok(), "valid input must serialize cleanly"); } @@ -3973,7 +4117,7 @@ src/archive.txt binary // emitter used `unwrap_or_default` and silently wrote an empty // JSON document to stdout when serde_json failed. let diffs: Vec = vec![]; - let res: Result<(), Box> = print_json(&diffs, None, &[]); + let res: Result<(), Box> = print_json(&diffs, None, &[], &[]); assert!(res.is_ok()); } diff --git a/docs/commands/diff.mdx b/docs/commands/diff.mdx index 8b42b6302..b7e30a68b 100644 --- a/docs/commands/diff.mdx +++ b/docs/commands/diff.mdx @@ -24,6 +24,7 @@ mehen diff [OPTIONS] | `--show-unchanged` | Show files where every metric is unchanged. | | `--ignore-git-attributes[=]` | Skip files marked `linguist-generated`, `linguist-vendored`, or `binary` (default `true`; `--ignore-generated` remains an alias). | | `--fail-on ` | Exit non-zero when named thresholds are crossed. Comma-separated: `dmi-drop`, `new-broken-link`, `filler-high`, `all`. | +| `--fail-on-analysis-error` | Exit `1` after rendering when any analyzed side has an error- or fatal-severity diagnostic. Without this flag, the diagnostic is advisory and other files remain reportable. | | `--coverage[=PATH\|auto\|off]` | Head-side coverage input, with the same semantics as on [`mehen metrics`](/commands/metrics): explicit report paths (repeatable), `auto` to [discover reports](/metrics/coverage/auto-discovery), `off` to disable. Bare `--coverage` means `auto`. When omitted, coverage loads lazily — when a `coverage.*` metric or configured threshold asks for it, or when `--base-coverage` is given. | | `--base-coverage=` | Coverage report(s) for the **base** revision (repeatable). Enriches the baseline side of the diff so `coverage.*` columns carry real trend arrows instead of "new measurement". Explicit paths only — no discovery, since the working tree holds *head* artifacts. See [Coverage trend columns](#coverage-trend-columns). | | `--config ` | Pin a [configuration file](/configuration) instead of discovering `mehen.toml` upward from the working directory. | @@ -146,6 +147,19 @@ and a configured coverage threshold is itself a lazy ingestion trigger — no fl The rules align with the severity-1 / severity-2 indicators on the [PR comment design](/guides/pr-comment-design) page. +## Analysis diagnostics + +A parser or analyzer can reject one side of one file without invalidating the whole comparison. +`mehen diff` records the affected path, side (`base` or `head`), severity, reason code, and message +under the top-level JSON `analysis_errors` array. Markdown output adds the same information in an +**Analysis diagnostics** table. + +Static metrics from an error- or fatal-severity side are withheld rather than computed from a +partial syntax tree. Parser-independent measurements such as repository history remain available, +and parseable files continue through normal metric and threshold evaluation. The command exits `0` +by default when diagnostics are the only issue. Add `--fail-on-analysis-error` when incomplete +analysis should remain a strict CI gate; the complete report is still emitted before exit `1`. + ## Configured thresholds When a [`mehen.toml`](/configuration) is present, the **head side** of every changed source-code @@ -164,8 +178,8 @@ See [Concepts → Thresholds and diffs](/concepts/thresholds-and-diffs) for how | Code | Meaning | |---|---| -| 0 | Success — the comment is advisory. | -| 1 | IO, git, parser-fatal, or unsupported-language error. Also: a [configured metric threshold](/configuration) crossed, after the report is emitted. | +| 0 | Success — including advisory per-file analysis diagnostics. | +| 1 | IO, git, or serialization failure; `--fail-on-analysis-error` found a blocking diagnostic; or a [configured metric threshold](/configuration) crossed. Reports are emitted first for the latter two cases. | | 2 | One or more `--fail-on` rules crossed (gating exit). | ## See also diff --git a/docs/concepts/output-formats.mdx b/docs/concepts/output-formats.mdx index 441fb771c..0dd6c48ea 100644 --- a/docs/concepts/output-formats.mdx +++ b/docs/concepts/output-formats.mdx @@ -82,6 +82,9 @@ Markdown files emit a top-level `markdown:` block with the ``` The GitHub Action consumes JSON for decisions and the Markdown shape for the comment body. +`analysis_errors` is always an array. Each record names the affected path and `base`/`head` side +and carries structured diagnostics; error- or fatal-severity records make that side's static +metrics unavailable without invalidating other files. ## Pretty-printing diff --git a/docs/configuration.mdx b/docs/configuration.mdx index fd2d14db1..e9cc03879 100644 --- a/docs/configuration.mdx +++ b/docs/configuration.mdx @@ -102,10 +102,10 @@ Two consequences worth knowing: [`--fail-on` gates](/commands/diff#fail-on-thresholds). Files whose parse produces a blocking diagnostic are never threshold-gated on partial metrics: -`mehen metrics` and `mehen diff` fail with exit `1` through their existing analysis-error -contract, while `mehen top-offenders` keeps its contract of skipping unparsable files from the -ranking (so a broken file cannot fabricate a passing measurement — but it does not fail the run -by itself either). +`mehen metrics` still exits `1` because its only requested file is unavailable; `mehen diff` +records the affected side under `analysis_errors`, keeps evaluating parseable files, and is +advisory unless `--fail-on-analysis-error` is set. `mehen top-offenders` keeps its contract of +skipping unparsable files from static ranking while preserving parser-independent history values. ## The violation report @@ -128,8 +128,8 @@ Each line names the measured value, the crossed limit, and the configuration pat Violations are sorted by path, then metric, for deterministic CI logs. Colors engage only on a terminal with `NO_COLOR` unset. With `--output-format json`, `mehen diff` additionally embeds the violations as a top-level -`threshold_violations` array (present only when the gate fired), so machine consumers can tell a -quality-gate exit from an analysis failure. +`threshold_violations` array (present only when the gate fired). Per-file analyzer diagnostics use +the separate, always-present `analysis_errors` array. ## Validation @@ -203,6 +203,7 @@ instrumented-but-uncovered code can. | Gate | Where it runs | What it checks | Exit code | |---|---|---|---| | `mehen.toml` thresholds | Inside the binary, every command | Absolute values at head | 1 | +| `mehen diff --fail-on-analysis-error` | Inside the binary, diff only | Error/fatal diagnostics on either analyzed side | 1 | | [`mehen diff --fail-on`](/commands/diff#fail-on-thresholds) | Inside the binary, diff only | Markdown documentation band crossings | 2 | | [GitHub Action `thresholds`](/guides/github-action) | Action post-processing | Adverse per-file **deltas** | Action failure | diff --git a/docs/guides/github-action.mdx b/docs/guides/github-action.mdx index ba5b77ba8..f26aae83e 100644 --- a/docs/guides/github-action.mdx +++ b/docs/guides/github-action.mdx @@ -80,6 +80,7 @@ trees: | `github-token` | (workflow token) | GitHub token used to update PR comments. | | `thresholds` | `""` | Adverse per-file delta limits, e.g. `cyclomatic=5,cognitive=3,loc.lloc=100`. | | `fail-on-threshold` | `true` | Fail the action when any configured threshold is exceeded. | +| `fail-on-analysis-error` | `false` | Fail after publishing the report when any file has an error- or fatal-severity analysis diagnostic. | | `comment-title` | `## 📊 Source Code Metrics` | Markdown heading used for the sticky comment. | | `coverage-files` | `""` | Coverage report paths produced by an earlier test step (newline, comma, or semicolon separated; literal paths). Enables the [coverage trend columns](#coverage-trends). | | `coverage-base-source` | `auto` | Where base-revision coverage comes from on PRs: `auto`, `cache`, `artifact`, `codecov`, or `off`. | @@ -91,6 +92,7 @@ trees: | Output | Description | |---|---| | `violations` | Number of threshold violations. | +| `analysis_errors` | Number of unique error- or fatal-severity analysis diagnostics. | | `report_json` | Path to the JSON diff report produced by mehen. | | `report_markdown` | Path to the rendered Markdown report. | @@ -133,6 +135,16 @@ columns its analyzer emits, and the rest render as `–`. This keeps SQL changes action path instead of silently dropping out. See [`mehen diff`](/commands/diff) for how the default metric set is resolved per file language. +## Analysis diagnostics + +An unsupported language feature or malformed file no longer prevents the Action from publishing +the rest of the report. The sticky comment adds an **Analysis diagnostics** table naming the file, +revision side, severity, reason code, and message. Static cells for a blocked side render as +unavailable; parser-independent values and other files remain valid. + +Diagnostics are advisory by default. Set `fail-on-analysis-error: true` to publish the complete +comment and then fail the workflow step. + ## Coverage trends Point `coverage-files` at the report(s) your test step already writes and the sticky comment gains a @@ -259,6 +271,7 @@ Notes for shared setups: cyclomatic=3 cognitive=3 fail-on-threshold: true + fail-on-analysis-error: true ``` diff --git a/docs/guides/pr-comment-design.mdx b/docs/guides/pr-comment-design.mdx index 5b941746d..ab2e44f02 100644 --- a/docs/guides/pr-comment-design.mdx +++ b/docs/guides/pr-comment-design.mdx @@ -14,6 +14,23 @@ the [GitHub Action](/guides/github-action). This page is the user-facing referen When a PR touches one or more Markdown files, `mehen diff` emits a **Documentation Metrics** section below the existing source-code table. +When an analyzer reports a diagnostic, the Action also appends an **Analysis diagnostics** table. +This section is factual and mechanically derived: file, revision side, severity, stable reason +code, and analyzer message. Exact duplicate diagnostics are collapsed so parser recovery does not +repeat the same row. + +```markdown +### Analysis diagnostics + +| File | Side | Severity | Diagnostic | +|---|---|---|---| +| [internal/config/rules.go](…) | head | error | `go.syntax_error`: tree-sitter error node at line 347 | +``` + +An error or fatal diagnostic makes that side's static metrics unavailable, but does not hide +parseable files or fail the Action by default. Strict repositories opt in with +`fail-on-analysis-error: true`. + The non-negotiable constraint: **every character of output is mechanically derivable from the AST, the metric tables, and the threshold bands defined in [Markdown metrics](/metrics/markdown/overview) and [Markdown prose metrics](/metrics/markdown/prose/overview).** No LLM call, no "likely cause" inference, diff --git a/scripts/github-action.mjs b/scripts/github-action.mjs index 645fdd692..1a4a4e0f4 100644 --- a/scripts/github-action.mjs +++ b/scripts/github-action.mjs @@ -127,6 +127,9 @@ async function main() { fs.writeFileSync(reportJson, `${diff.stdout.trim()}\n`, "utf8"); const diffs = parseDiffJson(diff.stdout); + const analysisErrors = parseAnalysisErrors(diff.stdout); + const blockingAnalysisErrors = + countBlockingAnalysisDiagnostics(analysisErrors); const gateViolations = parseGateViolations(diff.stdout); const violations = collectThresholdViolations(diffs, thresholds); let markdown = renderMarkdown( @@ -137,6 +140,7 @@ async function main() { version, gateViolations, baseCoverage.disclosure, + analysisErrors, ); // Phase F (§39): `mehen diff --output-format markdown` emits a @@ -166,14 +170,16 @@ async function main() { // delta thresholds and repository `mehen.toml` breaches parsed from // the diff report. setOutput("violations", String(violations.length + gateViolations.length)); + setOutput("analysis_errors", String(blockingAnalysisErrors)); setOutput("report_json", reportJson); setOutput("report_markdown", reportMarkdown); + let shouldFail = false; if (violations.length > 0 && boolInput("FAIL_ON_THRESHOLD", true)) { console.error( `Mehen threshold check failed with ${violations.length} violation(s).`, ); - process.exit(1); + shouldFail = true; } // A deferred quality-gate failure from the diff itself (configured @@ -183,6 +189,20 @@ async function main() { console.error( `mehen diff exited with status ${diff.gateStatus}: a configured quality gate failed (see the report above).`, ); + shouldFail = true; + } + + if ( + blockingAnalysisErrors > 0 && + boolInput("FAIL_ON_ANALYSIS_ERROR", false) + ) { + console.error( + `Mehen analysis check failed with ${blockingAnalysisErrors} blocking diagnostic(s) (see the report above).`, + ); + shouldFail = true; + } + + if (shouldFail) { process.exit(1); } } @@ -941,9 +961,8 @@ function runMehen(cli, args, options = {}) { * Whether a failing `mehen diff --output-format json` invocation is a * configured quality-gate exit: the payload must be complete AND carry * the explicit `threshold_violations` signal the CLI emits only when a - * `mehen.toml` gate fired. An analysis failure also exits 1 with - * well-formed JSON but without that key — it must keep failing fast - * instead of publishing a partial report under the wrong reason. + * `mehen.toml` gate fired. Per-file analysis diagnostics are advisory in + * current CLIs and live in their own `analysis_errors` array. */ function isGateFailureReport(stdout) { try { @@ -1125,6 +1144,73 @@ function parseGateViolations(stdout) { } } +function parseAnalysisErrors(stdout) { + try { + const parsed = JSON.parse(typeof stdout === "string" ? stdout : ""); + return parsed && Array.isArray(parsed.analysis_errors) + ? parsed.analysis_errors + : []; + } catch { + return []; + } +} + +function analysisDiagnosticRows(records) { + const rows = []; + const seen = new Set(); + for (const record of Array.isArray(records) ? records : []) { + const path = typeof record?.path === "string" ? record.path : ""; + const side = record?.side === "base" ? "base" : "head"; + for (const diagnostic of Array.isArray(record?.diagnostics) + ? record.diagnostics + : []) { + const severity = + typeof diagnostic?.severity === "string" + ? diagnostic.severity.toLowerCase() + : "warning"; + const code = + typeof diagnostic?.code === "string" + ? diagnostic.code + : "analysis.diagnostic"; + const message = + typeof diagnostic?.message === "string" + ? diagnostic.message + : "Analyzer diagnostic"; + const line = Number.isInteger(diagnostic?.span?.start_line) + ? diagnostic.span.start_line + : null; + const spanKey = diagnostic?.span + ? [ + diagnostic.span.start_byte, + diagnostic.span.end_byte, + diagnostic.span.start_line, + diagnostic.span.end_line, + ] + : null; + const key = JSON.stringify([ + path, + side, + severity, + code, + message, + spanKey, + ]); + if (seen.has(key)) { + continue; + } + seen.add(key); + rows.push({ path, side, severity, code, message, line }); + } + } + return rows; +} + +function countBlockingAnalysisDiagnostics(records) { + return analysisDiagnosticRows(records).filter( + (row) => row.severity === "error" || row.severity === "fatal", + ).length; +} + function renderMarkdown( diffs, context, @@ -1133,6 +1219,7 @@ function renderMarkdown( version = "", gateViolations = [], coverageNote = null, + analysisErrors = [], ) { const title = input("COMMENT_TITLE", DEFAULT_TITLE).trim() || DEFAULT_TITLE; const scope = @@ -1198,6 +1285,22 @@ function renderMarkdown( } } + const analysisRows = analysisDiagnosticRows(analysisErrors); + if (analysisRows.length > 0) { + body += "\n### Analysis diagnostics\n\n"; + body += + "Mehen reported analyzer diagnostics. Sides with error or fatal diagnostics use only metrics that remain valid without the failed parse.\n\n"; + body += "| File | Side | Severity | Diagnostic |\n"; + body += "|---|---|---|---|\n"; + for (const row of analysisRows) { + const revision = row.side === "base" ? context.baseSha : context.sha; + const side = + row.side === "base" ? context.baseLabel || "base" : "head"; + const line = row.line ? ` at line ${row.line}` : ""; + body += `| ${renderFile(row.path, context, revision)} | ${escapeCell(side)} | ${escapeCell(row.severity)} | \`${escapeCell(row.code)}\`${line}: ${escapeCell(row.message)} |\n`; + } + } + if (sawNotApplicable) { body += "\n> `—` indicates the metric does not apply to the file's language.\n"; @@ -1220,16 +1323,16 @@ function renderFooter(version) { return `${FOOTER_PREFIX}${versionSuffix} ${FOOTER_SUFFIX}`; } -function renderFile(filePath, context) { +function renderFile(filePath, context, revision = context.sha) { const escaped = escapeCell(filePath); - if (!context.repository || !context.sha) { + if (!context.repository || !revision) { return escaped; } const urlPath = String(filePath) .split("/") .map((part) => encodeURIComponent(part)) .join("/"); - return `[${escaped}](https://github.com/${context.repository}/blob/${context.sha}/${urlPath})`; + return `[${escaped}](https://github.com/${context.repository}/blob/${revision}/${urlPath})`; } // Build the master header as the union of metrics across all files, preserving @@ -1575,11 +1678,14 @@ export { isGateFailureReport, isNotApplicable, listFilesRecursively, + countBlockingAnalysisDiagnostics, + parseAnalysisErrors, parseGateViolations, parseList, parseThresholds, parseVersionOutput, pickBaseArtifact, + renderMarkdown, renderFooter, unionMetricColumns, }; diff --git a/scripts/github-action.test.mjs b/scripts/github-action.test.mjs index 8d228c314..d46102073 100644 --- a/scripts/github-action.test.mjs +++ b/scripts/github-action.test.mjs @@ -12,6 +12,7 @@ import { canonicalMetricName, codecovToLcov, collectThresholdViolations, + countBlockingAnalysisDiagnostics, diffJsonHasDocs, extractMarkdownDocsSection, extractZip, @@ -21,12 +22,14 @@ import { isGateFailureReport, isNotApplicable, listFilesRecursively, + parseAnalysisErrors, parseGateViolations, parseList, parseThresholds, parseVersionOutput, pickBaseArtifact, renderFooter, + renderMarkdown, unionMetricColumns, } from "./github-action.mjs"; @@ -60,6 +63,100 @@ test("parseGateViolations is empty for passing runs and older CLIs", () => { assert.deepEqual(parseGateViolations(undefined), []); }); +test("parseAnalysisErrors extracts structured per-side diagnostics", () => { + const errors = parseAnalysisErrors( + JSON.stringify({ + source_code: [], + analysis_errors: [ + { + path: "internal/config/rules.go", + side: "head", + diagnostics: [ + { + severity: "error", + code: "go.syntax_error", + message: "tree-sitter error node at line 347", + span: null, + }, + ], + }, + ], + }), + ); + assert.equal(errors.length, 1); + assert.equal(errors[0].path, "internal/config/rules.go"); +}); + +test("parseAnalysisErrors is empty for older or malformed output", () => { + assert.deepEqual(parseAnalysisErrors('{"source_code": []}'), []); + assert.deepEqual(parseAnalysisErrors("not json"), []); + assert.deepEqual(parseAnalysisErrors(undefined), []); +}); + +test("blocking analysis diagnostic count deduplicates parser recovery noise", () => { + const duplicate = { + severity: "error", + code: "go.syntax_error", + message: "tree-sitter error node at line 347", + span: null, + }; + const records = [ + { + path: "internal/config/rules.go", + side: "head", + diagnostics: [duplicate, { ...duplicate }], + }, + { + path: "README.md", + side: "head", + diagnostics: [ + { + severity: "warning", + code: "markdown.reference", + message: "reference could not be resolved", + }, + ], + }, + ]; + assert.equal(countBlockingAnalysisDiagnostics(records), 1); +}); + +test("renderMarkdown reports analysis diagnostics without hiding other results", () => { + const markdown = renderMarkdown( + [], + { + eventName: "pull_request", + repository: "wharflab/tally", + sha: "head-sha", + baseSha: "base-sha", + baseLabel: "main", + }, + new Map(), + [], + "1.11.0", + [], + null, + [ + { + path: "internal/config/rules.go", + side: "head", + diagnostics: [ + { + severity: "error", + code: "go.syntax_error", + message: "tree-sitter error node at line 347", + }, + ], + }, + ], + ); + assert.ok(markdown.includes("### Analysis diagnostics")); + assert.ok(markdown.includes("go.syntax_error")); + assert.ok(markdown.includes("internal/config/rules.go")); + assert.ok(markdown.includes("head-sha")); + assert.ok(markdown.includes("No metric changes detected.")); +}); + test("isGateFailureReport requires the explicit threshold_violations signal", () => { assert.equal( isGateFailureReport( @@ -76,10 +173,16 @@ test("isGateFailureReport requires the explicit threshold_violations signal", () }); test("isGateFailureReport rejects reports without a fired gate", () => { - // An analysis failure also exits 1 with well-formed JSON — without - // the threshold_violations key it must keep failing fast. + // Analysis diagnostics use their own advisory array and are not a + // repository-threshold gate signal. assert.equal(isGateFailureReport('{"source_code": [], "markdown": []}'), false); assert.equal(isGateFailureReport('{"source_code": [{"path": "a.py"}]}'), false); + assert.equal( + isGateFailureReport( + '{"source_code": [], "analysis_errors": [{"path": "a.py"}]}', + ), + false, + ); assert.equal( isGateFailureReport('{"source_code": [], "threshold_violations": []}'), false, From 60fe1cfc1a47c4316ca00e3105f700d4916ce5f5 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Wed, 26 Aug 2026 00:34:12 +0200 Subject: [PATCH 2/3] fix: address PR review comment Use revision-specific paths for renamed baseline diagnostics, preserve unavailable history and coverage sides, and align the diagnostics documentation with both renderers. Addresses: https://github.com/ophi-dev/mehen/pull/261#discussion_r3837426059 Addresses: https://github.com/ophi-dev/mehen/pull/261#discussion_r3837430365 Addresses: https://github.com/ophi-dev/mehen/pull/261#discussion_r3837430367 Addresses: https://github.com/ophi-dev/mehen/pull/261#discussion_r3837430368 --- crates/mehen-cli/tests/cli_smoke.rs | 71 +++++++++++++++++++++++++ crates/mehen-cli/tests/coverage_cli.rs | 72 ++++++++++++++++++++++++++ crates/mehen-engine/src/diff.rs | 38 +++++++------- docs/guides/pr-comment-design.mdx | 16 +++--- 4 files changed, 173 insertions(+), 24 deletions(-) diff --git a/crates/mehen-cli/tests/cli_smoke.rs b/crates/mehen-cli/tests/cli_smoke.rs index 7b27d961c..63b66bbe7 100644 --- a/crates/mehen-cli/tests/cli_smoke.rs +++ b/crates/mehen-cli/tests/cli_smoke.rs @@ -2089,6 +2089,67 @@ fn diff_reports_history_only_for_unparsable_files() { ); } +#[test] +fn renamed_base_diagnostics_use_the_source_path() { + let dir = tempfile::tempdir().expect("tempdir"); + init_git_repo(dir.path()); + + let broken = "def f(x):\n if x:\n return 1\n return 0\n\n\ndef broken(:\n"; + write_python(dir.path(), "before.py", broken); + commit_all(dir.path(), "base"); + git_ok(dir.path(), &["tag", "rename-diagnostic-base"]); + + git_ok(dir.path(), &["mv", "before.py", "after.py"]); + write_python( + dir.path(), + "after.py", + "def f(x):\n if x:\n return 1\n return 0\n\n\ndef repaired():\n return 2\n", + ); + commit_all(dir.path(), "rename and repair"); + git_ok(dir.path(), &["tag", "rename-diagnostic-head"]); + + let output = Command::new(env!("CARGO_BIN_EXE_mehen")) + .current_dir(dir.path()) + .args([ + "diff", + "--from", + "rename-diagnostic-base", + "--to", + "rename-diagnostic-head", + "--metrics", + "cognitive", + "--output-format", + "json", + ]) + .env_remove("GITHUB_ACTIONS") + .env_remove("GITHUB_EVENT_NAME") + .env_remove("GITHUB_BASE_REF") + .env_remove("GITHUB_SHA") + .env_remove("GITHUB_REPOSITORY") + .output() + .expect("failed to run mehen diff"); + assert!( + output.status.success(), + "base diagnostics are advisory: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let value: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("diff output must be JSON"); + let errors = value["analysis_errors"] + .as_array() + .expect("analysis_errors array"); + let base_error = errors + .iter() + .find(|record| record["side"].as_str() == Some("base")) + .unwrap_or_else(|| panic!("expected base diagnostic: {errors:?}")); + assert_eq!( + base_error["path"].as_str(), + Some("before.py"), + "the base-side diagnostic must link to the path that exists at the base revision" + ); +} + #[test] fn split_rename_baselines_reject_blocked_source_analyses() { // The split-rename baseline staging re-analyzes the *source* blob @@ -2156,4 +2217,14 @@ fn split_rename_baselines_reject_blocked_source_analyses() { Some(0.0), "partial source statics staged into the synthetic baseline: {metric:?}" ); + assert_eq!( + metric["baseline_unavailable"].as_bool(), + Some(true), + "a blocked source parse must not turn the synthetic baseline into a measured zero: {metric:?}" + ); + assert_eq!( + metric["delta"].as_f64(), + Some(0.0), + "an unavailable synthetic baseline cannot support a trend: {metric:?}" + ); } diff --git a/crates/mehen-cli/tests/coverage_cli.rs b/crates/mehen-cli/tests/coverage_cli.rs index 55055686f..78f00b8f7 100644 --- a/crates/mehen-cli/tests/coverage_cli.rs +++ b/crates/mehen-cli/tests/coverage_cli.rs @@ -679,6 +679,78 @@ fn diff_renders_one_sided_coverage_as_measurement_change_and_omits_unmeasured() assert!(lone_row.contains('\u{2013}'), "expected – cell: {lone_row}"); } +#[test] +fn diff_marks_missing_coverage_unavailable_when_the_other_side_cannot_parse() { + let dir = tempfile::tempdir().unwrap(); + init_git_repo(dir.path()); + + write( + dir.path(), + "head-broken.py", + "HEAD_MARKER = 1\n\ndef head_ok():\n return 1\n", + ); + write( + dir.path(), + "base-broken.py", + "BASE_MARKER = 2\n\ndef base_broken(:\n", + ); + commit_all(dir.path(), "base"); + git_ok(dir.path(), &["tag", "parse-coverage-base"]); + + write( + dir.path(), + "head-broken.py", + "HEAD_MARKER = 1\n\ndef head_broken(:\n", + ); + write( + dir.path(), + "base-broken.py", + "BASE_MARKER = 2\n\ndef base_ok():\n return 2\n", + ); + commit_all(dir.path(), "head"); + git_ok(dir.path(), &["tag", "parse-coverage-head"]); + + write( + dir.path(), + "base.info", + "TN:\nSF:head-broken.py\nDA:1,1\nend_of_record\n", + ); + write( + dir.path(), + "head.info", + "TN:\nSF:base-broken.py\nDA:1,1\nend_of_record\n", + ); + + let output = mehen_diff( + dir.path(), + &[ + "--from", + "parse-coverage-base", + "--to", + "parse-coverage-head", + "--metrics", + "cognitive,coverage.line", + "--coverage=head.info", + "--base-coverage=base.info", + "--output-format", + "json", + ], + ); + let json = json_stdout(&output); + + let head_broken = coverage_line_metric(&json, "head-broken.py") + .unwrap_or_else(|| panic!("missing base coverage measurement: {json}")); + assert_eq!(head_broken["baseline"], 100.0, "{head_broken}"); + assert_eq!(head_broken["current_unavailable"], true); + assert_eq!(head_broken["delta"], 0.0); + + let base_broken = coverage_line_metric(&json, "base-broken.py") + .unwrap_or_else(|| panic!("missing head coverage measurement: {json}")); + assert_eq!(base_broken["current"], 100.0, "{base_broken}"); + assert_eq!(base_broken["baseline_unavailable"], true); + assert_eq!(base_broken["delta"], 0.0); +} + #[test] fn diff_new_and_deleted_files_keep_honest_coverage_cells() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/mehen-engine/src/diff.rs b/crates/mehen-engine/src/diff.rs index 36c31adca..989a243d5 100644 --- a/crates/mehen-engine/src/diff.rs +++ b/crates/mehen-engine/src/diff.rs @@ -1175,6 +1175,7 @@ fn run_diff_inner( // Renamed files carry the baseline under their old path — both // the baseline blob and the baseline history live there. let base_path = cf.source_path.as_deref().unwrap_or(cf.path.as_path()); + let base_utf8_path = Utf8PathBuf::try_from(base_path.to_path_buf()).ok(); // No analyzer for a recognized language (the owning crate is // feature-gated off in this build): static columns are @@ -1196,8 +1197,12 @@ fn run_diff_inner( .into_iter() .flatten() { + let diagnostic_path = match side { + DiffSide::Base => base_utf8_path.as_ref().unwrap_or(utf8_path), + DiffSide::Head => utf8_path, + }; analysis_errors.push(AnalysisErrorRecord { - path: utf8_path.clone(), + path: diagnostic_path.clone(), side, diagnostics: vec![ParseDiagnostic::warning( "engine.analyzer_unavailable", @@ -1233,9 +1238,9 @@ fn run_diff_inner( let mut analyze = |bytes: Vec, side: DiffSide| -> Option { let analyzer = analyzer.as_deref()?; - let side_label = match side { - DiffSide::Base => "baseline", - DiffSide::Head => "current", + let (side_label, side_path) = match side { + DiffSide::Base => ("baseline", base_utf8_path.as_ref().unwrap_or(utf8_path)), + DiffSide::Head => ("current", utf8_path), }; let text = match String::from_utf8(bytes) { Ok(text) => text, @@ -1246,31 +1251,31 @@ fn run_diff_inner( ); log::warn!( "{} ({side_label}): {}: {}", - cf.path.display(), + side_path, diagnostic.code, diagnostic.message ); analysis_errors.push(AnalysisErrorRecord { - path: utf8_path.clone(), + path: side_path.clone(), side, diagnostics: vec![diagnostic], }); return None; } }; - let source = SourceFile::new(utf8_path.clone(), *language, text); + let source = SourceFile::new(side_path.clone(), *language, text); let analysis = match analyzer.analyze(&source, &analysis_config) { Ok(a) => a, Err(err) => { let diagnostic = ParseDiagnostic::error("analysis.error", err.to_string()); log::error!( "{} ({side_label}): {}: {}", - cf.path.display(), + side_path, diagnostic.code, diagnostic.message ); analysis_errors.push(AnalysisErrorRecord { - path: utf8_path.clone(), + path: side_path.clone(), side, diagnostics: vec![diagnostic], }); @@ -1281,13 +1286,13 @@ fn run_diff_inner( match diag.severity { DiagnosticSeverity::Warning => log::warn!( "{} ({side_label}): {}: {}", - cf.path.display(), + side_path, diag.code, diag.message ), DiagnosticSeverity::Error | DiagnosticSeverity::Fatal => log::error!( "{} ({side_label}): {}: {}", - cf.path.display(), + side_path, diag.code, diag.message ), @@ -1295,7 +1300,7 @@ fn run_diff_inner( } if !analysis.diagnostics.is_empty() { analysis_errors.push(AnalysisErrorRecord { - path: utf8_path.clone(), + path: side_path.clone(), side, diagnostics: analysis.diagnostics.clone(), }); @@ -1628,16 +1633,13 @@ fn run_diff_inner( if !base_measured && !head_measured { return None; } - ( - baseline_space.is_some() && !base_measured, - current_space.is_some() && !head_measured, - ) + (!is_new && !base_measured, !is_deleted && !head_measured) } else if sel.name.starts_with("history.") { let baseline_history_missing = - matches!(histories.as_ref(), Some((None, _))) && !is_new; + matches!(histories.as_ref(), Some((None, _))) && !is_new_row; ( baseline_history_missing - || (!is_new + || (baseline_space.is_some() && !history_metrics::selector_available( sel.name, baseline_composites, diff --git a/docs/guides/pr-comment-design.mdx b/docs/guides/pr-comment-design.mdx index ab2e44f02..8de99e909 100644 --- a/docs/guides/pr-comment-design.mdx +++ b/docs/guides/pr-comment-design.mdx @@ -14,19 +14,23 @@ the [GitHub Action](/guides/github-action). This page is the user-facing referen When a PR touches one or more Markdown files, `mehen diff` emits a **Documentation Metrics** section below the existing source-code table. -When an analyzer reports a diagnostic, the Action also appends an **Analysis diagnostics** table. -This section is factual and mechanically derived: file, revision side, severity, stable reason -code, and analyzer message. Exact duplicate diagnostics are collapsed so parser recovery does not -repeat the same row. +When an analyzer reports a diagnostic, `mehen diff --output-format markdown` and the Action append +an **Analysis diagnostics** table. This section is factual and mechanically derived: file, +revision side, severity, stable reason code, and analyzer message. Exact duplicate diagnostics are +collapsed so parser recovery does not repeat the same row. ```markdown -### Analysis diagnostics +## Analysis diagnostics | File | Side | Severity | Diagnostic | |---|---|---|---| -| [internal/config/rules.go](…) | head | error | `go.syntax_error`: tree-sitter error node at line 347 | +| internal/config/rules.go | head | error | `go.syntax_error`: tree-sitter error node at line 347 | ``` +The direct CLI renderer uses the level-2 heading and escaped plain paths shown above. The Action +nests the section under its source-code report with a level-3 heading and links paths to the +corresponding base or head revision. + An error or fatal diagnostic makes that side's static metrics unavailable, but does not hide parseable files or fail the Action by default. Strict repositories opt in with `fail-on-analysis-error: true`. From 2f0d4464e76904c782e75776d6029557b1d140fd Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Thu, 27 Aug 2026 14:10:28 +0200 Subject: [PATCH 3/3] fix: address PR review comment Resolve explicit diff refs to commit IDs for report links while retaining event SHAs for PR coverage retrieval. Addresses: https://github.com/ophi-dev/mehen/pull/261#discussion_r3858081109 --- scripts/github-action.mjs | 59 +++++++++++++++++++--- scripts/github-action.test.mjs | 90 ++++++++++++++++++++++++++++++++-- 2 files changed, 140 insertions(+), 9 deletions(-) diff --git a/scripts/github-action.mjs b/scripts/github-action.mjs index 1a4a4e0f4..7d139ac56 100644 --- a/scripts/github-action.mjs +++ b/scripts/github-action.mjs @@ -1108,7 +1108,34 @@ function parseDiffJson(stdout) { } } -function readGithubContext() { +function resolveGitCommit(revision, cwd = process.cwd()) { + const trimmed = String(revision ?? "").trim(); + if (!trimmed) { + return ""; + } + const result = spawnSync( + "git", + [ + "rev-parse", + "--verify", + "--quiet", + "--end-of-options", + `${trimmed}^{commit}`, + ], + { + cwd, + encoding: "utf8", + maxBuffer: 1024 * 1024, + }, + ); + if (result.error || result.status !== 0) { + return ""; + } + const resolved = result.stdout.trim(); + return /^[0-9a-f]{40,64}$/i.test(resolved) ? resolved : ""; +} + +function readGithubContext(cwd = process.cwd()) { let payload = {}; const eventPath = process.env.GITHUB_EVENT_PATH; if (eventPath && fs.existsSync(eventPath)) { @@ -1116,12 +1143,24 @@ function readGithubContext() { } const pullRequest = payload.pull_request; + const eventHeadSha = pullRequest?.head?.sha || process.env.GITHUB_SHA || ""; + const eventBaseSha = pullRequest?.base?.sha || ""; + const configuredFrom = input("FROM").trim(); + const configuredTo = input("TO").trim(); return { eventName: process.env.GITHUB_EVENT_NAME || "", repository: process.env.GITHUB_REPOSITORY || "", - sha: pullRequest?.head?.sha || process.env.GITHUB_SHA || "", - baseSha: pullRequest?.base?.sha || "", - baseLabel: pullRequest?.base?.ref || input("FROM").trim() || "base", + // Event SHAs stay authoritative for PR-scoped coverage retrieval. + sha: eventHeadSha, + baseSha: eventBaseSha, + // Report links follow the exact refs passed to `mehen diff`. + headRevision: configuredTo + ? resolveGitCommit(configuredTo, cwd) + : eventHeadSha, + baseRevision: configuredFrom + ? resolveGitCommit(configuredFrom, cwd) + : eventBaseSha, + baseLabel: configuredFrom || pullRequest?.base?.ref || "base", prNumber: pullRequest?.number || payload.number || null, token: input("GITHUB_TOKEN").trim() || process.env.GITHUB_TOKEN || "", }; @@ -1293,7 +1332,10 @@ function renderMarkdown( body += "| File | Side | Severity | Diagnostic |\n"; body += "|---|---|---|---|\n"; for (const row of analysisRows) { - const revision = row.side === "base" ? context.baseSha : context.sha; + const revision = + row.side === "base" + ? context.baseRevision || context.baseSha + : context.headRevision || context.sha; const side = row.side === "base" ? context.baseLabel || "base" : "head"; const line = row.line ? ` at line ${row.line}` : ""; @@ -1323,7 +1365,11 @@ function renderFooter(version) { return `${FOOTER_PREFIX}${versionSuffix} ${FOOTER_SUFFIX}`; } -function renderFile(filePath, context, revision = context.sha) { +function renderFile( + filePath, + context, + revision = context.headRevision || context.sha, +) { const escaped = escapeCell(filePath); if (!context.repository || !revision) { return escaped; @@ -1685,6 +1731,7 @@ export { parseThresholds, parseVersionOutput, pickBaseArtifact, + readGithubContext, renderMarkdown, renderFooter, unionMetricColumns, diff --git a/scripts/github-action.test.mjs b/scripts/github-action.test.mjs index d46102073..eca6dd65f 100644 --- a/scripts/github-action.test.mjs +++ b/scripts/github-action.test.mjs @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -28,6 +29,7 @@ import { parseThresholds, parseVersionOutput, pickBaseArtifact, + readGithubContext, renderFooter, renderMarkdown, unionMetricColumns, @@ -127,8 +129,10 @@ test("renderMarkdown reports analysis diagnostics without hiding other results", { eventName: "pull_request", repository: "wharflab/tally", - sha: "head-sha", - baseSha: "base-sha", + sha: "event-head-sha", + baseSha: "event-base-sha", + headRevision: "analyzed-head-sha", + baseRevision: "analyzed-base-sha", baseLabel: "main", }, new Map(), @@ -148,15 +152,95 @@ test("renderMarkdown reports analysis diagnostics without hiding other results", }, ], }, + { + path: "internal/config/old-rules.go", + side: "base", + diagnostics: [ + { + severity: "error", + code: "go.syntax_error", + message: "tree-sitter error node at line 12", + }, + ], + }, ], ); assert.ok(markdown.includes("### Analysis diagnostics")); assert.ok(markdown.includes("go.syntax_error")); assert.ok(markdown.includes("internal/config/rules.go")); - assert.ok(markdown.includes("head-sha")); + assert.ok(markdown.includes("analyzed-head-sha")); + assert.ok(markdown.includes("analyzed-base-sha")); + assert.ok(!markdown.includes("event-head-sha")); + assert.ok(!markdown.includes("event-base-sha")); assert.ok(markdown.includes("No metric changes detected.")); }); +test("readGithubContext resolves explicit analysis refs separately from event SHAs", () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "mehen-action-context-")); + const gitEnv = { + ...process.env, + GIT_AUTHOR_NAME: "Mehen Test", + GIT_AUTHOR_EMAIL: "test@mehen.invalid", + GIT_COMMITTER_NAME: "Mehen Test", + GIT_COMMITTER_EMAIL: "test@mehen.invalid", + }; + const git = (...args) => + execFileSync("git", args, { cwd: repo, env: gitEnv, encoding: "utf8" }).trim(); + git("init", "-q", "-b", "main"); + fs.writeFileSync(path.join(repo, "sample.txt"), "base\n", "utf8"); + git("add", "sample.txt"); + git("commit", "-q", "-m", "base"); + const base = git("rev-parse", "HEAD"); + fs.writeFileSync(path.join(repo, "sample.txt"), "head\n", "utf8"); + git("commit", "-q", "-am", "head"); + const head = git("rev-parse", "HEAD"); + + const eventPath = path.join(repo, "event.json"); + fs.writeFileSync( + eventPath, + JSON.stringify({ + number: 261, + pull_request: { + number: 261, + base: { ref: "main", sha: "event-base-sha" }, + head: { sha: "event-head-sha" }, + }, + }), + "utf8", + ); + + const names = [ + "GHA_MEHEN_FROM", + "GHA_MEHEN_TO", + "GITHUB_EVENT_PATH", + "GITHUB_EVENT_NAME", + "GITHUB_REPOSITORY", + ]; + const saved = new Map(names.map((name) => [name, process.env[name]])); + try { + process.env.GHA_MEHEN_FROM = "HEAD~1"; + process.env.GHA_MEHEN_TO = "HEAD"; + process.env.GITHUB_EVENT_PATH = eventPath; + process.env.GITHUB_EVENT_NAME = "pull_request"; + process.env.GITHUB_REPOSITORY = "ophi-dev/mehen"; + + const context = readGithubContext(repo); + assert.equal(context.baseSha, "event-base-sha"); + assert.equal(context.sha, "event-head-sha"); + assert.equal(context.baseRevision, base); + assert.equal(context.headRevision, head); + assert.equal(context.baseLabel, "HEAD~1"); + } finally { + for (const [name, value] of saved) { + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } + } + } +}); + test("isGateFailureReport requires the explicit threshold_violations signal", () => { assert.equal( isGateFailureReport(