Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,10 @@ inputs:
description: "Fail the action when any configured threshold is exceeded."
required: false
default: "true"
fail-on-analysis-error:
description: "Fail the action after publishing the report when any file has an error- or fatal-severity analysis diagnostic."
required: false
default: "false"
comment-title:
description: "Markdown heading used for the sticky pull request comment."
required: false
Expand Down Expand Up @@ -113,6 +117,9 @@ outputs:
violations:
description: "Number of threshold violations (delta thresholds from the `thresholds` input plus repository `mehen.toml` breaches)."
value: ${{ steps.run.outputs.violations }}
analysis_errors:
description: "Number of unique error- or fatal-severity analysis diagnostics."
value: ${{ steps.run.outputs.analysis_errors }}
report_json:
description: "Path to the JSON diff report produced by mehen."
value: ${{ steps.run.outputs.report_json }}
Expand Down Expand Up @@ -206,6 +213,7 @@ runs:
GHA_MEHEN_GITHUB_TOKEN: ${{ inputs.github-token || github.token }}
GHA_MEHEN_THRESHOLDS: ${{ inputs.thresholds }}
GHA_MEHEN_FAIL_ON_THRESHOLD: ${{ inputs.fail-on-threshold }}
GHA_MEHEN_FAIL_ON_ANALYSIS_ERROR: ${{ inputs.fail-on-analysis-error }}
GHA_MEHEN_COMMENT_TITLE: ${{ inputs.comment-title }}
GHA_MEHEN_COVERAGE_FILES: ${{ inputs.coverage-files }}
GHA_MEHEN_COVERAGE_BASE_SOURCE: ${{ inputs.coverage-base-source }}
Expand Down
136 changes: 133 additions & 3 deletions crates/mehen-cli/tests/cli_smoke.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1962,7 +1962,7 @@ fn diff_reports_history_only_for_unparsable_files() {
// A malformed head file has no trustworthy static metrics: the
// row must show zeros for statics (not partial values blended
// into history composites) while history reads real values. The
// run still exits non-zero for the blocking diagnostics.
// diagnostic is advisory unless strict analysis gating is requested.
let dir = tempfile::tempdir().expect("tempdir");
init_git_repo(dir.path());

Expand Down Expand Up @@ -1994,8 +1994,9 @@ fn diff_reports_history_only_for_unparsable_files() {
.output()
.expect("failed to run mehen diff");
assert!(
!output.status.success(),
"blocking diagnostics must fail the run"
output.status.success(),
"per-file diagnostics are advisory by default: stderr={}",
String::from_utf8_lossy(&output.stderr)
);

let value: serde_json::Value =
Expand Down Expand Up @@ -2028,6 +2029,125 @@ fn diff_reports_history_only_for_unparsable_files() {
metric("history.churn.relative")["current"].as_f64(),
Some(0.0)
);
assert_eq!(
metric("cognitive")["current_unavailable"].as_bool(),
Some(true)
);
let analysis_errors = value["analysis_errors"]
.as_array()
.expect("analysis_errors must be an array");
assert_eq!(analysis_errors.len(), 1, "{analysis_errors:?}");
assert_eq!(analysis_errors[0]["path"].as_str(), Some("broken.py"));
assert_eq!(analysis_errors[0]["side"].as_str(), Some("head"));
assert!(
analysis_errors[0]["diagnostics"]
.as_array()
.expect("diagnostics array")
.iter()
.any(|diagnostic| {
matches!(diagnostic["severity"].as_str(), Some("error" | "fatal"))
&& diagnostic["code"]
.as_str()
.is_some_and(|code| code.contains("syntax") || code.contains("parse"))
}),
"{analysis_errors:?}"
);

let strict = Command::new(env!("CARGO_BIN_EXE_mehen"))
.current_dir(dir.path())
.args([
"diff",
"--from",
"broken-cli-base",
"--to",
"broken-cli-head",
"--metrics",
"cognitive",
"--output-format",
"json",
"--fail-on-analysis-error",
])
.env_remove("GITHUB_ACTIONS")
.env_remove("GITHUB_EVENT_NAME")
.env_remove("GITHUB_BASE_REF")
.env_remove("GITHUB_SHA")
.env_remove("GITHUB_REPOSITORY")
.output()
.expect("failed to run strict mehen diff");
assert_eq!(
strict.status.code(),
Some(1),
"strict analysis gating must fail after emitting the report"
);
let strict_value: serde_json::Value =
serde_json::from_slice(&strict.stdout).expect("strict diff output must be JSON");
assert!(
strict_value["analysis_errors"]
.as_array()
.is_some_and(|errors| !errors.is_empty()),
"{strict_value}"
);
}

#[test]
fn renamed_base_diagnostics_use_the_source_path() {
let dir = tempfile::tempdir().expect("tempdir");
init_git_repo(dir.path());

let broken = "def f(x):\n if x:\n return 1\n return 0\n\n\ndef broken(:\n";
write_python(dir.path(), "before.py", broken);
commit_all(dir.path(), "base");
git_ok(dir.path(), &["tag", "rename-diagnostic-base"]);

git_ok(dir.path(), &["mv", "before.py", "after.py"]);
write_python(
dir.path(),
"after.py",
"def f(x):\n if x:\n return 1\n return 0\n\n\ndef repaired():\n return 2\n",
);
commit_all(dir.path(), "rename and repair");
git_ok(dir.path(), &["tag", "rename-diagnostic-head"]);

let output = Command::new(env!("CARGO_BIN_EXE_mehen"))
.current_dir(dir.path())
.args([
"diff",
"--from",
"rename-diagnostic-base",
"--to",
"rename-diagnostic-head",
"--metrics",
"cognitive",
"--output-format",
"json",
])
.env_remove("GITHUB_ACTIONS")
.env_remove("GITHUB_EVENT_NAME")
.env_remove("GITHUB_BASE_REF")
.env_remove("GITHUB_SHA")
.env_remove("GITHUB_REPOSITORY")
.output()
.expect("failed to run mehen diff");
assert!(
output.status.success(),
"base diagnostics are advisory: {}",
String::from_utf8_lossy(&output.stderr)
);

let value: serde_json::Value =
serde_json::from_slice(&output.stdout).expect("diff output must be JSON");
let errors = value["analysis_errors"]
.as_array()
.expect("analysis_errors array");
let base_error = errors
.iter()
.find(|record| record["side"].as_str() == Some("base"))
.unwrap_or_else(|| panic!("expected base diagnostic: {errors:?}"));
assert_eq!(
base_error["path"].as_str(),
Some("before.py"),
"the base-side diagnostic must link to the path that exists at the base revision"
);
}

#[test]
Expand Down Expand Up @@ -2097,4 +2217,14 @@ fn split_rename_baselines_reject_blocked_source_analyses() {
Some(0.0),
"partial source statics staged into the synthetic baseline: {metric:?}"
);
assert_eq!(
metric["baseline_unavailable"].as_bool(),
Some(true),
"a blocked source parse must not turn the synthetic baseline into a measured zero: {metric:?}"
);
assert_eq!(
metric["delta"].as_f64(),
Some(0.0),
"an unavailable synthetic baseline cannot support a trend: {metric:?}"
);
}
24 changes: 14 additions & 10 deletions crates/mehen-cli/tests/config_thresholds.rs
Original file line number Diff line number Diff line change
Expand Up @@ -488,11 +488,10 @@ fn diff_passes_when_head_within_thresholds() {
}

#[test]
fn diff_analysis_failure_outranks_the_threshold_gate() {
fn diff_analysis_errors_do_not_suppress_the_threshold_gate() {
// One file crosses the threshold, another has a hard syntax error:
// the run must fail as an analysis failure — JSON without the
// machine-readable gate signal — so CI consumers do not publish a
// partial report as an ordinary gate failure.
// the parseable file's gate remains authoritative, while the broken
// file is disclosed separately and contributes no partial statics.
let dir = tempfile::tempdir().expect("tempdir");
git_ok(dir.path(), &["init", "-q", "-b", "main"]);
git_ok(dir.path(), &["config", "commit.gpgsign", "false"]);
Expand Down Expand Up @@ -525,10 +524,16 @@ fn diff_analysis_failure_outranks_the_threshold_gate() {
assert_eq!(output.status.code(), Some(1));
let value: serde_json::Value =
serde_json::from_slice(&output.stdout).expect("JSON still emitted");
assert!(
value.get("threshold_violations").is_none(),
"an analysis failure must withhold the gate signal: {value}"
);
let violations = value["threshold_violations"]
.as_array()
.expect("parseable threshold breaches must remain visible");
assert_eq!(violations.len(), 1, "{value}");
assert_eq!(violations[0]["path"].as_str(), Some("sample.py"));
let errors = value["analysis_errors"]
.as_array()
.expect("analysis_errors must be present");
assert_eq!(errors.len(), 1, "{value}");
assert_eq!(errors[0]["path"].as_str(), Some("broken.py"));
}

#[test]
Expand Down Expand Up @@ -565,8 +570,7 @@ fn diff_json_output_still_emitted_before_threshold_failure() {
.expect("machine output must stay parseable when the gate fails");
assert!(value["source_code"].is_array());
// The explicit gate signal machine consumers (e.g. the GitHub
// Action) use to distinguish a quality-gate exit from an analysis
// failure, which also exits 1 but without this key.
// Action) use to identify a configured quality-gate exit.
let violations = value["threshold_violations"]
.as_array()
.expect("gate failures must carry threshold_violations");
Expand Down
72 changes: 72 additions & 0 deletions crates/mehen-cli/tests/coverage_cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -679,6 +679,78 @@ fn diff_renders_one_sided_coverage_as_measurement_change_and_omits_unmeasured()
assert!(lone_row.contains('\u{2013}'), "expected – cell: {lone_row}");
}

#[test]
fn diff_marks_missing_coverage_unavailable_when_the_other_side_cannot_parse() {
let dir = tempfile::tempdir().unwrap();
init_git_repo(dir.path());

write(
dir.path(),
"head-broken.py",
"HEAD_MARKER = 1\n\ndef head_ok():\n return 1\n",
);
write(
dir.path(),
"base-broken.py",
"BASE_MARKER = 2\n\ndef base_broken(:\n",
);
commit_all(dir.path(), "base");
git_ok(dir.path(), &["tag", "parse-coverage-base"]);

write(
dir.path(),
"head-broken.py",
"HEAD_MARKER = 1\n\ndef head_broken(:\n",
);
write(
dir.path(),
"base-broken.py",
"BASE_MARKER = 2\n\ndef base_ok():\n return 2\n",
);
commit_all(dir.path(), "head");
git_ok(dir.path(), &["tag", "parse-coverage-head"]);

write(
dir.path(),
"base.info",
"TN:\nSF:head-broken.py\nDA:1,1\nend_of_record\n",
);
write(
dir.path(),
"head.info",
"TN:\nSF:base-broken.py\nDA:1,1\nend_of_record\n",
);

let output = mehen_diff(
dir.path(),
&[
"--from",
"parse-coverage-base",
"--to",
"parse-coverage-head",
"--metrics",
"cognitive,coverage.line",
"--coverage=head.info",
"--base-coverage=base.info",
"--output-format",
"json",
],
);
let json = json_stdout(&output);

let head_broken = coverage_line_metric(&json, "head-broken.py")
.unwrap_or_else(|| panic!("missing base coverage measurement: {json}"));
assert_eq!(head_broken["baseline"], 100.0, "{head_broken}");
assert_eq!(head_broken["current_unavailable"], true);
assert_eq!(head_broken["delta"], 0.0);

let base_broken = coverage_line_metric(&json, "base-broken.py")
.unwrap_or_else(|| panic!("missing head coverage measurement: {json}"));
assert_eq!(base_broken["current"], 100.0, "{base_broken}");
assert_eq!(base_broken["baseline_unavailable"], true);
assert_eq!(base_broken["delta"], 0.0);
}

#[test]
fn diff_new_and_deleted_files_keep_honest_coverage_cells() {
let dir = tempfile::tempdir().unwrap();
Expand Down
8 changes: 5 additions & 3 deletions crates/mehen-core/src/diagnostic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ use crate::span::SourceSpan;
///
/// Per the rewrite plan §9.3:
/// - `Warning`: recoverable, exit 0 unless thresholds fail.
/// - `Error`: analysis incomplete; `mehen metrics` exits 1, `mehen diff`
/// records under `analysis_errors`.
/// - `Fatal`: IO/toolchain/invariant failure; exit 1.
/// - `Error`: analysis incomplete; `mehen metrics` exits 1, while
/// `mehen diff` records the affected side under `analysis_errors`
/// and only exits 1 when strict analysis gating is requested.
/// - `Fatal`: toolchain/invariant failure; follows the same command-level
/// policy as `Error`.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum DiagnosticSeverity {
Expand Down
Loading
Loading