diff --git a/README.md b/README.md index 333a742..f072518 100644 --- a/README.md +++ b/README.md @@ -263,13 +263,18 @@ $ agentwise scan . --supply-chain Use the bundled composite action for release installs, safer argument handling, and source-mode self-tests: ```yaml -- uses: brandonwise/agentwise@v1 +- id: agentwise + uses: brandonwise/agentwise@v1 with: path: ./Agent Configs/.mcp.json format: sarif output: ./reports/agentwise.sarif fail-on: high +- run: | + echo "Score: ${{ steps.agentwise.outputs.score }} (${{ steps.agentwise.outputs.grade }})" + echo "Critical findings: ${{ steps.agentwise.outputs.critical_count }}" + - uses: github/codeql-action/upload-sarif@v3 with: sarif_file: ./reports/agentwise.sarif @@ -279,6 +284,7 @@ Notes: - Paths and output files with spaces are supported. - `install-mode: source` builds the checked-out action source instead of downloading a release. Useful for testing action changes in pull requests. - The `--fail-on` threshold still gates the job when findings meet or exceed the selected severity. +- The action emits structured outputs for `score`, `grade`, severity counts, scan counts, and requested report path/format, and it writes a Markdown job summary to the workflow run. If you prefer the manual path, this still works: @@ -299,6 +305,7 @@ If you prefer the manual path, this still works: ```bash agentwise scan . # Colorized terminal output (default) agentwise scan . --format json # JSON for scripting and pipelines +agentwise scan . --summary-output scan-summary.json # JSON summary sidecar without full finding bodies agentwise scan . --format sarif # SARIF for GitHub Code Scanning agentwise scan . --format html --output report.html # Dark-themed HTML report agentwise scan . --format markdown # Markdown for PRs/Notion/Confluence diff --git a/action.yml b/action.yml index 2fbb190..cacff80 100644 --- a/action.yml +++ b/action.yml @@ -5,6 +5,44 @@ branding: icon: 'shield' color: 'blue' +outputs: + score: + description: 'Numeric security score (0-100)' + value: ${{ steps.scan.outputs.score }} + grade: + description: 'Letter grade derived from the score' + value: ${{ steps.scan.outputs.grade }} + findings_total: + description: 'Total findings across all severities' + value: ${{ steps.scan.outputs.findings_total }} + critical_count: + description: 'Critical finding count' + value: ${{ steps.scan.outputs.critical_count }} + high_count: + description: 'High finding count' + value: ${{ steps.scan.outputs.high_count }} + medium_count: + description: 'Medium finding count' + value: ${{ steps.scan.outputs.medium_count }} + low_count: + description: 'Low finding count' + value: ${{ steps.scan.outputs.low_count }} + configs_scanned: + description: 'Configuration files scanned' + value: ${{ steps.scan.outputs.configs_scanned }} + servers_scanned: + description: 'Servers scanned' + value: ${{ steps.scan.outputs.servers_scanned }} + suppressed_count: + description: 'Findings suppressed by baseline rules' + value: ${{ steps.scan.outputs.suppressed_count }} + report_format: + description: 'Requested report format' + value: ${{ steps.scan.outputs.report_format }} + report_path: + description: 'Requested report output path, if one was provided' + value: ${{ steps.scan.outputs.report_path }} + inputs: path: description: 'Path to scan (file or directory)' @@ -49,7 +87,8 @@ runs: INPUT_VERSION: ${{ inputs.version }} run: bash "$GITHUB_ACTION_PATH/scripts/action-install.sh" - - name: Run agentwise scan + - id: scan + name: Run agentwise scan shell: bash env: INPUT_PATH: ${{ inputs.path }} diff --git a/scripts/action-scan.sh b/scripts/action-scan.sh index c96a397..3bed8d0 100644 --- a/scripts/action-scan.sh +++ b/scripts/action-scan.sh @@ -9,6 +9,7 @@ set -euo pipefail : "${INPUT_SUPPLY_CHAIN:=false}" cmd=(agentwise scan "$INPUT_PATH" --format "$INPUT_FORMAT") +summary_path="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/agentwise-action-summary-$$.json" if [[ -n "$INPUT_FAIL_ON" ]]; then cmd+=(--fail-on "$INPUT_FAIL_ON") @@ -42,7 +43,73 @@ case "$INPUT_SUPPLY_CHAIN" in ;; esac +cmd+=(--summary-output "$summary_path") + printf 'Running:' printf ' %q' "${cmd[@]}" printf '\n' + +set +e "${cmd[@]}" +status=$? +set -e + +if [[ -f "$summary_path" ]]; then + python3 - "$summary_path" "$status" <<'PY' +import json +import os +import sys + +summary_path = sys.argv[1] +status = int(sys.argv[2]) + +with open(summary_path, encoding="utf-8") as fh: + data = json.load(fh) + +summary = data.get("summary", {}) +outputs = { + "score": str(data.get("score", "")), + "grade": str(data.get("grade", "")), + "findings_total": str(summary.get("total", "")), + "critical_count": str(summary.get("critical", "")), + "high_count": str(summary.get("high", "")), + "medium_count": str(summary.get("medium", "")), + "low_count": str(summary.get("low", "")), + "configs_scanned": str(data.get("configs_scanned", "")), + "servers_scanned": str(data.get("servers_scanned", "")), + "suppressed_count": str(data.get("suppressed_count", "")), + "report_format": os.environ.get("INPUT_FORMAT", ""), + "report_path": os.environ.get("INPUT_OUTPUT", ""), +} + +github_output = os.environ.get("GITHUB_OUTPUT") +if github_output: + with open(github_output, "a", encoding="utf-8") as fh: + for key, value in outputs.items(): + fh.write(f"{key}={value}\n") + +github_summary = os.environ.get("GITHUB_STEP_SUMMARY") +if github_summary: + status_label = "passed" if status == 0 else "failed" + with open(github_summary, "a", encoding="utf-8") as fh: + fh.write("### agentwise summary\n\n") + fh.write(f"- Status: {status_label}\n") + fh.write(f"- Score: {outputs['score']} ({outputs['grade']})\n") + fh.write(f"- Findings: {outputs['findings_total']} total\n\n") + fh.write("| Metric | Value |\n| --- | ---: |\n") + for label, key in [ + ("Configs scanned", "configs_scanned"), + ("Servers scanned", "servers_scanned"), + ("Critical", "critical_count"), + ("High", "high_count"), + ("Medium", "medium_count"), + ("Low", "low_count"), + ("Suppressed", "suppressed_count"), + ]: + fh.write(f"| {label} | {outputs[key]} |\n") + if outputs["report_path"]: + fh.write(f"\nReport path: `{outputs['report_path']}`\n") +PY +fi + +exit "$status" diff --git a/src/main.rs b/src/main.rs index fd014dd..6d4718b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,6 +47,10 @@ enum Commands { #[arg(long)] output: Option, + /// Optional JSON summary file path (counts, score, grade, no finding bodies) + #[arg(long)] + summary_output: Option, + /// Exit with code 1 if findings at this severity or above are found #[arg(long, value_parser = ["critical", "high", "medium", "low"])] fail_on: Option, @@ -187,6 +191,7 @@ async fn main() { path, format, output, + summary_output, fail_on, live, offline, @@ -247,6 +252,15 @@ async fn main() { let rendered_output = report::render(&result, output_format); + if let Some(summary_path) = summary_output { + let summary_output = report::render_summary(&result); + if let Err(e) = std::fs::write(&summary_path, summary_output) { + eprintln!("Failed to write summary to {}: {}", summary_path, e); + std::process::exit(1); + } + eprintln!("Summary written to {}", summary_path); + } + if let Some(output_path) = output { if let Err(e) = std::fs::write(&output_path, &rendered_output) { eprintln!("Failed to write report to {}: {}", output_path, e); diff --git a/src/report/json.rs b/src/report/json.rs index a84a331..03bebb9 100644 --- a/src/report/json.rs +++ b/src/report/json.rs @@ -15,18 +15,30 @@ struct JsonReport<'a> { } #[derive(Serialize)] -struct Summary { - critical: usize, - high: usize, - medium: usize, - low: usize, - total: usize, +struct SummaryReport<'a> { + version: &'static str, + configs_scanned: usize, + servers_scanned: usize, + score: i32, + grade: &'a str, + duration_ms: u64, + suppressed_count: usize, + summary: Summary, } -pub fn render(result: &ScanResult) -> String { +#[derive(Serialize, Clone, Copy)] +pub struct Summary { + pub critical: usize, + pub high: usize, + pub medium: usize, + pub low: usize, + pub total: usize, +} + +fn build_summary(result: &ScanResult) -> Summary { use crate::rules::Severity; - let summary = Summary { + Summary { critical: result .findings .iter() @@ -48,8 +60,10 @@ pub fn render(result: &ScanResult) -> String { .filter(|f| f.severity == Severity::Low) .count(), total: result.findings.len(), - }; + } +} +pub fn render(result: &ScanResult) -> String { let report = JsonReport { version: env!("CARGO_PKG_VERSION"), configs_scanned: result.configs_scanned, @@ -59,7 +73,22 @@ pub fn render(result: &ScanResult) -> String { duration_ms: result.duration_ms, suppressed_count: result.suppressed_count, findings: &result.findings, - summary, + summary: build_summary(result), + }; + + serde_json::to_string_pretty(&report).unwrap() +} + +pub fn render_summary(result: &ScanResult) -> String { + let report = SummaryReport { + version: env!("CARGO_PKG_VERSION"), + configs_scanned: result.configs_scanned, + servers_scanned: result.servers_scanned, + score: result.score, + grade: &result.grade, + duration_ms: result.duration_ms, + suppressed_count: result.suppressed_count, + summary: build_summary(result), }; serde_json::to_string_pretty(&report).unwrap() @@ -117,4 +146,34 @@ mod tests { let parsed: serde_json::Value = serde_json::from_str(&output).unwrap(); assert_eq!(parsed["summary"]["total"], 0); } + + #[test] + fn test_summary_output_excludes_findings() { + let result = ScanResult { + findings: vec![Finding { + rule_id: "AW-004".to_string(), + severity: Severity::High, + title: "Secret".to_string(), + message: "Plaintext secret".to_string(), + fix: "Move to env vars".to_string(), + config_file: "test.json".to_string(), + server_name: "secrets".to_string(), + source: None, + epss: None, + sub_items: None, + }], + configs_scanned: 1, + servers_scanned: 1, + score: 90, + grade: "A".to_string(), + duration_ms: 2, + osv_stats: None, + suppressed_count: 0, + }; + + let output = render_summary(&result); + let parsed: serde_json::Value = serde_json::from_str(&output).unwrap(); + assert_eq!(parsed["summary"]["high"], 1); + assert!(parsed.get("findings").is_none()); + } } diff --git a/src/report/mod.rs b/src/report/mod.rs index 0341421..eafff4e 100644 --- a/src/report/mod.rs +++ b/src/report/mod.rs @@ -38,3 +38,7 @@ pub fn render(result: &ScanResult, format: OutputFormat) -> String { OutputFormat::Html => html::render(result), } } + +pub fn render_summary(result: &ScanResult) -> String { + json::render_summary(result) +} diff --git a/tests/action_smoke.sh b/tests/action_smoke.sh index 0b5afb7..1bfdec8 100644 --- a/tests/action_smoke.sh +++ b/tests/action_smoke.sh @@ -10,6 +10,10 @@ export RUNNER_TEMP="$tmpdir/runner temp" mkdir -p "$RUNNER_TEMP" export GITHUB_PATH="$tmpdir/github-path" : > "$GITHUB_PATH" +export GITHUB_OUTPUT="$tmpdir/github-output" +: > "$GITHUB_OUTPUT" +export GITHUB_STEP_SUMMARY="$tmpdir/github-step-summary.md" +: > "$GITHUB_STEP_SUMMARY" export INPUT_INSTALL_MODE="source" export INPUT_VERSION="latest" @@ -41,6 +45,29 @@ export INPUT_SUPPLY_CHAIN="false" bash "$repo_root/scripts/action-scan.sh" -python3 -c 'import json,sys; data=json.load(open(sys.argv[1])); assert data["servers_scanned"] == 1, data; assert data["score"] > 50, data' "$report_path" +python3 -c ' +import json, pathlib, sys +data = json.load(open(sys.argv[1])) +assert data["servers_scanned"] == 1, data +assert data["score"] > 50, data +outputs = {} +for line in pathlib.Path(sys.argv[2]).read_text().splitlines(): + if "=" in line: + key, value = line.split("=", 1) + outputs[key] = value +assert outputs["report_format"] == "json", outputs +assert outputs["report_path"] == sys.argv[1], outputs +assert outputs["configs_scanned"] == "1", outputs +assert outputs["servers_scanned"] == "1", outputs +assert int(outputs["score"]) == data["score"], outputs +assert int(outputs["findings_total"]) == data["summary"]["total"], outputs +assert int(outputs["critical_count"]) == data["summary"]["critical"], outputs +assert int(outputs["high_count"]) == data["summary"]["high"], outputs +assert int(outputs["medium_count"]) == data["summary"]["medium"], outputs +assert int(outputs["low_count"]) == data["summary"]["low"], outputs +summary = pathlib.Path(sys.argv[3]).read_text() +assert "agentwise summary" in summary.lower(), summary +assert "Score:" in summary, summary +' "$report_path" "$GITHUB_OUTPUT" "$GITHUB_STEP_SUMMARY" echo "action smoke passed" diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index b3f8857..4873016 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -56,6 +56,7 @@ fn test_scan_help() { let stdout = String::from_utf8_lossy(&output.stdout); assert!(stdout.contains("--format")); assert!(stdout.contains("--fail-on")); + assert!(stdout.contains("--summary-output")); } // ── Scanning files ────────────────────────────────────────── @@ -188,6 +189,48 @@ fn test_json_output() { assert!(parsed["score"].is_number()); } +#[test] +fn test_summary_output_writes_json_metadata() { + let summary_path = temp_path("summary", "json"); + let output = agentwise() + .args([ + "scan", + "testdata/vulnerable-mcp.json", + "--summary-output", + summary_path.to_str().unwrap(), + ]) + .output() + .unwrap(); + assert!(output.status.success()); + + let raw = fs::read_to_string(&summary_path).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&raw).unwrap(); + assert!(parsed["score"].is_number()); + assert!(parsed["summary"]["total"].as_u64().unwrap() > 0); + assert!(parsed.get("findings").is_none()); +} + +#[test] +fn test_summary_output_persists_when_fail_on_triggers() { + let summary_path = temp_path("summary-fail", "json"); + let output = agentwise() + .args([ + "scan", + "testdata/vulnerable-mcp.json", + "--summary-output", + summary_path.to_str().unwrap(), + "--fail-on", + "critical", + ]) + .output() + .unwrap(); + assert!(!output.status.success()); + + let raw = fs::read_to_string(&summary_path).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&raw).unwrap(); + assert!(parsed["summary"]["critical"].as_u64().unwrap() > 0); +} + #[test] fn test_inspect_json_output() { let output = agentwise()