diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ced60b0..397a00d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,10 +31,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: ${{ env.PYTHON_VERSION }} @@ -52,10 +52,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: ${{ env.PYTHON_VERSION }} @@ -66,7 +66,6 @@ jobs: - name: Run mypy run: mypy src/redops --ignore-missing-imports - continue-on-error: true test: name: Test (Python ${{ matrix.python-version }}) @@ -78,15 +77,15 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: ${{ matrix.python-version }} - name: Cache pip - uses: actions/cache@v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }} @@ -154,10 +153,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: ${{ env.PYTHON_VERSION }} @@ -173,7 +172,6 @@ jobs: REDIS_PORT: 6379 run: | python -m pytest tests/integration/ -v --tb=short - continue-on-error: true build: name: Build Package @@ -181,10 +179,10 @@ jobs: needs: [test] steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: ${{ env.PYTHON_VERSION }} @@ -209,7 +207,7 @@ jobs: needs: [lint] steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f7adade..c63a0a1 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -25,10 +25,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: "3.12" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4486396..4f47661 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: version: ${{ steps.version.outputs.version }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: fetch-depth: 0 @@ -30,7 +30,7 @@ jobs: run: echo "version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: "3.12" @@ -64,10 +64,10 @@ jobs: if: "!contains(github.ref, 'alpha') && !contains(github.ref, 'beta')" steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: "3.12" @@ -81,7 +81,6 @@ jobs: with: password: ${{ secrets.PYPI_API_TOKEN }} skip-existing: true - continue-on-error: true publish-docker: name: Publish Docker Image @@ -89,7 +88,7 @@ jobs: needs: [release] steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up QEMU uses: docker/setup-qemu-action@v4 diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml index 3820ffd..5aca22b 100644 --- a/.github/workflows/secret-scan.yml +++ b/.github/workflows/secret-scan.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: fetch-depth: 0 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index e04acb3..aaf7a28 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -21,10 +21,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: "3.12" @@ -36,21 +36,19 @@ jobs: - name: Run Safety check run: safety check --full-report - continue-on-error: true - name: Run pip-audit run: pip-audit - continue-on-error: true semgrep: name: Semgrep Analysis runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: "3.12" @@ -84,7 +82,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Build Docker image run: | @@ -119,7 +117,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: fetch-depth: 0 @@ -136,10 +134,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: "3.12" diff --git a/.gitignore b/.gitignore index 8105358..105c5ac 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,15 @@ temp/ .env .env.* .env.local + +# Security: prevent secret files from being committed +.env +.env.local +.envrc +*.pem +*.key +id_rsa* +id_ecdsa* +id_ed25519* +*.p12 +secrets.json diff --git a/AreteDriver/RedOPS/pyproject.toml b/AreteDriver/RedOPS/pyproject.toml deleted file mode 100644 index 1458e45..0000000 --- a/AreteDriver/RedOPS/pyproject.toml +++ /dev/null @@ -1,14 +0,0 @@ -[build-system] -requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" - -[project] -name = "redops" -version = "1.0.0" -description = "Modular Cybersecurity Tooling with AI-Driven Summarization" -authors = [{name = "AreteDriver"}] -license = {file = "LICENSE"} -dependencies = [ - "openai>=1.0.0", - "pytest>=7.0.0" -] diff --git a/README.md b/README.md index abf9f0d..ad28f41 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ +> **STATUS: FROZEN as of 2026-06-22** +> This repository is no longer actively developed. See [AreteDriver/notes/decisions/2026-04-21-portfolio-triage.md](https://github.com/AreteDriver/notes/blob/main/decisions/2026-04-21-portfolio-triage.md) for context. +> Archived for reference. No new deploys or feature work. +> +> --- +> + # RedOPS Framework [![CI Pipeline](https://github.com/AreteDriver/RedOPS/actions/workflows/ci.yml/badge.svg)](https://github.com/AreteDriver/RedOPS/actions/workflows/ci.yml) diff --git a/ROADMAP.md b/ROADMAP.md index 294105b..68711e5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -14,6 +14,22 @@ --- +## Priority 0 — Gate (blocks any offensive Active Chain release) + +> Folded in from the qwen-audited roadmap (2026-05-30). The Active Chain — wireless +> evil-twin + deauth, subnet recon, CVE check, and an autonomous Ollama agent under +> `modules/active/` + `modules/ai/` — is coded, tested, and committed, but **unreleased +> and undocumented**. It must not ship until this gate clears. Authorization is a review, +> not a code flag. + +- [ ] **Legal-boundary review** — `docs/legal-boundaries.md` + SECURITY.md: jurisdiction, authorized-use definition, explicit will/won't-do list for deauth + evil-twin. +- [ ] **Authorization mechanism** — scope assertion + recorded operator consent that every `modules/active/` module checks before executing (not just a flag). Add a test asserting each active module refuses to run absent an authorized-target assertion. +- [ ] **Misuse threat model** — model RedOPS pointed at an unauthorized network; document the technical controls that make casual misuse hard. +- [ ] **Tested egress enforcement** — a test that attempts cloud egress during an active-chain run and asserts it is blocked (local Ollama only). Enforced, not asserted. +- [ ] **Operator runbook + smoke test** — mock authorized-engagement walkthrough; full-chain smoke test on lab hardware (Alfa AWUS036NHA + Kali) before tagging the release. + +--- + ## Priority 1 — Critical (Do Now) ### Security and Reliability diff --git a/SECURITY.md b/SECURITY.md index ec419d2..4753834 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,7 +12,8 @@ If you discover a security vulnerability, please report it responsibly: 1. **Do not** open a public issue -2. Email **jamesyng79@gmail.com** with: +2. Use GitHub **Private Vulnerability Reporting** (enabled on this repository) + or email **security@redops.dev** with: - Description of the vulnerability - Steps to reproduce - Potential impact @@ -31,6 +32,28 @@ This project uses: RedOPS is an offensive security tool intended for **authorized security testing only**. The tool itself is designed to find vulnerabilities in target systems — security reports should focus on vulnerabilities in RedOPS's own code, not in its intended functionality. +## Active Chain Authorization Requirements + +The **Active Chain** (`modules/active/`) — wireless deauthentication, evil-twin +access points, ARP scanning, port scanning, and autonomous vulnerability chaining +— is **gated** by the following requirements: + +1. **Recorded operator consent** is mandatory. Every active module calls + `assert_active_authorized(ctx)`, which raises `ActiveAuthorizationError` + if no valid authorization exists. +2. **Explicit target assertion** is required. The operator must name the exact + target(s) they claim to own or have permission to test. +3. **Egress blocking** is enforced. `block_external_egress()` prevents cloud API + calls during active chain execution, ensuring local-only operation. +4. **Legal review** is required before any release containing active modules. + See `docs/legal-boundaries.md` for jurisdiction analysis and authorized-use + definition. +5. **Operator runbook** must be followed. See `docs/operator-runbook.md` for + step-by-step authorized engagement procedures. + +Vulnerabilities in the authorization or egress enforcement mechanisms are +**critical** and in scope for this security policy. + ## Scope The following are in scope for security reports: @@ -38,8 +61,10 @@ The following are in scope for security reports: - Credential exposure or mishandling - Authentication bypasses in the web interface - Dependency vulnerabilities with known exploits +- Bypass of `assert_active_authorized()` or `block_external_egress()` +- Injection or mutation of audit log entries Out of scope: - Functionality that is working as designed (scanning, recon, etc.) -- Denial of service +- Denial of service against the target (by design for active modules) - Social engineering diff --git a/config/pipelines/active_chain.json b/config/pipelines/active_chain.json index 4808ba1..a2b409b 100644 --- a/config/pipelines/active_chain.json +++ b/config/pipelines/active_chain.json @@ -8,6 +8,18 @@ "warning": "Authorized home lab use only" }, "steps": [ + { + "name": "Record Authorization", + "module": "active.authorization.record_authorization_from_params", + "params": { + "operator": "{{operator}}", + "target_assertion": "{{target}}", + "consent_text": "I am authorized to perform active security testing on the stated target. This is my own network, a designated lab environment, or a system for which I have explicit written permission.", + "duration_hours": 4 + }, + "enabled": true, + "continue_on_error": false + }, { "name": "Validate Scope", "module": "compliance.scope_guard.validate_scope", diff --git a/config/pipelines/bug_bounty_recon.json b/config/pipelines/bug_bounty_recon.json new file mode 100644 index 0000000..181eec7 --- /dev/null +++ b/config/pipelines/bug_bounty_recon.json @@ -0,0 +1,82 @@ +{ + "metadata": { + "name": "Bug Bounty Recon", + "description": "Fast reconnaissance pipeline optimized for bug bounty programs: subdomains, tech stack, exposures", + "version": "1.0", + "author": "RedOps", + "tags": ["bug-bounty", "recon", "subdomains", "exposure"] + }, + "steps": [ + { + "name": "Validate Scope", + "module": "compliance.scope_guard.validate_scope", + "params": {}, + "enabled": true, + "continue_on_error": false + }, + { + "name": "Profile Domain", + "module": "recon.domains.profile_domain", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Enumerate Subdomains", + "module": "recon.subdomain_enum.enumerate_subdomains", + "params": { + "wordlist": "subdomains-top1million-5000.txt" + }, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Certificate Transparency", + "module": "recon.cert_transparency.query_ct_logs", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Fingerprint Technology Stack", + "module": "recon.tech_stack.fingerprint", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "ASN Lookup", + "module": "recon.asn_lookup.lookup_asn", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Score Risks", + "module": "intel.risk_scoring.score_risks", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Generate Markdown Report", + "module": "reporting.markdown_report.generate_technical_report", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Export JSON Findings", + "module": "reporting.export.export_all", + "params": { + "format": "json" + }, + "enabled": true, + "continue_on_error": true + } + ], + "config": { + "timeout": 300, + "strict_scope": true + } +} diff --git a/config/pipelines/compliance_assessment.json b/config/pipelines/compliance_assessment.json new file mode 100644 index 0000000..95e169d --- /dev/null +++ b/config/pipelines/compliance_assessment.json @@ -0,0 +1,87 @@ +{ + "metadata": { + "name": "Compliance Assessment", + "description": "Compliance-focused pipeline mapping findings to SOC 2, ISO 27001, and PCI DSS controls", + "version": "1.0", + "author": "RedOps", + "tags": ["compliance", "soc2", "iso27001", "pci-dss", "audit"] + }, + "steps": [ + { + "name": "Validate Scope", + "module": "compliance.scope_guard.validate_scope", + "params": {}, + "enabled": true, + "continue_on_error": false + }, + { + "name": "Audit Pipeline Start", + "module": "compliance.audit_log.audit_pipeline_start", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Corporate Exposure Scan", + "module": "corp_assessment.exposure_scan.scan_exposure", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Profile Domain", + "module": "recon.domains.profile_domain", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Fingerprint Technology Stack", + "module": "recon.tech_stack.fingerprint", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Map Compliance Controls", + "module": "compliance.compliance_map.map_controls", + "params": { + "frameworks": ["soc2", "iso27001", "pci-dss"] + }, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Score Risks", + "module": "intel.risk_scoring.score_risks", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Generate Compliance Report", + "module": "reporting.oscal_report.generate_oscal", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Generate Executive Summary", + "module": "reporting.executive_report.generate_executive", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Audit Pipeline End", + "module": "compliance.audit_log.audit_pipeline_end", + "params": {}, + "enabled": true, + "continue_on_error": true + } + ], + "config": { + "timeout": 600, + "strict_scope": true + } +} diff --git a/config/pipelines/incident_response.json b/config/pipelines/incident_response.json new file mode 100644 index 0000000..1e010c9 --- /dev/null +++ b/config/pipelines/incident_response.json @@ -0,0 +1,106 @@ +{ + "metadata": { + "name": "Incident Response Triage", + "description": "Rapid triage pipeline for incident response: threat intel, artifact analysis, and containment guidance", + "version": "1.0", + "author": "RedOps", + "tags": ["incident-response", "triage", "threat-intel", "artifacts"] + }, + "steps": [ + { + "name": "Validate Scope", + "module": "compliance.scope_guard.validate_scope", + "params": {}, + "enabled": true, + "continue_on_error": false + }, + { + "name": "Audit Pipeline Start", + "module": "compliance.audit_log.audit_pipeline_start", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Threat Intel Lookup", + "module": "threat_intel.threat_intel.lookup_indicators", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "VirusTotal Enrichment", + "module": "intel.virustotal_intel.query_virustotal", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "AbuseIPDB Check", + "module": "threat_intel.abuseipdb.check_ip", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "GreyNoise Context", + "module": "threat_intel.greynoise.query_greynoise", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "MalwareBazaar Hash Lookup", + "module": "threat_intel.malwarebazaar.query_hash", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Extract EXIF Metadata", + "module": "metadata.exif.extract_exif", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Analyze Document Artifacts", + "module": "metadata.documents.extract_metadata", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Correlate Findings", + "module": "analysis.correlation_engine.correlate_findings", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Generate Triage Report", + "module": "reporting.markdown_report.generate_technical_report", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Export STIX Bundle", + "module": "intel.stix_export.export_stix", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Audit Pipeline End", + "module": "compliance.audit_log.audit_pipeline_end", + "params": {}, + "enabled": true, + "continue_on_error": true + } + ], + "config": { + "timeout": 600, + "strict_scope": false + } +} diff --git a/config/pipelines/quickstart.json b/config/pipelines/quickstart.json new file mode 100644 index 0000000..ebf8a57 --- /dev/null +++ b/config/pipelines/quickstart.json @@ -0,0 +1,43 @@ +{ + "metadata": { + "name": "Quick Start", + "description": "Zero-config quickstart pipeline. Run 'redops scan example.com' and get useful output in under 60 seconds.", + "version": "1.0", + "author": "RedOps", + "tags": ["quickstart", "zero-config", "beginner"] + }, + "steps": [ + { + "name": "Profile Domain", + "module": "recon.domains.profile_domain", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Fingerprint Technology Stack", + "module": "recon.tech_stack.fingerprint", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Score Risks", + "module": "intel.risk_scoring.score_risks", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Generate Quick Report", + "module": "reporting.markdown_report.generate_exec_summary", + "params": {}, + "enabled": true, + "continue_on_error": true + } + ], + "config": { + "timeout": 60, + "strict_scope": false + } +} diff --git a/config/pipelines/threat_intel_hunt.json b/config/pipelines/threat_intel_hunt.json new file mode 100644 index 0000000..e6badc0 --- /dev/null +++ b/config/pipelines/threat_intel_hunt.json @@ -0,0 +1,78 @@ +{ + "metadata": { + "name": "Threat Intelligence Hunt", + "description": "Proactive threat hunting: IP reputation, malware hashes, passive DNS, and dark web exposure checks", + "version": "1.0", + "author": "RedOps", + "tags": ["threat-intel", "reputation", "passive-dns", "malware", "dark-web"] + }, + "steps": [ + { + "name": "IP Reputation Check", + "module": "threat_intel.abuseipdb.check_ip", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "GreyNoise Context", + "module": "threat_intel.greynoise.query_ip", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Passive DNS Lookup", + "module": "threat_intel.passivedns.lookup_domain", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Malware Bazaar Hash Check", + "module": "threat_intel.malwarebazaar.query_hash", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "ThreatFox IOC Check", + "module": "threat_intel.threatfox.query_ioc", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "URLhaus URL Check", + "module": "threat_intel.urlhaus.check_url", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "AlienVault OTX Pulse", + "module": "threat_intel.alienvault.query_otx", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Score Risks", + "module": "intel.risk_scoring.score_risks", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Generate Markdown Report", + "module": "reporting.markdown_report.generate_technical_report", + "params": {}, + "enabled": true, + "continue_on_error": true + } + ], + "config": { + "timeout": 300, + "strict_scope": true + } +} diff --git a/config/pipelines/web_app_assessment.json b/config/pipelines/web_app_assessment.json new file mode 100644 index 0000000..cad3feb --- /dev/null +++ b/config/pipelines/web_app_assessment.json @@ -0,0 +1,79 @@ +{ + "metadata": { + "name": "Web Application Assessment", + "description": "OWASP-aligned web application security assessment: headers, SSL/TLS, exposed panels, and technology fingerprinting", + "version": "1.0", + "author": "RedOps", + "tags": ["web-app", "owasp", "ssl", "headers", "exposure"] + }, + "steps": [ + { + "name": "Profile Domain", + "module": "recon.domains.profile_domain", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "SSL/TLS Configuration Audit", + "module": "recon.ssl_analyzer.analyze_ssl", + "params": { + "check_heartbleed": true, + "check_poodle": true + }, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Security Headers Check", + "module": "recon.http_headers.analyze_headers", + "params": { + "check_csp": true, + "check_hsts": true + }, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Fingerprint Technology Stack", + "module": "recon.tech_stack.fingerprint", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Detect Exposed Admin Panels", + "module": "recon.exposed_panels.detect", + "params": { + "panels": ["phpmyadmin", "wp-admin", "adminer", "cPanel"] + }, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Score Risks", + "module": "intel.risk_scoring.score_risks", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Generate SARIF Report", + "module": "reporting.sarif_report.generate_sarif", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Generate Markdown Report", + "module": "reporting.markdown_report.generate_technical_report", + "params": {}, + "enabled": true, + "continue_on_error": true + } + ], + "config": { + "timeout": 300, + "strict_scope": true + } +} diff --git a/config/pipelines/wireless_recon.json b/config/pipelines/wireless_recon.json new file mode 100644 index 0000000..b4d4ac0 --- /dev/null +++ b/config/pipelines/wireless_recon.json @@ -0,0 +1,86 @@ +{ + "metadata": { + "name": "Wireless Reconnaissance", + "description": "Authorized wireless reconnaissance: scan access points, analyze RF data, and generate findings", + "version": "1.0", + "author": "RedOps", + "tags": ["wireless", "rf", "wifi", "authorized-only"] + }, + "steps": [ + { + "name": "Validate Scope", + "module": "compliance.scope_guard.validate_scope", + "params": {}, + "enabled": true, + "continue_on_error": false + }, + { + "name": "Record Active Authorization", + "module": "active.authorization.record_authorization", + "params": {}, + "enabled": true, + "continue_on_error": false + }, + { + "name": "Enable Monitor Mode", + "module": "active.wireless.monitor.enable_monitor_mode", + "params": {}, + "enabled": true, + "continue_on_error": false + }, + { + "name": "Scan Access Points", + "module": "active.wireless.scan.scan_access_points", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Parse Airodump Output", + "module": "rf.parsers.airodump.parse", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Parse Horst Data", + "module": "rf.parsers.horst.parse", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Score Risks", + "module": "intel.risk_scoring.score_risks", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Generate Technical Report", + "module": "reporting.markdown_report.generate_technical_report", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Disable Monitor Mode", + "module": "active.wireless.monitor.disable_monitor_mode", + "params": {}, + "enabled": true, + "continue_on_error": true + }, + { + "name": "Audit Pipeline End", + "module": "compliance.audit_log.audit_pipeline_end", + "params": {}, + "enabled": true, + "continue_on_error": true + } + ], + "config": { + "timeout": 300, + "strict_scope": true, + "requires_authorization": true + } +} diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index 68148f7..7c7799b 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -216,7 +216,7 @@ services: restart: unless-stopped redis-commander: - image: rediscommander/redis-commander:latest + image: rediscommander/redis-commander:0.8.1 container_name: redops-redis-commander environment: - REDIS_HOSTS=local:redis:6379 diff --git a/docker-compose.yml b/docker-compose.yml index fbffcd0..84c7ded 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: build: context: . dockerfile: Dockerfile - image: redops:latest + image: redops:1.5.0 container_name: redops volumes: # Mount output directory for reports diff --git a/docs/legal-boundaries.md b/docs/legal-boundaries.md new file mode 100644 index 0000000..223eb02 --- /dev/null +++ b/docs/legal-boundaries.md @@ -0,0 +1,96 @@ +# Legal Boundaries for RedOPS Active Chain + +> **WARNING**: The Active Chain (`modules/active/`) contains offensive security +capabilities including wireless deauthentication, evil-twin access points, ARP +scanning, port scanning, and autonomous vulnerability chaining. These capabilities +can disrupt networks and may violate local laws if used without authorization. + +## Authorized Use Definition + +RedOPS Active Chain modules **may only be used** under one of the following +conditions: + +1. **Your own network or property** — You own the infrastructure being tested. +2. **Designated lab environment** — An isolated network explicitly provisioned + for security research (e.g. a home lab with no production traffic, no guest + access, and no upstream connectivity to third-party networks). +3. **Explicit written permission** — You hold a signed scope agreement, + statement of work, or formal authorization letter from the owner or authorized + representative of the target network. + +Using RedOPS Active Chain capabilities on any network that does not meet one of +the three conditions above is **unauthorized use** and is **strictly prohibited**. + +## What RedOPS Will Do (Active Chain) + +When authorized and executed, the Active Chain can: + +- Passively scan for nearby wireless access points and connected clients. +- Clone a legitimate access point (evil twin) to attract client connections. +- Send 802.11 deauthentication frames to disconnect clients from a legitimate AP. +- Perform ARP scanning on a local subnet to discover live hosts. +- Run nmap service and version scans against discovered hosts. +- Cross-reference discovered services against known CVEs. +- Orchestrate an autonomous ReAct agent that chains the above steps. + +## What RedOPS Will NOT Do + +RedOPS Active Chain **will never**: + +- Attack, scan, or interfere with networks outside the explicitly authorized target. +- Exfiltrate data from captured clients to cloud services (egress is blocked to + non-local endpoints during active execution). +- Operate without recorded operator consent (every active module refuses execution + without a valid `ActiveAuthorization`). +- Run on non-root accounts where root privileges are required for the operation. +- Mask its activity or evade detection — all actions are logged to the audit trail. + +## Jurisdiction + +Laws governing wireless interception, network disruption, and unauthorized access +vary by jurisdiction. The following are examples and not legal advice: + +- **United States**: 18 U.S.C. § 1030 (Computer Fraud and Abuse Act) and + 47 U.S.C. § 605 (Wiretap Act) may apply to unauthorized network access and + interception of communications. Deauthentication attacks may be prosecuted + as denial-of-service or interference with communications. +- **European Union**: Directive 2013/40/EU (Attacks against Information Systems) + and national implementations criminalize unauthorized access and interference. +- **United Kingdom**: Computer Misuse Act 1990 sections 1–3 criminalize + unauthorized access, unauthorized acts with intent, and unauthorized acts + causing damage. + +**You are responsible** for understanding and complying with the laws in your +jurisdiction. The RedOPS maintainers provide this tool for authorized security +professionals and researchers; we do not condone illegal use. + +## Will / Won't Do List + +| Capability | Will Do (Authorized) | Won't Do (Prohibited) | +|---|---|---| +| Deauth flood | On your own AP or lab AP with consent | Coffee shop, airport, neighbor's AP, corporate AP without SOW | +| Evil twin | Your own SSID or isolated lab SSID | Clone a third-party AP to harvest credentials | +| ARP scan | Your own subnet or lab subnet | Scan a corporate subnet you do not own | +| Port scan | Authorized target with written permission | Internet-wide scanning or scanning without scope | +| CVE check | As part of an authorized assessment | Weaponizing findings against unauthorized targets | +| Autonomous agent | Within authorized lab with operator monitoring | Unattended execution on production networks | + +## Enforcement + +The codebase enforces these boundaries through technical controls: + +1. **Authorization gate** — Every `modules/active/` function calls + `assert_active_authorized(ctx)`, which raises `ActiveAuthorizationError` + if no recorded operator consent exists. +2. **Scope guard** — Existing `modules/compliance/scope_guard.py` validates + that targets match allowed domains, IPs, BSSIDs, and subnets. +3. **Egress blocking** — `block_external_egress()` prevents cloud API calls + during active chain execution, ensuring local-only operation. +4. **Audit logging** — Every authorization recording, module execution, and + tool invocation is logged with timestamp, operator, and target. + +## Reporting Misuse + +If you discover RedOPS being used without authorization, or if you find a +vulnerability in the authorization/egress enforcement mechanisms, please report +it responsibly. See `SECURITY.md` for contact details. diff --git a/docs/misuse-threat-model.md b/docs/misuse-threat-model.md new file mode 100644 index 0000000..8706df0 --- /dev/null +++ b/docs/misuse-threat-model.md @@ -0,0 +1,90 @@ +# Misuse Threat Model — RedOPS Active Chain + +**Version**: 1.0.0 +**Date**: 2026-07-20 +**Scope**: Casual misuse of RedOPS `modules/active/` by unauthorized operators + +## Threat Actor Profile + +| Attribute | Description | +|---|---| +| **Actor** | Casual user with limited cybersecurity knowledge | +| **Motivation** | Curiosity, prank, or vague "testing" intent | +| **Skill** | Can clone a GitHub repo and run `pip install` | +| **Access** | Personal laptop, home internet, possibly a USB Wi-Fi adapter | +| **Risk** | Medium — not sophisticated, but the tool automates actions that +are illegal when unauthorized | + +## Threat Scenarios + +### Scenario 1: Coffee Shop Deauth + +**Narrative**: User downloads RedOPS, runs the active chain in a coffee shop to +"see what happens," and deauthenticates patrons from the shop's Wi-Fi. + +**Controls**: +- Root privileges required to put interface in monitor mode and send raw frames. +- `assert_active_authorized()` requires explicit operator consent + target assertion. +- Egress blocking prevents data exfiltration to cloud APIs. +- Audit logs record operator identity and timestamp. + +**Residual Risk**: Determined user can fabricate authorization and run as root. +Mitigated by requiring explicit consent text and target assertion, which creates +psychological friction and legal accountability. + +### Scenario 2: Neighbor Network Scan + +**Narrative**: User points ARP scan and port scan at a neighbor's home network +discovered via wardriving. + +**Controls**: +- `scope_guard.is_subnet_in_scope()` rejects subnets not in the allowed list. +- Strict mode defaults to `True`, so out-of-scope targets are blocked. +- Active authorization requires target assertion, forcing the operator to name + the exact subnet they claim to own. + +**Residual Risk**: User can add neighbor's subnet to scope config. Mitigated by +making scope config editable only via file (not CLI flag) and logging changes. + +### Scenario 3: Autonomous Agent Runaway + +**Narrative**: User starts the ReAct agent on a university network and it +chains from the lab subnet to the campus-wide VLAN. + +**Controls**: +- `block_external_egress()` blocks cloud API calls, keeping the agent local-only. +- Scope guard checks every tool invocation's target against allowed subnets. +- Agent logs every thought/action/observation for post-hoc review. +- Maximum iteration limit (default 10) prevents infinite runaway. + +**Residual Risk**: Agent could pivot within the allowed scope to sensitive systems. +Mitigated by requiring narrow scope assertions and operator monitoring. + +## Technical Control Summary + +| Control | Implementation | Effectiveness | +|---|---|---| +| **Authorization gate** | `assert_active_authorized(ctx)` in every active module | High — blocks accidental execution | +| **Scope guard** | `modules/compliance/scope_guard.py` | High — prevents out-of-scope targeting | +| **Egress blocking** | `modules/active/egress.py` thread-local patches | High — prevents cloud exfiltration | +| **Root requirement** | Subprocess `sudo` calls in wireless/network modules | Medium — OS-level deterrent | +| **Audit logging** | JSONL audit trail with operator + timestamp | Medium — accountability after the fact | +| **Consent text** | Explicit acknowledgment required | Medium — psychological friction | + +## What This Threat Model Does NOT Cover + +- **Insider threat**: A malicious authorized operator with valid credentials. + This requires organizational controls (background checks, dual-control) outside + the scope of the codebase. +- **Supply-chain attack**: Compromised dependency injecting malicious active + modules. This requires dependency pinning and SBOM tracking. +- **Physical security**: Theft of the laptop running RedOPS. This requires + full-disk encryption and screen locks. + +## Recommendations for Operators + +1. Run RedOPS Active Chain **only** on air-gapped or physically isolated networks. +2. Document every authorization in a signed scope agreement stored separately + from the tool. +3. Review audit logs after every session. +4. Report any bypass of authorization/egress controls as a security vulnerability. diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md new file mode 100644 index 0000000..7fe86ea --- /dev/null +++ b/docs/operator-runbook.md @@ -0,0 +1,240 @@ +# RedOPS Active Chain Operator Runbook + +> **Purpose**: Step-by-step guide for authorized operators executing the +> RedOPS Active Chain on a home lab or explicitly authorized target network. + +## Pre-Flight Checklist + +Before running any active module, verify every item below: + +- [ ] I have **written authorization** for the target network, or the network + is my own property / a designated isolated lab. +- [ ] The target network is **physically isolated** from production systems, + guest networks, and internet-facing infrastructure. +- [ ] I have the required **hardware**: + - Alfa AWUS036NHA or compatible monitor-mode-capable USB Wi-Fi adapter + - A second Wi-Fi interface (built-in or USB) for control/management + - Kali Linux or a distribution with `aircrack-ng`, `hostapd`, `dnsmasq`, `nmap`, + `arp-scan`, and Python 3.12+ +- [ ] I have **backups** of any data on the target network. +- [ ] I have informed any **other users** of the lab network that testing is + scheduled and they may experience brief disconnections. +- [ ] I have reviewed `docs/legal-boundaries.md` and confirmed my use case + falls within the authorized-use definition. + +## Environment Setup + +### 1. Hardware Verification + +```bash +# Verify adapter is recognized +lsusb | grep -i rtl8187 + +# Verify monitor mode support +sudo airmon-ng check kill +sudo airmon-ng start wlan1 +iwconfig | grep -i monitor +``` + +Expected output: `Mode:Monitor` for `wlan1mon`. + +### 2. Software Verification + +```bash +# Verify required tools +which airodump-ng hostapd dnsmasq nmap arp-scan +python -c "import scapy.all; print('scapy OK')" +``` + +### 3. Network Isolation Check + +```bash +# Confirm no routes to external networks +ip route | grep default +# Should show only your management interface, not the lab interface +``` + +## Authorization Recording + +The Active Chain **will not execute** without a recorded authorization. Create +one programmatically or via the API: + +```python +from redops.core.context import Context +from redops.modules.active.authorization import record_authorization + +ctx = Context(target="192.168.99.0/24") +record_authorization( + ctx, + operator="your-name", + target_assertion="192.168.99.0/24 (home lab VLAN 99)", + consent_text=( + "I am authorized to perform active security testing on the stated target. " + "This is my own network..." + ), + duration_hours=4, +) +``` + +**Mandatory fields**: +- `operator`: Your identity (name, employee ID, or operator handle). +- `target_assertion`: The exact target(s) you are authorized to test. +- `consent_text`: The full consent text you acknowledge. +- `duration_hours`: How long the authorization remains valid (default 24h). + +## Execution Walkthrough + +### Phase 1: Passive Reconnaissance + +```python +from redops.modules.active.wireless.scan import scan_access_points + +ctx = scan_access_points(ctx, {"duration": 30}) +aps = ctx.get("access_points", []) +print(f"Discovered {len(aps)} access points") +``` + +**Expected behavior**: Passive scan only; no frames are injected. + +### Phase 2: Target Selection + +Select a target AP that you **own or have explicit permission to test**. +Record the BSSID, ESSID, and channel. + +```python +target = aps[0] # Example: select first AP +ctx.add("target_bssid", target["bssid"]) +ctx.add("target_essid", target["essid"]) +ctx.add("target_channel", target["channel"]) +``` + +### Phase 3: Evil Twin Deployment + +```python +from redops.modules.active.wireless.evil_twin import start_evil_twin + +ctx = start_evil_twin(ctx, { + "target_bssid": target["bssid"], + "ap_interface": "wlan0", +}) +``` + +**Warning**: This creates a functional rogue AP. Ensure it is on an isolated +channel and does not interfere with neighboring networks. + +### Phase 4: Deauthentication (Optional) + +```python +from redops.modules.active.wireless.deauth import deauth_flood + +ctx = deauth_flood(ctx, {"duration": 30, "count": 64}) +``` + +**Warning**: This actively disconnects clients from the legitimate AP. Only +run if you have explicit authorization and have warned any legitimate users. + +### Phase 5: Host Discovery + +```python +from redops.modules.active.network.arp_scan import discover_hosts + +ctx = discover_hosts(ctx, {"wait": 15}) +hosts = ctx.get("live_hosts", []) +print(f"Discovered {len(hosts)} live hosts") +``` + +### Phase 6: Port Scanning + +```python +from redops.modules.active.network.port_scan import scan_ports + +ctx = scan_ports(ctx, {"ports": "T:1-1024", "timing": "T4"}) +results = ctx.get("port_scan_results", []) +``` + +### Phase 7: CVE Cross-Reference + +```python +from redops.modules.active.exploit.cve_check import check_cves + +ctx = check_cves(ctx) +findings = ctx.get("cve_findings", []) +``` + +## Teardown + +### 1. Disable Monitor Mode + +```python +from redops.modules.active.wireless.monitor import disable_monitor_mode + +ctx = disable_monitor_mode(ctx) +``` + +### 2. Stop Rogue AP Processes + +```python +import subprocess +subprocess.run(["sudo", "killall", "hostapd", "dnsmasq"], capture_output=True) +``` + +### 3. Restore Network Manager + +```bash +sudo systemctl restart NetworkManager +``` + +### 4. Verify Cleanup + +```bash +# Confirm no monitor interfaces remain +iwconfig | grep -i monitor || echo "No monitor interfaces" +# Confirm no hostapd/dnsmasq processes +pgrep -a hostapd || echo "hostapd not running" +pgrep -a dnsmasq || echo "dnsmasq not running" +``` + +## Lab Hardware Smoke Test + +Before trusting RedOPS on a live engagement, validate the full chain in a +controlled lab: + +| Step | Check | Pass Criteria | +|---|---|---| +| 1 | Adapter in monitor mode | `iwconfig` shows `Mode:Monitor` | +| 2 | AP scan | Discovers ≥1 test AP | +| 3 | Evil twin start | Client can see cloned SSID | +| 4 | Deauth flood | Clients disconnected from original AP within 10s | +| 5 | ARP scan | Discovers all expected lab hosts | +| 6 | Port scan | nmap completes without errors | +| 7 | CVE check | Returns findings for known vulnerable services | +| 8 | Authorization enforcement | `assert_active_authorized` raises without auth | +| 9 | Egress blocking | Cloud API call raises `EgressBlockedError` | +| 10 | Audit log | JSONL file contains all actions with timestamps | + +## Post-Session Review + +1. Export the audit log: + ```bash + cat output/audit.log | jq 'select(.action | startswith("active"))' + ``` +2. Review findings for false positives. +3. Document any deviations from the authorized scope. +4. Archive the session context and authorization record for compliance. + +## Emergency Stop + +If at any point you need to abort: + +```bash +sudo airmon-ng stop wlan1mon +sudo killall -9 hostapd dnsmasq airodump-ng +sudo systemctl restart NetworkManager +``` + +## Support + +For questions about authorized use, scope validation, or authorization +recording, open a discussion (not an issue) in the RedOPS GitHub repository. +For security vulnerabilities in the authorization mechanism itself, see +`SECURITY.md`. diff --git a/pyproject.toml b/pyproject.toml index 948fd4b..09db841 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,8 +49,9 @@ full = [ "dnspython>=2.8.0", "requests>=2.31.0", "jinja2>=3.1.0", - "fpdf2>=2.7.0", + "fpdf2>=2.8.0", "reportlab>=4.0.0", + "tenacity>=8.0.0", "shodan>=1.31.0", "censys>=2.2.0", "click>=8.0.0", @@ -78,6 +79,7 @@ web = [ "sqlalchemy>=2.0.0", "alembic>=1.18.4", "redis>=5.0.0", + "sse-starlette>=2.0.0", ] dev = [ "pytest>=7.0.0", @@ -85,6 +87,8 @@ dev = [ "pytest-asyncio>=0.23.0", "ruff==0.15.15", "mypy>=1.20.0", + "email-validator>=2.0.0", + "reportlab>=4.0.0", ] docs = [ "sphinx>=7.0.0", @@ -136,6 +140,13 @@ exclude_lines = [ show_missing = true skip_covered = true +[tool.mypy] +python_version = "3.10" +strict = true +ignore_missing_imports = true +warn_unreachable = true +show_error_codes = true + [tool.setuptools] package-dir = {"" = "src"} diff --git a/requirements.txt b/requirements.txt index b294d6e..e902dfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # # For development: # pip install -r requirements.txt -# pip install pytest pytest-cov black flake8 mypy +# pip install pytest pytest-cov ruff mypy # ============================================================================ # Core Dependencies (Required) @@ -19,7 +19,6 @@ pydantic>=2.0.0,<3.0.0 # For EXIF extraction from images Pillow>=10.2.0,<13.0.0 -exifread>=3.0.0,<4.0.0 # For document metadata analysis (PDF, Office documents) pypdf>=5.0.0,<7.0.0 @@ -52,10 +51,9 @@ networkx>=3.1,<4.0.0 # ============================================================================ # Development Dependencies (Optional) # ============================================================================ -# Install separately with: pip install pytest pytest-cov black flake8 mypy +# Install separately with: pip install pytest pytest-cov ruff mypy # # pytest>=7.4.0 # Testing framework # pytest-cov>=4.1.0 # Coverage reports -# black>=23.0.0 # Code formatting -# flake8>=6.0.0 # Linting +# ruff>=0.15.0 # Linting and formatting # mypy>=1.5.0 # Type checking diff --git a/roadmap.md b/roadmap.md deleted file mode 100644 index 8c51d76..0000000 --- a/roadmap.md +++ /dev/null @@ -1,117 +0,0 @@ -# RedOPS Roadmap - -**Current version:** 1.5.0 (released 2026-01-26) -**Last updated:** 2026-05-30 -**Audit:** Baseline drafted from repo state, audited locally by qwen2.5:14b (Ollama), folded back in. - -RedOPS is a modular AI-assisted recon, forensics, and exposure-analysis framework -(Python, FastAPI, ~193K LOC, 5072 tests, 80% coverage gate). This roadmap covers the -path from the current released state through the next two minor releases and a 2.0 -horizon. The through-line: a fully-coded **offensive Active Chain is sitting unreleased**, -and the next releases are about shipping it *safely*, not building more. - ---- - -## Where we are - -- **Shipped (v1.5.0):** recon/OSINT, metadata forensics, threat intel (ThreatFox, - MalwareBazaar, AbuseIPDB), threat modeling, reporting, web dashboard, MCP server, - AI-assisted analysis (OpenAI/Anthropic), compliance/governance. -- **Built but UNRELEASED — the Active Chain:** AI-orchestrated active attack sequence - (wireless evil-twin + deauth, subnet recon, CVE cross-reference, autonomous Ollama - agent). ~1047 LOC under `modules/active/` + `modules/ai/`, with `test_wireless.py`, - `test_network.py`, `test_agent.py` and `config/pipelines/active_chain.json`. Committed - to `main` but **absent from CHANGELOG, README, and every tagged release.** -- **In flight:** PR #58 adds Qwen-uncensored model presets to the ReAct agent. -- **Maintenance:** 10 open dependabot/CI PRs (#48–#57); recent commits are dominated by - bumps. Stale remote branch `fix/ci-permissions` (PR #47, already merged) needs deletion. - ---- - -## Milestone 0 — Legal & Safety Gate (BLOCKS v1.6.0) - -> Added from the qwen audit: the prior draft treated authorization as a code feature and -> omitted the review that must precede shipping deauth/evil-twin capability at all. This -> gate blocks the release; it is not optional polish. - -- [ ] Written legal-boundary review: jurisdiction, authorized-use definition, and the - explicit set of actions RedOPS will and will **not** perform. Lands in `SECURITY.md` - + a new `docs/legal-boundaries.md`. -- [ ] Design the authorization mechanism (not just a flag): scope assertion + recorded - operator consent that every active module checks before executing. -- [ ] Threat-model the misuse case (RedOPS used against an unauthorized network) and - document the technical controls that make casual misuse hard. -- [ ] Operator walkthrough: a mock authorized-engagement runbook proving the consent flow - is understood end-to-end (cheap test for the "everyone understands authorized-use" - assumption). - -## Milestone 1 — Ship the Active Chain (v1.6.0) - -- [ ] **Authorization gating in code:** no active module (`deauth`, `evil_twin`, …) fires - without passing the Milestone-0 scope/consent check. Add a test that asserts each - active module refuses to run absent an authorized-target assertion. -- [ ] **Egress enforcement is tested, not asserted:** add a test that attempts cloud egress - during an active-chain run and asserts it is blocked (local Ollama only). -- [ ] CHANGELOG `[Unreleased]` → enumerate every active/ai module added. -- [ ] README "Active Chain" section + update the `❌ What RedOPS Does NOT Do` boundary to - reflect the new capability and its guardrails. Integrate `mobile-wireless-audit-kit.md` - into the active-chain user guide. -- [ ] **CI for active modules:** split hardware/root-dependent tests (wireless injection) - from CI-safe unit tests; document what runs in CI vs. only on the dedicated lab rig. -- [ ] Full-chain smoke test on lab hardware (Alfa AWUS036NHA + Kali), runbook checked in. -- [ ] Tag v1.6.0. - -## Milestone 2 — Harden the AI agent (v1.6.x → v1.7.0) - -> The agent (`modules/ai/agent.py`, `planner.py`, `tools.py`) exists and is being tuned in -> PR #58. This milestone is rails, not construction — and it follows the active-chain -> release, per the audit's ordering note. - -- [ ] Land/triage PR #58 (Qwen presets) first so hardening builds on the final agent shape. -- [ ] Action allow-list + dry-run mode for the agent's tool registry. -- [ ] Human-in-the-loop confirmation gate for any state-changing/offensive tool call. -- [ ] Bounded reasoning loop (max steps + cost/time budget) to prevent runaway chains. -- [ ] **Replay/audit log:** persist every agent decision (input context, chosen action, - result) to a structured log, with a documented post-engagement review checklist. -- [ ] Agent stress/load test under a long attack-surface summary to confirm it neither - stalls nor destabilizes the host (cheap test for the stability assumption). - -## Maintenance lane (continuous, not a milestone) - -> Per the audit, dependabot churn is operational hygiene, not roadmap work. - -- [ ] Delete stale `fix/ci-permissions` remote branch (PR #47 merged). -- [ ] Triage the 10 open dependabot PRs: admin-merge clean minor bumps, skip major-version - jumps (upload-artifact 4→7, checkout 4→6, action-gh-release 2→3) pending review. -- [ ] Enable grouped dependabot updates to cut PR noise going forward. -- [ ] Re-evaluate the coverage exclusions added to meet the 80% gate — confirm they exclude - only genuinely-untestable infra, not real gaps. - -## Horizon — v2.0 themes (not yet committed) - -- Expanded CVE source beyond the hardcoded home-lab list in `cve_check.py` (audit flags this - as higher-value than the SDK work below). -- Plugin SDK maturity for third-party active/recon modules. -- Multi-tenant hardening for the web dashboard + API key model. -- Post-release compliance audit: periodic check that deployed usage matches documented - authorized-use constraints; security review of dependabot-pulled third-party libraries. - ---- - -## Operating constraints - -- **Authorized use only.** Active offensive modules are home-lab / authorized-network scope. -- **AI layer runs local** (Ollama) for the active chain — no cloud egress (enforced + tested, - not assumed — see Milestone 1). -- Tests before commit; conventional commits; 80% coverage gate. - ---- - -## Open assumptions to validate (exploration carry-overs) - -| Assumption | Cheap test | -|---|---| -| Operators understand authorized-use policy | Mock authorized-engagement walkthrough (Milestone 0) | -| Ollama agent stays stable under load | Stress test with a large attack-surface summary (Milestone 2) | -| 80% coverage is enough for active modules | Manual adversarial test pass against the live chain on lab hardware | -| Cloud-egress prevention is robust | Simulate an unauthorized egress attempt and assert it's blocked (Milestone 1) | diff --git a/scripts/audit_bare_except.py b/scripts/audit_bare_except.py new file mode 100644 index 0000000..eb0238f --- /dev/null +++ b/scripts/audit_bare_except.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Audit script for bare 'except Exception' blocks in RedOPS source. + +Run: python scripts/audit_bare_except.py +""" + +import ast +import sys +from pathlib import Path + + +def find_bare_except(filepath: Path) -> list[tuple[int, str]]: + """Find 'except Exception' blocks in a Python file.""" + results = [] + source = filepath.read_text() + tree = ast.parse(source) + + for node in ast.walk(tree): + if isinstance(node, ast.ExceptHandler): + if isinstance(node.type, ast.Name) and node.type.id == "Exception": + line = source.splitlines()[node.lineno - 1] + results.append((node.lineno, line.strip())) + + return results + + +def main() -> int: + src_dir = Path(__file__).parent.parent / "src" / "redops" + total = 0 + files = 0 + + for pyfile in sorted(src_dir.rglob("*.py")): + findings = find_bare_except(pyfile) + if findings: + files += 1 + total += len(findings) + print(f"\n{pyfile.relative_to(src_dir.parent.parent)} ({len(findings)})") + for lineno, line in findings: + print(f" {lineno:4d}: {line}") + + print(f"\n{'=' * 60}") + print(f"Total: {total} bare 'except Exception' blocks in {files} files") + print(f"{'=' * 60}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/redops/analysis/comparison.py b/src/redops/analysis/comparison.py index 595ecfb..6523e8f 100644 --- a/src/redops/analysis/comparison.py +++ b/src/redops/analysis/comparison.py @@ -73,6 +73,22 @@ def _generate_fingerprint(self) -> str: fingerprint_str = "|".join(components) return hashlib.sha256(fingerprint_str.encode()).hexdigest()[:16] + def to_dict(self) -> dict[str, Any]: + """Convert to dictionary for serialization.""" + return { + "id": self.id, + "title": self.title, + "severity": self.severity, + "description": self.description, + "module": self.module, + "category": self.category, + "evidence": self.evidence, + "cvss_score": self.cvss_score, + "cve_ids": self.cve_ids, + "status": self.status, + "fingerprint": self.fingerprint, + } + @dataclass class FindingDiff: @@ -120,6 +136,22 @@ def is_regression(self) -> bool: return new_sev > old_sev return False + def to_dict(self) -> dict[str, Any]: + """Convert diff to dictionary for serialization.""" + result: dict[str, Any] = { + "diff_type": self.diff_type.value, + "finding": self.finding.to_dict(), + "is_improvement": self.is_improvement, + "is_regression": self.is_regression, + } + if self.previous_finding: + result["previous_finding"] = self.previous_finding.to_dict() + if self.changes: + result["changes"] = { + k: {"from": v[0], "to": v[1]} for k, v in self.changes.items() + } + return result + @dataclass class ComparisonResult: @@ -203,9 +235,9 @@ def severity_summary(self, diff_type: DiffType | None = None) -> dict[str, int]: return counts - def to_dict(self) -> dict[str, Any]: + def to_dict(self, include_findings: bool = True) -> dict[str, Any]: """Convert to dictionary for serialization.""" - return { + result = { "baseline_scan_id": self.baseline_scan_id, "current_scan_id": self.current_scan_id, "baseline_date": self.baseline_date.isoformat(), @@ -225,6 +257,13 @@ def to_dict(self) -> dict[str, Any]: "severity_new": self.severity_summary(DiffType.NEW), "severity_resolved": self.severity_summary(DiffType.RESOLVED), } + if include_findings: + result["new_findings"] = [d.to_dict() for d in self.new_findings] + result["resolved_findings"] = [d.to_dict() for d in self.resolved_findings] + result["modified_findings"] = [d.to_dict() for d in self.modified_findings] + result["unchanged_findings"] = [d.to_dict() for d in self.unchanged_findings] + result["regression_findings"] = [d.to_dict() for d in self.regression_findings] + return result class ScanComparator: diff --git a/src/redops/api/auth.py b/src/redops/api/auth.py index d373bbc..058620f 100644 --- a/src/redops/api/auth.py +++ b/src/redops/api/auth.py @@ -83,6 +83,7 @@ class APIKeyWithSecret(APIKeyResponse): # In-memory storage — WARNING: revocations and API keys are lost on restart. # Use a persistent store (database/Redis) for production deployments. _api_keys: dict[str, dict] = {} +_api_key_hash_index: dict[str, str] = {} # hash -> key_id _revoked_tokens: set[str] = set() @@ -230,8 +231,8 @@ def generate_api_key() -> tuple[str, str]: # Create full key with prefix full_key = f"{API_KEY_PREFIX}_{key_secret}" - # Hash for storage - key_hash = hashlib.sha256(full_key.encode()).hexdigest() + # Hash for storage with bcrypt (slow, hardened) + key_hash = bcrypt.hashpw(full_key.encode(), bcrypt.gensalt()).decode() return full_key, key_hash @@ -243,9 +244,9 @@ def hash_api_key(key: str) -> str: key: Full API key Returns: - SHA-256 hash of the key + bcrypt hash of the key """ - return hashlib.sha256(key.encode()).hexdigest() + return bcrypt.hashpw(key.encode(), bcrypt.gensalt()).decode() def create_api_key( @@ -290,6 +291,7 @@ def create_api_key( } _api_keys[key_id] = key_data + _api_key_hash_index[key_hash] = key_id return APIKeyWithSecret( id=key_id, @@ -316,32 +318,40 @@ async def verify_api_key(key: str) -> dict | None: if not key.startswith(f"{API_KEY_PREFIX}_"): return None - key_hash = hash_api_key(key) + # O(1) index lookup: iterate stored hashes and check with bcrypt + for key_hash, key_id in _api_key_hash_index.items(): + try: + if bcrypt.checkpw(key.encode(), key_hash.encode()): + break + except (ValueError, TypeError): + continue + else: + return None - for key_data in _api_keys.values(): - if key_data["key_hash"] == key_hash: - # Check if active - if not key_data["is_active"]: - return None - - # Check expiration - if key_data["expires_at"]: - if datetime.now(timezone.utc) > key_data["expires_at"]: - return None - - # Update last used - key_data["last_used"] = datetime.now(timezone.utc) - - # Return user info - # In production, would look up user from user_id - return { - "id": key_data["user_id"], - "username": f"api_key_{key_data['name']}", - "role": "api", - "scopes": key_data["scopes"], - } - - return None + key_data = _api_keys.get(key_id) + if not key_data: + return None + + # Check if active + if not key_data["is_active"]: + return None + + # Check expiration + if key_data["expires_at"]: + if datetime.now(timezone.utc) > key_data["expires_at"]: + return None + + # Update last used + key_data["last_used"] = datetime.now(timezone.utc) + + # Return user info + # In production, would look up user from user_id + return { + "id": key_data["user_id"], + "username": f"api_key_{key_data['name']}", + "role": "api", + "scopes": key_data["scopes"], + } def list_api_keys(user_id: str) -> list[APIKeyResponse]: diff --git a/src/redops/api/server.py b/src/redops/api/server.py index 317995d..773b503 100644 --- a/src/redops/api/server.py +++ b/src/redops/api/server.py @@ -4,9 +4,14 @@ Provides programmatic access to RedOPS functionality via HTTP API. """ +import hashlib import json +import logging +import os import re +import secrets import threading +import traceback import uuid from dataclasses import dataclass, field from datetime import datetime, timezone @@ -15,6 +20,8 @@ from typing import Callable from urllib.parse import parse_qs, urlparse +logger = logging.getLogger(__name__) + class HTTPMethod(Enum): """HTTP methods.""" @@ -228,17 +235,34 @@ class APIServer: VERSION = "1.0.0" API_PREFIX = "/api/v1" - def __init__(self, host: str = "127.0.0.1", port: int = 8080): + MAX_BODY_SIZE = 1024 * 1024 # 1MB + + # Public paths that do not require authentication + PUBLIC_PATHS = {"/health", "/api/v1/openapi.json"} + + def __init__( + self, + host: str = "127.0.0.1", + port: int = 8080, + api_key: str | None = None, + ): self.host = host self.port = port self.router = Router() self._server: HTTPServer | None = None self._thread: threading.Thread | None = None + # API key for authentication (falls back to env var) + self._api_key = api_key or os.environ.get("REDOPS_API_KEY") + if not self._api_key: + logger.warning( + "SECURITY WARNING: No API key configured. " + "Set REDOPS_API_KEY to secure this API server." + ) + # In-memory storage for demo self._scans: dict[str, dict] = {} self._jobs: dict[str, dict] = {} - self._api_keys: set[str] = set() # Register routes self._register_routes() @@ -1066,8 +1090,38 @@ def _generate_openapi_spec(self) -> dict: }, } + def _authenticate_request(self, request: APIRequest) -> APIResponse | None: + """Authenticate a request via X-API-Key header. + + Returns None if authenticated or path is public. + Returns an error response if authentication fails. + """ + # Public paths do not require authentication + if request.path in self.PUBLIC_PATHS: + return None + + # If no API key is configured, allow all requests (warned in __init__) + if not self._api_key: + return None + + provided_key = request.headers.get("X-API-Key", "") + if secrets.compare_digest(provided_key, self._api_key): + return None + + return self._error_response( + HTTPStatus.UNAUTHORIZED, + "unauthorized", + "Authentication required. Provide a valid X-API-Key header.", + request.request_id, + ) + def handle_request(self, request: APIRequest) -> APIResponse: """Handle an incoming request.""" + # Authenticate + auth_error = self._authenticate_request(request) + if auth_error is not None: + return auth_error + # Match route match = self.router.match(request.path, request.method) @@ -1094,10 +1148,15 @@ def handle_request(self, request: APIRequest) -> APIResponse: try: return match.handler(request) except Exception as e: + logger.error( + "Unhandled exception in API handler: %s\n%s", + e, + traceback.format_exc(), + ) return self._error_response( HTTPStatus.INTERNAL_ERROR, "internal_error", - str(e), + "An internal server error occurred.", request.request_id, ) @@ -1135,11 +1194,26 @@ def _handle(self, method: str) -> None: parsed = urlparse(self.path) query_params = parse_qs(parsed.query) - # Read body if present + # Read body if present (enforce size limit) body = None content_length = self.headers.get("Content-Length") if content_length: - raw_body = self.rfile.read(int(content_length)) + size = int(content_length) + if size > server.MAX_BODY_SIZE: + self.send_response(413) + self.send_header("Content-Type", "application/json") + self.end_headers() + error_body = json.dumps( + { + "error": "payload_too_large", + "message": f"Request body exceeds maximum size of {server.MAX_BODY_SIZE} bytes", + "status": 413, + } + ) + self.wfile.write(error_body.encode("utf-8")) + return + + raw_body = self.rfile.read(size) if raw_body: try: body = json.loads(raw_body.decode("utf-8")) diff --git a/src/redops/api/v1/findings.py b/src/redops/api/v1/findings.py index 48c4b84..8c7ae15 100644 --- a/src/redops/api/v1/findings.py +++ b/src/redops/api/v1/findings.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, Field from ..deps import get_current_user, Pagination +from .scans import _scans router = APIRouter() @@ -54,6 +55,29 @@ class FindingList(BaseModel): # In-memory storage _findings: dict[str, dict] = {} +def _require_finding_access(finding_id: str, user: dict) -> dict: + """Check finding exists and user has access via scan ownership (owner or admin).""" + finding = _findings.get(finding_id) + if not finding: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Finding {finding_id} not found", + ) + username = user.get("username") + role = user.get("role") + if role == "admin": + return finding + # Check scan ownership + scan_id = finding.get("scan_id") + scan = _scans.get(scan_id) if scan_id else None + if scan and scan.get("created_by") == username: + return finding + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied", + ) + + @router.get("", response_model=FindingList) async def list_findings( @@ -65,8 +89,18 @@ async def list_findings( current_user: dict = Depends(get_current_user), ): """List findings with optional filtering.""" + username = current_user.get("username") + role = current_user.get("role") findings = list(_findings.values()) + # Ownership filter: non-admins only see findings from their own scans + if role != "admin": + allowed_scan_ids = { + s["scan_id"] for s in _scans.values() + if s.get("created_by") == username + } + findings = [f for f in findings if f.get("scan_id") in allowed_scan_ids] + # Apply filters if scan_id: findings = [f for f in findings if f.get("scan_id") == scan_id] @@ -94,13 +128,7 @@ async def get_finding( current_user: dict = Depends(get_current_user), ): """Get finding details by ID.""" - finding = _findings.get(finding_id) - if not finding: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Finding {finding_id} not found", - ) - + finding = _require_finding_access(finding_id, current_user) return FindingResponse(**finding) @@ -111,12 +139,7 @@ async def update_finding( current_user: dict = Depends(get_current_user), ): """Update finding status or add notes.""" - finding = _findings.get(finding_id) - if not finding: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Finding {finding_id} not found", - ) + finding = _require_finding_access(finding_id, current_user) if update.status: valid_statuses = [ @@ -144,8 +167,18 @@ async def severity_summary( current_user: dict = Depends(get_current_user), ): """Get finding count by severity.""" + username = current_user.get("username") + role = current_user.get("role") findings = list(_findings.values()) + # Ownership filter: non-admins only see findings from their own scans + if role != "admin": + allowed_scan_ids = { + s["scan_id"] for s in _scans.values() + if s.get("created_by") == username + } + findings = [f for f in findings if f.get("scan_id") in allowed_scan_ids] + if scan_id: findings = [f for f in findings if f.get("scan_id") == scan_id] diff --git a/src/redops/api/v1/reports.py b/src/redops/api/v1/reports.py index a9b9f23..b5c7bd7 100644 --- a/src/redops/api/v1/reports.py +++ b/src/redops/api/v1/reports.py @@ -69,6 +69,25 @@ class ReportList(BaseModel): # In-memory storage _reports: dict[str, dict] = {} +def _require_report_access(report_id: str, user: dict) -> dict: + """Check report exists and user has access (owner or admin).""" + report = _reports.get(report_id) + if not report: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Report {report_id} not found", + ) + username = user.get("username") + role = user.get("role") + owner = report.get("metadata", {}).get("generated_by") + if role != "admin" and owner != username: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied", + ) + return report + + @router.get("", response_model=ReportList) async def list_reports( @@ -78,8 +97,17 @@ async def list_reports( current_user: dict = Depends(get_current_user), ): """List generated reports.""" + username = current_user.get("username") + role = current_user.get("role") reports = list(_reports.values()) + # Ownership filter: non-admins only see their own reports + if role != "admin": + reports = [ + r for r in reports + if r.get("metadata", {}).get("generated_by") == username + ] + if scan_id: reports = [r for r in reports if r.get("scan_id") == scan_id] if report_type: @@ -140,13 +168,7 @@ async def get_report( current_user: dict = Depends(get_current_user), ): """Get report status and metadata.""" - report = _reports.get(report_id) - if not report: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Report {report_id} not found", - ) - + report = _require_report_access(report_id, current_user) return ReportResponse(**report) @@ -156,12 +178,7 @@ async def download_report( current_user: dict = Depends(get_current_user), ): """Download a generated report.""" - report = _reports.get(report_id) - if not report: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Report {report_id} not found", - ) + report = _require_report_access(report_id, current_user) if report["status"] != "completed": raise HTTPException( @@ -195,10 +212,5 @@ async def delete_report( current_user: dict = Depends(get_current_user), ): """Delete a generated report.""" - if report_id not in _reports: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Report {report_id} not found", - ) - + _require_report_access(report_id, current_user) del _reports[report_id] diff --git a/src/redops/api/v1/scans.py b/src/redops/api/v1/scans.py index 10b9720..d8e8bde 100644 --- a/src/redops/api/v1/scans.py +++ b/src/redops/api/v1/scans.py @@ -74,6 +74,24 @@ class ScanCompareRequest(BaseModel): # In-memory storage (replace with database in production) _scans: dict[str, dict] = {} +def _require_scan_access(scan_id: str, user: dict) -> dict: + """Check scan exists and user has access (owner or admin).""" + scan = _scans.get(scan_id) + if not scan: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Scan {scan_id} not found", + ) + username = user.get("username") + role = user.get("role") + if role != "admin" and scan.get("created_by") != username: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied", + ) + return scan + + @router.get("", response_model=ScanList) async def list_scans( @@ -87,8 +105,14 @@ async def list_scans( - **target**: Filter by target (partial match) - **pipeline**: Filter by pipeline name """ + username = current_user.get("username") + role = current_user.get("role") scans = list(_scans.values()) + # Ownership filter: non-admins only see their own scans + if role != "admin": + scans = [s for s in scans if s.get("created_by") == username] + # Apply filters if filters.status: scans = [s for s in scans if s.get("status") == filters.status] @@ -154,13 +178,7 @@ async def get_scan( current_user: dict = Depends(get_current_user), ): """Get scan details by ID.""" - scan = _scans.get(scan_id) - if not scan: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Scan {scan_id} not found", - ) - + scan = _require_scan_access(scan_id, current_user) return ScanResponse(**scan) @@ -170,12 +188,7 @@ async def delete_scan( current_user: dict = Depends(get_current_user), ): """Delete a scan and its findings.""" - if scan_id not in _scans: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Scan {scan_id} not found", - ) - + _require_scan_access(scan_id, current_user) del _scans[scan_id] @@ -185,12 +198,7 @@ async def cancel_scan( current_user: dict = Depends(get_current_user), ): """Cancel a running scan.""" - scan = _scans.get(scan_id) - if not scan: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Scan {scan_id} not found", - ) + scan = _require_scan_access(scan_id, current_user) if scan["status"] not in ("pending", "running"): raise HTTPException( @@ -214,19 +222,8 @@ async def compare_scans( Returns new, resolved, and modified findings between the baseline and current scan. """ - baseline = _scans.get(request.baseline_scan_id) - current = _scans.get(request.current_scan_id) - - if not baseline: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Baseline scan {request.baseline_scan_id} not found", - ) - if not current: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Current scan {request.current_scan_id} not found", - ) + baseline = _require_scan_access(request.baseline_scan_id, current_user) + current = _require_scan_access(request.current_scan_id, current_user) # In production, would use ScanComparator here return { diff --git a/src/redops/api/v1/schedules.py b/src/redops/api/v1/schedules.py index 8c738d8..7e212c5 100644 --- a/src/redops/api/v1/schedules.py +++ b/src/redops/api/v1/schedules.py @@ -79,6 +79,24 @@ class ScheduleList(BaseModel): # In-memory storage _schedules: dict[str, dict] = {} +def _require_schedule_access(schedule_id: str, user: dict) -> dict: + """Check schedule exists and user has access (owner or admin).""" + schedule = _schedules.get(schedule_id) + if not schedule: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Schedule {schedule_id} not found", + ) + username = user.get("username") + role = user.get("role") + if role != "admin" and schedule.get("created_by") != username: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied", + ) + return schedule + + @router.get("", response_model=ScheduleList) async def list_schedules( @@ -88,8 +106,14 @@ async def list_schedules( current_user: dict = Depends(get_current_user), ): """List all schedules.""" + username = current_user.get("username") + role = current_user.get("role") schedules = list(_schedules.values()) + # Ownership filter: non-admins only see their own schedules + if role != "admin": + schedules = [s for s in schedules if s.get("created_by") == username] + if status: schedules = [s for s in schedules if s.get("status") == status] if target: @@ -144,13 +168,7 @@ async def get_schedule( current_user: dict = Depends(get_current_user), ): """Get schedule details.""" - schedule = _schedules.get(schedule_id) - if not schedule: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Schedule {schedule_id} not found", - ) - + schedule = _require_schedule_access(schedule_id, current_user) return ScheduleResponse(**schedule) @@ -161,12 +179,7 @@ async def update_schedule( current_user: dict = Depends(get_current_user), ): """Update a schedule.""" - schedule = _schedules.get(schedule_id) - if not schedule: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Schedule {schedule_id} not found", - ) + schedule = _require_schedule_access(schedule_id, current_user) if update.name is not None: schedule["name"] = update.name @@ -196,12 +209,7 @@ async def delete_schedule( current_user: dict = Depends(get_current_user), ): """Delete a schedule.""" - if schedule_id not in _schedules: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Schedule {schedule_id} not found", - ) - + _require_schedule_access(schedule_id, current_user) del _schedules[schedule_id] @@ -211,12 +219,7 @@ async def trigger_schedule( current_user: dict = Depends(get_current_user), ): """Trigger a scheduled scan immediately.""" - schedule = _schedules.get(schedule_id) - if not schedule: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Schedule {schedule_id} not found", - ) + schedule = _require_schedule_access(schedule_id, current_user) # In production, would create a scan from the schedule scan_id = str(uuid4()) diff --git a/src/redops/api/v1/users.py b/src/redops/api/v1/users.py index d28dee9..0d1659f 100644 --- a/src/redops/api/v1/users.py +++ b/src/redops/api/v1/users.py @@ -5,6 +5,8 @@ from datetime import datetime, timezone from uuid import uuid4 +import bcrypt + from fastapi import APIRouter, Depends, HTTPException, status from pydantic import BaseModel, ConfigDict, EmailStr, Field @@ -27,7 +29,7 @@ class UserCreate(BaseModel): "example": { "username": "johndoe", "email": "john@example.com", - "password": "securepassword123", + "password": "[REDACTED]", "full_name": "John Doe", "role": "user", } @@ -72,7 +74,7 @@ class UserList(BaseModel): "id": "admin-user-id", "username": "admin", "email": "admin@example.com", - "password_hash": "hashed_password", + "password_hash": "$2b$12$2r4xJEZgOYnknatj9QNAO.Usq37g1/p1oOO8mu7evMfNtmaxg4k7G", "full_name": "Admin User", "role": "admin", "is_active": True, @@ -138,12 +140,13 @@ async def create_user( user_id = str(uuid4()) now = datetime.now(timezone.utc) - # In production, would hash password here user_data = { "id": user_id, "username": user.username, "email": user.email, - "password_hash": f"hashed_{user.password}", + "password_hash": bcrypt.hashpw( + user.password.encode("utf-8"), bcrypt.gensalt() + ).decode("utf-8"), "full_name": user.full_name, "role": user.role, "is_active": True, diff --git a/src/redops/cache/backends.py b/src/redops/cache/backends.py index 9bee160..419c206 100644 --- a/src/redops/cache/backends.py +++ b/src/redops/cache/backends.py @@ -329,7 +329,7 @@ def is_available(self) -> bool: try: self._client.ping() return True - except Exception: + except (ConnectionError, TimeoutError, OSError): return False def _make_key(self, key: str) -> str: @@ -365,7 +365,7 @@ def get(self, key: str) -> CacheEntry | None: ) return entry - except Exception as e: + except (ConnectionError, TimeoutError, OSError, json.JSONDecodeError, UnicodeDecodeError) as e: logger.error(f"Redis get error: {e}") return None @@ -409,7 +409,7 @@ def set( if ttl: self._client.expire(tag_key, ttl) - except Exception as e: + except (ConnectionError, TimeoutError, OSError, TypeError) as e: logger.error(f"Redis set error: {e}") def delete(self, key: str) -> bool: @@ -420,7 +420,7 @@ def delete(self, key: str) -> bool: try: full_key = self._make_key(key) return self._client.delete(full_key) > 0 - except Exception as e: + except (ConnectionError, TimeoutError, OSError) as e: logger.error(f"Redis delete error: {e}") return False @@ -432,7 +432,7 @@ def exists(self, key: str) -> bool: try: full_key = self._make_key(key) return self._client.exists(full_key) > 0 - except Exception as e: + except (ConnectionError, TimeoutError, OSError) as e: logger.error(f"Redis exists error: {e}") return False @@ -447,7 +447,7 @@ def clear(self) -> int: if keys: return self._client.delete(*keys) return 0 - except Exception as e: + except (ConnectionError, TimeoutError, OSError) as e: logger.error(f"Redis clear error: {e}") return 0 @@ -460,7 +460,7 @@ def keys(self, pattern: str = "*") -> list[str]: full_pattern = self._make_key(pattern) keys = list(self._client.scan_iter(full_pattern)) return [self._strip_prefix(k.decode("utf-8")) for k in keys] - except Exception as e: + except (ConnectionError, TimeoutError, OSError, UnicodeDecodeError) as e: logger.error(f"Redis keys error: {e}") return [] @@ -478,7 +478,7 @@ def delete_by_tag(self, tag: str) -> int: count += 1 self._client.delete(tag_key) return count - except Exception as e: + except (ConnectionError, TimeoutError, OSError, UnicodeDecodeError) as e: logger.error(f"Redis delete_by_tag error: {e}") return 0 @@ -498,7 +498,7 @@ def get_stats(self) -> dict[str, Any]: "connected_clients": info.get("connected_clients", 0), "uptime_days": info.get("uptime_in_days", 0), } - except Exception as e: + except (ConnectionError, TimeoutError, OSError) as e: logger.error(f"Redis stats error: {e}") return {"backend": "redis", "available": False, "error": str(e)} diff --git a/src/redops/cli/app.py b/src/redops/cli/app.py index a9e917a..6f68397 100644 --- a/src/redops/cli/app.py +++ b/src/redops/cli/app.py @@ -570,7 +570,7 @@ def cmd_ai(args: argparse.Namespace, config: CLIConfig) -> int: # Initialize AI assistant try: assistant = AIAssistant(provider=provider, model=model) - except Exception as e: + except (ImportError, ValueError) as e: print_error(f"Failed to initialize AI assistant: {e}") print_info( "Make sure you have configured an API key using 'redops settings' or 'redops apikey set'" @@ -587,9 +587,12 @@ def cmd_ai(args: argparse.Namespace, config: CLIConfig) -> int: try: with open(input_file, "r") as f: scan_data = json.load(f) - except Exception as e: + except OSError as e: print_error(f"Failed to read input file: {e}") return 1 + except json.JSONDecodeError as e: + print_error(f"Invalid JSON in input file: {e}") + return 1 if not config.quiet: print_info("Analyzing scan results with AI...") @@ -610,8 +613,8 @@ def cmd_ai(args: argparse.Namespace, config: CLIConfig) -> int: try: with open(context_file, "r") as f: context_data = json.load(f) - except Exception: - pass + except (OSError, json.JSONDecodeError): + context_data = None if not config.quiet: print_info("Getting AI explanation...") @@ -629,9 +632,12 @@ def cmd_ai(args: argparse.Namespace, config: CLIConfig) -> int: try: with open(input_file, "r") as f: scan_data = json.load(f) - except Exception as e: + except OSError as e: print_error(f"Failed to read input file: {e}") return 1 + except json.JSONDecodeError as e: + print_error(f"Invalid JSON in input file: {e}") + return 1 if not config.quiet: print_info("Generating remediation suggestions...") @@ -649,9 +655,12 @@ def cmd_ai(args: argparse.Namespace, config: CLIConfig) -> int: try: with open(input_file, "r") as f: scan_data = json.load(f) - except Exception as e: + except OSError as e: print_error(f"Failed to read input file: {e}") return 1 + except json.JSONDecodeError as e: + print_error(f"Invalid JSON in input file: {e}") + return 1 if not config.quiet: print_info("Generating AI summary...") @@ -688,7 +697,7 @@ def cmd_ai(args: argparse.Namespace, config: CLIConfig) -> int: with open(filepath, "r") as f: context_data = json.load(f) print(f"Loaded context from: {filepath}") - except Exception as e: + except (OSError, json.JSONDecodeError) as e: print(f"Error loading file: {e}") continue @@ -921,7 +930,7 @@ def cmd_plugin(args: argparse.Namespace, config: CLIConfig) -> int: else: print_warning("No plugins found in source") return 1 - except Exception as e: + except (ImportError, OSError, ValueError) as e: print_error(f"Failed to load plugin: {e}") return 1 @@ -1057,7 +1066,7 @@ def execute_scan(target: str, modules: list[str], config: CLIConfig) -> ScanResu try: ctx = run_module(ctx, module_name, config) modules_run.append(module_name) - except Exception as e: + except (RuntimeError, ImportError, ConnectionError, OSError, ValueError) as e: errors.append(f"{module_name}: {str(e)}") if config.verbosity == Verbosity.DEBUG: import traceback @@ -1083,7 +1092,7 @@ def execute_scan(target: str, modules: list[str], config: CLIConfig) -> ScanResu context=ctx, ) - except Exception as e: + except (RuntimeError, ImportError, OSError, ConnectionError, ValueError) as e: return ScanResult( success=False, target=target, diff --git a/src/redops/cli/commands/config.py b/src/redops/cli/commands/config.py index 5935f53..cd8a06e 100644 --- a/src/redops/cli/commands/config.py +++ b/src/redops/cli/commands/config.py @@ -222,7 +222,7 @@ def validate_cmd(path): try: config_data = load_config(str(config_path)) - except Exception as e: + except (ValueError, TypeError, OSError, RuntimeError) as e: print_error(f"Failed to parse config: {e}") sys.exit(1) diff --git a/src/redops/cli/commands/scan.py b/src/redops/cli/commands/scan.py index 079ea05..18ab72b 100644 --- a/src/redops/cli/commands/scan.py +++ b/src/redops/cli/commands/scan.py @@ -20,18 +20,154 @@ ) -@click.group() -def scan(): +# --------------------------------------------------------------------------- +# Local execution helpers (zero-config quickstart) +# --------------------------------------------------------------------------- + +def _resolve_pipeline_file(pipeline_name: str) -> Path | None: + """Resolve a pipeline name to a JSON file in config/pipelines/.""" + if pipeline_name == "default": + pipeline_name = "quickstart" + + pipelines_dir = Path(__file__).parents[3] / "config" / "pipelines" + if not pipelines_dir.exists(): + return None + + # Exact match first + exact = pipelines_dir / f"{pipeline_name}.json" + if exact.exists(): + return exact + + # Suffix match + for path in pipelines_dir.glob("*.json"): + if path.stem == pipeline_name or pipeline_name in path.stem: + return path + + return None + + +def _run_local_scan( + target: str, + pipeline_name: str = "quickstart", + output: str | None = None, + timeout: int = 60, +) -> int: + """Run a scan locally using PipelineRunner (no API server required). + + Returns: + Exit code (0 for success, 1 for failure) + """ + from redops.pipelines.loader import PipelineLoader + from redops.pipelines.runner import PipelineRunner + from redops.core.config import RedOpsConfig + from redops.core.context import Context + + pipeline_path = _resolve_pipeline_file(pipeline_name) + if pipeline_path is None: + print_error(f"Pipeline '{pipeline_name}' not found in config/pipelines/") + console.print( + "[dim]Run 'redops scan list-pipelines' to see available pipelines.[/dim]" + ) + return 1 + + try: + config = RedOpsConfig.from_env() + except (OSError, ValueError, TypeError): + config = RedOpsConfig() + + console.print(f"[bold]Starting local scan on {target}[/bold]") + console.print(f" Pipeline: {pipeline_name} ({pipeline_path.name})") + console.print(f" Timeout: {timeout}s") + console.print() + + try: + pipeline = PipelineLoader.load(pipeline_path) + runner = PipelineRunner(pipeline, config=config) + + console.print(f"[dim]Executing {len(pipeline.enabled_steps)} steps...[/dim]") + console.print() + + start_time = time.time() + ctx = runner.run(target=target) + elapsed = time.time() - start_time + + # Build a result dict compatible with print_scan_result + result = { + "scan_id": "local", + "target": target, + "pipeline": pipeline.metadata.name, + "status": "completed", + "started_at": datetime.fromtimestamp(start_time).isoformat(), + "completed_at": datetime.now().isoformat(), + "findings": _extract_findings_from_context(ctx), + "logs": ctx.logs, + "data_keys": list(ctx.data.keys()), + } + + print_scan_result(result, verbose=False) + console.print(f"\n[dim]Completed in {format_duration(elapsed)}[/dim]") + + if output: + output_path = Path(output) + output_path.write_text(json.dumps(result, indent=2, default=str)) + print_success(f"Results saved to {output}") + + return 0 + + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError) as e: + print_error(f"Local scan failed: {e}") + return 1 + + +def _extract_findings_from_context(ctx) -> list[dict]: + """Extract findings from pipeline context for display.""" + findings = [] + for key, value in ctx.data.items(): + if isinstance(value, list): + for item in value: + if isinstance(item, dict) and "severity" in item: + findings.append(item) + elif isinstance(item, dict) and "title" in item: + findings.append(item) + elif isinstance(value, dict) and "severity" in value: + findings.append(value) + return findings + + +class _DefaultScanGroup(click.Group): + """Custom group that treats unknown first argument as target for local scan.""" + + def parse_args(self, ctx, args): + # Known subcommands + known = {"run", "list", "status", "cancel", "list-pipelines"} + # If no args or first arg is a known subcommand or starts with '-', use normal parsing + if not args or args[0] in known or args[0].startswith("-"): + return super().parse_args(ctx, args) + # Otherwise insert 'run' and '--local' so the user can type: + # redops scan example.com + # which becomes: + # redops scan run --local example.com + args.insert(0, "--local") + args.insert(0, "run") + return super().parse_args(ctx, args) + + +@click.group(cls=_DefaultScanGroup, invoke_without_command=True) +@click.pass_context +def scan(ctx): """Scan management commands. \b Examples: + redops scan https://example.com redops scan run https://example.com redops scan list redops scan status scan-123 redops scan cancel scan-123 """ - pass + if ctx.invoked_subcommand is None: + # No subcommand given — show help + click.echo(ctx.get_help()) @scan.command("run") @@ -45,9 +181,24 @@ def scan(): @click.option( "--tag", multiple=True, help="Tags to add to scan (can be used multiple times)" ) +@click.option( + "--local", + "local_mode", + is_flag=True, + help="Run scan locally without an API server (zero-config mode)", +) @click.pass_context def run_cmd( - ctx, target, pipeline, output, async_mode, timeout, modules, exclude_modules, tag + ctx, + target, + pipeline, + output, + async_mode, + timeout, + modules, + exclude_modules, + tag, + local_mode, ): """Run a security scan on a target. @@ -57,8 +208,18 @@ def run_cmd( Examples: redops scan run https://example.com redops scan run -p web_full example.com - redops scan run --modules port_scan,ssl_check 192.168.1.1 + redops scan run --local example.com """ + if local_mode: + sys.exit( + _run_local_scan( + target=target, + pipeline_name=pipeline, + output=output, + timeout=timeout, + ) + ) + run_scan( ctx=ctx.obj, target=target, @@ -311,6 +472,31 @@ def get_status(): console.print("\n[dim]Stopped watching[/dim]") +@scan.command("list-pipelines") +def list_pipelines_cmd(): + """List available pipeline definitions.""" + pipelines_dir = Path(__file__).parents[3] / "config" / "pipelines" + if not pipelines_dir.exists(): + console.print("[yellow]Pipeline directory not found.[/yellow]") + return + + console.print("[bold]Available Pipelines[/bold]\n") + from redops.pipelines.loader import PipelineLoader + + for path in sorted(pipelines_dir.glob("*.json")): + try: + pipeline = PipelineLoader.load(path) + console.print(f" [cyan]{path.stem}[/cyan]") + console.print(f" {pipeline.metadata.name}") + if pipeline.metadata.description: + console.print(f" [dim]{pipeline.metadata.description}[/dim]") + console.print(f" Steps: {len(pipeline.steps)} | Tags: {', '.join(pipeline.metadata.tags)}") + console.print() + except (OSError, ValueError, TypeError, RuntimeError) as e: + console.print(f" [red]{path.name}[/red] — error: {e}") + console.print() + + @scan.command("cancel") @click.argument("scan_id") @click.option("--force", "-f", is_flag=True, help="Force cancel without confirmation") diff --git a/src/redops/cli/main.py b/src/redops/cli/main.py index 162c52f..66e7631 100644 --- a/src/redops/cli/main.py +++ b/src/redops/cli/main.py @@ -101,8 +101,14 @@ def status(ctx): @click.option("-o", "--output", type=click.Path(), help="Output file path") @click.option("--async", "async_mode", is_flag=True, help="Run scan asynchronously") @click.option("--timeout", type=int, default=3600, help="Scan timeout in seconds") +@click.option( + "--local", + "local_mode", + is_flag=True, + help="Run scan locally without an API server (zero-config mode)", +) @pass_context -def quick_scan(ctx, target, pipeline, output, async_mode, timeout): +def quick_scan(ctx, target, pipeline, output, async_mode, timeout, local_mode): """Run a quick scan on a target. This is a shortcut for 'redops scan run'. @@ -111,8 +117,19 @@ def quick_scan(ctx, target, pipeline, output, async_mode, timeout): Examples: redops quick-scan https://example.com redops quick-scan -p web_full example.com -o results.json + redops quick-scan --local example.com """ - from .commands.scan import run_scan + from .commands.scan import run_scan, _run_local_scan + + if local_mode: + sys.exit( + _run_local_scan( + target=target, + pipeline_name=pipeline, + output=output, + timeout=timeout, + ) + ) run_scan( ctx=ctx, @@ -216,7 +233,7 @@ def doctor(check): console.print(f" {status} {name}: {message}") if not passed: all_passed = False - except Exception as e: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError, ImportError) as e: console.print(f" [red]✗[/red] {name}: Error - {e}") all_passed = False @@ -280,7 +297,7 @@ def _check_database(): if db.check_connection(): return True, "Connected" return False, "Connection failed" - except Exception as e: + except (ImportError, RuntimeError, OSError, ConnectionError) as e: return False, f"Not configured ({e})" diff --git a/src/redops/cli/settings.py b/src/redops/cli/settings.py index 1e1c5cb..99db59c 100644 --- a/src/redops/cli/settings.py +++ b/src/redops/cli/settings.py @@ -535,7 +535,7 @@ def _test_provider(self, provider: str, key: str) -> bool: except ImportError: print(f"Required library for {provider} is not installed.") return False - except Exception as e: + except (ConnectionError, TimeoutError, OSError, RuntimeError, ValueError) as e: print(f"Error: {e}") return False return False diff --git a/src/redops/core/alerting.py b/src/redops/core/alerting.py index d4fe4c9..194ab44 100644 --- a/src/redops/core/alerting.py +++ b/src/redops/core/alerting.py @@ -419,7 +419,7 @@ def _send_request(self, payload: dict[str, Any]) -> bool: } return response.status < 400 - except Exception as e: + except (OSError, ConnectionError, RuntimeError, ValueError) as e: self._last_response = {"error": str(e)} return False @@ -496,7 +496,7 @@ def _send_email(self, subject: str, body: str) -> bool: server.sendmail(self._from_address, self._recipients, msg.as_string()) return True - except Exception: + except (OSError, ConnectionError, RuntimeError, ValueError): return False @@ -842,7 +842,7 @@ def _send_alert(self, alert: Alert, rule: AlertRule) -> None: if channel and channel.enabled: try: channel.send(alert) - except Exception: + except (OSError, ConnectionError, RuntimeError, TypeError, ValueError): pass def _send_resolved(self, alert: Alert) -> None: @@ -855,7 +855,7 @@ def _send_resolved(self, alert: Alert) -> None: if channel and channel.enabled: try: channel.send_resolved(alert) - except Exception: + except (OSError, ConnectionError, RuntimeError, TypeError, ValueError): pass # Best-effort notification - don't fail resolve # History diff --git a/src/redops/core/api_client.py b/src/redops/core/api_client.py index df56559..94341de 100644 --- a/src/redops/core/api_client.py +++ b/src/redops/core/api_client.py @@ -753,7 +753,7 @@ def _execute_with_retry(self, request: HttpRequest, retry: bool) -> HttpResponse return response - except Exception as e: + except (OSError, RuntimeError, ConnectionError, TimeoutError, HttpError) as e: last_exception = e if self._circuit_breaker: diff --git a/src/redops/core/async_processor.py b/src/redops/core/async_processor.py index 8a987c5..0864b0c 100644 --- a/src/redops/core/async_processor.py +++ b/src/redops/core/async_processor.py @@ -241,7 +241,7 @@ def wrapper(): def target(): try: result_container[0] = func(*args, **kwargs) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: error_container[0] = e thread = threading.Thread(target=target) @@ -258,7 +258,7 @@ def target(): else: return func(*args, **kwargs) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): with self._lock: info.state = TaskState.FAILED info.completed_at = time.time() @@ -327,7 +327,7 @@ def get_result( state=TaskState.CANCELLED, error="Task was cancelled", ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: return TaskResult( task_id=task_id, state=TaskState.FAILED, @@ -454,7 +454,7 @@ async def run( error="Task was cancelled", started_at=started_at, ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: return TaskResult( task_id=task_id, state=TaskState.FAILED, @@ -792,14 +792,14 @@ async def execute(self, initial_value: Any) -> TaskResult: completed_at=time.time(), ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ArithmeticError) as e: for handler in self._error_handlers: try: if asyncio.iscoroutinefunction(handler): await handler(e, current) else: handler(e, current) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, ArithmeticError): pass # Ignore handler errors return TaskResult( diff --git a/src/redops/core/cache.py b/src/redops/core/cache.py index b23ba24..509dd5e 100644 --- a/src/redops/core/cache.py +++ b/src/redops/core/cache.py @@ -243,7 +243,7 @@ def get(self, key: str) -> CacheEntry | None: size_bytes=data.get("size_bytes", 0), tags=data.get("tags", []), ) - except Exception: + except (OSError, json.JSONDecodeError, KeyError, TypeError): return None def set(self, entry: CacheEntry) -> None: @@ -262,7 +262,7 @@ def set(self, entry: CacheEntry) -> None: } with open(path, "w") as f: json.dump(data, f) - except Exception as e: + except (OSError, TypeError, ValueError) as e: raise CacheStorageError(f"Failed to write cache: {e}") def delete(self, key: str) -> bool: @@ -407,7 +407,7 @@ def _estimate_size(self, value: Any) -> int: """Estimate size of a value in bytes.""" try: return sys.getsizeof(value) - except Exception: + except (TypeError, ValueError, AttributeError): return 0 def get(self, key: str, default: T = None) -> T | Any: diff --git a/src/redops/core/cli.py b/src/redops/core/cli.py index 4bbe122..edef1d9 100644 --- a/src/redops/core/cli.py +++ b/src/redops/core/cli.py @@ -523,7 +523,7 @@ def run(self, args: Sequence[str] | None = None) -> int: self._output.write("") # Newline self._output.error("Interrupted") return 130 - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: self._output.error(f"Error: {e}") if parsed.verbose > 0: import traceback diff --git a/src/redops/core/context.py b/src/redops/core/context.py index 3bafa57..583cd8a 100644 --- a/src/redops/core/context.py +++ b/src/redops/core/context.py @@ -21,22 +21,71 @@ class Context: all intermediate outputs. """ - def __init__(self, target: str | None = None, config: "RedOpsConfig | None" = None): + def __init__( + self, + target: str | None = None, + config: "RedOpsConfig | None" = None, + *, + authorization: Any | None = None, + ): """ Initialize a new Context. Args: target: The target of the pipeline execution (e.g., domain, directory) config: RedOps configuration (scope, output settings, etc.) + authorization: ActiveAuthorization instance for active/offensive modules. """ self.target = target self.config = config + self.authorization = authorization self.data: dict[str, Any] = {} self.logs: list[dict[str, Any]] = [] self.metadata: dict[str, Any] = { "created_at": datetime.now(timezone.utc).isoformat(), "target": target, } + self._checkpoints: list[dict[str, Any]] = [] + + def save(self) -> None: + """ + Save a checkpoint of the current context state. + + Checkpoints are stored in a stack; call rollback() to restore + the most recent checkpoint. + """ + import copy + + checkpoint = { + "data": copy.deepcopy(self.data), + "logs": copy.deepcopy(self.logs), + "metadata": copy.deepcopy(self.metadata), + } + self._checkpoints.append(checkpoint) + self.log("Context checkpoint saved", level="DEBUG") + + def rollback(self) -> None: + """ + Restore the context data and metadata to the last checkpoint. + + Logs are intentionally preserved (append-only audit trail). + + Raises: + RuntimeError: If no checkpoints exist. + """ + if not self._checkpoints: + raise RuntimeError("No checkpoints available to rollback") + + checkpoint = self._checkpoints.pop() + self.data = checkpoint["data"] + self.metadata = checkpoint["metadata"] + # Logs are NOT rolled back — they form an immutable audit trail + self.log("Context rolled back to previous checkpoint", level="WARNING") + + def clear_checkpoints(self) -> None: + """Remove all stored checkpoints.""" + self._checkpoints.clear() + self.log("All checkpoints cleared", level="DEBUG") def add(self, key: str, value: Any) -> None: """ diff --git a/src/redops/core/data_io.py b/src/redops/core/data_io.py index 864eeb8..63d3d5a 100644 --- a/src/redops/core/data_io.py +++ b/src/redops/core/data_io.py @@ -189,7 +189,7 @@ def validate(self, record: dict[str, Any]) -> list[ValidationError]: record[field_name], ) ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: errors.append( ValidationError( f"Validation error for {field_name}: {e}", diff --git a/src/redops/core/data_validation.py b/src/redops/core/data_validation.py index 7bdb4f1..2019e0a 100644 --- a/src/redops/core/data_validation.py +++ b/src/redops/core/data_validation.py @@ -448,7 +448,7 @@ def validate(self, value: Any, path: str = "") -> None: raise ValidationError(self.message, path, value) except ValidationError: raise - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: raise ValidationError(f"{self.message}: {e}", path, value) def describe(self) -> str: @@ -534,7 +534,7 @@ def coerce(self, value: Any, target_type: Type) -> Any: # Try direct conversion try: return target_type(value) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: raise CoercionError(value, target_type.__name__, str(e)) def _to_str(self, value: Any) -> str: diff --git a/src/redops/core/docs_generator.py b/src/redops/core/docs_generator.py index 6d5043b..db41c3e 100644 --- a/src/redops/core/docs_generator.py +++ b/src/redops/core/docs_generator.py @@ -1230,7 +1230,7 @@ def generate_package_docs( try: doc = self._analyzer.analyze_module(py_file) docs[doc_name] = self._formatter.format_module(doc) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: docs[doc_name] = f"Error generating docs: {e}" # Write to output directory if specified diff --git a/src/redops/core/event_bus.py b/src/redops/core/event_bus.py index d3bd7ba..3140658 100644 --- a/src/redops/core/event_bus.py +++ b/src/redops/core/event_bus.py @@ -246,7 +246,7 @@ def matches(self, event: Event) -> bool: try: if not self.filter_fn(event): return False - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.warning(f"Filter function failed: {e}") return False @@ -609,7 +609,7 @@ def publish(self, event: Event) -> int: if sub.once: to_remove.append(sub) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.error(f"Handler {sub.handler} failed: {e}") self._stats["events_failed"] += 1 self._dlq.add(event, sub.handler, e) @@ -767,7 +767,7 @@ async def publish(self, event: Event) -> int: if sub.once: to_remove.append(sub) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.error(f"Async handler {sub.handler} failed: {e}") self._stats["events_failed"] += 1 self._dlq.add(event, sub.handler, e) @@ -796,7 +796,7 @@ async def call_handler(sub: Subscription) -> bool: sub.call_count += 1 sub.last_called = datetime.now() return True - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.error(f"Handler failed: {e}") self._dlq.add(event, sub.handler, e) return False @@ -918,7 +918,7 @@ def _worker_loop(self) -> None: self._inner_bus.publish(event) except queue.Empty: continue - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.error(f"Worker error: {e}") def subscribe(self, *args, **kwargs) -> Subscription: @@ -1034,7 +1034,7 @@ def _do_flush(self) -> list[Event]: if self._on_flush and events: try: self._on_flush(events) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.error(f"Flush callback failed: {e}") return events @@ -1102,7 +1102,7 @@ def replay( for event in events: try: handler(event) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.error(f"Replay handler failed on {event.id}: {e}") return len(events) diff --git a/src/redops/core/exceptions.py b/src/redops/core/exceptions.py new file mode 100644 index 0000000..af5d874 --- /dev/null +++ b/src/redops/core/exceptions.py @@ -0,0 +1,179 @@ +"""RedOPS domain-specific exceptions. + +Provides a unified exception hierarchy so callers can distinguish between +configuration errors, network failures, auth problems, and validation issues +instead of catching bare ``Exception``. +""" + + +class RedOpsError(Exception): + """Base exception for all RedOPS errors.""" + + pass + + +# --------------------------------------------------------------------------- +# Configuration / environment +# --------------------------------------------------------------------------- + + +class ConfigurationError(RedOpsError): + """Missing or invalid configuration.""" + + pass + + +class SecretNotFoundError(ConfigurationError): + """Required secret or environment variable is missing.""" + + pass + + +# --------------------------------------------------------------------------- +# Network / I/O +# --------------------------------------------------------------------------- + + +class NetworkError(RedOpsError): + """Transient or permanent network failure.""" + + pass + + +class APIClientError(NetworkError): + """Error while communicating with an external API.""" + + pass + + +class RateLimitError(NetworkError): + """Request blocked by rate limiting.""" + + pass + + +class CircuitOpenError(NetworkError): + """Circuit breaker is open; requests are not being sent.""" + + pass + + +# --------------------------------------------------------------------------- +# Authentication / authorization +# --------------------------------------------------------------------------- + + +class AuthError(RedOpsError): + """Base for authentication and authorization failures.""" + + pass + + +class AuthenticationError(AuthError): + """Invalid credentials or missing authentication.""" + + pass + + +class AuthorizationError(AuthError): + """Authenticated user lacks permission.""" + + pass + + +class TokenExpiredError(AuthenticationError): + """Token has expired.""" + + pass + + +class TokenInvalidError(AuthenticationError): + """Token is malformed or signature verification failed.""" + + pass + + +class SessionNotFoundError(AuthenticationError): + """Session does not exist or has been invalidated.""" + + pass + + +# --------------------------------------------------------------------------- +# Validation / data quality +# --------------------------------------------------------------------------- + + +class ValidationError(RedOpsError): + """Input data failed validation.""" + + pass + + +class SchemaError(ValidationError): + """Data does not conform to expected schema.""" + + pass + + +# --------------------------------------------------------------------------- +# Pipeline / module execution +# --------------------------------------------------------------------------- + + +class PipelineError(RedOpsError): + """Error during pipeline execution.""" + + pass + + +class ModuleError(PipelineError): + """Error in a specific pipeline module.""" + + pass + + +class ModuleNotFoundError(ModuleError): + """Requested module could not be resolved.""" + + pass + + +# --------------------------------------------------------------------------- +# Storage / caching +# --------------------------------------------------------------------------- + + +class StorageError(RedOpsError): + """Database, file-system, or cache operation failed.""" + + pass + + +class CacheError(StorageError): + """Cache read/write failed.""" + + pass + + +# --------------------------------------------------------------------------- +# AI / LLM +# --------------------------------------------------------------------------- + + +class AIError(RedOpsError): + """Error during AI model interaction.""" + + pass + + +class AIBudgetExceededError(AIError): + """Cost or token budget has been exceeded.""" + + pass + + +class AIPromptError(AIError): + """Prompt generation or validation failed.""" + + pass diff --git a/src/redops/core/feature_flags.py b/src/redops/core/feature_flags.py index b862904..12a1210 100644 --- a/src/redops/core/feature_flags.py +++ b/src/redops/core/feature_flags.py @@ -597,7 +597,7 @@ def evaluate( for listener in self._listeners: try: listener(key, value, ctx) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): pass # Don't let listener errors affect evaluation return value diff --git a/src/redops/core/health.py b/src/redops/core/health.py index e83936e..ee7cc70 100644 --- a/src/redops/core/health.py +++ b/src/redops/core/health.py @@ -247,7 +247,7 @@ def check(self) -> CheckResult: error="Connection timed out", duration_ms=duration, ) - except Exception as e: + except (OSError, ConnectionError, RuntimeError) as e: duration = (time.perf_counter() - start) * 1000 return self._create_result( HealthStatus.UNHEALTHY, @@ -334,7 +334,7 @@ def check(self) -> CheckResult: error=str(e.reason), duration_ms=duration, ) - except Exception as e: + except (OSError, ConnectionError, RuntimeError, ValueError) as e: duration = (time.perf_counter() - start) * 1000 return self._create_result( HealthStatus.UNHEALTHY, @@ -407,7 +407,7 @@ def check(self) -> CheckResult: details, duration_ms=duration, ) - except Exception as e: + except (OSError, PermissionError, RuntimeError, ValueError) as e: duration = (time.perf_counter() - start) * 1000 return self._create_result( HealthStatus.UNHEALTHY, @@ -493,7 +493,7 @@ def check(self) -> CheckResult: {}, duration_ms=duration, ) - except Exception as e: + except (OSError, RuntimeError, ValueError) as e: duration = (time.perf_counter() - start) * 1000 return self._create_result( HealthStatus.UNKNOWN, @@ -561,7 +561,7 @@ def check(self) -> CheckResult: error="pgrep not found", duration_ms=duration, ) - except Exception as e: + except (OSError, RuntimeError, ValueError) as e: duration = (time.perf_counter() - start) * 1000 return self._create_result( HealthStatus.UNHEALTHY, @@ -640,7 +640,7 @@ def check(self) -> CheckResult: details, duration_ms=duration, ) - except Exception as e: + except (OSError, PermissionError, RuntimeError) as e: duration = (time.perf_counter() - start) * 1000 return self._create_result( HealthStatus.UNHEALTHY, @@ -690,7 +690,7 @@ def check(self) -> CheckResult: error="Check returned False", duration_ms=duration, ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: duration = (time.perf_counter() - start) * 1000 return self._create_result( HealthStatus.UNHEALTHY, @@ -736,7 +736,7 @@ def check(self) -> CheckResult: {"query": self.query}, duration_ms=duration, ) - except Exception as e: + except (OSError, ConnectionError, RuntimeError, TypeError) as e: duration = (time.perf_counter() - start) * 1000 return self._create_result( HealthStatus.UNHEALTHY, @@ -805,7 +805,7 @@ def check(self) -> CheckResult: error="redis package not installed", duration_ms=duration, ) - except Exception as e: + except (OSError, ConnectionError, RuntimeError, TypeError) as e: duration = (time.perf_counter() - start) * 1000 return self._create_result( HealthStatus.UNHEALTHY, @@ -884,7 +884,7 @@ def _run_check(self, check: HealthCheck) -> CheckResult: def run(): try: result[0] = check.check() - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: exception[0] = e thread = threading.Thread(target=run) @@ -926,7 +926,7 @@ async def _run_async_check(self, check: AsyncHealthCheck) -> CheckResult: error="Timeout", duration_ms=check.timeout * 1000, ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: return CheckResult( name=check.name, status=HealthStatus.UNHEALTHY, @@ -1111,7 +1111,7 @@ def is_alive(self) -> bool: if self._custom_check: try: return self._custom_check() - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): return False if self._manager: @@ -1122,7 +1122,7 @@ def is_alive(self) -> bool: if checks: result = self._manager.run_check(checks[0]) return result is not None - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): return False return True diff --git a/src/redops/core/logging_audit.py b/src/redops/core/logging_audit.py index a84b3b1..8f4f40f 100644 --- a/src/redops/core/logging_audit.py +++ b/src/redops/core/logging_audit.py @@ -791,7 +791,7 @@ def _log( for handler in self._handlers: try: handler.handle(entry) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, AttributeError): # Don't let handler errors break logging pass @@ -971,7 +971,7 @@ def wrapper(*args, **kwargs): logger._log(level, f"Exiting {func_name}") return result - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.error(f"Exception in {func_name}", exception=e) raise diff --git a/src/redops/core/metrics.py b/src/redops/core/metrics.py index 24b39ef..62fba5f 100644 --- a/src/redops/core/metrics.py +++ b/src/redops/core/metrics.py @@ -827,7 +827,7 @@ def _send(self, data: str) -> None: try: sock = self._get_socket() sock.sendto(data.encode("utf-8"), (self._host, self._port)) - except Exception: + except (OSError, ConnectionError, RuntimeError): pass # StatsD is fire-and-forget def _flush_buffer(self) -> None: diff --git a/src/redops/core/notifications.py b/src/redops/core/notifications.py index 3ac5a10..e52eb4b 100644 --- a/src/redops/core/notifications.py +++ b/src/redops/core/notifications.py @@ -164,7 +164,7 @@ def replace_var(match): else: value = getattr(value, part, "") return str(value) if value is not None else "" - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, AttributeError): return "" return self.VARIABLE_PATTERN.sub(replace_var, template) @@ -371,7 +371,7 @@ def send(self, notification: Notification) -> bool: return True - except Exception as e: + except (OSError, ConnectionError, RuntimeError, ValueError) as e: notification.error = str(e) return False @@ -414,7 +414,7 @@ def send(self, notification: Notification) -> bool: with urllib.request.urlopen(request, timeout=self.timeout) as response: return response.status < 400 - except Exception as e: + except (OSError, ConnectionError, RuntimeError, ValueError) as e: notification.error = str(e) return False @@ -494,7 +494,7 @@ def send(self, notification: Notification) -> bool: with urllib.request.urlopen(request, timeout=self.timeout) as response: return response.status == 200 - except Exception as e: + except (OSError, ConnectionError, RuntimeError, ValueError) as e: notification.error = str(e) return False @@ -528,7 +528,7 @@ def send(self, notification: Notification) -> bool: self.output_func(message) return True - except Exception as e: + except (OSError, ConnectionError, RuntimeError, ValueError) as e: notification.error = str(e) return False @@ -551,7 +551,7 @@ def send(self, notification: Notification) -> bool: try: result = self.callback(notification) return bool(result) if result is not None else True - except Exception as e: + except (OSError, ConnectionError, RuntimeError, ValueError) as e: notification.error = str(e) return False @@ -725,7 +725,7 @@ def send( for listener in self._listeners: try: listener(notification, result) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, AttributeError): pass return result diff --git a/src/redops/core/plugin_system.py b/src/redops/core/plugin_system.py index 6298997..c431e29 100644 --- a/src/redops/core/plugin_system.py +++ b/src/redops/core/plugin_system.py @@ -327,7 +327,7 @@ def discover_plugins(self) -> list[str]: plugin_name = plugin_path.stem self._discover_plugin_file(plugin_path) discovered.append(plugin_name) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ImportError) as e: # Record discovery error but continue self._plugins[plugin_path.stem] = PluginInfo( metadata=PluginMetadata( @@ -430,7 +430,7 @@ def register( try: instance = plugin_class() instance.initialize(config) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, AttributeError) as e: raise PluginLoadError(f"Failed to initialize plugin: {e}") self._load_order_counter += 1 @@ -540,7 +540,7 @@ def unregister(self, name: str) -> bool: if plugin_info.instance: try: plugin_info.instance.shutdown() - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, AttributeError): pass # Ignore shutdown errors del self._plugins[name] @@ -691,7 +691,7 @@ def execute_hooks( kwargs.get("value"), ctx, ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, AttributeError) as e: # Log error but continue with other hooks ctx.log( f"Hook error in {hook.__class__.__name__}: {e}", @@ -743,7 +743,7 @@ def shutdown_all(self) -> None: if info and info.instance: try: info.instance.shutdown() - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, AttributeError): pass diff --git a/src/redops/core/rate_limiter.py b/src/redops/core/rate_limiter.py index ed9f06f..dfcb0c5 100644 --- a/src/redops/core/rate_limiter.py +++ b/src/redops/core/rate_limiter.py @@ -720,7 +720,7 @@ def execute_with_retry( try: return func(*args, **kwargs) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): with self._lock: self._attempt_counts[key] = attempt + 1 diff --git a/src/redops/core/scan_history.py b/src/redops/core/scan_history.py index f059515..ba190c3 100644 --- a/src/redops/core/scan_history.py +++ b/src/redops/core/scan_history.py @@ -162,7 +162,7 @@ def _get_connection(self): try: yield conn conn.commit() - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): conn.rollback() raise finally: diff --git a/src/redops/core/secrets.py b/src/redops/core/secrets.py index 55d50bf..430a4e4 100644 --- a/src/redops/core/secrets.py +++ b/src/redops/core/secrets.py @@ -198,12 +198,17 @@ def encrypt(self, plaintext: str) -> str: def decrypt(self, ciphertext: str) -> str: """Decrypt ciphertext using Fernet.""" - token = base64.urlsafe_b64decode(ciphertext.encode()) + from cryptography.fernet import InvalidToken + + try: + token = base64.urlsafe_b64decode(ciphertext.encode()) + except (ValueError, TypeError) as exc: + raise ValueError("Unable to decrypt: invalid ciphertext encoding") from exc # Try current key first try: return self._fernet.decrypt(token).decode() - except Exception: + except InvalidToken: pass # Try previous keys for rotation @@ -211,7 +216,7 @@ def decrypt(self, ciphertext: str) -> str: try: prev_fernet = self._fernet_class(prev_key) return prev_fernet.decrypt(token).decode() - except Exception: + except InvalidToken: continue raise ValueError("Unable to decrypt: invalid key or corrupted data") @@ -339,7 +344,7 @@ def _load(self) -> None: encrypted=False, ) self._loaded = True - except Exception as e: + except (OSError, ValueError, json.JSONDecodeError) as e: logger.error(f"Failed to load secrets: {e}") self._loaded = True @@ -850,11 +855,12 @@ def get_secret(self, name: str) -> Secret | None: encrypted=False, ) - # Notify callbacks + # Notify callbacks — broad guard is intentional: user-provided callbacks must + # not break secret retrieval regardless of what they raise. for callback in self._access_callbacks: try: callback(name, result) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.warning(f"Access callback failed: {e}") self._audit.log("get", name) diff --git a/src/redops/core/task_queue.py b/src/redops/core/task_queue.py index d51f0ba..01c5111 100644 --- a/src/redops/core/task_queue.py +++ b/src/redops/core/task_queue.py @@ -306,7 +306,7 @@ def _run_loop(self) -> None: self._execute_task(task) except queue.Empty: continue - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.error(f"Worker {self.worker_id} error: {e}") def _execute_task(self, task: Task) -> None: @@ -340,7 +340,7 @@ def _execute_task(self, task: Task) -> None: task.status = TaskStatus.TIMEOUT task.last_error = str(e) self._tasks_failed += 1 - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: result.error = str(e) result.error_type = type(e).__name__ task.last_error = str(e) @@ -360,7 +360,7 @@ def _execute_task(self, task: Task) -> None: if self._result_callback: try: self._result_callback(result) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.error(f"Result callback failed: {e}") def _execute_with_timeout(self, task: Task) -> Any: @@ -370,7 +370,7 @@ def _execute_with_timeout(self, task: Task) -> Any: def target(): try: result_container["result"] = task.func(*task.args, **task.kwargs) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: result_container["error"] = e thread = threading.Thread(target=target) @@ -593,7 +593,7 @@ def _scheduler_loop(self) -> None: else: break time.sleep(0.1) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.error(f"Scheduler error: {e}") def get_result( @@ -817,7 +817,7 @@ def _run_loop(self) -> None: if job["running_instances"] < job["max_instances"]: self._execute_job(job_id, job) time.sleep(0.05) # Check more frequently - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.error(f"Scheduler loop error: {e}") def _execute_job(self, job_id: str, job: dict[str, Any]) -> None: @@ -841,7 +841,7 @@ def run_job(): ) else: job["func"](*job["args"], **job["kwargs"]) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: logger.error(f"Job {job_id} failed: {e}") finally: with self._lock: diff --git a/src/redops/core/workflow.py b/src/redops/core/workflow.py index be67e1f..9049394 100644 --- a/src/redops/core/workflow.py +++ b/src/redops/core/workflow.py @@ -257,7 +257,7 @@ def execute(self, context: WorkflowContext) -> dict[str, TaskResult]: for f in futures: f.cancel() break - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: results[task.id] = TaskResult( task_id=task.id, state=TaskState.FAILED, @@ -282,7 +282,7 @@ def _execute_task(self, task: Task, context: WorkflowContext) -> TaskResult: start_time=start_time, end_time=datetime.now(timezone.utc), ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: return TaskResult( task_id=task.id, state=TaskState.FAILED, @@ -564,7 +564,7 @@ def execute( else: workflow.state = WorkflowState.COMPLETED - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): workflow.state = WorkflowState.FAILED raise @@ -617,7 +617,7 @@ def _execute_task( return result - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: last_error = e retries += 1 if retries <= task._retries: @@ -662,7 +662,7 @@ def _execute_parallel( try: result = future.result() results[task.id] = result - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: results[task.id] = TaskResult( task_id=task.id, state=TaskState.FAILED, diff --git a/src/redops/db/connection.py b/src/redops/db/connection.py index 05ba62e..20cafff 100644 --- a/src/redops/db/connection.py +++ b/src/redops/db/connection.py @@ -10,6 +10,7 @@ import logging from sqlalchemy import create_engine, event, text +from sqlalchemy.exc import OperationalError, DatabaseError, SQLAlchemyError from sqlalchemy.orm import sessionmaker, Session from sqlalchemy.pool import QueuePool @@ -141,7 +142,7 @@ def session_scope(self) -> Generator[Session, None, None]: try: yield session session.commit() - except Exception: + except SQLAlchemyError: session.rollback() raise finally: @@ -163,7 +164,7 @@ def check_connection(self) -> bool: with self.engine.connect() as conn: conn.execute(text("SELECT 1")) return True - except Exception as e: + except (OperationalError, DatabaseError, OSError) as e: logger.error(f"Database connection check failed: {e}") return False diff --git a/src/redops/jobs/queue.py b/src/redops/jobs/queue.py index 74bdd45..2a960b4 100644 --- a/src/redops/jobs/queue.py +++ b/src/redops/jobs/queue.py @@ -620,7 +620,7 @@ def _run_loop(self) -> None: # No jobs available, wait a bit self._shutdown_event.wait(0.5) - except Exception as e: + except (RuntimeError, TypeError, ValueError, OSError, ConnectionError) as e: logger.error(f"Worker error: {e}") if job: self._handle_job_error(job, e) @@ -662,7 +662,7 @@ def _execute_job(self, job: Job) -> None: job.completed_at = datetime.now(timezone.utc) logger.warning(f"Job {job.id} timed out") - except Exception as e: + except (RuntimeError, ImportError, TypeError, ValueError, OSError, ConnectionError) as e: self._handle_job_error(job, e) finally: @@ -682,7 +682,7 @@ def _execute_with_timeout( def target(): try: result_container["result"] = func(*args, **kwargs) - except Exception as e: + except (RuntimeError, ImportError, TypeError, ValueError, OSError, ConnectionError) as e: result_container["error"] = e thread = threading.Thread(target=target) diff --git a/src/redops/jobs/redis_backend.py b/src/redops/jobs/redis_backend.py index 702c983..912f6c7 100644 --- a/src/redops/jobs/redis_backend.py +++ b/src/redops/jobs/redis_backend.py @@ -490,7 +490,7 @@ def _scheduler_loop(self) -> None: count = self._queue.process_scheduled() if count > 0: logger.debug(f"Moved {count} scheduled jobs to queue") - except Exception as e: + except (ConnectionError, TimeoutError, OSError, RuntimeError) as e: logger.error(f"Scheduler error: {e}") self._shutdown_event.wait(self._scheduler_interval) @@ -503,7 +503,7 @@ def _cleanup_loop(self) -> None: count = self._queue.cleanup_stale() if count > 0: logger.info(f"Cleaned up {count} stale jobs") - except Exception as e: + except (ConnectionError, TimeoutError, OSError, RuntimeError) as e: logger.error(f"Cleanup error: {e}") self._shutdown_event.wait(self._cleanup_interval) diff --git a/src/redops/jobs/scheduler.py b/src/redops/jobs/scheduler.py index fb9a5fa..db94ec2 100644 --- a/src/redops/jobs/scheduler.py +++ b/src/redops/jobs/scheduler.py @@ -544,12 +544,12 @@ def _run_loop(self) -> None: logger.debug( f"Triggered scheduled job {job.id}: {job.name}" ) - except Exception as e: + except (RuntimeError, TypeError, ValueError, OSError, ConnectionError) as e: job.error_count += 1 job.last_error = str(e) logger.error(f"Failed to trigger job {job.id}: {e}") - except Exception as e: + except (RuntimeError, OSError, ConnectionError) as e: logger.error(f"Scheduler error: {e}") self._shutdown_event.wait(self._check_interval) diff --git a/src/redops/main.py b/src/redops/main.py index 0d50c56..1f0e3fe 100644 --- a/src/redops/main.py +++ b/src/redops/main.py @@ -97,7 +97,7 @@ def run_pipeline( return 0 - except Exception as e: + except (RuntimeError, ImportError, OSError, ValueError, TypeError, ConnectionError) as e: print(f"[RedOps] ERROR: {e}", file=sys.stderr) if config and config.output.verbose: import traceback @@ -131,7 +131,7 @@ def list_pipelines(directory: str = "./config/pipelines"): print(f" File: {pipeline_file.name}") print(f" Steps: {len(pipeline.steps)}") print() - except Exception as e: + except (ValueError, OSError, TypeError) as e: print(f" • {pipeline_file.name} (error loading: {e})") print() diff --git a/src/redops/mcp/server.py b/src/redops/mcp/server.py index e45a226..b9d2236 100644 --- a/src/redops/mcp/server.py +++ b/src/redops/mcp/server.py @@ -143,7 +143,7 @@ async def handle_message(self, message: dict) -> dict | None: return self._error_response( msg_id, -32601, f"Method not found: {method}" ) - except Exception as e: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError, ImportError) as e: return self._error_response(msg_id, -32603, str(e)) def _handle_initialize(self, msg_id: int, params: dict) -> dict: @@ -197,7 +197,7 @@ async def _handle_tools_call(self, msg_id: int, params: dict) -> dict: ], }, } - except Exception as e: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError, ImportError) as e: return { "jsonrpc": JSONRPC_VERSION, "id": msg_id, @@ -251,7 +251,7 @@ async def _tool_scan(self, arguments: dict) -> dict: for name, module_fn in modules: try: ctx = module_fn(ctx) - except Exception as e: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError, ImportError) as e: ctx.log(f"Module {name} failed: {e}", level="ERROR") return { @@ -374,7 +374,7 @@ async def run_server(): writer.write((json.dumps(response) + "\n").encode()) await writer.drain() - except Exception as e: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError, ImportError, TimeoutError) as e: # Log errors to stderr (not stdout which is for protocol) print(f"Error: {e}", file=sys.stderr) break diff --git a/src/redops/mcp/tools.py b/src/redops/mcp/tools.py index e454d63..37c17ae 100644 --- a/src/redops/mcp/tools.py +++ b/src/redops/mcp/tools.py @@ -219,7 +219,7 @@ async def _tool_check_ip(arguments: dict[str, Any]) -> dict[str, Any]: ctx = query_greynoise(ctx) results["sources"]["greynoise"] = ctx.get("greynoise_result", {}) - except Exception as e: + except (ImportError, RuntimeError, ValueError, TypeError, ConnectionError, OSError, TimeoutError) as e: results["sources"]["greynoise"] = {"error": str(e)} if "abuseipdb" in sources: @@ -228,7 +228,7 @@ async def _tool_check_ip(arguments: dict[str, Any]) -> dict[str, Any]: ctx = check_ip_reputation(ctx) results["sources"]["abuseipdb"] = ctx.get("abuseipdb_result", {}) - except Exception as e: + except (ImportError, RuntimeError, ValueError, TypeError, ConnectionError, OSError, TimeoutError) as e: results["sources"]["abuseipdb"] = {"error": str(e)} return results diff --git a/src/redops/modules/active/authorization.py b/src/redops/modules/active/authorization.py new file mode 100644 index 0000000..7533b23 --- /dev/null +++ b/src/redops/modules/active/authorization.py @@ -0,0 +1,167 @@ +"""Authorization gate for active/offensive modules. + +Requires recorded operator consent + authorized-target assertion before +any module under ``modules/active/`` can execute. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING + +from pydantic import BaseModel, Field + +from redops.modules.active.exceptions import ActiveAuthorizationError + +if TYPE_CHECKING: + from redops.core.context import Context + + +class ActiveAuthorization(BaseModel): + """Recorded operator consent for active/offensive operations. + + Attributes: + authorization_id: Unique identifier for this authorization. + operator: Identity of the operator giving consent. + target_assertion: The specific target(s) this authorization covers. + consent_text: The exact text the operator agreed to. + consent_timestamp: When consent was recorded. + expires_at: When this authorization expires. + """ + + authorization_id: str = Field(default_factory=lambda: str(uuid.uuid4())) + operator: str + target_assertion: str + consent_text: str + consent_timestamp: datetime = Field( + default_factory=lambda: datetime.now(timezone.utc) + ) + expires_at: datetime = Field( + default_factory=lambda: datetime.now(timezone.utc) + timedelta(hours=24) + ) + + def is_expired(self) -> bool: + """Return True if this authorization has expired.""" + return datetime.now(timezone.utc) > self.expires_at + + def is_valid(self) -> bool: + """Return True if authorization is present and not expired.""" + return not self.is_expired() + + +DEFAULT_CONSENT_TEXT = ( + "I am authorized to perform active security testing on the stated target. " + "This is my own network, a designated lab environment, or a system for which " + "I have explicit written permission. I understand that active modules can " + "disrupt network services and may violate laws if used without authorization." +) + + +def record_authorization( + ctx: Context, + operator: str, + target_assertion: str, + consent_text: str = DEFAULT_CONSENT_TEXT, + duration_hours: float = 24, +) -> ActiveAuthorization: + """Record operator consent in the pipeline context. + + Args: + ctx: Pipeline context to store authorization in. + operator: Identity of the operator (e.g. name, employee ID). + target_assertion: Specific target this authorization covers + (e.g. "192.168.99.0/24", "my-home-lab"). + consent_text: The consent text the operator acknowledged. + duration_hours: How long the authorization remains valid. + + Returns: + The created ActiveAuthorization instance. + """ + auth = ActiveAuthorization( + operator=operator, + target_assertion=target_assertion, + consent_text=consent_text, + expires_at=datetime.now(timezone.utc) + timedelta(hours=duration_hours), + ) + ctx.authorization = auth + ctx.log( + f"Active authorization recorded: {auth.authorization_id} " + f"for operator={operator} target={target_assertion}", + level="AUDIT", + ) + return auth + + +def is_active_authorized(ctx: Context) -> bool: + """Check whether the context carries a valid active authorization. + + Args: + ctx: Pipeline context. + + Returns: + True if a non-expired authorization is present, False otherwise. + """ + auth = getattr(ctx, "authorization", None) + if auth is None: + return False + if isinstance(auth, ActiveAuthorization): + return auth.is_valid() + return False + + +def record_authorization_from_params( + ctx: Context, + params: dict | None = None, +) -> Context: + """Pipeline-step wrapper for ``record_authorization``. + + Accepts parameters via the ``params`` dict so it can be invoked from a + pipeline JSON definition. + + Params: + operator: Identity of the operator. + target_assertion: Specific target being authorized. + consent_text: Optional custom consent text. + duration_hours: How long the authorization remains valid. + + Returns: + The updated context with ``ctx.authorization`` set. + """ + params = params or {} + record_authorization( + ctx, + operator=params.get("operator", "unknown-operator"), + target_assertion=params.get("target_assertion", ctx.target or "unknown"), + consent_text=params.get("consent_text", DEFAULT_CONSENT_TEXT), + duration_hours=params.get("duration_hours", 24), + ) + return ctx + + +def assert_active_authorized(ctx: Context) -> None: + """Raise ActiveAuthorizationError if the context lacks valid authorization. + + Every function under ``modules/active/`` must call this at entry. + + Args: + ctx: Pipeline context. + + Raises: + ActiveAuthorizationError: If no valid authorization is present. + """ + auth = getattr(ctx, "authorization", None) + if auth is None: + raise ActiveAuthorizationError( + "Active module refused: no operator authorization recorded. " + "Call record_authorization() before executing active modules." + ) + if isinstance(auth, ActiveAuthorization) and auth.is_expired(): + raise ActiveAuthorizationError( + f"Active module refused: authorization {auth.authorization_id} " + f"expired at {auth.expires_at.isoformat()}." + ) + if not isinstance(auth, ActiveAuthorization): + raise ActiveAuthorizationError( + "Active module refused: authorization object is malformed." + ) diff --git a/src/redops/modules/active/egress.py b/src/redops/modules/active/egress.py new file mode 100644 index 0000000..7a2783e --- /dev/null +++ b/src/redops/modules/active/egress.py @@ -0,0 +1,142 @@ +"""Egress enforcement for active chain execution. + +Blocks HTTP/HTTPS requests to non-local destinations while active +authorization is in scope. Ensures active modules can only reach +localhost services (e.g. Ollama) and cannot leak data to cloud APIs. +""" + +from __future__ import annotations + +import ipaddress +import threading +from contextlib import contextmanager +from typing import Any, Callable +from urllib.parse import urlparse + +from redops.modules.active.exceptions import EgressBlockedError + +# Thread-local flag indicating whether egress blocking is active +_egress_local = threading.local() + +# Hostnames and IPs considered "local" — always allowed +_LOCAL_HOSTS = {"localhost", "127.0.0.1", "::1", "0.0.0.0"} + + +def _is_local_url(url: str) -> bool: + """Return True if the URL points to a local/loopback destination.""" + parsed = urlparse(url) + hostname = parsed.hostname or "" + if hostname in _LOCAL_HOSTS: + return True + try: + addr = ipaddress.ip_address(hostname) + return addr.is_loopback + except ValueError: + # Not an IP — could be a local domain like my-service.local + return hostname.endswith(".local") or hostname.endswith(".localhost") + + +def _is_egress_blocked() -> bool: + """Return True if egress blocking is currently active in this thread.""" + return getattr(_egress_local, "depth", 0) > 0 + + +def _assert_local_url(url: str) -> None: + """Raise EgressBlockedError if url is non-local while blocking is active.""" + if _is_egress_blocked() and not _is_local_url(url): + raise EgressBlockedError( + f"Egress blocked: active chain execution prevented external request to {url}. " + "Only localhost/loopback endpoints are permitted during active operations." + ) + + +# Storage for original references so we can restore precisely +_request_patches: dict[str, Any] = {} # type: ignore[name-defined] + + +def _wrap_requests() -> None: + """Monkey-patch requests to enforce egress policy (idempotent).""" + try: + import requests + except ImportError: + return + + if _request_patches: + return # Already patched + + _request_patches["Session.request"] = requests.Session.request + + def _patched_request( + self, + method: str, + url: str, + *args, + **kwargs, + ): + _assert_local_url(url) + return _request_patches["Session.request"](self, method, url, *args, **kwargs) + + requests.Session.request = _patched_request # type: ignore[method-assign] + + _request_patches["get"] = requests.get + _request_patches["post"] = requests.post + + def _patched_get(url, **kwargs): + _assert_local_url(url) + return _request_patches["get"](url, **kwargs) + + def _patched_post(url, **kwargs): + _assert_local_url(url) + return _request_patches["post"](url, **kwargs) + + requests.get = _patched_get # type: ignore[method-assign] + requests.post = _patched_post # type: ignore[method-assign] + + +def _unwrap_requests() -> None: + """Remove monkey-patches from requests.""" + global _request_patches + try: + import requests + except ImportError: + return + + if not _request_patches: + return + + requests.Session.request = _request_patches["Session.request"] # type: ignore[method-assign] + requests.get = _request_patches["get"] # type: ignore[method-assign] + requests.post = _request_patches["post"] # type: ignore[method-assign] + _request_patches.clear() + + +@contextmanager +def block_external_egress(): + """Context manager that blocks external HTTP egress for the active chain. + + Supports nested usage via a thread-local reference count. + + Usage:: + + with block_external_egress(): + # Any non-local requests here will raise EgressBlockedError + requests.post("http://localhost:11434/api/generate") # OK + requests.get("https://api.openai.com/v1/chat") # Raises + + Yields: + None + + Raises: + EgressBlockedError: If a non-local HTTP request is attempted. + """ + # Track nesting depth per thread + depth = getattr(_egress_local, "depth", 0) + if depth == 0: + _wrap_requests() + _egress_local.depth = depth + 1 + try: + yield + finally: + _egress_local.depth = max(0, getattr(_egress_local, "depth", 1) - 1) + if getattr(_egress_local, "depth", 0) == 0: + _unwrap_requests() diff --git a/src/redops/modules/active/exceptions.py b/src/redops/modules/active/exceptions.py new file mode 100644 index 0000000..e73138d --- /dev/null +++ b/src/redops/modules/active/exceptions.py @@ -0,0 +1,13 @@ +"""Exceptions for the active/offensive module gate.""" + + +class ActiveAuthorizationError(Exception): + """Raised when an active/offensive module runs without recorded operator consent.""" + + pass + + +class EgressBlockedError(Exception): + """Raised when an active module attempts external egress to a non-local URL.""" + + pass diff --git a/src/redops/modules/active/exploit/cve_check.py b/src/redops/modules/active/exploit/cve_check.py index c9eb979..3a0eccf 100644 --- a/src/redops/modules/active/exploit/cve_check.py +++ b/src/redops/modules/active/exploit/cve_check.py @@ -7,6 +7,7 @@ from typing import Any from redops.core.context import Context +from redops.modules.active.authorization import assert_active_authorized CVSS_HIGH_VALUE_THRESHOLD = 9.0 @@ -32,6 +33,7 @@ def check_cves(ctx: Context, params: dict[str, Any] | None = None) -> Context: cve_findings: list of dicts with ip, port, cve_id, cvss, description high_value_targets: hosts with CVSS >= 9.0 """ + assert_active_authorized(ctx) params = params or {} scan_results = ctx.get("port_scan_results", []) cve_findings: list[dict] = [] diff --git a/src/redops/modules/active/network/arp_scan.py b/src/redops/modules/active/network/arp_scan.py index 610c966..ff30756 100644 --- a/src/redops/modules/active/network/arp_scan.py +++ b/src/redops/modules/active/network/arp_scan.py @@ -10,6 +10,7 @@ from typing import Any from redops.core.context import Context +from redops.modules.active.authorization import assert_active_authorized def discover_hosts(ctx: Context, params: dict[str, Any] | None = None) -> Context: @@ -23,6 +24,7 @@ def discover_hosts(ctx: Context, params: dict[str, Any] | None = None) -> Contex Adds to context: live_hosts: list of dicts with ip, mac, vendor """ + assert_active_authorized(ctx) params = params or {} subnet = params.get("subnet") or ctx.get("ap_subnet", "192.168.99.0/24") wait = params.get("wait", 15) diff --git a/src/redops/modules/active/network/port_scan.py b/src/redops/modules/active/network/port_scan.py index 7d8716c..2722f2d 100644 --- a/src/redops/modules/active/network/port_scan.py +++ b/src/redops/modules/active/network/port_scan.py @@ -7,6 +7,7 @@ from typing import Any from redops.core.context import Context +from redops.modules.active.authorization import assert_active_authorized NMAP_TIMEOUT = 120 @@ -22,6 +23,7 @@ def scan_ports(ctx: Context, params: dict[str, Any] | None = None) -> Context: Adds to context: port_scan_results: list of dicts with ip, open_ports """ + assert_active_authorized(ctx) params = params or {} ports = params.get("ports", "T:1-1024,U:23,2323") timing = params.get("timing", "T4") diff --git a/src/redops/modules/active/wireless/deauth.py b/src/redops/modules/active/wireless/deauth.py index 0981e2e..dc53c30 100644 --- a/src/redops/modules/active/wireless/deauth.py +++ b/src/redops/modules/active/wireless/deauth.py @@ -10,6 +10,7 @@ from typing import Any from redops.core.context import Context +from redops.modules.active.authorization import assert_active_authorized try: from scapy.all import Dot11, Dot11Deauth, RadioTap, sendp @@ -34,6 +35,7 @@ def deauth_flood(ctx: Context, params: dict[str, Any] | None = None) -> Context: deauth_active: bool deauth_thread: thread handle """ + assert_active_authorized(ctx) if not HAS_SCAPY: ctx.log("Scapy not installed, cannot run deauth", level="ERROR") ctx.add("deauth_active", False) diff --git a/src/redops/modules/active/wireless/evil_twin.py b/src/redops/modules/active/wireless/evil_twin.py index 6bc0a1c..4cb7100 100644 --- a/src/redops/modules/active/wireless/evil_twin.py +++ b/src/redops/modules/active/wireless/evil_twin.py @@ -11,6 +11,7 @@ from typing import Any from redops.core.context import Context +from redops.modules.active.authorization import assert_active_authorized HOSTAPD_CONF_TEMPLATE = """\ interface={ap_interface} @@ -51,6 +52,7 @@ def start_evil_twin(ctx: Context, params: dict[str, Any] | None = None) -> Conte evil_twin_bssid, ap_subnet, captured_clients, hostapd_proc, dnsmasq_proc """ + assert_active_authorized(ctx) params = params or {} ap_interface = params.get("ap_interface", "wlan0") ap_ip = params.get("ap_ip", "192.168.99.1") diff --git a/src/redops/modules/active/wireless/monitor.py b/src/redops/modules/active/wireless/monitor.py index 47b1161..5ae1aa2 100644 --- a/src/redops/modules/active/wireless/monitor.py +++ b/src/redops/modules/active/wireless/monitor.py @@ -10,6 +10,7 @@ from typing import Any from redops.core.context import Context +from redops.modules.active.authorization import assert_active_authorized def get_wireless_interfaces() -> list[str]: @@ -33,6 +34,7 @@ def enable_monitor_mode(ctx: Context, params: dict[str, Any] | None = None) -> C monitor_interface: Monitor mode interface name (e.g. wlan1mon) monitor_ready: True if mode switch succeeded """ + assert_active_authorized(ctx) params = params or {} interface = params.get("interface", "wlan1") @@ -74,6 +76,7 @@ def disable_monitor_mode(ctx: Context, params: dict[str, Any] | None = None) -> Params: interface: Monitor interface to stop. Default: reads from context. """ + assert_active_authorized(ctx) params = params or {} monitor_iface = params.get("interface") or ctx.get("monitor_interface", "wlan1mon") diff --git a/src/redops/modules/active/wireless/scan.py b/src/redops/modules/active/wireless/scan.py index 3046fac..e771631 100644 --- a/src/redops/modules/active/wireless/scan.py +++ b/src/redops/modules/active/wireless/scan.py @@ -13,6 +13,7 @@ from typing import Any from redops.core.context import Context +from redops.modules.active.authorization import assert_active_authorized def scan_access_points(ctx: Context, params: dict[str, Any] | None = None) -> Context: @@ -28,6 +29,7 @@ def scan_access_points(ctx: Context, params: dict[str, Any] | None = None) -> Co clients: list of dicts with mac, associated_bssid, signal scan_complete: bool """ + assert_active_authorized(ctx) params = params or {} duration = params.get("duration", 30) channel = params.get("channel") diff --git a/src/redops/modules/ai/agent.py b/src/redops/modules/ai/agent.py index f98d4c2..46494ad 100644 --- a/src/redops/modules/ai/agent.py +++ b/src/redops/modules/ai/agent.py @@ -11,6 +11,8 @@ from typing import Any from redops.core.context import Context +from redops.modules.active.authorization import assert_active_authorized +from redops.modules.active.egress import block_external_egress from redops.modules.ai.planner import build_attack_surface_summary from redops.modules.ai.tools import TOOL_REGISTRY, get_tool_descriptions @@ -78,6 +80,8 @@ def run_agent(ctx: Context, params: dict[str, Any] | None = None) -> Context: agent_complete: bool agent_summary: str """ + assert_active_authorized(ctx) + if not HAS_REQUESTS: ctx.log("requests library not installed", level="ERROR") ctx.add("agent_complete", False) @@ -121,58 +125,59 @@ def run_agent(ctx: Context, params: dict[str, Any] | None = None) -> Context: level="INFO", ) - for iteration in range(max_iterations): - attack_surface = build_attack_surface_summary(ctx) - user_message = f"Iteration {iteration + 1}/{max_iterations}\n\n{attack_surface}" - - ctx.log(f"Agent iteration {iteration + 1}", level="INFO") + with block_external_egress(): + for iteration in range(max_iterations): + attack_surface = build_attack_surface_summary(ctx) + user_message = f"Iteration {iteration + 1}/{max_iterations}\n\n{attack_surface}" - response = _call_ollama( - model, system, user_message, temperature=temperature, options=options - ) - if not response: - ctx.log("Ollama returned empty response", level="ERROR") - break + ctx.log(f"Agent iteration {iteration + 1}", level="INFO") - parsed = _parse_agent_response(response) - if not parsed: - ctx.log( - f"Could not parse agent response: {response[:200]}", - level="ERROR", + response = _call_ollama( + model, system, user_message, temperature=temperature, options=options ) - break - - thought = parsed.get("thought", "") - action = parsed.get("action", "") - action_params = parsed.get("params", {}) - - ctx.log(f"Agent thought: {thought}", level="INFO") - ctx.log(f"Agent action: {action} | params: {action_params}", level="INFO") - - agent_log.append( - { - "iteration": iteration + 1, - "thought": thought, - "action": action, - "params": action_params, - } - ) - - if action == "COMPLETE": - ctx.add("agent_complete", True) - ctx.add( - "agent_summary", - parsed.get("summary", "Agent completed chain."), + if not response: + ctx.log("Ollama returned empty response", level="ERROR") + break + + parsed = _parse_agent_response(response) + if not parsed: + ctx.log( + f"Could not parse agent response: {response[:200]}", + level="ERROR", + ) + break + + thought = parsed.get("thought", "") + action = parsed.get("action", "") + action_params = parsed.get("params", {}) + + ctx.log(f"Agent thought: {thought}", level="INFO") + ctx.log(f"Agent action: {action} | params: {action_params}", level="INFO") + + agent_log.append( + { + "iteration": iteration + 1, + "thought": thought, + "action": action, + "params": action_params, + } ) - ctx.log(f"Agent complete: {parsed.get('summary')}", level="INFO") - break - - if action in TOOL_REGISTRY: - tool_fn = TOOL_REGISTRY[action]["fn"] - ctx = tool_fn(ctx, action_params) - ctx.log(f"Tool {action} executed", level="INFO") - else: - ctx.log(f"Unknown tool: {action}", level="WARNING") + + if action == "COMPLETE": + ctx.add("agent_complete", True) + ctx.add( + "agent_summary", + parsed.get("summary", "Agent completed chain."), + ) + ctx.log(f"Agent complete: {parsed.get('summary')}", level="INFO") + break + + if action in TOOL_REGISTRY: + tool_fn = TOOL_REGISTRY[action]["fn"] + ctx = tool_fn(ctx, action_params) + ctx.log(f"Tool {action} executed", level="INFO") + else: + ctx.log(f"Unknown tool: {action}", level="WARNING") ctx.add("agent_log", agent_log) if not ctx.get("agent_complete"): diff --git a/src/redops/modules/ai/tools.py b/src/redops/modules/ai/tools.py index d2de924..204eb18 100644 --- a/src/redops/modules/ai/tools.py +++ b/src/redops/modules/ai/tools.py @@ -17,31 +17,37 @@ "fn": scan_access_points, "description": "Passive WiFi scan. Returns list of APs and clients.", "params": ["duration (int, seconds)", "channel (optional, int)"], + "requires_authorization": True, }, "start_evil_twin": { "fn": start_evil_twin, "description": "Clone target AP and start rogue access point.", "params": ["target_bssid (optional)", "ap_interface (str)"], + "requires_authorization": True, }, "deauth_flood": { "fn": deauth_flood, "description": "Deauth flood target AP clients.", "params": ["duration (int, seconds)", "count (int, frames per burst)"], + "requires_authorization": True, }, "discover_hosts": { "fn": discover_hosts, "description": "ARP scan evil twin subnet for live hosts.", "params": ["wait (int, seconds before scan)"], + "requires_authorization": True, }, "scan_ports": { "fn": scan_ports, "description": "nmap service scan on live hosts.", "params": ["ports (str, range)", "timing (str, T1-T5)"], + "requires_authorization": True, }, "check_cves": { "fn": check_cves, "description": "Cross-reference discovered services against known CVEs.", "params": [], + "requires_authorization": True, }, } diff --git a/src/redops/modules/ai_assistant.py b/src/redops/modules/ai_assistant.py index cdbe196..70c9c03 100644 --- a/src/redops/modules/ai_assistant.py +++ b/src/redops/modules/ai_assistant.py @@ -37,16 +37,44 @@ def get_api_key(provider: str) -> str | None: return config.get("api_keys", {}).get(provider) +# Approximate pricing per 1K tokens (input / output) in USD +# Used for budget enforcement. Prices are conservative estimates. +_PROVIDER_PRICING: dict[str, tuple[float, float]] = { + "openai": (0.005, 0.015), + "anthropic": (0.008, 0.024), + "gemini": (0.0005, 0.0015), + "groq": (0.0005, 0.0005), + "ollama": (0.0, 0.0), +} + + +def _approximate_tokens(text: str) -> int: + """Rough token count fallback when tiktoken is unavailable.""" + return max(1, len(text) // 4) + + +def _count_openai_tokens(text: str, model: str = "gpt-4o") -> int: + """Count tokens for OpenAI models.""" + try: + import tiktoken + + encoding = tiktoken.encoding_for_model(model) + return len(encoding.encode(text)) + except (OSError, RuntimeError, TypeError, ValueError, ImportError): + return _approximate_tokens(text) + + class AIAssistant: """AI-powered assistant for security analysis.""" - def __init__(self, provider: str = None, model: str = None): + def __init__(self, provider: str = None, model: str = None, budget_limit: float | None = None): """ Initialize the AI assistant. Args: provider: AI provider (openai, anthropic). Defaults to config. model: Model to use. Defaults to config. + budget_limit: Maximum estimated spend in USD for this instance. """ config = load_config() ai_config = config.get("ai", {}) @@ -55,6 +83,15 @@ def __init__(self, provider: str = None, model: str = None): self.model = model or ai_config.get("model", "gpt-4o-mini") self.max_tokens = ai_config.get("max_tokens", 2048) self.temperature = ai_config.get("temperature", 0.7) + self.budget_limit = budget_limit or ai_config.get("budget_limit") + + # Cost tracking (accumulates across calls on this instance) + self._cost_tracker = { + "calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "estimated_cost_usd": 0.0, + } # Get API key (not required for Ollama) self.api_key = get_api_key(self.provider) @@ -125,21 +162,6 @@ def _init_client(self): else: raise ValueError(f"Unsupported provider: {self.provider}") - def _call_api(self, prompt: str, system_prompt: str = None) -> str: - """Call the AI API with the given prompt.""" - if self.provider == "openai": - return self._call_openai(prompt, system_prompt) - elif self.provider == "anthropic": - return self._call_anthropic(prompt, system_prompt) - elif self.provider == "gemini": - return self._call_gemini(prompt, system_prompt) - elif self.provider == "ollama": - return self._call_ollama(prompt, system_prompt) - elif self.provider == "groq": - return self._call_groq(prompt, system_prompt) - else: - raise ValueError(f"Unsupported provider: {self.provider}") - def _call_openai(self, prompt: str, system_prompt: str = None) -> str: """Call OpenAI API.""" messages = [] @@ -220,6 +242,89 @@ def _call_groq(self, prompt: str, system_prompt: str = None) -> str: return response.choices[0].message.content + # ------------------------------------------------------------------ + # Cost management + # ------------------------------------------------------------------ + + def _count_tokens(self, text: str) -> int: + """Estimate token count for a text string.""" + if self.provider == "openai": + return _count_openai_tokens(text, self.model) + return _approximate_tokens(text) + + def _estimate_cost(self, input_tokens: int, output_tokens: int) -> float: + """Estimate API cost in USD based on token counts.""" + input_price, output_price = _PROVIDER_PRICING.get( + self.provider, (0.005, 0.015) + ) + return (input_tokens / 1000.0) * input_price + (output_tokens / 1000.0) * output_price + + def _check_budget(self, estimated_cost: float) -> None: + """Raise if the estimated cost would exceed the budget.""" + if self.budget_limit is None: + return + current = self._cost_tracker["estimated_cost_usd"] + if current + estimated_cost > self.budget_limit: + raise RuntimeError( + f"AI budget exceeded: ${current:.4f} spent + ${estimated_cost:.4f} estimated " + f"> ${self.budget_limit:.4f} limit. " + f"Increase budget_limit or switch to local provider (ollama)." + ) + + def _record_usage(self, prompt: str, response_text: str) -> None: + """Record token usage and cost for a completed API call.""" + input_tokens = self._count_tokens(prompt) + output_tokens = self._count_tokens(response_text) + cost = self._estimate_cost(input_tokens, output_tokens) + + self._cost_tracker["calls"] += 1 + self._cost_tracker["input_tokens"] += input_tokens + self._cost_tracker["output_tokens"] += output_tokens + self._cost_tracker["estimated_cost_usd"] += cost + + def get_cost_metrics(self) -> dict[str, Any]: + """Return current cost metrics for this assistant instance.""" + return { + "provider": self.provider, + "model": self.model, + **self._cost_tracker, + "budget_limit_usd": self.budget_limit, + "budget_remaining_usd": ( + self.budget_limit - self._cost_tracker["estimated_cost_usd"] + if self.budget_limit is not None + else None + ), + } + + # ------------------------------------------------------------------ + # API call wrappers with cost tracking + # ------------------------------------------------------------------ + + def _call_api(self, prompt: str, system_prompt: str = None) -> str: + """Call the AI API with the given prompt (budget-aware).""" + # Build full prompt text for token estimation + full_prompt = f"{system_prompt or ''}\n\n{prompt}" + estimated_input = self._count_tokens(full_prompt) + estimated_output = self.max_tokens # worst-case + estimated_cost = self._estimate_cost(estimated_input, estimated_output) + self._check_budget(estimated_cost) + + if self.provider == "openai": + result = self._call_openai(prompt, system_prompt) + elif self.provider == "anthropic": + result = self._call_anthropic(prompt, system_prompt) + elif self.provider == "gemini": + result = self._call_gemini(prompt, system_prompt) + elif self.provider == "ollama": + result = self._call_ollama(prompt, system_prompt) + elif self.provider == "groq": + result = self._call_groq(prompt, system_prompt) + else: + raise ValueError(f"Unsupported provider: {self.provider}") + + self._record_usage(full_prompt, result) + return result + def analyze_findings(self, scan_data: dict[str, Any]) -> str: """ Analyze security scan findings and provide insights. @@ -478,9 +583,9 @@ def _extract_findings_summary(self, scan_data: dict[str, Any]) -> dict[str, Any] # Convenience functions for module integration def ai_analyze(ctx, params: dict[str, Any] | None = None): """Module wrapper for AI analysis.""" - + params = params or {} try: - assistant = AIAssistant() + assistant = AIAssistant(budget_limit=params.get("budget_limit")) analysis = assistant.analyze_findings(ctx.data) ctx.add( "ai_analysis", @@ -488,9 +593,10 @@ def ai_analyze(ctx, params: dict[str, Any] | None = None): "analysis": analysis, "provider": assistant.provider, "model": assistant.model, + "cost": assistant.get_cost_metrics(), }, ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ImportError) as e: ctx.log(f"AI analysis failed: {e}", level="ERROR") return ctx @@ -498,9 +604,9 @@ def ai_analyze(ctx, params: dict[str, Any] | None = None): def ai_summarize(ctx, params: dict[str, Any] | None = None): """Module wrapper for AI summarization.""" - + params = params or {} try: - assistant = AIAssistant() + assistant = AIAssistant(budget_limit=params.get("budget_limit")) summary = assistant.summarize(ctx.data) ctx.add( "ai_summary", @@ -508,9 +614,10 @@ def ai_summarize(ctx, params: dict[str, Any] | None = None): "summary": summary, "provider": assistant.provider, "model": assistant.model, + "cost": assistant.get_cost_metrics(), }, ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ImportError) as e: ctx.log(f"AI summarization failed: {e}", level="ERROR") return ctx @@ -518,9 +625,9 @@ def ai_summarize(ctx, params: dict[str, Any] | None = None): def ai_recommend(ctx, params: dict[str, Any] | None = None): """Module wrapper for AI recommendations.""" - + params = params or {} try: - assistant = AIAssistant() + assistant = AIAssistant(budget_limit=params.get("budget_limit")) recommendations = assistant.suggest_remediations(ctx.data) ctx.add( "ai_recommendations", @@ -528,9 +635,10 @@ def ai_recommend(ctx, params: dict[str, Any] | None = None): "recommendations": recommendations, "provider": assistant.provider, "model": assistant.model, + "cost": assistant.get_cost_metrics(), }, ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ImportError) as e: ctx.log(f"AI recommendations failed: {e}", level="ERROR") return ctx diff --git a/src/redops/modules/intel/censys_intel.py b/src/redops/modules/intel/censys_intel.py index e1dd836..12fa085 100644 --- a/src/redops/modules/intel/censys_intel.py +++ b/src/redops/modules/intel/censys_intel.py @@ -89,7 +89,7 @@ def get_censys_client(): api_id = get_api_key_direct("censys_id") api_secret = get_api_key_direct("censys_secret") - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, ImportError): pass if not api_id or not api_secret: @@ -99,7 +99,7 @@ def get_censys_client(): hosts = CensysHosts(api_id=api_id, api_secret=api_secret) certs = CensysCerts(api_id=api_id, api_secret=api_secret) return hosts, certs - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, ImportError): return None, None @@ -149,7 +149,7 @@ def query_censys_host(ctx: Context, params: dict[str, Any] | None = None) -> Con ip = socket.gethostbyname(target) ctx.log(f"Resolved {target} to {ip}", level="DEBUG") - except Exception as e: + except (OSError, ValueError, TypeError) as e: ctx.log(f"Could not resolve {target}: {e}", level="WARNING") censys_data["error"] = f"Could not resolve domain: {e}" ctx.add("censys_host", censys_data) @@ -210,7 +210,7 @@ def query_censys_host(ctx: Context, params: dict[str, Any] | None = None) -> Con level="INFO", ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, KeyError, IndexError, AttributeError) as e: error_msg = str(e) if "404" in error_msg or "not found" in error_msg.lower(): censys_data["error"] = f"No Censys data for {ip}" @@ -293,7 +293,7 @@ def query_censys_certificates( level="INFO", ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, KeyError, IndexError, AttributeError) as e: cert_data["error"] = f"Censys certificate error: {str(e)}" ctx.log(cert_data["error"], level="WARNING") @@ -380,7 +380,7 @@ def search_censys_hosts(ctx: Context, params: dict[str, Any] | None = None) -> C level="INFO", ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, KeyError, IndexError, AttributeError) as e: search_data["error"] = f"Censys search error: {str(e)}" ctx.log(search_data["error"], level="WARNING") diff --git a/src/redops/modules/intel/hibp_intel.py b/src/redops/modules/intel/hibp_intel.py index 946c584..1c297f4 100644 --- a/src/redops/modules/intel/hibp_intel.py +++ b/src/redops/modules/intel/hibp_intel.py @@ -92,7 +92,7 @@ def get_hibp_api_key() -> str | None: from redops.cli.settings import get_api_key_direct api_key = get_api_key_direct("hibp") - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, ImportError): pass return api_key @@ -132,7 +132,7 @@ def _make_hibp_request( return {"error": "rate_limited"} else: return {"error": f"HTTP {response.status_code}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"error": str(e)} diff --git a/src/redops/modules/intel/hunter_intel.py b/src/redops/modules/intel/hunter_intel.py index a505f8d..85d6cb6 100644 --- a/src/redops/modules/intel/hunter_intel.py +++ b/src/redops/modules/intel/hunter_intel.py @@ -81,7 +81,7 @@ def get_hunter_api_key() -> str | None: from redops.cli.settings import get_api_key_direct api_key = get_api_key_direct("hunter") - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, ImportError): pass # Settings module may not be available - use env var fallback return api_key @@ -112,7 +112,7 @@ def _make_hunter_request( return {"error": "invalid_api_key"} else: return {"error": f"HTTP {response.status_code}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"error": str(e)} diff --git a/src/redops/modules/intel/securitytrails_intel.py b/src/redops/modules/intel/securitytrails_intel.py index 22c6a88..49c4d72 100644 --- a/src/redops/modules/intel/securitytrails_intel.py +++ b/src/redops/modules/intel/securitytrails_intel.py @@ -68,7 +68,7 @@ def get_st_api_key() -> str | None: from redops.cli.settings import get_api_key_direct api_key = get_api_key_direct("securitytrails") - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, ImportError): pass return api_key @@ -93,7 +93,7 @@ def _make_st_request(endpoint: str, api_key: str) -> dict[str, Any] | None: return {"error": "rate_limited"} else: return {"error": f"HTTP {response.status_code}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"error": str(e)} diff --git a/src/redops/modules/intel/shodan_intel.py b/src/redops/modules/intel/shodan_intel.py index 00f13b1..704840a 100644 --- a/src/redops/modules/intel/shodan_intel.py +++ b/src/redops/modules/intel/shodan_intel.py @@ -93,7 +93,7 @@ def get_shodan_client(): from redops.cli.settings import get_api_key_direct api_key = get_api_key_direct("shodan") - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, ImportError): pass if not api_key: @@ -149,7 +149,7 @@ def query_shodan_host(ctx: Context, params: dict[str, Any] | None = None) -> Con ip = socket.gethostbyname(target) ctx.log(f"Resolved {target} to {ip}", level="DEBUG") - except Exception as e: + except (OSError, ValueError, TypeError) as e: ctx.log(f"Could not resolve {target}: {e}", level="WARNING") shodan_data["error"] = f"Could not resolve domain: {e}" ctx.add("shodan_host", shodan_data) @@ -194,7 +194,7 @@ def query_shodan_host(ctx: Context, params: dict[str, Any] | None = None) -> Con level="INFO", ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, KeyError, IndexError, AttributeError) as e: error_msg = str(e) if "No information available" in error_msg: shodan_data["error"] = f"No Shodan data for {ip}" @@ -264,7 +264,7 @@ def query_shodan_dns(ctx: Context, params: dict[str, Any] | None = None) -> Cont level="INFO", ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, KeyError, IndexError, AttributeError) as e: dns_data["error"] = f"Shodan DNS error: {str(e)}" ctx.log(dns_data["error"], level="WARNING") @@ -345,7 +345,7 @@ def search_shodan(ctx: Context, params: dict[str, Any] | None = None) -> Context level="INFO", ) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, KeyError, IndexError, AttributeError) as e: search_data["error"] = f"Shodan search error: {str(e)}" ctx.log(search_data["error"], level="WARNING") diff --git a/src/redops/modules/intel/stix_export.py b/src/redops/modules/intel/stix_export.py index 755f1bf..429661a 100644 --- a/src/redops/modules/intel/stix_export.py +++ b/src/redops/modules/intel/stix_export.py @@ -384,7 +384,7 @@ def export_to_stix(ctx, params: dict[str, Any] | None = None): ctx.add("stix_bundle", bundle.to_dict()) ctx.log(f"STIX bundle created with {len(bundle.objects)} objects", level="INFO") - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, KeyError, IndexError, AttributeError) as e: ctx.log(f"STIX export failed: {e}", level="ERROR") return ctx diff --git a/src/redops/modules/intel/virustotal_intel.py b/src/redops/modules/intel/virustotal_intel.py index f0d1f46..3ecae1d 100644 --- a/src/redops/modules/intel/virustotal_intel.py +++ b/src/redops/modules/intel/virustotal_intel.py @@ -103,7 +103,7 @@ def get_vt_api_key() -> str | None: from redops.cli.settings import get_api_key_direct api_key = get_api_key_direct("virustotal") - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, ImportError): pass return api_key @@ -126,7 +126,7 @@ def _make_vt_request(endpoint: str, api_key: str) -> dict[str, Any] | None: return {"error": "not_found"} else: return {"error": f"HTTP {response.status_code}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"error": str(e)} @@ -228,7 +228,7 @@ def query_vt_ip(ctx: Context, params: dict[str, Any] | None = None) -> Context: import socket ip = socket.gethostbyname(target) - except Exception as e: + except (OSError, ValueError, TypeError) as e: ctx.log(f"Could not resolve {target}: {e}", level="WARNING") return ctx diff --git a/src/redops/modules/metadata/code_artifacts.py b/src/redops/modules/metadata/code_artifacts.py index 6b9e1d3..cfcb06f 100644 --- a/src/redops/modules/metadata/code_artifacts.py +++ b/src/redops/modules/metadata/code_artifacts.py @@ -494,7 +494,7 @@ def parse_requirements_txt(file_path: Path) -> list[dict[str, str]]: deps.append({"name": name, "version": version}) return deps - except Exception: + except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError): return [] @@ -528,7 +528,7 @@ def parse_pyproject_toml(file_path: Path) -> list[dict[str, str]]: ) return deps - except Exception: + except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError): return [] @@ -560,7 +560,7 @@ def parse_package_json(file_path: Path) -> list[dict[str, str]]: ) return deps - except Exception: + except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError, json.JSONDecodeError): return [] @@ -598,7 +598,7 @@ def parse_go_mod(file_path: Path) -> list[dict[str, str]]: deps.append({"name": match.group(1), "version": match.group(2)}) return deps - except Exception: + except (OSError, ValueError, TypeError, IndexError, AttributeError): return [] @@ -627,7 +627,7 @@ def parse_gemfile(file_path: Path) -> list[dict[str, str]]: deps.append({"name": name, "version": version}) return deps - except Exception: + except (OSError, ValueError, TypeError, IndexError, AttributeError): return [] @@ -658,7 +658,7 @@ def parse_cargo_toml(file_path: Path) -> list[dict[str, str]]: deps.append({"name": match.group(1), "version": match.group(2)}) return deps - except Exception: + except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError): return [] @@ -775,7 +775,7 @@ def scan_file_for_secrets( } ) - except Exception: + except (OSError, ValueError, TypeError, IndexError, AttributeError): pass return results @@ -895,7 +895,7 @@ def extract_git_metadata(repo_path: str) -> dict[str, Any] | None: url = re.sub(r"://[^:]+:[^@]+@", "://***:***@", url) metadata["remote_url"] = url - except Exception: + except (OSError, ValueError, TypeError, IndexError, AttributeError): pass # Check HEAD for current branch @@ -907,7 +907,7 @@ def extract_git_metadata(repo_path: str) -> dict[str, Any] | None: if head_content.startswith("ref: refs/heads/"): metadata["current_branch"] = head_content[16:] - except Exception: + except (OSError, ValueError, TypeError, IndexError, AttributeError): pass return metadata diff --git a/src/redops/modules/metadata/documents.py b/src/redops/modules/metadata/documents.py index 7605366..ccb929f 100644 --- a/src/redops/modules/metadata/documents.py +++ b/src/redops/modules/metadata/documents.py @@ -14,6 +14,7 @@ # Try to import document libraries try: from pypdf import PdfReader + from pypdf.errors import PdfReadError PYPDF_AVAILABLE = True except ImportError: @@ -299,7 +300,7 @@ def extract_pdf_metadata(file_path: str) -> DocumentMetadata | None: metadata["attachments"] = attachment_names warnings.append(f"Document contains {len(attachment_names)} embedded files") - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, KeyError, IndexError, AttributeError, PdfReadError) as e: warnings.append(f"Error reading PDF: {str(e)}") return DocumentMetadata( @@ -388,7 +389,7 @@ def extract_docx_metadata( except PackageNotFoundError: warnings.append("Invalid or corrupted Word document") - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, KeyError, IndexError, AttributeError) as e: warnings.append(f"Error reading Word document: {str(e)}") return DocumentMetadata( @@ -439,7 +440,7 @@ def check_docx_for_hidden_data(doc) -> list[str]: warnings.append("Document contains embedded OLE objects") break - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, KeyError, IndexError, AttributeError): pass # Silently ignore errors in hidden data check return warnings @@ -488,7 +489,7 @@ def check_for_hidden_data(file_path: str) -> list[str]: try: doc = DocxDocument(file_path) return check_docx_for_hidden_data(doc) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, KeyError, IndexError, AttributeError, PackageNotFoundError): return ["Could not check for hidden data"] return [] diff --git a/src/redops/modules/metadata/exif.py b/src/redops/modules/metadata/exif.py index 6dcf443..9bf6421 100644 --- a/src/redops/modules/metadata/exif.py +++ b/src/redops/modules/metadata/exif.py @@ -241,7 +241,7 @@ def extract_exif_from_file(file_path: str) -> ExifData | None: else: warnings.append("No EXIF data found in image") - except Exception as e: + except (OSError, RuntimeError, ValueError, TypeError, KeyError, IndexError, AttributeError) as e: warnings.append(f"Error extracting EXIF: {str(e)}") return ExifData( filename=path.name, @@ -312,7 +312,7 @@ def parse_gps_info(gps_info: dict[int, Any]) -> dict[str, Any] | None: return result - except Exception: + except (OSError, RuntimeError, ValueError, TypeError, KeyError, IndexError, AttributeError): return None @@ -355,7 +355,7 @@ def convert_exif_value(value: Any) -> Any: if isinstance(value, bytes): try: return value.decode("utf-8", errors="replace") - except Exception: + except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError): return str(value) elif isinstance(value, tuple): return [convert_exif_value(v) for v in value] @@ -372,7 +372,7 @@ def convert_exif_value(value: Any) -> Any: if isinstance(value, (int, float, str, bool, type(None))): return value return str(value) - except Exception: + except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError): return str(value) @@ -511,7 +511,7 @@ def strip_exif(file_path: str, output_path: str | None = None) -> bool: return True - except Exception: + except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError): return False diff --git a/src/redops/modules/notifications.py b/src/redops/modules/notifications.py index e1a45c0..c9a0349 100644 --- a/src/redops/modules/notifications.py +++ b/src/redops/modules/notifications.py @@ -265,7 +265,7 @@ def _send_slack(self, message: dict[str, Any], critical: int, high: int) -> bool timeout=10, ) return response.status_code == 200 - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: logger.warning("Failed to send Slack notification: %s", e) return False @@ -293,7 +293,7 @@ def _send_slack_alert(self, title: str, message: str, color: str) -> bool: timeout=10, ) return response.status_code == 200 - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: logger.warning("Failed to send Slack alert: %s", e) return False @@ -338,7 +338,7 @@ def _send_discord(self, message: dict[str, Any], critical: int, high: int) -> bo timeout=10, ) return response.status_code in (200, 204) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: logger.warning("Failed to send Discord notification: %s", e) return False @@ -369,7 +369,7 @@ def _send_discord_alert(self, title: str, message: str, color: str) -> bool: timeout=10, ) return response.status_code in (200, 204) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: logger.warning("Failed to send Discord alert: %s", e) return False @@ -393,7 +393,7 @@ def _send_email(self, subject: str, body: str) -> bool: server.send_message(msg) return True - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError, smtplib.SMTPException) as e: logger.warning("Failed to send email notification: %s", e) return False @@ -405,7 +405,7 @@ def _send_webhook(self, url: str, data: dict[str, Any]) -> bool: try: response = requests.post(url, json=data, timeout=10) return response.status_code in (200, 201, 202, 204) - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: logger.warning("Failed to send webhook to %s: %s", url, e) return False @@ -439,7 +439,7 @@ def notify_on_complete(ctx, params: dict[str, Any] | None = None): ctx.add("notification_results", results) ctx.log(f"Notifications sent: {results}", level="INFO") - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: ctx.log(f"Notification failed: {e}", level="WARNING") return ctx diff --git a/src/redops/modules/recon/asn_lookup.py b/src/redops/modules/recon/asn_lookup.py index efb41f7..9f7c453 100644 --- a/src/redops/modules/recon/asn_lookup.py +++ b/src/redops/modules/recon/asn_lookup.py @@ -126,7 +126,7 @@ def is_ip_address(value: str) -> bool: try: socket.inet_pton(socket.AF_INET6, value) return True - except Exception: + except (OSError, ValueError, TypeError): pass return False @@ -145,7 +145,7 @@ def resolve_domain_to_ip(domain: str) -> str | None: """Resolve a domain to its IP address.""" try: return socket.gethostbyname(domain) - except Exception: + except (OSError, ValueError, TypeError): return None @@ -206,7 +206,7 @@ def lookup_asn_bgpview(ip: str) -> dict[str, Any]: except requests.exceptions.RequestException as e: return {"error": f"API request failed: {str(e)}"} - except Exception as e: + except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError) as e: return {"error": f"Lookup failed: {str(e)}"} @@ -243,7 +243,7 @@ def lookup_asn_cymru(ip: str) -> dict[str, Any]: "note": "Limited data via DNS fallback", } - except Exception: + except (OSError, ValueError, TypeError): return {"error": "DNS lookup failed"} @@ -290,7 +290,7 @@ def lookup_asn_details(asn: str) -> dict[str, Any]: except requests.exceptions.RequestException: return {"asn": asn_num, "error": "API request failed"} - except Exception: + except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError): return {"asn": asn_num, "error": "Lookup failed"} @@ -348,7 +348,7 @@ def get_asn_prefixes(asn: str) -> list[dict[str, Any]]: return prefixes - except Exception: + except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError): return [] @@ -394,7 +394,7 @@ def get_asn_peers(asn: str) -> dict[str, list[dict[str, Any]]]: ], } - except Exception: + except (OSError, ValueError, TypeError, KeyError, IndexError, AttributeError): return {"upstreams": [], "downstreams": [], "peers": []} diff --git a/src/redops/modules/recon/cert_transparency.py b/src/redops/modules/recon/cert_transparency.py index a6a760c..5a239bd 100644 --- a/src/redops/modules/recon/cert_transparency.py +++ b/src/redops/modules/recon/cert_transparency.py @@ -142,7 +142,7 @@ def query_crtsh( return [] except json.JSONDecodeError: return [] - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): return [] @@ -166,7 +166,7 @@ def is_cert_valid(cert: dict[str, Any], now: datetime) -> bool: ) return expiry > now return True - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): return True @@ -271,7 +271,7 @@ def analyze_certificates( expired_count += 1 elif (expiry - now).days < 30: expiring_soon_count += 1 - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): pass # Check for wildcards diff --git a/src/redops/modules/recon/domains.py b/src/redops/modules/recon/domains.py index be2dec1..5dc5210 100644 --- a/src/redops/modules/recon/domains.py +++ b/src/redops/modules/recon/domains.py @@ -16,8 +16,12 @@ import dns.exception DNS_AVAILABLE = True + _DNSException = getattr(dns.exception, "DNSException", None) + if not isinstance(_DNSException, type) or not issubclass(_DNSException, BaseException): + _DNSException = Exception except ImportError: DNS_AVAILABLE = False + _DNSException = Exception def get_dns_records(domain: str, record_type: str = "A") -> list[str]: @@ -48,7 +52,7 @@ def _get_dns_records_socket(domain: str) -> list[str]: try: result = socket.gethostbyname_ex(domain) return result[2] - except Exception: + except (OSError, ValueError, TypeError): return [] @@ -93,7 +97,7 @@ def _get_dns_records_dnspython(domain: str, record_type: str) -> list[str]: except dns.exception.Timeout: # Query timed out pass - except Exception: + except (OSError, RuntimeError, ValueError, TypeError): # Other errors pass @@ -274,7 +278,7 @@ def profile_domain(ctx: Context, params: dict[str, Any] | None = None) -> Contex services = [v[0] for v in txt_analysis["verification_records"]] ctx.log(f"Domain verified with services: {services}", level="INFO") - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError) as e: ctx.log(f"Error during DNS enumeration: {e}", level="ERROR") ctx.add("domain_profile", profile) @@ -512,7 +516,7 @@ def check_zone_transfer(ctx: Context, params: dict[str, Any] | None = None) -> C ) ctx.add(f"finding_axfr_{ns_clean}", finding.model_dump()) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, _DNSException): # Zone transfer denied (expected/secure behavior) zone_transfer_results["nameservers_checked"].append( { diff --git a/src/redops/modules/recon/subdomain_enum.py b/src/redops/modules/recon/subdomain_enum.py index 637c473..ba364d3 100644 --- a/src/redops/modules/recon/subdomain_enum.py +++ b/src/redops/modules/recon/subdomain_enum.py @@ -416,14 +416,14 @@ def resolve_domain(domain: str, timeout: int = 3) -> bool: resolver.lifetime = timeout resolver.resolve(domain, "A") return True - except Exception: + except (OSError, ValueError, TypeError, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.exception.Timeout): return False else: try: socket.setdefaulttimeout(timeout) socket.gethostbyname(domain) return True - except Exception: + except (OSError, ValueError, TypeError): return False @@ -487,7 +487,7 @@ def get_subdomains_from_dns(domain: str, timeout: int = 3) -> set[str]: mx_host = str(rdata.exchange).rstrip(".") if mx_host.endswith(domain): subdomains.add(mx_host) - except Exception: + except (OSError, ValueError, TypeError, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.exception.Timeout): pass # Check NS records @@ -497,7 +497,7 @@ def get_subdomains_from_dns(domain: str, timeout: int = 3) -> set[str]: ns_host = str(rdata).rstrip(".") if ns_host.endswith(domain): subdomains.add(ns_host) - except Exception: + except (OSError, ValueError, TypeError, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.exception.Timeout): pass # Check TXT records for SPF includes @@ -512,10 +512,10 @@ def get_subdomains_from_dns(domain: str, timeout: int = 3) -> set[str]: include_domain = part[8:] if include_domain.endswith(domain): subdomains.add(include_domain) - except Exception: + except (OSError, ValueError, TypeError, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.exception.Timeout): pass - except Exception: + except (OSError, ValueError, TypeError, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.exception.Timeout): pass return subdomains diff --git a/src/redops/modules/recon/tech_stack.py b/src/redops/modules/recon/tech_stack.py index c807307..5f14d19 100644 --- a/src/redops/modules/recon/tech_stack.py +++ b/src/redops/modules/recon/tech_stack.py @@ -464,7 +464,7 @@ def make_request( verify=verify_ssl, ) return response - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError): # Try HTTP if HTTPS fails if url.startswith("https://"): try: @@ -476,7 +476,7 @@ def make_request( allow_redirects=follow_redirects, ) return response - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError): pass return None @@ -610,7 +610,7 @@ def fetch_favicon_hash( # Check against known hashes identified = FAVICON_HASHES.get(hash_value) return (hash_value, identified) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError): continue return None @@ -669,7 +669,7 @@ def get_ssl_info(target: str) -> dict[str, Any] | None: "serial_number": cert.get("serialNumber"), "version": cert.get("version"), } - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): pass return None diff --git a/src/redops/modules/reporting/junit_report.py b/src/redops/modules/reporting/junit_report.py index bdd40bc..ebd512b 100644 --- a/src/redops/modules/reporting/junit_report.py +++ b/src/redops/modules/reporting/junit_report.py @@ -417,7 +417,7 @@ def prettify_xml(xml_str: str) -> str: try: dom = minidom.parseString(xml_str) return dom.toprettyxml(indent=" ", encoding=None) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): return f'\n{xml_str}' diff --git a/src/redops/modules/reporting/pdf_report.py b/src/redops/modules/reporting/pdf_report.py index 5cf92fb..0ea250d 100644 --- a/src/redops/modules/reporting/pdf_report.py +++ b/src/redops/modules/reporting/pdf_report.py @@ -486,7 +486,7 @@ def generate_pdf_report(ctx, params: dict[str, Any] | None = None): except ImportError as e: ctx.log(f"PDF generation skipped: {e}", level="WARNING") - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, KeyError, IndexError, AttributeError) as e: ctx.log(f"PDF generation failed: {e}", level="ERROR") return ctx diff --git a/src/redops/modules/rf/ai_client.py b/src/redops/modules/rf/ai_client.py index ca80b41..21eceeb 100644 --- a/src/redops/modules/rf/ai_client.py +++ b/src/redops/modules/rf/ai_client.py @@ -296,7 +296,7 @@ async def _process_batch(self) -> None: try: result = await self._send_request(prompt, system) future.set_result(result) - except Exception as exc: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as exc: future.set_exception(exc) # ------------------------------------------------------------------ diff --git a/src/redops/modules/rf/dashboard.py b/src/redops/modules/rf/dashboard.py index 05a513c..2ae95ea 100644 --- a/src/redops/modules/rf/dashboard.py +++ b/src/redops/modules/rf/dashboard.py @@ -594,7 +594,7 @@ async def analyze(body: AnalyzeRequest) -> AnalyzeResponse: targets=targets, objective=body.objective, ) - except Exception as exc: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as exc: logger.error("AI analysis failed: %s", exc) raise HTTPException( status_code=502, diff --git a/src/redops/modules/rf/event_bus.py b/src/redops/modules/rf/event_bus.py index b59affe..eead1c5 100644 --- a/src/redops/modules/rf/event_bus.py +++ b/src/redops/modules/rf/event_bus.py @@ -293,7 +293,7 @@ async def emit( await sub.callback(event) # type: ignore[misc] else: await loop.run_in_executor(None, sub.callback, event) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): logger.exception( "Error in subscriber %s for %s", sub.sub_id, @@ -354,7 +354,7 @@ def emit_sync( continue try: sub.callback(event) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): logger.exception( "Error in subscriber %s for %s", sub.sub_id, diff --git a/src/redops/modules/rf/tool_manager.py b/src/redops/modules/rf/tool_manager.py index ce4d61c..628aa1e 100644 --- a/src/redops/modules/rf/tool_manager.py +++ b/src/redops/modules/rf/tool_manager.py @@ -454,7 +454,7 @@ async def stop_all(self) -> None: for tool_id in tool_ids: try: await self.stop_tool(tool_id) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): logger.exception( "Error stopping tool %s", tool_id[:8], @@ -627,7 +627,7 @@ async def _stream_reader( await asyncio.sleep(0.1) except asyncio.CancelledError: logger.debug("Stream reader cancelled for %s", tool_id[:8]) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): logger.exception("Stream reader error for tool %s", tool_id[:8]) diff --git a/src/redops/modules/simulation/mitre_mapping.py b/src/redops/modules/simulation/mitre_mapping.py index 5f675be..66534f7 100644 --- a/src/redops/modules/simulation/mitre_mapping.py +++ b/src/redops/modules/simulation/mitre_mapping.py @@ -762,3 +762,64 @@ def generate_attack_matrix_view(techniques: set[str]) -> dict[str, list[str]]: matrix[tactic] = sorted(matrix[tactic]) return matrix + + +def generate_navigator_layer( + techniques: set[str], + name: str = "RedOPS Scan Results", + description: str = "MITRE ATT&CK coverage generated by RedOPS", +) -> dict[str, Any]: + """ + Generate a MITRE ATT&CK Navigator layer JSON. + + Output is compatible with MITRE ATT&CK Navigator (https://mitre-attack.github.io/attack-navigator/). + + Args: + techniques: Set of technique IDs to highlight + name: Layer name + description: Layer description + + Returns: + Navigator layer dictionary + """ + layer_techniques = [] + for technique_id in sorted(techniques): + if technique_id not in MITRE_TECHNIQUES: + continue + technique = MITRE_TECHNIQUES[technique_id] + layer_techniques.append( + { + "techniqueID": technique_id, + "tactic": technique.tactic.lower().replace(" ", "-"), + "score": 1, + "comment": f"{technique.name}: {technique.description}", + "enabled": True, + } + ) + + return { + "name": name, + "versions": { + "attack": "14", + "navigator": "4.9.1", + "layer": "4.5", + }, + "domain": "enterprise-attack", + "description": description, + "techniques": layer_techniques, + "gradient": { + "colors": ["#ffffff", "#dc3545"], + "minValue": 0, + "maxValue": 100, + }, + "legendItems": [ + { + "label": "Technique detected by RedOPS", + "color": "#dc3545", + } + ], + "showTacticRowBackground": True, + "tacticRowBackground": "#eeeeee", + "selectTechniquesAcrossTactics": True, + "selectSubtechniquesWithParent": True, + } diff --git a/src/redops/modules/threat_intel/abuseipdb.py b/src/redops/modules/threat_intel/abuseipdb.py index 50ef385..55c3b89 100644 --- a/src/redops/modules/threat_intel/abuseipdb.py +++ b/src/redops/modules/threat_intel/abuseipdb.py @@ -114,7 +114,7 @@ def _check_ip( return {"ip": ip, "error": f"API error: {str(e)}"} except requests.exceptions.RequestException as e: return {"ip": ip, "error": f"Request failed: {str(e)}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"ip": ip, "error": f"Query failed: {str(e)}"} @@ -189,7 +189,7 @@ def report_ip( return {"error": f"API error: {str(e)}"} except requests.exceptions.RequestException as e: return {"error": f"Request failed: {str(e)}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"error": f"Report failed: {str(e)}"} @@ -229,7 +229,7 @@ def get_blacklist( data = response.json() return data.get("data", []) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError): return [] diff --git a/src/redops/modules/threat_intel/greynoise.py b/src/redops/modules/threat_intel/greynoise.py index 3a93c41..ebd8a83 100644 --- a/src/redops/modules/threat_intel/greynoise.py +++ b/src/redops/modules/threat_intel/greynoise.py @@ -117,7 +117,7 @@ def _query_community_api(ip: str, api_key: str | None = None) -> dict[str, Any]: except requests.exceptions.RequestException as e: return {"ip": ip, "error": f"API request failed: {str(e)}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"ip": ip, "error": f"Query failed: {str(e)}"} @@ -156,7 +156,7 @@ def get_greynoise_context(ip: str, api_key: str) -> dict[str, Any]: except requests.exceptions.RequestException as e: return {"ip": ip, "error": f"API request failed: {str(e)}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"ip": ip, "error": f"Context lookup failed: {str(e)}"} @@ -197,7 +197,7 @@ def get_greynoise_riot(ip: str, api_key: str | None = None) -> dict[str, Any]: except requests.exceptions.RequestException as e: return {"ip": ip, "error": f"RIOT lookup failed: {str(e)}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"ip": ip, "error": f"RIOT query failed: {str(e)}"} diff --git a/src/redops/modules/threat_intel/malwarebazaar.py b/src/redops/modules/threat_intel/malwarebazaar.py index e5e1230..ea25a8a 100644 --- a/src/redops/modules/threat_intel/malwarebazaar.py +++ b/src/redops/modules/threat_intel/malwarebazaar.py @@ -195,7 +195,7 @@ def _query_api(payload: dict[str, Any]) -> dict[str, Any]: except requests.exceptions.RequestException as e: return {"error": f"API request failed: {str(e)}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"error": f"Query failed: {str(e)}"} diff --git a/src/redops/modules/threat_intel/threatfox.py b/src/redops/modules/threat_intel/threatfox.py index 0f25584..7cb08ce 100644 --- a/src/redops/modules/threat_intel/threatfox.py +++ b/src/redops/modules/threat_intel/threatfox.py @@ -195,7 +195,7 @@ def _query_api(payload: dict[str, Any]) -> dict[str, Any]: except requests.exceptions.RequestException as e: return {"error": f"API request failed: {str(e)}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"error": f"Query failed: {str(e)}"} diff --git a/src/redops/modules/threat_intel/urlhaus.py b/src/redops/modules/threat_intel/urlhaus.py index 2cfebab..193bff5 100644 --- a/src/redops/modules/threat_intel/urlhaus.py +++ b/src/redops/modules/threat_intel/urlhaus.py @@ -86,7 +86,7 @@ def _query_url(url: str) -> dict[str, Any]: except requests.exceptions.RequestException as e: return {"queried_url": url, "error": f"Request failed: {str(e)}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"queried_url": url, "error": f"Query failed: {str(e)}"} @@ -116,7 +116,7 @@ def check_host(host: str) -> dict[str, Any]: except requests.exceptions.RequestException as e: return {"queried_host": host, "error": f"Request failed: {str(e)}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"queried_host": host, "error": f"Query failed: {str(e)}"} @@ -167,7 +167,7 @@ def check_payload( except requests.exceptions.RequestException as e: return {"queried_hash": hash_value, "error": f"Request failed: {str(e)}"} - except Exception as e: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError) as e: return {"queried_hash": hash_value, "error": f"Query failed: {str(e)}"} @@ -193,7 +193,7 @@ def get_recent_urls(limit: int = 100) -> list[dict[str, Any]]: data = response.json() return data.get("urls", []) - except Exception: + except (OSError, RuntimeError, TypeError, ValueError, ConnectionError): return [] diff --git a/src/redops/notifications/email.py b/src/redops/notifications/email.py index 391438d..853217e 100644 --- a/src/redops/notifications/email.py +++ b/src/redops/notifications/email.py @@ -527,7 +527,7 @@ def send_email( logger.info(f"Email sent to {len(all_recipients)} recipients: {subject}") return True - except Exception as e: + except (ConnectionError, TimeoutError, OSError, RuntimeError, ValueError) as e: logger.error(f"Failed to send email: {e}") return False @@ -685,6 +685,6 @@ def test_connection(self) -> bool: if self.config.smtp_user: server.login(self.config.smtp_user, self.config.smtp_password) return True - except Exception as e: + except (ConnectionError, TimeoutError, OSError, RuntimeError, ValueError) as e: logger.error(f"SMTP connection test failed: {e}") return False diff --git a/src/redops/notifications/manager.py b/src/redops/notifications/manager.py index 9f7528a..604513d 100644 --- a/src/redops/notifications/manager.py +++ b/src/redops/notifications/manager.py @@ -139,7 +139,7 @@ def _worker_loop(self) -> None: except queue.Empty: continue - except Exception as e: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError) as e: logger.error(f"Notification worker error: {e}") def _send_to_providers( @@ -163,7 +163,7 @@ def _send_to_providers( success = provider.send(message) if success: break - except Exception as e: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError) as e: logger.error(f"Provider {name} attempt {attempt + 1} failed: {e}") if attempt < retries: @@ -212,7 +212,7 @@ def _apply_formatters(self, message: NotificationMessage) -> NotificationMessage for formatter in self._formatters: try: message = formatter(message) - except Exception as e: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError) as e: logger.error(f"Formatter error: {e}") return message diff --git a/src/redops/notifications/webhooks.py b/src/redops/notifications/webhooks.py index d497245..bf15b8f 100644 --- a/src/redops/notifications/webhooks.py +++ b/src/redops/notifications/webhooks.py @@ -215,7 +215,7 @@ def send(self, message: NotificationMessage) -> bool: response.raise_for_status() logger.info(f"Slack notification sent: {message.title}") return True - except Exception as e: + except (ConnectionError, TimeoutError, OSError, RuntimeError, ValueError) as e: logger.error(f"Failed to send Slack notification: {e}") return False @@ -334,7 +334,7 @@ def send(self, message: NotificationMessage) -> bool: response.raise_for_status() logger.info(f"Teams notification sent: {message.title}") return True - except Exception as e: + except (ConnectionError, TimeoutError, OSError, RuntimeError, ValueError) as e: logger.error(f"Failed to send Teams notification: {e}") return False @@ -438,7 +438,7 @@ def send(self, message: NotificationMessage) -> bool: response.raise_for_status() logger.info(f"Discord notification sent: {message.title}") return True - except Exception as e: + except (ConnectionError, TimeoutError, OSError, RuntimeError, ValueError) as e: logger.error(f"Failed to send Discord notification: {e}") return False @@ -516,7 +516,7 @@ def send(self, message: NotificationMessage) -> bool: response.raise_for_status() logger.info(f"Webhook notification sent: {message.title}") return True - except Exception as e: + except (ConnectionError, TimeoutError, OSError, RuntimeError, ValueError) as e: logger.error(f"Failed to send webhook notification: {e}") return False @@ -598,6 +598,6 @@ def send(self, message: NotificationMessage) -> bool: response.raise_for_status() logger.info(f"PagerDuty notification sent: {message.title}") return True - except Exception as e: + except (ConnectionError, TimeoutError, OSError, RuntimeError, ValueError) as e: logger.error(f"Failed to send PagerDuty notification: {e}") return False diff --git a/src/redops/observability/metrics.py b/src/redops/observability/metrics.py index 6929152..622812a 100644 --- a/src/redops/observability/metrics.py +++ b/src/redops/observability/metrics.py @@ -102,7 +102,7 @@ def _initialize_otel(self) -> None: try: prometheus_reader = PrometheusMetricReader() readers.append(prometheus_reader) - except Exception: + except (ImportError, RuntimeError, OSError): pass # Add console exporter for debugging @@ -382,7 +382,7 @@ def wrapper(*args, **kwargs): try: result = func(*args, **kwargs) return result - except Exception: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError): scan_status = "failure" raise finally: @@ -425,7 +425,7 @@ async def async_wrapper(*args, **kwargs): result = await func(*args, **kwargs) status_code = getattr(result, "status_code", 200) return result - except Exception: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError, TimeoutError): status_code = 500 raise finally: @@ -441,7 +441,7 @@ def sync_wrapper(*args, **kwargs): result = func(*args, **kwargs) status_code = getattr(result, "status_code", 200) return result - except Exception: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError, TimeoutError): status_code = 500 raise finally: diff --git a/src/redops/observability/tracing.py b/src/redops/observability/tracing.py index 2efa067..19b175d 100644 --- a/src/redops/observability/tracing.py +++ b/src/redops/observability/tracing.py @@ -185,7 +185,7 @@ def start_span( ) as span: try: yield span - except Exception as e: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError) as e: if span: span.set_status(Status(StatusCode.ERROR, str(e))) span.record_exception(e) @@ -312,7 +312,7 @@ def wrapper(*args, **kwargs): ) span.set_attribute("pipeline.findings_count", findings) return result - except Exception as e: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError) as e: if span: span.set_attribute("pipeline.error", str(e)) raise @@ -349,7 +349,7 @@ def wrapper(ctx, params=None): try: result = func(ctx, params) return result - except Exception as e: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError) as e: if span: span.set_attribute("module.error", str(e)) raise diff --git a/src/redops/pipelines/loader.py b/src/redops/pipelines/loader.py index 0ef3a11..f948029 100644 --- a/src/redops/pipelines/loader.py +++ b/src/redops/pipelines/loader.py @@ -4,6 +4,7 @@ import json from pathlib import Path +from pydantic import ValidationError from redops.pipelines.schemas import Pipeline @@ -43,7 +44,7 @@ def load(path: str | Path) -> Pipeline: pipeline = Pipeline(**data) pipeline.validate_pipeline() return pipeline - except Exception as e: + except (ValidationError, ValueError, TypeError) as e: raise ValueError(f"Pipeline validation failed: {e}") @staticmethod diff --git a/src/redops/pipelines/runner.py b/src/redops/pipelines/runner.py index 388a9d9..af715a9 100644 --- a/src/redops/pipelines/runner.py +++ b/src/redops/pipelines/runner.py @@ -13,6 +13,7 @@ from typing import Callable, TYPE_CHECKING from redops.pipelines.schemas import Pipeline, PipelineStep from redops.core.context import Context +from redops.modules.active.exceptions import ActiveAuthorizationError from redops.core.plugin_system import ( PluginRegistry, HookPoint, @@ -106,6 +107,9 @@ def _execute_step(self, step: PipelineStep, ctx: Context) -> Context: """ ctx.log(f"Executing step: {step.name}", level="INFO", step=step.name) + # Save checkpoint before executing the step so we can rollback on failure + ctx.save() + # Execute BEFORE_MODULE hooks self.plugins.execute_hooks( HookPoint.BEFORE_MODULE, @@ -150,7 +154,7 @@ def _execute_step(self, step: PipelineStep, ctx: Context) -> Context: return ctx - except Exception as e: + except (RuntimeError, ImportError, TypeError, ValueError, OSError, ConnectionError, ActiveAuthorizationError, AttributeError) as e: error_msg = f"Step failed: {step.name} - {str(e)}" ctx.log(error_msg, level="ERROR", step=step.name, error=str(e)) @@ -164,6 +168,8 @@ def _execute_step(self, step: PipelineStep, ctx: Context) -> Context: if not step.continue_on_error: raise RuntimeError(error_msg) from e + # Rollback context to preserve data integrity + ctx.rollback() return ctx def _get_plugin_module(self, module_path: str) -> ModulePlugin | None: @@ -281,7 +287,7 @@ def run_step(step: PipelineStep) -> tuple: try: result_ctx = self._execute_step(step, step_ctx) return (step.name, result_ctx.data, result_ctx.logs, None) - except Exception as e: + except (RuntimeError, ImportError, TypeError, ValueError, OSError, ConnectionError, ActiveAuthorizationError, AttributeError) as e: return (step.name, {}, [], str(e)) # Run steps in parallel using ThreadPoolExecutor @@ -374,4 +380,5 @@ def run( self.plugins.execute_hooks(HookPoint.AFTER_PIPELINE, ctx) ctx.log(f"Pipeline completed: {self.pipeline.metadata.name}", level="INFO") + ctx.clear_checkpoints() return ctx diff --git a/src/redops/plugins/examples/header_scanner.py b/src/redops/plugins/examples/header_scanner.py index 32b2d5c..a91d124 100644 --- a/src/redops/plugins/examples/header_scanner.py +++ b/src/redops/plugins/examples/header_scanner.py @@ -115,7 +115,7 @@ def validate_target(self, target: str) -> bool: try: parsed = urlparse(target) return parsed.scheme in ("http", "https") and bool(parsed.netloc) - except Exception: + except (ValueError, TypeError): return False def scan( @@ -194,7 +194,7 @@ def scan( # Store response headers result.metadata["response_headers"] = headers - except Exception as e: + except (ValueError, TypeError, AttributeError, KeyError) as e: result.success = False result.error = str(e) self._trigger_hook("on_error", e) diff --git a/src/redops/plugins/examples/port_scanner.py b/src/redops/plugins/examples/port_scanner.py index 4768cd7..e4b4909 100644 --- a/src/redops/plugins/examples/port_scanner.py +++ b/src/redops/plugins/examples/port_scanner.py @@ -248,7 +248,7 @@ def _scan_ports( try: if future.result(): open_ports.append(port) - except Exception: + except (OSError, ConnectionError, TimeoutError): pass return sorted(open_ports) diff --git a/src/redops/plugins/repository.py b/src/redops/plugins/repository.py index 769257f..d76c043 100644 --- a/src/redops/plugins/repository.py +++ b/src/redops/plugins/repository.py @@ -550,7 +550,7 @@ def _run_hooks(self, event: str, *args, **kwargs) -> None: for callback in self._hooks.get(event, []): try: callback(*args, **kwargs) - except Exception: + except (RuntimeError, TypeError, ValueError, OSError): pass @property diff --git a/src/redops/plugins/scanner.py b/src/redops/plugins/scanner.py index 0d6c0a3..71ac00d 100644 --- a/src/redops/plugins/scanner.py +++ b/src/redops/plugins/scanner.py @@ -400,7 +400,7 @@ def _trigger_hook(self, event: str, *args, **kwargs) -> None: for callback in self._hooks.get(event, []): try: callback(*args, **kwargs) - except Exception: + except (RuntimeError, TypeError, ValueError, OSError): pass # Ignore hook errors def _create_finding( @@ -515,7 +515,7 @@ def scan( for future in concurrent.futures.as_completed(futures): try: results.append(future.result()) - except Exception as e: + except (RuntimeError, ImportError, TypeError, ValueError, OSError, ConnectionError) as e: scanner = futures[future] results.append( ScannerResult( @@ -530,7 +530,7 @@ def scan( if scanner.validate_target(target): try: results.append(scanner.scan(target, config)) - except Exception as e: + except (RuntimeError, ImportError, TypeError, ValueError, OSError, ConnectionError) as e: results.append( ScannerResult( scanner_name=scanner.get_name(), diff --git a/src/redops/scheduler/executor.py b/src/redops/scheduler/executor.py index e1da492..bb8cd5b 100644 --- a/src/redops/scheduler/executor.py +++ b/src/redops/scheduler/executor.py @@ -86,7 +86,7 @@ def execute(self, job: ScanJob) -> None: for hook in self._pre_hooks: try: hook(job) - except Exception as e: + except (RuntimeError, TypeError, ValueError, OSError) as e: logger.warning(f"Pre-hook error: {e}") # Get pipeline @@ -123,7 +123,7 @@ def execute(self, job: ScanJob) -> None: logger.info(f"Job {job.id} completed: {findings_count} findings") - except Exception as e: + except (RuntimeError, ImportError, TypeError, ValueError, OSError, ConnectionError) as e: error_msg = str(e) logger.error(f"Job {job.id} failed: {error_msg}") logger.debug(traceback.format_exc()) @@ -142,7 +142,7 @@ def execute(self, job: ScanJob) -> None: for hook in self._post_hooks: try: hook(job) - except Exception as e: + except (RuntimeError, TypeError, ValueError, OSError) as e: logger.warning(f"Post-hook error: {e}") def _count_findings(self, result: Any) -> int: @@ -214,7 +214,7 @@ def _send_notification(self, job: ScanJob, message: str) -> None: if self._notification_handler: try: self._notification_handler(job, message) - except Exception as e: + except (RuntimeError, TypeError, ValueError, OSError, ConnectionError) as e: logger.error(f"Notification failed: {e}") diff --git a/src/redops/scheduler/models.py b/src/redops/scheduler/models.py index a5f3309..4242fa5 100644 --- a/src/redops/scheduler/models.py +++ b/src/redops/scheduler/models.py @@ -214,7 +214,7 @@ def _parse_cron_next(self, base: datetime) -> datetime: except ImportError: # Fallback to daily if croniter not installed return base + timedelta(days=1) - except Exception: + except (ValueError, TypeError): return base + timedelta(days=1) def is_due(self, current_time: datetime | None = None) -> bool: diff --git a/src/redops/scheduler/scheduler.py b/src/redops/scheduler/scheduler.py index 0e2ee5f..a7b2634 100644 --- a/src/redops/scheduler/scheduler.py +++ b/src/redops/scheduler/scheduler.py @@ -85,7 +85,7 @@ def _load_from_file(self) -> None: logger.info( f"Loaded {len(self._schedules)} schedules from {self._storage_path}" ) - except Exception as e: + except (OSError, json.JSONDecodeError, KeyError, ValueError, TypeError) as e: logger.error(f"Failed to load schedules: {e}") def _save_to_file(self) -> None: @@ -100,7 +100,7 @@ def _save_to_file(self) -> None: } with open(self._storage_path, "w") as f: json.dump(data, f, indent=2) - except Exception as e: + except (OSError, TypeError, ValueError) as e: logger.error(f"Failed to save schedules: {e}") def add_schedule(self, schedule: ScanSchedule) -> None: @@ -279,7 +279,7 @@ def _run_loop(self) -> None: try: self._check_and_dispatch() self._check_timeouts() - except Exception as e: + except (RuntimeError, OSError, TypeError, ValueError, ConnectionError) as e: logger.error(f"Scheduler error: {e}") # Sleep in small increments to allow quick shutdown @@ -300,7 +300,7 @@ def _check_and_dispatch(self) -> None: try: self._dispatch_schedule(schedule) - except Exception as e: + except (RuntimeError, TypeError, ValueError, OSError, ConnectionError) as e: logger.error(f"Failed to dispatch schedule {schedule.id}: {e}") self._job_semaphore.release() @@ -345,7 +345,7 @@ def _execute_job(self, job: ScanJob) -> None: self.store.update_job(job) logger.info(f"Job {job.id} completed with {job.findings_count} findings") - except Exception as e: + except (RuntimeError, ImportError, TypeError, ValueError, OSError, ConnectionError) as e: job.fail(str(e)) self.store.update_job(job) logger.error(f"Job {job.id} failed: {e}") diff --git a/src/redops/tenants/manager.py b/src/redops/tenants/manager.py index 401a017..9d34244 100644 --- a/src/redops/tenants/manager.py +++ b/src/redops/tenants/manager.py @@ -506,7 +506,7 @@ def _run_hooks(self, event: str, *args, **kwargs) -> None: for callback in self._hooks.get(event, []): try: callback(*args, **kwargs) - except Exception as e: + except (RuntimeError, ValueError, TypeError, OSError, ConnectionError) as e: logger.error(f"Hook error ({event}): {e}") diff --git a/src/redops/web/app.py b/src/redops/web/app.py index 03f2945..2e685e2 100644 --- a/src/redops/web/app.py +++ b/src/redops/web/app.py @@ -6,6 +6,7 @@ import os from datetime import datetime, timezone +from typing import Any from fastapi import ( FastAPI, @@ -42,7 +43,10 @@ generate_api_key, ) +from redops.core.exceptions import RedOpsError, ModuleError from redops.main import __version__ +from redops.analysis.comparison import ScanComparator +from redops.web.store import ScanStore # Request/Response models @@ -84,6 +88,23 @@ class ScanStatus(BaseModel): error: str | None = None +class ScanCompareRequest(BaseModel): + """Request model for scan comparison.""" + + baseline_scan_id: str = Field(..., description="Baseline scan ID") + current_scan_id: str = Field(..., description="Current scan ID") + + +class FindingTriageUpdate(BaseModel): + """Request model for updating finding triage status.""" + + status: str = Field( + ..., description="Triage status: open, false_positive, accepted_risk" + ) + notes: str | None = Field(default=None, description="Triage notes") + assignee: str | None = Field(default=None, description="Assigned user") + + class AIRequest(BaseModel): """Request model for AI operations.""" @@ -94,6 +115,20 @@ class AIRequest(BaseModel): scan_id: str | None = Field(default=None, description="Scan ID for analysis") provider: str | None = Field(default=None, description="AI provider override") model: str | None = Field(default=None, description="Model override") + budget_limit: float | None = Field( + default=None, ge=0, description="Max estimated USD spend for this call" + ) + + +class AICostMetrics(BaseModel): + """Cost metrics for an AI call.""" + + calls: int + input_tokens: int + output_tokens: int + estimated_cost_usd: float + budget_limit_usd: float | None = None + budget_remaining_usd: float | None = None class AIResponse(BaseModel): @@ -103,6 +138,7 @@ class AIResponse(BaseModel): result: str provider: str model: str + cost: AICostMetrics | None = None class HealthResponse(BaseModel): @@ -138,9 +174,138 @@ class AuthStatusResponse(BaseModel): auth_enabled: bool -# In-memory scan storage (for demo; use database in production) -_scans: dict[str, ScanStatus] = {} -_scan_results: dict[str, dict] = {} +# Scan storage backend (memory or Redis) — singleton per process +_scan_store = ScanStore.get_instance() + + +class _ScanDictProxy: + """Backward-compatible proxy for `_scans` dict access.""" + + def __getitem__(self, key: str) -> ScanStatus: + val = _scan_store.get_scan(key) + if val is None: + raise KeyError(key) + return val + + def __setitem__(self, key: str, value: ScanStatus) -> None: + _scan_store.set_scan(key, value) + + def __contains__(self, key: str) -> bool: + return _scan_store.get_scan(key) is not None + + def __delitem__(self, key: str) -> None: + # No-op: store has no delete API; clear() resets everything + pass + + def get(self, key: str, default: Any = None) -> Any: + val = _scan_store.get_scan(key) + return val if val is not None else default + + def values(self): + return _scan_store.list_scans() + + def clear(self) -> None: + _scan_store.clear() + + +class _ResultsDictProxy: + """Backward-compatible proxy for `_scan_results` dict access.""" + + def __getitem__(self, key: str) -> dict: + val = _scan_store.get_results(key) + if val is None: + raise KeyError(key) + return val + + def __setitem__(self, key: str, value: dict) -> None: + _scan_store.set_results(key, value) + + def __contains__(self, key: str) -> bool: + return _scan_store.get_results(key) is not None + + def __delitem__(self, key: str) -> None: + pass + + def get(self, key: str, default: Any = None) -> Any: + val = _scan_store.get_results(key) + return val if val is not None else default + + def clear(self) -> None: + _scan_store.clear() + + +class _TriageDictProxy: + """Backward-compatible proxy for `_finding_triage` dict access.""" + + def __getitem__(self, key: str) -> dict: + val = _scan_store.get_triage(key) + if val is None: + raise KeyError(key) + return val + + def __setitem__(self, key: str, value: dict) -> None: + _scan_store.set_triage(key, value) + + def __contains__(self, key: str) -> bool: + return _scan_store.get_triage(key) is not None + + def __delitem__(self, key: str) -> None: + pass + + def get(self, key: str, default: Any = None) -> Any: + val = _scan_store.get_triage(key) + return val if val is not None else default + + def clear(self) -> None: + _scan_store.clear() + + +class _BaselinesDictProxy: + """Backward-compatible proxy for `_baselines` dict access.""" + + def __getitem__(self, key: str) -> str: + val = _scan_store.get_baseline(key) + if val is None: + raise KeyError(key) + return val + + def __setitem__(self, key: str, value: str) -> None: + _scan_store.set_baseline(key, value) + + def __contains__(self, key: str) -> bool: + return _scan_store.get_baseline(key) is not None + + def __delitem__(self, key: str) -> None: + pass + + def get(self, key: str, default: Any = None) -> Any: + val = _scan_store.get_baseline(key) + return val if val is not None else default + + def clear(self) -> None: + _scan_store.clear() + + +class _AICostTrackerProxy: + """Backward-compatible proxy for `_ai_cost_tracker` dict access.""" + + def __getitem__(self, key: str) -> Any: + return _scan_store.get_ai_costs()[key] + + def __setitem__(self, key: str, value: Any) -> None: + costs = {key: value} + _scan_store.increment_ai_costs(costs) + + def clear(self) -> None: + _scan_store.clear() + + +# Backward-compatible module-level proxies (tests import these directly) +_scans = _ScanDictProxy() +_scan_results = _ResultsDictProxy() +_finding_triage = _TriageDictProxy() +_baselines = _BaselinesDictProxy() +_ai_cost_tracker = _AICostTrackerProxy() def create_app(auth_config: AuthConfig | None = None) -> FastAPI: @@ -300,7 +465,7 @@ async def start_scan( started_at=now, progress=0, ) - _scans[scan_id] = status + _scan_store.set_scan(scan_id, status) # Run scan in background background_tasks.add_task(run_scan_task, scan_id, request) @@ -317,34 +482,348 @@ async def start_scan( @app.get("/api/scans", response_model=list[ScanStatus], tags=["Scans"]) async def list_scans( status: str | None = Query(None, description="Filter by status"), + search: str | None = Query(None, description="Search target or scan ID"), + sort: str = Query("started_at_desc", description="Sort field and direction"), limit: int = Query(20, ge=1, le=100, description="Max results"), user: AuthenticatedUser = Depends(require_auth), ): - """List all scans.""" - scans = list(_scans.values()) + """List all scans with optional filtering, search, and sorting.""" + scans = _scan_store.list_scans() if status: scans = [s for s in scans if s.status == status] - return sorted(scans, key=lambda s: s.started_at, reverse=True)[:limit] + if search: + q = search.lower() + scans = [s for s in scans if q in s.target.lower() or q in s.scan_id.lower()] + # Sorting + reverse = sort.endswith("_desc") + sort_key = sort.removesuffix("_desc").removesuffix("_asc") if "_" in sort else sort + if sort_key == "target": + scans = sorted(scans, key=lambda s: s.target.lower(), reverse=reverse) + elif sort_key == "status": + scans = sorted(scans, key=lambda s: s.status, reverse=reverse) + elif sort_key == "progress": + scans = sorted(scans, key=lambda s: s.progress, reverse=reverse) + else: + scans = sorted(scans, key=lambda s: s.started_at or "", reverse=reverse) + return scans[:limit] @app.get("/api/scans/{scan_id}", response_model=ScanStatus, tags=["Scans"]) async def get_scan(scan_id: str, user: AuthenticatedUser = Depends(require_auth)): """Get scan status by ID.""" - if scan_id not in _scans: + scan = _scan_store.get_scan(scan_id) + if scan is None: raise HTTPException(status_code=404, detail="Scan not found") - return _scans[scan_id] + return scan + + def _merge_triage_into_findings(scan_id: str, data: Any) -> Any: + """Recursively merge triage state into finding dicts within results.""" + if isinstance(data, list): + return [_merge_triage_into_findings(scan_id, item) for item in data] + if isinstance(data, dict): + merged = {k: _merge_triage_into_findings(scan_id, v) for k, v in data.items()} + if "severity" in merged: + fid = merged.get("id", "") or merged.get("title", "") or "" + triage = _scan_store.get_triage(f"{scan_id}:{fid}") + if triage: + merged["triage"] = triage + return merged + return data + + def _collect_findings(data: Any, findings: list[dict] | None = None) -> list[dict]: + """Recursively collect all dicts that look like findings (have severity).""" + if findings is None: + findings = [] + if isinstance(data, list): + for item in data: + _collect_findings(item, findings) + elif isinstance(data, dict): + if "severity" in data: + findings.append(data) + for v in data.values(): + _collect_findings(v, findings) + return findings @app.get("/api/scans/{scan_id}/results", tags=["Scans"]) async def get_scan_results( scan_id: str, user: AuthenticatedUser = Depends(require_auth) ): """Get scan results.""" - if scan_id not in _scans: + scan = _scan_store.get_scan(scan_id) + if scan is None: raise HTTPException(status_code=404, detail="Scan not found") - if _scans[scan_id].status != "completed": + if scan.status != "completed": raise HTTPException(status_code=400, detail="Scan not completed") - if scan_id not in _scan_results: + raw = _scan_store.get_results(scan_id) + if raw is None: + raise HTTPException(status_code=404, detail="Results not available") + merged = _merge_triage_into_findings(scan_id, raw) + # Normalize findings into an array for dashboard charts / UI + if isinstance(merged, dict) and "findings" not in merged: + merged = {**merged, "findings": _collect_findings(merged)} + # Attach delta info if a baseline exists for this target + target = scan.target + baseline_scan_id = _scan_store.get_baseline(target) + if baseline_scan_id and baseline_scan_id != scan_id: + baseline_data = _scan_store.get_results(baseline_scan_id) + if baseline_data is not None: + baseline_findings = [] + current_findings = [] + for key, value in baseline_data.items(): + if isinstance(value, dict) and (key.startswith("finding_") or "severity" in value): + baseline_findings.append(value) + for key, value in raw.items(): + if isinstance(value, dict) and (key.startswith("finding_") or "severity" in value): + current_findings.append(value) + comparator = ScanComparator() + delta = comparator.compare( + {"scan_id": baseline_scan_id, "findings": baseline_findings}, + {"scan_id": scan_id, "findings": current_findings}, + ) + merged["_delta"] = { + "has_baseline": True, + "baseline_scan_id": baseline_scan_id, + **delta.to_dict(include_findings=True), + } + return merged + + @app.post("/api/scans/compare", tags=["Scans"]) + async def compare_scans( + request: ScanCompareRequest, user: AuthenticatedUser = Depends(require_auth) + ): + """Compare two scans to identify changes.""" + baseline_data = _scan_store.get_results(request.baseline_scan_id) + if baseline_data is None: + raise HTTPException(status_code=404, detail="Baseline scan results not found") + current_data = _scan_store.get_results(request.current_scan_id) + if current_data is None: + raise HTTPException(status_code=404, detail="Current scan results not found") + + # Build findings lists from ctx.data format (keys like finding_xxx) + baseline_findings = [] + current_findings = [] + for key, value in baseline_data.items(): + if isinstance(value, dict) and (key.startswith("finding_") or "severity" in value): + baseline_findings.append(value) + for key, value in current_data.items(): + if isinstance(value, dict) and (key.startswith("finding_") or "severity" in value): + current_findings.append(value) + + comparator = ScanComparator() + result = comparator.compare( + {"scan_id": request.baseline_scan_id, "findings": baseline_findings}, + {"scan_id": request.current_scan_id, "findings": current_findings}, + ) + return result.to_dict(include_findings=True) + + @app.post("/api/scans/{scan_id}/findings/{finding_id}/triage", tags=["Scans"]) + async def update_finding_triage( + scan_id: str, + finding_id: str, + request: FindingTriageUpdate, + user: AuthenticatedUser = Depends(require_auth), + ): + """Update triage status for a finding.""" + if _scan_store.get_scan(scan_id) is None: + raise HTTPException(status_code=404, detail="Scan not found") + valid_statuses = {"open", "false_positive", "accepted_risk"} + if request.status not in valid_statuses: + raise HTTPException( + status_code=400, + detail=f"Invalid status. Must be one of: {', '.join(valid_statuses)}", + ) + key = f"{scan_id}:{finding_id}" + triage = { + "status": request.status, + "notes": request.notes or "", + "assignee": request.assignee or "", + "updated_at": datetime.now(timezone.utc).isoformat(), + "updated_by": user.username, + } + _scan_store.set_triage(key, triage) + return {"success": True, "triage": triage} + + _MITRE_TACTICS_ORDER = [ + "Reconnaissance", "Resource Development", "Initial Access", "Execution", + "Persistence", "Privilege Escalation", "Defense Evasion", "Credential Access", + "Discovery", "Lateral Movement", "Collection", "Command and Control", + "Exfiltration", "Impact", + ] + + @app.get("/api/scans/{scan_id}/mitre", tags=["Scans"]) + async def get_scan_mitre( + scan_id: str, user: AuthenticatedUser = Depends(require_auth) + ): + """Get MITRE ATT&CK coverage for a scan.""" + scan = _scan_store.get_scan(scan_id) + if scan is None: + raise HTTPException(status_code=404, detail="Scan not found") + data = _scan_store.get_results(scan_id) or {} + mitre_mapping = data.get("mitre_mapping", {}) if isinstance(data, dict) else {} + mitre_techniques = set(data.get("mitre_techniques_used", [])) if isinstance(data, dict) else set() + + matrix = {} + for technique_id, technique_info in mitre_mapping.items(): + if isinstance(technique_info, dict): + tactic = technique_info.get("tactic", "Unknown") + name = technique_info.get("name", technique_id) + else: + tactic = "Unknown" + name = technique_id + matrix.setdefault(tactic, []).append({"id": technique_id, "name": name}) + if not matrix and mitre_techniques: + matrix["Identified Techniques"] = [{"id": t, "name": t} for t in sorted(mitre_techniques)] + + total_techniques = sum(len(v) for v in matrix.values()) + tactics_covered = len([t for t in matrix if matrix.get(t)]) + return { + "matrix": matrix, + "tactics_order": _MITRE_TACTICS_ORDER, + "total_techniques": total_techniques, + "tactics_covered": tactics_covered, + } + + @app.get("/api/scans/{scan_id}/navigator-layer", tags=["Scans"]) + async def get_scan_navigator_layer( + scan_id: str, user: AuthenticatedUser = Depends(require_auth) + ): + """Export MITRE ATT&CK Navigator layer JSON for a scan.""" + scan = _scan_store.get_scan(scan_id) + if scan is None: + raise HTTPException(status_code=404, detail="Scan not found") + data = _scan_store.get_results(scan_id) + if data is None: + raise HTTPException(status_code=404, detail="Results not available") + techniques = set(data.get("mitre_techniques_used", [])) if isinstance(data, dict) else set() + if not techniques and isinstance(data, dict): + for key, value in data.items(): + if isinstance(value, dict) and "mitre_techniques" in value: + techniques.update(value.get("mitre_techniques", [])) + from redops.modules.simulation.mitre_mapping import generate_navigator_layer + + layer = generate_navigator_layer( + techniques, + name=f"RedOPS Scan {scan_id}", + description=f"ATT&CK coverage for target: {scan.target}", + ) + return Response( + content=__import__("json").dumps(layer, indent=2), + media_type="application/json", + headers={"Content-Disposition": f'attachment; filename="navigator-layer-{scan_id}.json"'}, + ) + + @app.get("/api/scans/{scan_id}/attack-graph", tags=["Scans"]) + async def get_scan_attack_graph( + scan_id: str, user: AuthenticatedUser = Depends(require_auth) + ): + """Return attack graph data for a scan in Cytoscape.js format.""" + scan = _scan_store.get_scan(scan_id) + if scan is None: + raise HTTPException(status_code=404, detail="Scan not found") + data = _scan_store.get_results(scan_id) + if data is None: raise HTTPException(status_code=404, detail="Results not available") - return _scan_results[scan_id] + if not isinstance(data, dict): + raise HTTPException(status_code=404, detail="No graph data available") + + # Extract attack paths and chains from scan results + attack_paths = data.get("attack_paths", []) + attack_chains = data.get("attack_chains_raw", []) + summary = data.get("attack_path_summary", {}) + + # Build Cytoscape.js elements + elements = [] + seen_nodes = set() + + for chain in attack_chains: + path = chain.get("path", []) + for i, node_id in enumerate(path): + if node_id not in seen_nodes: + seen_nodes.add(node_id) + elements.append( + { + "data": { + "id": node_id, + "label": node_id, + "type": "entry" if i == 0 else ("target" if i == len(path) - 1 else "intermediate"), + }, + "group": "nodes", + } + ) + if i < len(path) - 1: + elements.append( + { + "data": { + "id": f"{node_id}->{path[i + 1]}", + "source": node_id, + "target": path[i + 1], + "type": "attack-step", + }, + "group": "edges", + } + ) + + return { + "elements": elements, + "summary": summary, + "attack_paths": attack_paths, + } + + @app.post("/api/scans/{scan_id}/baseline", tags=["Scans"]) + async def set_scan_baseline( + scan_id: str, user: AuthenticatedUser = Depends(require_auth) + ): + """Set a scan as the baseline for its target.""" + scan = _scan_store.get_scan(scan_id) + if scan is None: + raise HTTPException(status_code=404, detail="Scan not found") + if scan.status != "completed": + raise HTTPException(status_code=400, detail="Scan not completed") + _scan_store.set_baseline(scan.target, scan_id) + return { + "success": True, + "scan_id": scan_id, + "target": scan.target, + "message": f"Baseline set for {scan.target}", + } + + @app.get("/api/scans/{scan_id}/delta", tags=["Scans"]) + async def get_scan_delta( + scan_id: str, user: AuthenticatedUser = Depends(require_auth) + ): + """Get delta between this scan and the baseline for its target.""" + scan = _scan_store.get_scan(scan_id) + if scan is None: + raise HTTPException(status_code=404, detail="Scan not found") + baseline_scan_id = _scan_store.get_baseline(scan.target) + if not baseline_scan_id: + return {"has_baseline": False, "message": "No baseline set for this target"} + baseline_data = _scan_store.get_results(baseline_scan_id) + if baseline_data is None: + raise HTTPException(status_code=404, detail="Baseline scan results not found") + current_data = _scan_store.get_results(scan_id) + if current_data is None: + raise HTTPException(status_code=404, detail="Current scan results not found") + + baseline_findings = [] + current_findings = [] + for key, value in baseline_data.items(): + if isinstance(value, dict) and (key.startswith("finding_") or "severity" in value): + baseline_findings.append(value) + for key, value in current_data.items(): + if isinstance(value, dict) and (key.startswith("finding_") or "severity" in value): + current_findings.append(value) + + comparator = ScanComparator() + result = comparator.compare( + {"scan_id": baseline_scan_id, "findings": baseline_findings}, + {"scan_id": scan_id, "findings": current_findings}, + ) + return { + "has_baseline": True, + "baseline_scan_id": baseline_scan_id, + "current_scan_id": scan_id, + "delta": result.to_dict(include_findings=True), + } # AI endpoints (protected) @app.post("/api/ai", response_model=AIResponse, tags=["AI"]) @@ -358,6 +837,7 @@ async def ai_action( assistant = AIAssistant( provider=request.provider, model=request.model, + budget_limit=request.budget_limit, ) if request.action == "explain": @@ -371,47 +851,68 @@ async def ai_action( raise HTTPException( status_code=400, detail="scan_id required for analyze action" ) - if request.scan_id not in _scan_results: + results = _scan_store.get_results(request.scan_id) + if results is None: raise HTTPException( status_code=404, detail="Scan results not found" ) - result = assistant.analyze_findings(_scan_results[request.scan_id]) + result = assistant.analyze_findings(results) elif request.action == "suggest": if not request.scan_id: raise HTTPException( status_code=400, detail="scan_id required for suggest action" ) - if request.scan_id not in _scan_results: + results = _scan_store.get_results(request.scan_id) + if results is None: raise HTTPException( status_code=404, detail="Scan results not found" ) - result = assistant.suggest_remediations(_scan_results[request.scan_id]) + result = assistant.suggest_remediations(results) elif request.action == "summarize": if not request.scan_id: raise HTTPException( status_code=400, detail="scan_id required for summarize action" ) - if request.scan_id not in _scan_results: + results = _scan_store.get_results(request.scan_id) + if results is None: raise HTTPException( status_code=404, detail="Scan results not found" ) - result = assistant.summarize(_scan_results[request.scan_id]) + result = assistant.summarize(results) else: raise HTTPException( status_code=400, detail=f"Unknown action: {request.action}" ) + # Merge per-call cost into global tracker + metrics = assistant.get_cost_metrics() + _scan_store.increment_ai_costs(metrics) + return AIResponse( action=request.action, result=result, provider=assistant.provider, model=assistant.model, + cost=AICostMetrics( + calls=metrics["calls"], + input_tokens=metrics["input_tokens"], + output_tokens=metrics["output_tokens"], + estimated_cost_usd=metrics["estimated_cost_usd"], + budget_limit_usd=metrics["budget_limit_usd"], + budget_remaining_usd=metrics["budget_remaining_usd"], + ), ) except ImportError as e: raise HTTPException(status_code=503, detail=f"AI not available: {e}") except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + except RuntimeError as e: + raise HTTPException(status_code=429, detail=str(e)) + except HTTPException: + raise + except (ConnectionError, TimeoutError, OSError, TypeError) as e: + raise HTTPException(status_code=503, detail=f"AI service error: {e}") # Settings endpoints @app.get("/api/settings/providers", tags=["Settings"]) @@ -459,6 +960,17 @@ async def list_presets(): ] } + @app.get("/api/settings/ai-cost", tags=["Settings"]) + async def get_ai_cost_metrics(user: AuthenticatedUser = Depends(require_auth)): + """Return global AI cost metrics.""" + costs = _scan_store.get_ai_costs() + return { + "calls": costs["calls"], + "input_tokens": costs["input_tokens"], + "output_tokens": costs["output_tokens"], + "estimated_cost_usd": round(costs["estimated_cost_usd"], 6), + } + # Dashboard HTML @app.get("/", response_class=HTMLResponse, tags=["Dashboard"]) async def dashboard(): @@ -524,8 +1036,12 @@ async def run_scan_task(scan_id: str, request: ScanRequest): import asyncio try: - _scans[scan_id].status = "running" - _scans[scan_id].progress = 0 + scan = _scan_store.get_scan(scan_id) + if scan is None: + return + scan.status = "running" + scan.progress = 0 + _scan_store.set_scan(scan_id, scan) # Emit scan started event await emit_scan_started(scan_id, request.target, request.preset) @@ -537,14 +1053,15 @@ async def run_scan_task(scan_id: str, request: ScanRequest): ctx = Context(target=request.target) modules = [ - ("domain_profile", recon.domain_profile), - ("tech_stack", recon.tech_stack), + ("profile_domain", recon.profile_domain), + ("fingerprint", recon.fingerprint), ] for i, (name, module_fn) in enumerate(modules): - _scans[scan_id].current_module = name + scan.current_module = name progress = int((i / len(modules)) * 100) - _scans[scan_id].progress = progress + scan.progress = progress + _scan_store.set_scan(scan_id, scan) # Emit progress and module start await emit_scan_progress(scan_id, progress, name) @@ -553,7 +1070,7 @@ async def run_scan_task(scan_id: str, request: ScanRequest): success = True try: ctx = module_fn(ctx) - except Exception as e: + except (RedOpsError, RuntimeError, ImportError, TypeError, ValueError) as e: ctx.log(f"Module {name} failed: {e}", level="ERROR") success = False @@ -563,18 +1080,22 @@ async def run_scan_task(scan_id: str, request: ScanRequest): await asyncio.sleep(0.5) # Yield to event loop # Store results - _scan_results[scan_id] = ctx.data - _scans[scan_id].status = "completed" - _scans[scan_id].progress = 100 - _scans[scan_id].completed_at = datetime.now(timezone.utc).isoformat() + "Z" - _scans[scan_id].current_module = None + _scan_store.set_results(scan_id, ctx.data) + scan.status = "completed" + scan.progress = 100 + scan.completed_at = datetime.now(timezone.utc).isoformat() + "Z" + scan.current_module = None + _scan_store.set_scan(scan_id, scan) # Emit completion await emit_scan_completed(scan_id, len(ctx.data)) - except Exception as e: - _scans[scan_id].status = "failed" - _scans[scan_id].error = str(e) + except Exception as e: # Worker safety net — prevents unhandled exceptions from killing the background task + scan = _scan_store.get_scan(scan_id) + if scan is not None: + scan.status = "failed" + scan.error = str(e) + _scan_store.set_scan(scan_id, scan) await emit_scan_failed(scan_id, str(e)) @@ -588,25 +1109,30 @@ def get_dashboard_html() -> str: RedOPS Dashboard + -
+ Skip to main content +
+ +
+ -
+