From 47fea23f57caab414b7b8070a0c215a1ab389326 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 18 Aug 2026 17:15:38 +0000 Subject: [PATCH 1/4] chore: ignore feature worktrees --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index e2c9cc7..7ff295e 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,6 @@ AGENTS.md # Personal AI workflow tooling -- not project convention, not for the public repo .claude/ + +# Isolated feature worktrees +.worktrees/ From 96e3e91cb5e0ef67cd47382a849c5f13efc98bad Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 18 Aug 2026 17:44:10 +0000 Subject: [PATCH 2/4] Add CVE snapshot distribution --- .github/workflows/update-cve-db.yml | 66 +++ CLAUDE.md | 4 +- README.md | 60 +-- bitprobe/bitprobe.py | 54 +- bitprobe/scanner/cve_db.py | 2 +- bitprobe/scanner/cve_db_bootstrap.py | 338 ++++++++++++ bitprobe/scanner/cve_db_manager.py | 488 ++++++++++++++++-- bitprobe/scanner/paths.py | 63 +++ bitprobe/scanner/update_lock.py | 79 +++ bitprobe/scanner/update_notifier.py | 40 +- bitsentry.py | 24 +- docker-compose.yml | 5 + .../2026-08-18-cve-database-distribution.md | 194 +++++++ scripts/build_cve_snapshot.py | 106 ++++ scripts/install_bitsentry.sh | 9 +- scripts/update_cve_snapshot_release.sh | 27 + tests/test_build_cve_snapshot.py | 55 ++ tests/test_cve_bootstrap_policy.py | 76 +++ tests/test_cve_cli.py | 70 +++ tests/test_cve_db_bootstrap.py | 173 +++++++ tests/test_cve_metadata.py | 82 +++ tests/test_cve_paths.py | 76 +++ tests/test_cve_resumability.py | 95 ++++ tests/test_cve_sync_windows.py | 96 ++++ tests/test_cve_workflow.py | 24 + tests/test_products_cli.py | 23 + tests/test_update_lock.py | 53 ++ tests/test_update_notifier_snapshot.py | 22 + 28 files changed, 2264 insertions(+), 140 deletions(-) create mode 100644 .github/workflows/update-cve-db.yml create mode 100644 bitprobe/scanner/cve_db_bootstrap.py create mode 100644 bitprobe/scanner/update_lock.py create mode 100644 docs/superpowers/plans/2026-08-18-cve-database-distribution.md create mode 100755 scripts/build_cve_snapshot.py create mode 100755 scripts/update_cve_snapshot_release.sh create mode 100644 tests/test_build_cve_snapshot.py create mode 100644 tests/test_cve_bootstrap_policy.py create mode 100644 tests/test_cve_cli.py create mode 100644 tests/test_cve_db_bootstrap.py create mode 100644 tests/test_cve_metadata.py create mode 100644 tests/test_cve_paths.py create mode 100644 tests/test_cve_resumability.py create mode 100644 tests/test_cve_sync_windows.py create mode 100644 tests/test_cve_workflow.py create mode 100644 tests/test_update_lock.py create mode 100644 tests/test_update_notifier_snapshot.py diff --git a/.github/workflows/update-cve-db.yml b/.github/workflows/update-cve-db.yml new file mode 100644 index 0000000..dea20ce --- /dev/null +++ b/.github/workflows/update-cve-db.yml @@ -0,0 +1,66 @@ +name: Update CVE Database + +on: + schedule: + - cron: "17 6 * * *" + workflow_dispatch: + inputs: + full_rebuild: + description: Ignore the existing snapshot and rebuild from NVD + required: false + type: boolean + default: false + +permissions: + contents: write + +concurrency: + group: cve-db-producer + cancel-in-progress: false + +jobs: + sync-and-publish: + runs-on: ubuntu-latest + env: + BITSENTRY_DATA_DIR: ${{ runner.temp }}/bitsentry-data + NVD_API_KEY: ${{ secrets.NVD_API_KEY }} + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Require NVD API key + run: test -n "$NVD_API_KEY" + + - name: Restore previous snapshot + id: restore + if: ${{ !inputs.full_rebuild }} + run: | + if gh release view cve-db-latest >/dev/null 2>&1; then + mkdir -p previous + gh release download cve-db-latest --dir previous --pattern manifest.json --pattern cve_db.sqlite.gz + PYTHONPATH=bitprobe python -m scanner.cve_db_bootstrap --manifest previous/manifest.json --artifact previous/cve_db.sqlite.gz + echo "restored=true" >> "$GITHUB_OUTPUT" + else + echo "restored=false" >> "$GITHUB_OUTPUT" + fi + + - name: Update canonical database + run: | + if [[ "${{ inputs.full_rebuild }}" == "true" || "${{ steps.restore.outputs.restored }}" != "true" ]]; then + PYTHONPATH=bitprobe python bitprobe/bitprobe.py update-cve-db --full --no-snapshot + else + PYTHONPATH=bitprobe python bitprobe/bitprobe.py update-cve-db --no-snapshot + fi + + - name: Build snapshot + run: python scripts/build_cve_snapshot.py + + - name: Publish releases + run: bash scripts/update_cve_snapshot_release.sh dist diff --git a/CLAUDE.md b/CLAUDE.md index 8e283ef..947c579 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,9 +30,9 @@ Network scanning has a fallback chain in `bitprobe/scanner/engines/network/__ini ## Data -- CVE data: `bitprobe/scanner/data/cve_db.sqlite` (primary) with `cve_db.json` as fallback/legacy (`cve_db.py`, `cve_db_manager.py`, `cve_updater.py`). Refresh via `bitprobe update-cve-db` (NVD API key recommended, see README). +- CVE data: `~/.bitsentry/data/cve_db.sqlite` (override with `BITSENTRY_DATA_DIR`), with the source-tree database used only as a one-time legacy migration source. Refresh via `bitsentry update-cve-db`; the default path installs a verified release snapshot before incremental NVD catch-up. - ASN/IP intel DB: `asn_db_updater.py`, refreshed via `bitsentry update-db`. -- Neither DB is checked into git (`bitprobe/data/cve_db.sqlite` is gitignored — a prior commit removed a large SQLite file from the repo for this reason). +- Neither generated database is checked into git. Published CVE snapshots live in reserved `cve-db-*` GitHub Releases. ## Commands diff --git a/README.md b/README.md index b454bd5..ea23e5d 100644 --- a/README.md +++ b/README.md @@ -54,14 +54,8 @@ Refresh local intelligence databases once so scans are useful. This is separate # 1) ASN database (fast; needed for ASN/IP intel plugins) bitsentry update-db -# 2) CVE database (choose one bootstrap — required for technology/CVE correlation) -export NVD_API_KEY="your-nvd-api-key" # optional but strongly recommended - -# Recommended: full local mirror (slow once; best coverage) -bitsentry update-cve-db --full - -# Alternative: smaller first-time bootstrap (~15 years of publications) -# bitsentry update-cve-db --years 15 +# 2) CVE database: verified snapshot, then incremental NVD catch-up +bitsentry update-cve-db # Check what was loaded bitsentry cve-stats @@ -70,14 +64,14 @@ bitsentry cve-stats bitsentry scan example.com ``` -**Ongoing maintenance** (after the one-time bootstrap above): +**Ongoing maintenance:** ```bash bitsentry update-db # refresh ASN data when stale -bitsentry update-cve-db # incremental CVE sync (fast) +bitsentry update-cve-db # snapshot if needed, otherwise incremental sync ``` -If you skip CVE bootstrap, the first scan may still run but will only auto-fetch a **short recent-publication window**—not enough for historical product/CVE exposure. See [CVE database](#cve-database) below for details. +If you skip this step, the first scan uses the same snapshot bootstrap automatically. See [CVE database](#cve-database) for direct-NVD and offline fallback behavior. ### Option 2: manual setup @@ -88,8 +82,7 @@ pip install -r requirements.txt # Same post-install DB steps as Option 1 (use python bitsentry.py if bitsentry is not on PATH) python bitsentry.py update-db -export NVD_API_KEY="your-nvd-api-key" # optional -python bitsentry.py update-cve-db --full # or: --years 15 +python bitsentry.py update-cve-db python bitsentry.py cve-stats # Full workflow (default): BitScope discovery -> BitProbe scan @@ -159,48 +152,49 @@ python bitsentry.py update-db # alias: update-asn-db ### CVE database -BitProbe stores CVEs in a local SQLite database (`bitprobe/data/cve_db.sqlite`) and matches them **by detected product and version** during scans—not by “CVEs published in the last N days.” +BitProbe stores mutable CVE data in `~/.bitsentry/data/cve_db.sqlite` and matches CVEs by detected product and version. Set `BITSENTRY_DATA_DIR` to use a different data directory. | Phase | What happens | |---|---| -| **Bootstrap** | Populates the local DB (one-time or after a wipe) | +| **Bootstrap** | Downloads and verifies the published full-corpus snapshot | | **Incremental sync** | Fetches only NVD records modified since the last cursor (fast) | | **Scan** | Fingerprints the target, then queries the DB for that product/CPE | -A short publication window (for example `--days 30`) only controls **what gets downloaded into the DB**. It does not limit scan logic. For real exposure coverage, bootstrap with a full or multi-year mirror first, then rely on incremental updates. - -**Recommended first-time setup:** +The default command installs a verified snapshot when the database is missing or incomplete, then fetches changes made after the snapshot cursor: ```bash -# Optional but strongly recommended (higher NVD rate limits) -export NVD_API_KEY="your-nvd-api-key" - -# One-time: build a complete local mirror (slow; ~350k CVEs) -python bitsentry.py update-cve-db --full - -# Alternative: compromise bootstrap (~15 years of publications) -python bitsentry.py update-cve-db --years 15 - -# Ongoing refresh (incremental when a sync cursor exists) python bitsentry.py update-cve-db -# Inspect local store +# Install the snapshot without an incremental NVD catch-up +python bitsentry.py update-cve-db --snapshot-only + +# Inspect local coverage and counts python bitsentry.py cve-stats ``` -**Other options:** +Direct-NVD modes skip the snapshot. BitSentry splits long NVD date ranges into 119-day windows: ```bash -# Quick bootstrap only (~recent publications; not sufficient alone for deep history) +# Rebuild the complete corpus directly from NVD +python bitsentry.py update-cve-db --full + +# Raw unfiltered crawl (best-effort offset resumption) +python bitsentry.py update-cve-db --raw-full + +# Build partial publication-window databases python bitsentry.py update-cve-db --days 30 +python bitsentry.py update-cve-db --years 15 + +# Synchronize directly without downloading a snapshot +python bitsentry.py update-cve-db --no-snapshot # Skip automatic CVE refresh at scan startup export BITSENTRY_SKIP_CVE_UPDATE=1 ``` -On scan startup, if the DB is empty, BitProbe may run a **7-day publication bootstrap** so the tool stays usable without blocking on a full NVD download. Run `update-cve-db --full` or `--years 15` before relying on CVE findings in production assessments. +Set `NVD_API_KEY` for the higher NVD request limit. Interrupted windowed updates resume from the last committed page. BitSentry checksum-verifies snapshots and installs them atomically. If the snapshot is unavailable on an empty installation, it falls back to a 30-day publication database and warns that coverage is partial. -Direct product commands are also available via `python bitprobe/bitprobe.py ...` (same flags: `--full`, `--years`, `--days`). +Direct product commands are also available via `python bitprobe/bitprobe.py ...` with the same flags. ### Other maintenance diff --git a/bitprobe/bitprobe.py b/bitprobe/bitprobe.py index 2da7d9c..2945dcb 100755 --- a/bitprobe/bitprobe.py +++ b/bitprobe/bitprobe.py @@ -16,6 +16,7 @@ from scanner.config import ScanConfig, SCAN_PROFILES from scanner.asn_db_updater import update_asn_db from scanner.cve_db_manager import update_cve_database, get_stats +from scanner.cve_db_bootstrap import update_with_snapshot_policy def cmd_scan(args) -> int: @@ -191,8 +192,8 @@ def main() -> int: cve_parser.add_argument( "--days", type=int, - default=30, - help="Publication window for bootstrap when DB is empty (default: 30)", + default=None, + help="Build a publication-window mirror directly from NVD", ) cve_parser.add_argument( "--years", @@ -203,7 +204,22 @@ def main() -> int: cve_parser.add_argument( "--full", action="store_true", - help="Build full local NVD mirror (~350k CVEs; first-time setup)", + help="Rebuild the full local mirror directly from NVD", + ) + cve_parser.add_argument( + "--raw-full", + action="store_true", + help="Best-effort unfiltered NVD crawl; offset resumption is not deterministic", + ) + cve_parser.add_argument( + "--snapshot-only", + action="store_true", + help="Install the published snapshot without contacting NVD afterward", + ) + cve_parser.add_argument( + "--no-snapshot", + action="store_true", + help="Use direct NVD synchronization without downloading a snapshot", ) cve_stats_parser = subparsers.add_parser( @@ -226,14 +242,28 @@ def main() -> int: elif args.command == "update-cve-db": try: - full_sync = getattr(args, "full", False) - count = update_cve_database( - days=args.days, - years=getattr(args, "years", None), - full_sync=full_sync, - force=full_sync, - verbose=verbose, - ) + raw_full = getattr(args, "raw_full", False) + full_sync = getattr(args, "full", False) or raw_full + years = getattr(args, "years", None) + days = getattr(args, "days", None) + snapshot_only = getattr(args, "snapshot_only", False) + direct = full_sync or years is not None or days is not None or getattr(args, "no_snapshot", False) + if snapshot_only and direct: + raise ValueError("--snapshot-only cannot be combined with direct-NVD options") + if direct: + count = update_cve_database( + days=days if days is not None else 30, + years=years, + full_sync=full_sync, + raw_full_sync=raw_full, + force=full_sync, + verbose=verbose, + ) + else: + count = update_with_snapshot_policy( + snapshot_only=snapshot_only, + verbose=verbose, + ) print(f"[+] CVE database updated with {count} entries") return 0 except Exception as e: @@ -247,6 +277,8 @@ def main() -> int: print("=" * 40) print(f"Total CVEs: {stats.get('total_cves', 0)}") print(f"Total Products: {stats.get('total_products', 0)}") + print(f"Coverage: {stats.get('coverage_mode', 'unknown')}") + print(f"NVD Cursor: {stats.get('nvd_cursor', 'Never')}") print(f"Last Updated: {stats.get('last_updated', 'Never')}") print("\nBy Severity:") for sev, count in stats.get('severity_counts', {}).items(): diff --git a/bitprobe/scanner/cve_db.py b/bitprobe/scanner/cve_db.py index 60e0868..63d0c22 100755 --- a/bitprobe/scanner/cve_db.py +++ b/bitprobe/scanner/cve_db.py @@ -2,10 +2,10 @@ import os from pathlib import Path from typing import List, Dict, Any +from scanner.paths import CVE_DB_PATH as CVE_SQLITE_PATH _DATA_DIR = Path(__file__).resolve().parents[1] / "data" CVE_DB_PATH = str(_DATA_DIR / "cve_db.json") -CVE_SQLITE_PATH = str(_DATA_DIR / "cve_db.sqlite") def sqlite_cve_db_available() -> bool: diff --git a/bitprobe/scanner/cve_db_bootstrap.py b/bitprobe/scanner/cve_db_bootstrap.py new file mode 100644 index 0000000..fe56808 --- /dev/null +++ b/bitprobe/scanner/cve_db_bootstrap.py @@ -0,0 +1,338 @@ +"""Download, validate, and atomically install published CVE snapshots.""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import os +import shutil +import sqlite3 +import tempfile +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import requests + +from scanner.cve_db_manager import CVE_SCHEMA_VERSION +from scanner.paths import CVE_DB_PATH +from scanner.update_lock import bitsentry_update_lock +from scanner.update_state import set_state_timestamp + + +SNAPSHOT_FORMAT_VERSION = 1 +SNAPSHOT_RELEASE_URL = ( + "https://github.com/rywils/BitSentry/releases/download/cve-db-latest" +) +MANIFEST_URL = f"{SNAPSHOT_RELEASE_URL}/manifest.json" +MAX_COMPRESSED_SIZE = 512 * 1024 * 1024 +MAX_UNCOMPRESSED_SIZE = 2 * 1024 * 1024 * 1024 +REQUIRED_TABLES = frozenset({"cve_entries", "cve_products", "cve_cpes", "metadata"}) +REQUIRED_MANIFEST_FIELDS = frozenset( + { + "format_version", + "schema_version", + "coverage_mode", + "coverage_start", + "coverage_end", + "built_at", + "nvd_cursor", + "cve_count", + "artifact", + "sha256_gz", + "compressed_size", + "uncompressed_size", + "source_commit", + } +) + + +class SnapshotError(RuntimeError): + pass + + +class SnapshotValidationError(SnapshotError): + pass + + +def validate_manifest(manifest: dict[str, Any]) -> dict[str, Any]: + if not isinstance(manifest, dict): + raise SnapshotValidationError("manifest must be a JSON object") + missing = REQUIRED_MANIFEST_FIELDS - set(manifest) + if missing: + raise SnapshotValidationError(f"manifest missing fields: {sorted(missing)}") + if manifest["format_version"] != SNAPSHOT_FORMAT_VERSION: + raise SnapshotValidationError("unsupported format_version") + if manifest["schema_version"] != CVE_SCHEMA_VERSION: + raise SnapshotValidationError("unsupported schema_version") + if manifest["coverage_mode"] != "full": + raise SnapshotValidationError("snapshot coverage_mode must be full") + for field in ("compressed_size", "uncompressed_size", "cve_count"): + if not isinstance(manifest[field], int) or manifest[field] < 0: + raise SnapshotValidationError(f"invalid {field}") + if manifest["compressed_size"] > MAX_COMPRESSED_SIZE: + raise SnapshotValidationError("compressed snapshot exceeds size limit") + if manifest["uncompressed_size"] > MAX_UNCOMPRESSED_SIZE: + raise SnapshotValidationError("uncompressed snapshot exceeds size limit") + digest = manifest["sha256_gz"] + if not isinstance(digest, str) or len(digest) != 64: + raise SnapshotValidationError("invalid sha256_gz") + if not isinstance(manifest["nvd_cursor"], str) or not manifest["nvd_cursor"]: + raise SnapshotValidationError("invalid nvd_cursor") + return manifest + + +def fetch_snapshot_manifest( + url: str = MANIFEST_URL, + *, + session: requests.Session | None = None, +) -> dict[str, Any]: + client = session or requests.Session() + try: + response = client.get(url, timeout=(10, 30)) + response.raise_for_status() + except requests.RequestException as exc: + raise SnapshotError("could not download snapshot manifest") from exc + try: + manifest = response.json() + except ValueError as exc: + raise SnapshotValidationError("manifest is not valid JSON") from exc + return validate_manifest(manifest) + + +def download_snapshot( + manifest: dict[str, Any], + destination: Path, + *, + session: requests.Session | None = None, + base_url: str = SNAPSHOT_RELEASE_URL, +) -> Path: + validate_manifest(manifest) + client = session or requests.Session() + url = f"{base_url}/{manifest['artifact']}" + written = 0 + try: + response = client.get(url, stream=True, timeout=(10, 120)) + response.raise_for_status() + with destination.open("wb") as output: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if not chunk: + continue + written += len(chunk) + if written > MAX_COMPRESSED_SIZE or written > manifest["compressed_size"]: + raise SnapshotValidationError("compressed snapshot exceeds declared size") + output.write(chunk) + except requests.RequestException as exc: + destination.unlink(missing_ok=True) + raise SnapshotError("could not download CVE snapshot") from exc + if written != manifest["compressed_size"]: + raise SnapshotValidationError("compressed snapshot size mismatch") + return destination + + +def verify_snapshot(path: Path, manifest: dict[str, Any]) -> None: + validate_manifest(manifest) + if path.stat().st_size != manifest["compressed_size"]: + raise SnapshotValidationError("compressed snapshot size mismatch") + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + if digest.hexdigest() != manifest["sha256_gz"]: + raise SnapshotValidationError("snapshot checksum mismatch") + + +def decompress_snapshot(source: Path, destination: Path, manifest: dict[str, Any]) -> Path: + verify_snapshot(source, manifest) + written = 0 + try: + with gzip.open(source, "rb") as compressed, destination.open("wb") as output: + while block := compressed.read(1024 * 1024): + written += len(block) + if written > MAX_UNCOMPRESSED_SIZE or written > manifest["uncompressed_size"]: + raise SnapshotValidationError("uncompressed snapshot exceeds declared size") + output.write(block) + except (gzip.BadGzipFile, EOFError) as exc: + raise SnapshotValidationError("invalid gzip snapshot") from exc + if written != manifest["uncompressed_size"]: + raise SnapshotValidationError("uncompressed snapshot size mismatch") + return destination + + +def validate_snapshot_database(path: Path, manifest: dict[str, Any]) -> dict[str, str | None]: + validate_manifest(manifest) + try: + with sqlite3.connect(f"file:{path}?mode=ro", uri=True) as conn: + if conn.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + raise SnapshotValidationError("SQLite integrity_check failed") + tables = { + row[0] + for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'") + } + if not REQUIRED_TABLES <= tables: + raise SnapshotValidationError("snapshot is missing required tables") + metadata = dict(conn.execute("SELECT key, value FROM metadata")) + count = conn.execute("SELECT COUNT(*) FROM cve_entries").fetchone()[0] + except sqlite3.Error as exc: + raise SnapshotValidationError("snapshot is not a valid SQLite database") from exc + + comparisons = { + "schema_version": str(manifest["schema_version"]), + "coverage_mode": manifest["coverage_mode"], + "coverage_start": manifest["coverage_start"], + "coverage_end": manifest["coverage_end"], + "nvd_cursor": manifest["nvd_cursor"], + } + for field, expected in comparisons.items(): + if metadata.get(field) != expected: + raise SnapshotValidationError(f"manifest/SQLite {field} mismatch") + if count != manifest["cve_count"]: + raise SnapshotValidationError("manifest/SQLite cve_count mismatch") + return {key: metadata.get(key) for key in comparisons} + + +def _checkpoint_existing_database(destination: Path) -> None: + sidecars = [Path(f"{destination}{suffix}") for suffix in ("-wal", "-shm")] + if not any(path.exists() for path in sidecars): + return + if not destination.exists(): + raise SnapshotValidationError("SQLite sidecar exists without the main database") + + try: + with sqlite3.connect(destination, timeout=0) as conn: + conn.execute("PRAGMA busy_timeout=0") + busy, _, _ = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + except sqlite3.Error as exc: + raise SnapshotValidationError( + "cannot checkpoint the current database before snapshot installation" + ) from exc + if busy: + raise SnapshotValidationError( + "current database is busy; snapshot installation was not attempted" + ) + + for sidecar in sidecars: + if not sidecar.exists(): + continue + if sidecar.stat().st_size: + raise SnapshotValidationError( + f"SQLite sidecar remains after checkpoint: {sidecar}" + ) + sidecar.unlink() + + +def install_snapshot_atomically( + snapshot_db: Path, + manifest: dict[str, Any], + *, + destination: Path | None = None, +) -> str: + destination = destination or Path(CVE_DB_PATH) + validate_snapshot_database(snapshot_db, manifest) + destination.parent.mkdir(parents=True, exist_ok=True) + temp_path: Path | None = None + with bitsentry_update_lock(): + _checkpoint_existing_database(destination) + try: + with tempfile.NamedTemporaryFile( + dir=destination.parent, + prefix=".cve-install-", + suffix=".sqlite", + delete=False, + ) as temp_file: + temp_path = Path(temp_file.name) + shutil.copyfile(snapshot_db, temp_path) + with sqlite3.connect(temp_path) as conn: + conn.execute("PRAGMA journal_mode=DELETE") + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + validate_snapshot_database(temp_path, manifest) + os.replace(temp_path, destination) + temp_path = None + finally: + if temp_path is not None: + temp_path.unlink(missing_ok=True) + return manifest["nvd_cursor"] + + +def bootstrap_from_snapshot() -> dict[str, Any]: + manifest = fetch_snapshot_manifest() + destination = Path(CVE_DB_PATH) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=destination.parent) as temp_dir: + temp_root = Path(temp_dir) + compressed = download_snapshot(manifest, temp_root / manifest["artifact"]) + database = decompress_snapshot(compressed, temp_root / "cve_db.sqlite", manifest) + cursor = install_snapshot_atomically(database, manifest, destination=destination) + set_state_timestamp("cve", "last_modified", cursor) + return manifest + + +def update_with_snapshot_policy( + *, + snapshot_only: bool = False, + verbose: bool = False, +) -> int: + """Bootstrap incomplete stores from a snapshot, then catch up from NVD.""" + from scanner import cve_db_manager as manager + + if snapshot_only: + bootstrap_from_snapshot() + return 0 + + complete = manager.cve_db_is_complete() + cursor = manager.read_cve_metadata().get("nvd_cursor") if complete else None + cursor_too_old = False + if cursor: + parsed = datetime.fromisoformat(cursor.replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + cursor_too_old = datetime.now(timezone.utc) - parsed > timedelta(days=119) + + if not complete or cursor_too_old: + try: + bootstrap_from_snapshot() + except SnapshotError as exc: + print(f"[!] CVE snapshot unavailable: {exc}") + if not complete: + print("[!] Falling back to a 30-day publication bootstrap; coverage is partial.") + return manager.update_cve_database( + days=30, + incremental=False, + verbose=verbose, + ) + print("[!] Falling back to chunked incremental NVD catch-up.") + return manager.update_cve_database(days=30, incremental=True, verbose=verbose) + + +def install_snapshot_artifact(manifest_path: Path, artifact_path: Path) -> dict[str, Any]: + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise SnapshotValidationError("cannot read local snapshot manifest") from exc + validate_manifest(manifest) + destination = Path(CVE_DB_PATH) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=destination.parent) as temp_dir: + database = decompress_snapshot( + artifact_path, + Path(temp_dir) / "cve_db.sqlite", + manifest, + ) + cursor = install_snapshot_atomically(database, manifest, destination=destination) + set_state_timestamp("cve", "last_modified", cursor) + return manifest + + +def main() -> int: + parser = argparse.ArgumentParser(description="Install a BitSentry CVE snapshot") + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--artifact", type=Path, required=True) + args = parser.parse_args() + install_snapshot_artifact(args.manifest, args.artifact) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bitprobe/scanner/cve_db_manager.py b/bitprobe/scanner/cve_db_manager.py index fdbd527..4819d45 100644 --- a/bitprobe/scanner/cve_db_manager.py +++ b/bitprobe/scanner/cve_db_manager.py @@ -10,16 +10,14 @@ import os import time import requests -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import List, Dict, Optional, Any from packaging import version from pathlib import Path +from scanner.paths import CVE_DB_PATH, CVE_META_PATH, migrate_legacy_cve_database +from scanner.update_lock import bitsentry_update_lock from scanner.update_state import get_state_timestamp, set_state_timestamp, merge_section - -_DATA_DIR = Path(__file__).resolve().parents[1] / "data" -CVE_DB_PATH = str(_DATA_DIR / "cve_db.sqlite") -CVE_META_PATH = str(_DATA_DIR / "cve_meta.json") NVD_API_URL = "https://services.nvd.nist.gov/rest/json/cves/2.0" DEFAULT_STALE_DAYS = 7 # Below this count, local DB is treated as a short publication window bootstrap, not full coverage. @@ -30,6 +28,19 @@ NVD_SLEEP_WITH_KEY = 0.65 NVD_MAX_RETRIES = 6 NVD_RETRY_HTTP = frozenset({404, 429, 500, 502, 503, 504}) +CVE_SCHEMA_VERSION = 1 +CVE_METADATA_KEYS = ( + "schema_version", + "coverage_mode", + "coverage_start", + "coverage_end", + "nvd_cursor", +) + + +def _utcnow() -> datetime: + """Naive UTC datetime for compatibility with existing NVD timestamps.""" + return datetime.now(timezone.utc).replace(tzinfo=None) def _format_nvd_datetime(dt: datetime) -> str: @@ -57,6 +68,24 @@ def _normalize_nvd_timestamp(value: str) -> str: return value +def iter_nvd_windows( + start: datetime, + end: datetime, + max_days: int = 119, +): + """Yield contiguous NVD-safe date windows with frozen boundaries.""" + if max_days <= 0: + raise ValueError("max_days must be positive") + if end < start: + raise ValueError("end must not precede start") + cursor = start + width = timedelta(days=max_days) + while cursor < end: + window_end = min(cursor + width, end) + yield cursor, window_end + cursor = window_end + + def _nvd_inter_request_sleep(api_key: Optional[str]) -> float: """ Seconds to wait between NVD requests (rate-limit safe). @@ -160,14 +189,15 @@ def bootstrap_cve_state() -> str | None: Seed ~/.bitsentry/state.json cve.last_modified from DB or legacy meta so incremental NVD sync works after upgrades or empty state files. """ - existing = get_state_timestamp("cve", "last_modified") - if existing: - return existing - if os.path.exists(CVE_DB_PATH): conn = _connect() try: cursor = conn.cursor() + cursor.execute("SELECT value FROM metadata WHERE key = 'nvd_cursor'") + sqlite_cursor = cursor.fetchone() + if sqlite_cursor and sqlite_cursor[0]: + set_state_timestamp("cve", "last_modified", sqlite_cursor[0]) + return sqlite_cursor[0] cursor.execute("SELECT COUNT(*) FROM cve_entries") if cursor.fetchone()[0] == 0: return None @@ -182,6 +212,10 @@ def bootstrap_cve_state() -> str | None: finally: conn.close() + existing = get_state_timestamp("cve", "last_modified") + if existing: + return existing + # Do not seed from legacy meta when SQLite is empty — that cursor is too # stale and triggers huge lastMod incremental pulls (~40k+ CVEs). meta = _load_legacy_meta() @@ -203,6 +237,105 @@ def bootstrap_cve_state() -> str | None: return None +def read_cve_metadata( + conn: sqlite3.Connection | None = None, +) -> Dict[str, Optional[str]]: + owns_conn = conn is None + conn = conn or _connect() + try: + rows = conn.execute( + "SELECT key, value FROM metadata WHERE key IN ({})".format( + ",".join("?" for _ in CVE_METADATA_KEYS) + ), + CVE_METADATA_KEYS, + ).fetchall() + values = {key: value for key, value in rows} + return {key: values.get(key) for key in CVE_METADATA_KEYS} + finally: + if owns_conn: + conn.close() + + +def write_cve_metadata( + updates: Dict[str, Optional[str]], + conn: sqlite3.Connection | None = None, +) -> None: + unknown = set(updates) - set(CVE_METADATA_KEYS) + if unknown: + raise ValueError(f"Unknown CVE metadata keys: {sorted(unknown)}") + owns_conn = conn is None + conn = conn or _connect() + try: + conn.executemany( + "INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)", + list(updates.items()), + ) + if owns_conn: + conn.commit() + finally: + if owns_conn: + conn.close() + + +def cve_db_is_complete() -> bool: + if not os.path.exists(CVE_DB_PATH): + return False + return read_cve_metadata().get("coverage_mode") == "full" + + +def mirror_sqlite_cursor_to_state() -> str | None: + if not os.path.exists(CVE_DB_PATH): + return None + cursor = read_cve_metadata().get("nvd_cursor") + if cursor: + set_state_timestamp("cve", "last_modified", cursor) + return cursor + + +def prepare_sync_window( + conn: sqlite3.Connection, + *, + mode: str, + window_start: str, + window_end: str, + results_per_page: int, +) -> int: + row = conn.execute( + "SELECT mode, window_start, window_end, results_per_page, " + "next_start_index, completed FROM sync_state WHERE id = 1" + ).fetchone() + signature = (mode, window_start, window_end, results_per_page) + if row and tuple(row[:4]) == signature and row[5] == 0: + return int(row[4]) + conn.execute( + "INSERT OR REPLACE INTO sync_state " + "(id, mode, window_start, window_end, results_per_page, " + "next_start_index, total_expected, started_at, completed) " + "VALUES (1, ?, ?, ?, ?, 0, NULL, ?, 0)", + (mode, window_start, window_end, results_per_page, _utcnow().isoformat()), + ) + conn.commit() + return 0 + + +def checkpoint_sync_page( + conn: sqlite3.Connection, + *, + next_start_index: int, + total_expected: int, +) -> None: + conn.execute( + "UPDATE sync_state SET next_start_index = ?, total_expected = ? WHERE id = 1", + (next_start_index, total_expected), + ) + conn.commit() + + +def complete_sync_window(conn: sqlite3.Connection) -> None: + conn.execute("UPDATE sync_state SET completed = 1 WHERE id = 1") + conn.commit() + + def cve_db_needs_update(stale_days: int = DEFAULT_STALE_DAYS) -> bool: """True when the local CVE store is missing or older than stale_days.""" if os.environ.get("BITSENTRY_SKIP_CVE_UPDATE", "").strip().lower() in { @@ -223,6 +356,11 @@ def cve_db_needs_update(stale_days: int = DEFAULT_STALE_DAYS) -> bool: if count == 0: return True + cursor.execute("SELECT value FROM metadata WHERE key = 'nvd_cursor'") + row = cursor.fetchone() + if row and row[0]: + last_dt = datetime.fromisoformat(_normalize_nvd_timestamp(row[0])) + return _utcnow() - last_dt > timedelta(days=stale_days) cursor.execute("SELECT value FROM metadata WHERE key = 'last_updated'") row = cursor.fetchone() if not row or not row[0]: @@ -236,7 +374,7 @@ def cve_db_needs_update(stale_days: int = DEFAULT_STALE_DAYS) -> bool: finally: conn.close() - return datetime.utcnow() - last_dt > timedelta(days=stale_days) + return _utcnow() - last_dt > timedelta(days=stale_days) def describe_cve_db_local_status() -> str: @@ -254,10 +392,9 @@ def describe_cve_db_local_status() -> str: conn.close() if count == 0: return "empty (0 CVEs; bootstrap required before reliable correlation)" - if count < MIN_PRODUCTION_CVE_COUNT: + if read_cve_metadata().get("coverage_mode") != "full": return ( - f"partial ({count} CVEs; run update-cve-db --full or --years 15 " - "for complete product coverage)" + f"partial ({count} CVEs; run update-cve-db to install full coverage)" ) if cve_db_needs_update(): return f"stale or incomplete ({count} CVEs; refresh recommended)" @@ -266,6 +403,7 @@ def describe_cve_db_local_status() -> str: def init_cve_database(): """Initialize SQLite database with CVE schema.""" + migrate_legacy_cve_database() os.makedirs(os.path.dirname(CVE_DB_PATH), exist_ok=True) conn = _connect() @@ -317,6 +455,29 @@ def init_cve_database(): value TEXT ) """) + cursor.execute(""" + CREATE TABLE IF NOT EXISTS sync_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + mode TEXT NOT NULL, + window_start TEXT, + window_end TEXT, + results_per_page INTEGER NOT NULL, + next_start_index INTEGER NOT NULL, + total_expected INTEGER, + started_at TEXT NOT NULL, + completed INTEGER NOT NULL DEFAULT 0 + ) + """) + cursor.executemany( + "INSERT OR IGNORE INTO metadata (key, value) VALUES (?, ?)", + [ + ("schema_version", str(CVE_SCHEMA_VERSION)), + ("coverage_mode", "windowed"), + ("coverage_start", None), + ("coverage_end", None), + ("nvd_cursor", None), + ], + ) # Create indexes for performance cursor.execute("CREATE INDEX IF NOT EXISTS idx_cve_severity ON cve_entries(severity)") @@ -330,14 +491,16 @@ def init_cve_database(): print(f"[+] CVE database initialized at {CVE_DB_PATH}") -def update_cve_database( +def _update_cve_database_unlocked( days: int = 30, years: Optional[int] = None, full_sync: bool = False, + raw_full_sync: bool = False, api_key: Optional[str] = None, incremental: bool = True, force: bool = False, verbose: bool = False, + _window: tuple[str, datetime, datetime] | None = None, ) -> int: """ Update CVE database from NVD feeds. @@ -376,23 +539,158 @@ def update_cve_database( finally: conn.close() - # Determine update strategy from persistent state - state_last_modified = get_state_timestamp("cve", "last_modified") - update_end = _format_nvd_datetime(datetime.utcnow()) + # Determine update strategy from persistent state. + state_last_modified = read_cve_metadata().get("nvd_cursor") or get_state_timestamp( + "cve", "last_modified" + ) + update_end = _format_nvd_datetime(_utcnow()) + + if _window is None: + now_dt = _utcnow() + if full_sync and not raw_full_sync: + metadata = read_cve_metadata() + with _connect() as state_conn: + resume_row = state_conn.execute( + "SELECT mode, window_start, window_end, completed " + "FROM sync_state WHERE id = 1" + ).fetchone() + resuming_full = bool( + resume_row + and str(resume_row[0]).startswith("full-") + and metadata.get("coverage_mode") == "windowed" + and metadata.get("coverage_end") + ) + if resuming_full: + build_started = datetime.fromisoformat(metadata["coverage_end"]) + else: + build_started = now_dt + with _connect() as reset_conn: + reset_conn.execute("DELETE FROM cve_products") + reset_conn.execute("DELETE FROM cve_cpes") + reset_conn.execute("DELETE FROM cve_entries") + write_cve_metadata( + { + "coverage_mode": "windowed", + "coverage_start": None, + "coverage_end": _format_nvd_datetime(build_started), + "nvd_cursor": None, + }, + conn=reset_conn, + ) + reset_conn.commit() + total = 0 + history_start = datetime(1999, 1, 1) + resume_catchup: tuple[datetime, datetime] | None = None + if resuming_full and resume_row: + if resume_row[0] == "full-publication": + history_start = datetime.fromisoformat( + resume_row[2] if resume_row[3] else resume_row[1] + ) + elif resume_row[0] == "full-catchup": + history_start = build_started + resume_catchup = ( + datetime.fromisoformat(resume_row[1]), + datetime.fromisoformat(resume_row[2]), + ) + for window_start, window_end in iter_nvd_windows(history_start, build_started): + total += _update_cve_database_unlocked( + api_key=api_key, + incremental=False, + force=True, + verbose=verbose, + _window=("full-publication", window_start, window_end), + ) + catchup_start, catchup_end = resume_catchup or ( + build_started, + _utcnow(), + ) + total += _update_cve_database_unlocked( + api_key=api_key, + incremental=True, + force=False, + verbose=verbose, + _window=("full-catchup", catchup_start, catchup_end), + ) + completed = _format_nvd_datetime(catchup_end) + write_cve_metadata( + { + "coverage_mode": "full", + "coverage_start": None, + "coverage_end": completed, + "nvd_cursor": completed, + } + ) + set_state_timestamp("cve", "last_modified", completed) + return total - if full_sync: + if incremental and state_last_modified and existing_count and not force: + cursor_dt = datetime.fromisoformat( + _normalize_nvd_timestamp(state_last_modified) + ) + if now_dt - cursor_dt > timedelta(days=119): + total = 0 + for window_start, window_end in iter_nvd_windows(cursor_dt, now_dt): + total += _update_cve_database_unlocked( + api_key=api_key, + incremental=True, + verbose=verbose, + _window=("modified", window_start, window_end), + ) + return total + elif not full_sync: + window_days = years * 365 if years is not None and years > 0 else days + start_dt = now_dt - timedelta(days=window_days) + if now_dt - start_dt > timedelta(days=119): + total = 0 + for window_start, window_end in iter_nvd_windows(start_dt, now_dt): + total += _update_cve_database_unlocked( + api_key=api_key, + incremental=False, + force=True, + verbose=verbose, + _window=("publication", window_start, window_end), + ) + coverage_start = _format_nvd_datetime(start_dt) + coverage_end = _format_nvd_datetime(now_dt) + write_cve_metadata( + { + "coverage_mode": "windowed", + "coverage_start": coverage_start, + "coverage_end": coverage_end, + "nvd_cursor": coverage_end, + } + ) + set_state_timestamp("cve", "last_modified", coverage_end) + return total + + if _window is not None: + window_kind, window_start_dt, window_end_dt = _window + window_start_text = _format_nvd_datetime(window_start_dt) + window_end_text = _format_nvd_datetime(window_end_dt) + use_incremental = window_kind in {"modified", "full-catchup"} + if use_incremental: + params["lastModStartDate"] = window_start_text + params["lastModEndDate"] = window_end_text + else: + params["pubStartDate"] = window_start_text + params["pubEndDate"] = window_end_text + elif full_sync: merge_section("cve", {"last_modified": None}) state_last_modified = None print("[*] Full sync requested: ignoring incremental cursor and date windows") elif existing_count == 0: merge_section("cve", {"last_modified": None}) state_last_modified = None + use_incremental = False - use_incremental = ( - incremental and state_last_modified and not force and not full_sync - ) + else: + use_incremental = ( + incremental and state_last_modified and not force and not full_sync + ) - if use_incremental: + if _window is not None: + pass + elif use_incremental: # Incremental: only fetch CVEs modified since last update mod_start = _normalize_nvd_timestamp(state_last_modified) params['lastModStartDate'] = mod_start @@ -411,7 +709,7 @@ def update_cve_database( ) print("[*] Full NVD corpus sync (no date filter)") else: - end_date = datetime.now() + end_date = _utcnow() if years is not None and years > 0: window_days = years * 365 label = f"{years} year(s)" @@ -442,7 +740,9 @@ def update_cve_database( api_failed = False expected_total: Optional[int] = None - if use_incremental: + if _window is not None: + update_type = f"{_window[0]} window" + elif use_incremental: update_type = "incremental" elif full_sync: update_type = "full corpus" @@ -453,6 +753,36 @@ def update_cve_database( print(f"[*] Fetching CVEs from NVD ({update_type})...") print(f"[*] Timeout per request: 60s | Results per page: {params['resultsPerPage']}") + if _window is not None: + checkpoint_mode = _window[0] + checkpoint_start = ( + params["lastModStartDate"] if use_incremental else params["pubStartDate"] + ) + checkpoint_end = ( + params["lastModEndDate"] if use_incremental else params["pubEndDate"] + ) + elif use_incremental: + checkpoint_mode = "modified" + checkpoint_start = params["lastModStartDate"] + checkpoint_end = params["lastModEndDate"] + elif full_sync: + checkpoint_mode = "raw-full" + checkpoint_start = "unbounded" + checkpoint_end = update_end + else: + checkpoint_mode = "publication" + checkpoint_start = params["pubStartDate"] + checkpoint_end = params["pubEndDate"] + start_index = prepare_sync_window( + store_conn, + mode=checkpoint_mode, + window_start=checkpoint_start, + window_end=checkpoint_end, + results_per_page=params["resultsPerPage"], + ) + if start_index: + print(f"[*] Resuming {checkpoint_mode} sync from startIndex={start_index}") + while True: batch_num += 1 params['startIndex'] = start_index @@ -486,6 +816,13 @@ def update_cve_database( saw_vulnerabilities = True store_start = time.time() + total_results = int(data.get("totalResults", 0)) + if expected_total is None: + expected_total = total_results + page_stride = int( + data.get("resultsPerPage") or params["resultsPerPage"] + ) + next_start_index = start_index + page_stride # Collect CVE data for batch processing for vuln in vulnerabilities: cve_data = vuln.get('cve', {}) @@ -500,11 +837,15 @@ def update_cve_database( if verbose and len(batch_cves) % 100 == 0: print(f"[VERBOSE] Collected {len(batch_cves)} CVEs in current batch") - # Batch insert every 1000 CVEs or at end - if len(batch_cves) >= 1000: + # Store and checkpoint each NVD page in one transaction. + if batch_cves: if verbose: print(f"[VERBOSE] Storing batch of {len(batch_cves)} CVEs...") - _store_cves_batch(batch_cves, conn=store_conn) + _store_cves_batch( + batch_cves, + conn=store_conn, + checkpoint=(next_start_index, total_results), + ) total_updated += len(batch_cves) if verbose: print(f"[VERBOSE] Batch stored. Total updated so far: {total_updated}") @@ -512,13 +853,7 @@ def update_cve_database( store_time = time.time() - store_start - total_results = int(data.get("totalResults", 0)) - if expected_total is None: - expected_total = total_results - page_stride = int( - data.get("resultsPerPage") or params["resultsPerPage"] - ) - start_index += page_stride + start_index = next_start_index elapsed = time.time() - overall_start_time rate = start_index / elapsed if elapsed > 0 else 0 @@ -548,18 +883,8 @@ def update_cve_database( api_failed = True break - # Store any remaining CVEs in the batch - if batch_cves: - if verbose: - print(f"[VERBOSE] Storing final batch of {len(batch_cves)} CVEs...") - _store_cves_batch(batch_cves, conn=store_conn) - total_updated += len(batch_cves) - if verbose: - print(f"[VERBOSE] Final batch stored.") - - store_conn.close() - if api_failed: + store_conn.close() raise RuntimeError("CVE update failed before completion; state not updated.") if ( @@ -573,6 +898,9 @@ def update_cve_database( f"{expected_total} CVEs. Re-run: bitsentry update-cve-db --full" ) + complete_sync_window(store_conn) + store_conn.close() + conn = _connect() cursor = conn.cursor() try: @@ -580,7 +908,7 @@ def update_cve_database( total_in_db = cursor.fetchone()[0] cursor.execute( "INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)", - ('last_updated', datetime.now().isoformat()), + ('last_updated', _utcnow().isoformat()), ) cursor.execute( "INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)", @@ -594,8 +922,8 @@ def update_cve_database( with open(CVE_META_PATH, "w", encoding="utf-8") as f: json.dump( { - "last_update": datetime.utcnow().isoformat(), - "updated": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "last_update": _utcnow().isoformat(), + "updated": _utcnow().strftime("%Y-%m-%d %H:%M:%S"), "entry_count": total_in_db, "incremental": bool(use_incremental), }, @@ -603,8 +931,25 @@ def update_cve_database( indent=2, ) - if saw_vulnerabilities: - set_state_timestamp("cve", "last_modified", latest_last_modified or update_end) + completed_window_end = ( + _format_nvd_datetime(_window[2]) if _window is not None else update_end + ) + if use_incremental: + write_cve_metadata({"nvd_cursor": completed_window_end}) + set_state_timestamp("cve", "last_modified", completed_window_end) + elif saw_vulnerabilities and not ( + _window is not None and _window[0] == "full-publication" + ): + cursor_value = latest_last_modified or update_end + write_cve_metadata( + { + "nvd_cursor": cursor_value, + "coverage_mode": "full" if full_sync else "windowed", + "coverage_start": None if full_sync else params.get("pubStartDate"), + "coverage_end": params.get("pubEndDate", update_end), + } + ) + set_state_timestamp("cve", "last_modified", cursor_value) print( f"[+] CVE database updated: {total_updated} CVEs added/updated " @@ -613,6 +958,30 @@ def update_cve_database( return total_updated +def update_cve_database( + days: int = 30, + years: Optional[int] = None, + full_sync: bool = False, + raw_full_sync: bool = False, + api_key: Optional[str] = None, + incremental: bool = True, + force: bool = False, + verbose: bool = False, +) -> int: + """Run a CVE update while holding the shared mutable-data lock.""" + with bitsentry_update_lock(): + return _update_cve_database_unlocked( + days=days, + years=years, + full_sync=full_sync, + raw_full_sync=raw_full_sync, + api_key=api_key, + incremental=incremental, + force=force, + verbose=verbose, + ) + + def _extract_cpe_matches_from_node(node: Dict) -> List[Dict]: """ Extract product entries from one NVD configuration node, recursing into @@ -701,7 +1070,11 @@ def _extract_cve_data(cve_data: Dict) -> Optional[Dict]: } -def _store_cves_batch(cve_data_list: List[Dict], conn: Optional[sqlite3.Connection] = None): +def _store_cves_batch( + cve_data_list: List[Dict], + conn: Optional[sqlite3.Connection] = None, + checkpoint: tuple[int, int] | None = None, +): """ Store multiple CVEs in a single batch transaction (much faster). @@ -772,6 +1145,12 @@ def _store_cves_batch(cve_data_list: List[Dict], conn: Optional[sqlite3.Connecti VALUES (?, ?, ?, ?, ?, ?, ?) """, unique_products) + if checkpoint is not None: + cursor.execute( + "UPDATE sync_state SET next_start_index = ?, total_expected = ? WHERE id = 1", + checkpoint, + ) + conn.commit() finally: if owns_conn: @@ -895,12 +1274,17 @@ def get_stats() -> Dict[str, Any]: cursor.execute("SELECT value FROM metadata WHERE key = 'last_updated'") last_updated = cursor.fetchone() + metadata = read_cve_metadata(conn) return { 'total_cves': total_cves, 'total_products': total_products, 'severity_counts': severity_counts, - 'last_updated': last_updated[0] if last_updated else None + 'last_updated': last_updated[0] if last_updated else None, + 'coverage_mode': metadata.get('coverage_mode'), + 'coverage_start': metadata.get('coverage_start'), + 'coverage_end': metadata.get('coverage_end'), + 'nvd_cursor': metadata.get('nvd_cursor'), } finally: diff --git a/bitprobe/scanner/paths.py b/bitprobe/scanner/paths.py index 591e5d9..07f73f1 100644 --- a/bitprobe/scanner/paths.py +++ b/bitprobe/scanner/paths.py @@ -2,10 +2,73 @@ from __future__ import annotations +import os +import shutil +import sqlite3 +import tempfile from pathlib import Path # bitprobe/ directory (parent of scanner/) BITPROBE_ROOT: Path = Path(__file__).resolve().parents[1] +BITSENTRY_STATE_DIR: Path = Path.home() / ".bitsentry" +BITSENTRY_DATA_DIR: Path = Path( + os.environ.get("BITSENTRY_DATA_DIR", BITSENTRY_STATE_DIR / "data") +).expanduser() + +LEGACY_CVE_DB_PATH: Path = BITPROBE_ROOT / "data" / "cve_db.sqlite" +CVE_DB_PATH: str = str(BITSENTRY_DATA_DIR / "cve_db.sqlite") +CVE_META_PATH: str = str(BITSENTRY_DATA_DIR / "cve_meta.json") + # Delegated-ASN JSON built by asn_db_updater ASN_DB_PATH: str = str(BITPROBE_ROOT / "data" / "asn_db.json") + +_CVE_REQUIRED_TABLES = frozenset( + {"cve_entries", "cve_products", "cve_cpes", "metadata"} +) + + +def _valid_cve_database(path: Path) -> bool: + try: + with sqlite3.connect(f"file:{path}?mode=ro", uri=True) as conn: + if conn.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + return False + tables = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + ) + } + return _CVE_REQUIRED_TABLES <= tables + except (OSError, sqlite3.Error): + return False + + +def migrate_legacy_cve_database( + legacy_path: Path = LEGACY_CVE_DB_PATH, + destination: Path | None = None, +) -> bool: + """Copy a valid legacy CVE DB once, leaving its source untouched.""" + destination = destination or Path(CVE_DB_PATH) + if destination.exists() or not legacy_path.is_file(): + return False + + destination.parent.mkdir(parents=True, exist_ok=True) + temp_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + dir=destination.parent, + prefix=".cve-migrate-", + suffix=".sqlite", + delete=False, + ) as temp_file: + temp_path = Path(temp_file.name) + shutil.copy2(legacy_path, temp_path) + if not _valid_cve_database(temp_path): + return False + os.replace(temp_path, destination) + temp_path = None + return True + finally: + if temp_path is not None: + temp_path.unlink(missing_ok=True) diff --git a/bitprobe/scanner/update_lock.py b/bitprobe/scanner/update_lock.py new file mode 100644 index 0000000..cf0a739 --- /dev/null +++ b/bitprobe/scanner/update_lock.py @@ -0,0 +1,79 @@ +"""Cross-process serialization for mutable BitSentry data updates.""" + +from __future__ import annotations + +import fcntl +import os +import threading +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator, TextIO + +from scanner.paths import BITSENTRY_STATE_DIR + + +class UpdateLockError(RuntimeError): + pass + + +_guard = threading.RLock() +_held: dict[Path, tuple[TextIO, int, int]] = {} + + +@contextmanager +def bitsentry_update_lock( + timeout: float = 0.0, + *, + lock_path: Path | None = None, + _allow_reentry: bool = True, +) -> Iterator[None]: + path = (lock_path or (BITSENTRY_STATE_DIR / ".update.lock")).resolve() + owner = threading.get_ident() + + with _guard: + current = _held.get(path) + if _allow_reentry and current and current[2] == owner: + _held[path] = (current[0], current[1] + 1, owner) + reentrant = True + else: + reentrant = False + + if reentrant: + try: + yield + finally: + with _guard: + handle, depth, held_owner = _held[path] + _held[path] = (handle, depth - 1, held_owner) + return + + path.parent.mkdir(parents=True, exist_ok=True) + handle = path.open("a+", encoding="utf-8") + deadline = time.monotonic() + max(timeout, 0.0) + while True: + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError as exc: + if time.monotonic() >= deadline: + handle.close() + raise UpdateLockError(f"BitSentry update already in progress: {path}") from exc + time.sleep(min(0.05, max(0.0, deadline - time.monotonic()))) + + handle.seek(0) + handle.truncate() + handle.write(f"pid={os.getpid()}\n") + handle.flush() + with _guard: + _held[path] = (handle, 1, owner) + + try: + yield + finally: + with _guard: + current = _held.get(path) + if current and current[0] is handle: + _held.pop(path, None) + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + handle.close() diff --git a/bitprobe/scanner/update_notifier.py b/bitprobe/scanner/update_notifier.py index b09e948..545f448 100755 --- a/bitprobe/scanner/update_notifier.py +++ b/bitprobe/scanner/update_notifier.py @@ -1,6 +1,5 @@ import json import os -from pathlib import Path from scanner.cve_db_manager import ( cve_db_needs_update, @@ -8,10 +7,10 @@ bootstrap_cve_state, get_stats, ) +from scanner.paths import CVE_META_PATH +from scanner.cve_db_bootstrap import update_with_snapshot_policy - -_DATA_DIR = Path(__file__).resolve().parents[1] / "data" -META_PATH = str(_DATA_DIR / "cve_meta.json") +META_PATH = CVE_META_PATH REMINDER_DAYS = 14 # Fast bootstrap on scan startup (not the full NVD corpus) SCAN_BOOTSTRAP_DAYS = 7 @@ -27,8 +26,8 @@ def _load_meta(): def check_and_notify(auto_update: bool = True, bootstrap_days: int = SCAN_BOOTSTRAP_DAYS): """ Runs at scan startup. - - Auto-updates via SQLite + incremental NVD (lastMod) when possible - - Empty DB: windowed bootstrap (last N days), never unfiltered 350k+ sync + - Installs a verified full snapshot when local coverage is incomplete + - Uses incremental NVD synchronization when a current full DB exists - Otherwise prints a loud reminder """ @@ -41,33 +40,8 @@ def check_and_notify(auto_update: bool = True, bootstrap_days: int = SCAN_BOOTST if auto_update: try: - stats = get_stats() - total = stats.get("total_cves", 0) if isinstance(stats, dict) else 0 - bootstrap_cve_state() - state_ts = None - try: - from scanner.update_state import get_state_timestamp - - state_ts = get_state_timestamp("cve", "last_modified") - except Exception: - pass - - if total == 0 or not state_ts: - print( - f"[*] Bootstrapping CVE DB (last {bootstrap_days} days of NVD publishes)..." - ) - update_cve_database( - days=bootstrap_days, - incremental=False, - force=False, - ) - else: - print("[*] Incremental CVE sync (modified since last cursor)...") - update_cve_database( - days=bootstrap_days, - incremental=True, - force=False, - ) + print("[*] Preparing CVE database snapshot and incremental updates...") + update_with_snapshot_policy(verbose=False) print("=" * 70 + "\n") return except Exception as exc: diff --git a/bitsentry.py b/bitsentry.py index c729981..907e57e 100755 --- a/bitsentry.py +++ b/bitsentry.py @@ -821,8 +821,8 @@ def main() -> int: cve_db_parser.add_argument( "--days", type=int, - default=30, - help="Publication window for bootstrap when DB is empty (default: 30)", + default=None, + help="Build a publication-window mirror directly from NVD", ) cve_db_parser.add_argument( "--years", @@ -833,7 +833,22 @@ def main() -> int: cve_db_parser.add_argument( "--full", action="store_true", - help="Build full local NVD mirror (~350k CVEs; first-time setup)", + help="Rebuild the full local mirror directly from NVD", + ) + cve_db_parser.add_argument( + "--raw-full", + action="store_true", + help="Best-effort unfiltered NVD crawl", + ) + cve_db_parser.add_argument( + "--snapshot-only", + action="store_true", + help="Install the published snapshot without contacting NVD afterward", + ) + cve_db_parser.add_argument( + "--no-snapshot", + action="store_true", + help="Use direct NVD synchronization without downloading a snapshot", ) subparsers.add_parser( "cve-stats", @@ -895,6 +910,9 @@ def main() -> int: cmd.extend(["--years", str(args.years)]) if getattr(args, "full", False): cmd.append("--full") + for flag in ("raw_full", "snapshot_only", "no_snapshot"): + if getattr(args, flag, False): + cmd.append(f"--{flag.replace('_', '-')}") return subprocess.run(cmd, cwd=str(BITPROBE_PATH.parent)).returncode if args.command == "cve-stats": cmd = [sys.executable, str(BITPROBE_PATH), "cve-stats"] diff --git a/docker-compose.yml b/docker-compose.yml index 998341a..36aa376 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,3 +6,8 @@ services: image: bitsentry:latest stdin_open: true tty: true + volumes: + - bitsentry-data:/home/bitsentry/.bitsentry + +volumes: + bitsentry-data: diff --git a/docs/superpowers/plans/2026-08-18-cve-database-distribution.md b/docs/superpowers/plans/2026-08-18-cve-database-distribution.md new file mode 100644 index 0000000..f7c986e --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-cve-database-distribution.md @@ -0,0 +1,194 @@ +# CVE Database Distribution Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Distribute a complete, verified CVE SQLite snapshot from scheduled GitHub Actions and use it to bootstrap BitSentry clients safely before incremental NVD synchronization. + +**Architecture:** Mutable CVE state moves to a configurable user-data directory. SQLite owns coverage and cursor metadata; all NVD date ranges are split into frozen 119-day windows. A dedicated bootstrap module validates and atomically installs GitHub Release snapshots under a shared process lock, while CI restores the prior snapshot, updates it, and publishes stable and dated assets. + +**Tech Stack:** Python 3.12+, SQLite, requests, pytest, GitHub Actions, GitHub CLI + +**Spec:** User-provided “BitSentry CVE Database Distribution — Implementation Contract” dated 2026-08-18 in this conversation. + +## Global Constraints + +- Snapshot releases stay in `rywils/BitSentry` under `cve-db-*` tags and use `--latest=false`. +- `cve-db-latest` is mutable; dated `cve-db-YYYY-MM-DD` releases are immutable. +- SQLite metadata is authoritative for `nvd_cursor`, coverage, and schema version. +- NVD date windows are at most 119 days. +- Snapshot failure never triggers an implicit full-corpus crawl. +- The updater, bootstrap installer, and scan-time updater share one cross-process lock. +- Production changes follow red-green-refactor; no commit is created unless the user asks. + +--- + +### Task 1: Mutable data paths and legacy migration + +**Files:** +- Modify: `bitprobe/scanner/paths.py` +- Modify: `bitprobe/scanner/cve_db_manager.py` +- Modify: `bitprobe/scanner/update_notifier.py` +- Test: `tests/test_cve_paths.py` + +**Interfaces:** +- Produces: `BITSENTRY_STATE_DIR: Path`, `BITSENTRY_DATA_DIR: Path`, `CVE_DB_PATH: str`, `CVE_META_PATH: str`, `LEGACY_CVE_DB_PATH: Path`, `migrate_legacy_cve_database() -> bool`. +- Consumes: `BITSENTRY_DATA_DIR` environment override. + +- [ ] Write tests that isolate `HOME`/`BITSENTRY_DATA_DIR`, assert the resolved user-data paths, copy a valid legacy SQLite database through a temporary file, reject an invalid legacy file, and never re-read legacy storage after the destination exists. +- [ ] Run `pytest tests/test_cve_paths.py -q` and confirm failures occur because the new paths and migration API do not exist. +- [ ] Add the path constants, create parent directories lazily, validate legacy databases with `PRAGMA integrity_check` and required-table checks, and install valid copies with `os.replace()`. +- [ ] Update CVE manager/notifier imports to use centralized paths. +- [ ] Run `pytest tests/test_cve_paths.py tests/test_cve_db_manager.py -q` and make the task green. + +### Task 2: Schema, coverage metadata, and authoritative cursor + +**Files:** +- Modify: `bitprobe/scanner/cve_db_manager.py` +- Modify: `bitprobe/scanner/update_state.py` +- Test: `tests/test_cve_metadata.py` + +**Interfaces:** +- Produces: `CVE_SCHEMA_VERSION = 1`, `read_cve_metadata(conn=None) -> dict[str, str | None]`, `write_cve_metadata(updates, conn=None) -> None`, `mirror_sqlite_cursor_to_state() -> str | None`. +- Metadata keys: `schema_version`, `coverage_mode`, `coverage_start`, `coverage_end`, `nvd_cursor`. + +- [ ] Write failing tests for schema initialization, full/windowed completeness, SQLite-over-state cursor precedence, and cursor mirroring only from SQLite to state. +- [ ] Run `pytest tests/test_cve_metadata.py -q` and confirm the missing metadata behavior fails. +- [ ] Add metadata helpers, initialize schema metadata, replace row-count completeness checks with `coverage_mode`, and make `bootstrap_cve_state()` migrate legacy cursor data into SQLite only when SQLite lacks a cursor. +- [ ] Run `pytest tests/test_cve_metadata.py tests/test_cve_db_manager.py -q`. + +### Task 3: Frozen 119-day windows and cursor advancement + +**Files:** +- Modify: `bitprobe/scanner/cve_db_manager.py` +- Test: `tests/test_cve_sync_windows.py` + +**Interfaces:** +- Produces: `iter_nvd_windows(start: datetime, end: datetime, max_days: int = 119) -> Iterator[tuple[datetime, datetime]]` and a window-fetch routine that receives explicit filter names, frozen bounds, and page offset. +- Consumes: metadata helpers from Task 2. + +- [ ] Write failing tests proving long `--days`, `--years`, and stale incremental ranges split into contiguous windows no longer than 119 days, including exact-boundary cases. +- [ ] Write failing tests proving a successful zero-result incremental window advances `nvd_cursor` to its frozen end and a failed window does not. +- [ ] Run `pytest tests/test_cve_sync_windows.py -q` and confirm expected failures. +- [ ] Extract one-window pagination from `update_cve_database()`, add the window iterator, route publication and modification modes through it, and advance SQLite cursor only after each complete incremental window. +- [ ] Mirror the final SQLite cursor to compatibility state after successful completion. +- [ ] Run `pytest tests/test_cve_sync_windows.py tests/test_cve_db_manager.py -q`. + +### Task 4: Shared cross-process update lock + +**Files:** +- Create: `bitprobe/scanner/update_lock.py` +- Modify: `bitprobe/scanner/cve_db_manager.py` +- Modify: `bitprobe/scanner/update_notifier.py` +- Test: `tests/test_update_lock.py` + +**Interfaces:** +- Produces: `bitsentry_update_lock(timeout: float = 0.0)` context manager and `UpdateLockError`. +- Lock path: `BITSENTRY_STATE_DIR / ".update.lock"`. + +- [ ] Write failing tests for acquisition, contention, release after exceptions, and re-entry behavior within one process. +- [ ] Run `pytest tests/test_update_lock.py -q` and confirm the API is missing. +- [ ] Implement a portable lock using atomic lock-file creation with PID metadata and bounded stale-lock recovery; keep lock mechanics isolated from CVE logic. +- [ ] Wrap explicit updates and scan-time auto-updates with the same context manager without double-acquiring it. +- [ ] Run `pytest tests/test_update_lock.py tests/test_cve_db_manager.py -q`. + +### Task 5: Snapshot validation and atomic installation + +**Files:** +- Create: `bitprobe/scanner/cve_db_bootstrap.py` +- Test: `tests/test_cve_db_bootstrap.py` + +**Interfaces:** +- Produces: `fetch_snapshot_manifest()`, `download_snapshot()`, `verify_snapshot()`, `validate_snapshot_database()`, `install_snapshot_atomically()`, and `bootstrap_from_snapshot()`. +- Manifest fields match §6 of the contract; default URLs use the `cve-db-latest` tag in `rywils/BitSentry`. + +- [ ] Write failing tests for malformed manifests, unsupported versions, checksum mismatch, compressed/uncompressed size limits, truncated gzip, corrupt SQLite, missing tables, non-full coverage, row-count mismatch, and cursor/coverage/schema disagreement. +- [ ] Write failing tests proving a rejected snapshot leaves the existing database and compatibility state unchanged. +- [ ] Write failing tests proving installation uses a destination-directory temporary file, closes validation handles, atomically replaces the DB under the shared lock, and mirrors the installed cursor afterward. +- [ ] Run `pytest tests/test_cve_db_bootstrap.py -q` and confirm failures are caused by the missing module. +- [ ] Implement streaming downloads with connect/read timeouts and byte limits; SHA-256 compressed bytes; bounded gzip decompression; read-only SQLite validation; `journal_mode=DELETE` normalization; and `os.replace()` installation. +- [ ] Checkpoint/close BitSentry-owned connections before replacement and refuse unsafe active sidecars rather than deleting live WAL state. +- [ ] Run `pytest tests/test_cve_db_bootstrap.py tests/test_update_lock.py -q`. + +### Task 6: Consumer selection policy and CLI + +**Files:** +- Modify: `bitprobe/scanner/cve_db_manager.py` +- Modify: `bitprobe/scanner/update_notifier.py` +- Modify: `bitprobe/bitprobe.py` +- Modify: `scripts/install_bitsentry.sh` +- Test: `tests/test_cve_bootstrap_policy.py` +- Test: `tests/test_products_cli.py` + +**Interfaces:** +- Produces: policy selection for snapshot, incremental, chunked fallback, explicit direct modes; CLI flags `--snapshot-only` and `--no-snapshot`. +- Consumes: snapshot API from Task 5 and sync API from Task 3. + +- [ ] Write failing table-driven tests for every row in §9, including missing/corrupt/windowed/full-fresh/full-stale states and snapshot failure fallbacks. +- [ ] Write failing CLI tests proving explicit `--full`, `--days`, and `--years` skip snapshots; `--snapshot-only` skips NVD; and `--no-snapshot` preserves direct behavior. +- [ ] Run the focused tests and confirm policy/flags fail before implementation. +- [ ] Implement the decision function, integrate it with CLI and scan-time bootstrap, and update installer messages. +- [ ] Run `pytest tests/test_cve_bootstrap_policy.py tests/test_products_cli.py -q`. + +### Task 7: Deterministic full builds and checkpoints + +**Files:** +- Modify: `bitprobe/scanner/cve_db_manager.py` +- Test: `tests/test_cve_resumability.py` + +**Interfaces:** +- Produces: `sync_state` schema, transactional page checkpointing, deterministic full build from the NVD epoch through a frozen build end, and final last-modified catch-up. +- Consumes: Task 3 window iterator and Task 2 coverage metadata. + +- [ ] Write failing tests for checkpoint creation, same-transaction page progress, exact-bound resumption, invalidation after mode/bound/page-size changes, and SIGINT-equivalent interruption. +- [ ] Write failing tests proving full coverage is withheld until every publication window and final catch-up completes. +- [ ] Run `pytest tests/test_cve_resumability.py -q` and confirm missing behavior. +- [ ] Add `sync_state`, refactor `_store_cves_batch()` to accept checkpoint updates in its transaction, and resume only exact matching windows. +- [ ] Implement deterministic `--full`; retain a separately named/documented raw unfiltered escape hatch if exposed by CLI. +- [ ] Run `pytest tests/test_cve_resumability.py tests/test_cve_sync_windows.py -q`. + +### Task 8: Snapshot builder + +**Files:** +- Create: `scripts/build_cve_snapshot.py` +- Test: `tests/test_build_cve_snapshot.py` + +**Interfaces:** +- Produces: `dist/cve_db.sqlite.gz` and `dist/manifest.json` with contract fields copied from SQLite metadata. +- Consumes: configurable `BITSENTRY_DATA_DIR` and `GITHUB_SHA`. + +- [ ] Write failing tests for WAL checkpointing, integrity/schema/coverage validation, SQLite/manifest field agreement, deterministic gzip creation, size/hash values, and failure before manifest publication. +- [ ] Run `pytest tests/test_build_cve_snapshot.py -q` and confirm the script is absent. +- [ ] Implement the builder with closed connections, `gzip.GzipFile(..., compresslevel=9, mtime=0)`, SHA-256, and atomic manifest output. +- [ ] Run `pytest tests/test_build_cve_snapshot.py -q`. + +### Task 9: GitHub Actions producer + +**Files:** +- Create: `.github/workflows/update-cve-db.yml` +- Create: `scripts/update_cve_snapshot_release.sh` +- Test: `tests/test_cve_workflow.py` + +**Interfaces:** +- Produces: scheduled/manual producer with `contents: write`, concurrency guard, Python 3.13, required NVD key, prior-snapshot restore, full first build, snapshot build, immutable dated publication, and mutable stable publication. + +- [ ] Write failing structural tests that parse the workflow and assert schedule, dispatch input, permissions, concurrency, Python version, secret check, restore-before-sync ordering, and `--latest=false` release behavior. +- [ ] Run `pytest tests/test_cve_workflow.py -q` and confirm the workflow is absent. +- [ ] Add the workflow and a strict shell helper implementing idempotent release existence checks, dated-release no-clobber semantics, and stable-release `--clobber` semantics. +- [ ] Run `pytest tests/test_cve_workflow.py -q` and `bash -n scripts/update_cve_snapshot_release.sh`. + +### Task 10: Documentation and complete verification + +**Files:** +- Modify: `README.md` +- Modify: `scripts/install_bitsentry.sh` +- Modify: Docker configuration if it currently persists `bitprobe/data` + +**Interfaces:** +- Documents: snapshot default, direct-NVD flags, user-data path, locking behavior, fallback coverage warning, release policy, and measured-values placeholder removal. + +- [ ] Update documentation and help text to match implemented behavior exactly, without claiming unmeasured transfer size or duration. +- [ ] Run `python -m compileall -q bitprobe scripts`. +- [ ] Run `pytest -q` and resolve every regression. +- [ ] Run `git diff --check`. +- [ ] Inspect `git status --short` and confirm only intended files changed; preserve the user’s pre-existing untracked `package.json` and `package-lock.json`. +- [ ] Re-read every contract verification item and map it to a passing automated test or explicitly documented manual GitHub/VM verification step. diff --git a/scripts/build_cve_snapshot.py b/scripts/build_cve_snapshot.py new file mode 100755 index 0000000..7f2565e --- /dev/null +++ b/scripts/build_cve_snapshot.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Build a validated, reproducible CVE database release artifact.""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import os +import sqlite3 +import tempfile +from datetime import datetime, timezone +from pathlib import Path + + +REQUIRED_TABLES = frozenset({"cve_entries", "cve_products", "cve_cpes", "metadata"}) +REQUIRED_METADATA = ( + "schema_version", + "coverage_mode", + "coverage_start", + "coverage_end", + "nvd_cursor", +) + + +def _inspect_database(path: Path) -> tuple[dict[str, str | None], int]: + with sqlite3.connect(path) as conn: + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + if conn.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + raise RuntimeError("CVE database failed integrity_check") + tables = { + row[0] + for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'") + } + if not REQUIRED_TABLES <= tables: + raise RuntimeError("CVE database is missing required tables") + metadata = dict(conn.execute("SELECT key, value FROM metadata")) + missing = [key for key in REQUIRED_METADATA if key not in metadata] + if missing: + raise RuntimeError(f"CVE database missing metadata: {missing}") + if metadata["coverage_mode"] != "full": + raise RuntimeError("refusing to publish a non-full CVE database") + if not metadata["nvd_cursor"]: + raise RuntimeError("refusing to publish without nvd_cursor") + count = conn.execute("SELECT COUNT(*) FROM cve_entries").fetchone()[0] + return {key: metadata[key] for key in REQUIRED_METADATA}, count + + +def build_snapshot( + database: Path, + output_dir: Path, + *, + source_commit: str, +) -> dict[str, object]: + metadata, count = _inspect_database(database) + output_dir.mkdir(parents=True, exist_ok=True) + artifact = output_dir / "cve_db.sqlite.gz" + with database.open("rb") as source, artifact.open("wb") as raw_output: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw_output, compresslevel=9, mtime=0) as output: + for block in iter(lambda: source.read(1024 * 1024), b""): + output.write(block) + + compressed = artifact.read_bytes() + manifest: dict[str, object] = { + "format_version": 1, + "schema_version": int(metadata["schema_version"] or 0), + "coverage_mode": metadata["coverage_mode"], + "coverage_start": metadata["coverage_start"], + "coverage_end": metadata["coverage_end"], + "built_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "nvd_cursor": metadata["nvd_cursor"], + "cve_count": count, + "artifact": artifact.name, + "sha256_gz": hashlib.sha256(compressed).hexdigest(), + "compressed_size": len(compressed), + "uncompressed_size": database.stat().st_size, + "source_commit": source_commit, + } + manifest_path = output_dir / "manifest.json" + with tempfile.NamedTemporaryFile( + dir=output_dir, prefix=".manifest-", suffix=".json", mode="w", encoding="utf-8", delete=False + ) as temp_file: + json.dump(manifest, temp_file, indent=2, sort_keys=True) + temp_file.write("\n") + temp_path = Path(temp_file.name) + os.replace(temp_path, manifest_path) + return manifest + + +def main() -> int: + parser = argparse.ArgumentParser() + default_data = Path(os.environ.get("BITSENTRY_DATA_DIR", Path.home() / ".bitsentry" / "data")) + parser.add_argument("--database", type=Path, default=default_data / "cve_db.sqlite") + parser.add_argument("--output-dir", type=Path, default=Path("dist")) + args = parser.parse_args() + build_snapshot( + args.database, + args.output_dir, + source_commit=os.environ.get("GITHUB_SHA", "unknown"), + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/install_bitsentry.sh b/scripts/install_bitsentry.sh index fc42bdc..1f901bf 100755 --- a/scripts/install_bitsentry.sh +++ b/scripts/install_bitsentry.sh @@ -67,11 +67,10 @@ print_before_first_scan_notice() { echo -e "${b} 1) bitsentry update-db${r} ${d}(ASN ok — optional refresh)${r}" fi - echo -e "${b} 2) export NVD_API_KEY=\"your-nvd-api-key\"${r} ${d}(optional, faster NVD sync)${r}" + echo -e "${b} 2) export NVD_API_KEY=\"your-nvd-api-key\"${r} ${d}(optional, faster incremental sync)${r}" if [[ "${CVE_NEEDS_BOOTSTRAP}" -eq 1 ]]; then - echo -e "${y} 3) bitsentry update-cve-db --full${r}" - echo -e "${b} ${d}# or: bitsentry update-cve-db --years 15${r}" + echo -e "${y} 3) bitsentry update-cve-db${r} ${d}(verified snapshot + catch-up)${r}" else echo -e "${b} 3) bitsentry update-cve-db${r} ${d}(CVE loaded — incremental refresh)${r}" fi @@ -197,10 +196,10 @@ if ready: else: if tty: print(f' Status: \\033[1;31m{status}\\033[0m') - print(' \\033[33mRun: bitsentry update-cve-db --full\\033[0m') + print(' \\033[33mRun: bitsentry update-cve-db\\033[0m') else: print(f' Status: {status}') - print(' Run: bitsentry update-cve-db --full') + print(' Run: bitsentry update-cve-db') sys.exit(0 if ready else 1) " cve_ready_rc=$? diff --git a/scripts/update_cve_snapshot_release.sh b/scripts/update_cve_snapshot_release.sh new file mode 100755 index 0000000..c89bf4c --- /dev/null +++ b/scripts/update_cve_snapshot_release.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +dist_dir="${1:-dist}" +artifact="${dist_dir}/cve_db.sqlite.gz" +manifest="${dist_dir}/manifest.json" +dated_tag="cve-db-$(date -u +%Y-%m-%d)" +stable_tag="cve-db-latest" + +test -f "${artifact}" +test -f "${manifest}" + +if ! gh release view "${dated_tag}" >/dev/null 2>&1; then + gh release create "${dated_tag}" "${artifact}" "${manifest}" \ + --title "CVE DB snapshot ${dated_tag}" \ + --notes "Automated daily CVE database snapshot" \ + --latest=false +fi + +if gh release view "${stable_tag}" >/dev/null 2>&1; then + gh release upload "${stable_tag}" "${artifact}" "${manifest}" --clobber +else + gh release create "${stable_tag}" "${artifact}" "${manifest}" \ + --title "Latest CVE database snapshot" \ + --notes "Mutable pointer used by BitSentry clients" \ + --latest=false +fi diff --git a/tests/test_build_cve_snapshot.py b/tests/test_build_cve_snapshot.py new file mode 100644 index 0000000..fda5b06 --- /dev/null +++ b/tests/test_build_cve_snapshot.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import gzip +import hashlib +import importlib.util +import json +import sqlite3 +from pathlib import Path + + +def _load_builder(): + path = Path(__file__).resolve().parents[1] / "scripts" / "build_cve_snapshot.py" + spec = importlib.util.spec_from_file_location("build_cve_snapshot", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def _database(path: Path) -> None: + with sqlite3.connect(path) as conn: + conn.execute("CREATE TABLE cve_entries (cve_id TEXT PRIMARY KEY)") + conn.execute("CREATE TABLE cve_products (id INTEGER PRIMARY KEY)") + conn.execute("CREATE TABLE cve_cpes (id INTEGER PRIMARY KEY)") + conn.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)") + conn.execute("INSERT INTO cve_entries VALUES ('CVE-2020-0001')") + conn.executemany( + "INSERT INTO metadata VALUES (?, ?)", + [ + ("schema_version", "1"), + ("coverage_mode", "full"), + ("coverage_start", None), + ("coverage_end", "2026-08-18T00:00:00.000"), + ("nvd_cursor", "2026-08-18T00:00:00.000"), + ], + ) + + +def test_build_snapshot_copies_sqlite_metadata_and_hashes_artifact(tmp_path: Path) -> None: + builder = _load_builder() + db = tmp_path / "cve.sqlite" + dist = tmp_path / "dist" + _database(db) + + manifest = builder.build_snapshot(db, dist, source_commit="deadbeef") + + artifact = dist / "cve_db.sqlite.gz" + on_disk = json.loads((dist / "manifest.json").read_text(encoding="utf-8")) + assert on_disk == manifest + assert manifest["coverage_mode"] == "full" + assert manifest["nvd_cursor"] == "2026-08-18T00:00:00.000" + assert manifest["cve_count"] == 1 + assert manifest["sha256_gz"] == hashlib.sha256(artifact.read_bytes()).hexdigest() + with gzip.open(artifact, "rb") as source: + assert source.read() == db.read_bytes() diff --git a/tests/test_cve_bootstrap_policy.py b/tests/test_cve_bootstrap_policy.py new file mode 100644 index 0000000..228c0f7 --- /dev/null +++ b/tests/test_cve_bootstrap_policy.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from unittest import mock + + +_BITPROBE = Path(__file__).resolve().parents[1] / "bitprobe" +if str(_BITPROBE) not in sys.path: + sys.path.insert(0, str(_BITPROBE)) + + +def test_incomplete_database_installs_snapshot_then_incremental(monkeypatch) -> None: + import scanner.cve_db_bootstrap as bootstrap + import scanner.cve_db_manager as manager + + monkeypatch.setattr(manager, "cve_db_is_complete", lambda: False) + install = mock.Mock(return_value={"nvd_cursor": "2026-08-18T00:00:00.000"}) + update = mock.Mock(return_value=7) + monkeypatch.setattr(bootstrap, "bootstrap_from_snapshot", install) + monkeypatch.setattr(manager, "update_cve_database", update) + + assert bootstrap.update_with_snapshot_policy() == 7 + install.assert_called_once_with() + update.assert_called_once_with(days=30, incremental=True, verbose=False) + + +def test_snapshot_failure_on_empty_db_uses_bounded_fallback(monkeypatch) -> None: + import scanner.cve_db_bootstrap as bootstrap + import scanner.cve_db_manager as manager + + monkeypatch.setattr(manager, "cve_db_is_complete", lambda: False) + monkeypatch.setattr( + bootstrap, + "bootstrap_from_snapshot", + mock.Mock(side_effect=bootstrap.SnapshotError("offline")), + ) + update = mock.Mock(return_value=3) + monkeypatch.setattr(manager, "update_cve_database", update) + + assert bootstrap.update_with_snapshot_policy() == 3 + update.assert_called_once_with(days=30, incremental=False, verbose=False) + + +def test_snapshot_only_never_contacts_nvd(monkeypatch) -> None: + import scanner.cve_db_bootstrap as bootstrap + import scanner.cve_db_manager as manager + + install = mock.Mock(return_value={}) + update = mock.Mock() + monkeypatch.setattr(bootstrap, "bootstrap_from_snapshot", install) + monkeypatch.setattr(manager, "update_cve_database", update) + + assert bootstrap.update_with_snapshot_policy(snapshot_only=True) == 0 + install.assert_called_once_with() + update.assert_not_called() + + +def test_full_database_over_119_days_stale_prefers_snapshot(monkeypatch) -> None: + import scanner.cve_db_bootstrap as bootstrap + import scanner.cve_db_manager as manager + + monkeypatch.setattr(manager, "cve_db_is_complete", lambda: True) + monkeypatch.setattr( + manager, + "read_cve_metadata", + lambda: {"nvd_cursor": "2020-01-01T00:00:00.000"}, + ) + install = mock.Mock(return_value={}) + update = mock.Mock(return_value=8) + monkeypatch.setattr(bootstrap, "bootstrap_from_snapshot", install) + monkeypatch.setattr(manager, "update_cve_database", update) + + assert bootstrap.update_with_snapshot_policy() == 8 + install.assert_called_once_with() + update.assert_called_once() diff --git a/tests/test_cve_cli.py b/tests/test_cve_cli.py new file mode 100644 index 0000000..424b801 --- /dev/null +++ b/tests/test_cve_cli.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[1] +BITPROBE = ROOT / "bitprobe" +if str(BITPROBE) not in sys.path: + sys.path.insert(0, str(BITPROBE)) + + +def _cli(): + spec = importlib.util.spec_from_file_location("bitprobe_cli_for_test", BITPROBE / "bitprobe.py") + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_default_update_uses_snapshot_policy(monkeypatch) -> None: + cli = _cli() + policy = mock.Mock(return_value=4) + direct = mock.Mock() + monkeypatch.setattr(cli, "update_with_snapshot_policy", policy) + monkeypatch.setattr(cli, "update_cve_database", direct) + monkeypatch.setattr(sys, "argv", ["bitprobe", "update-cve-db"]) + + assert cli.main() == 0 + policy.assert_called_once_with(snapshot_only=False, verbose=False) + direct.assert_not_called() + + +def test_explicit_years_skips_snapshot(monkeypatch) -> None: + cli = _cli() + policy = mock.Mock() + direct = mock.Mock(return_value=2) + monkeypatch.setattr(cli, "update_with_snapshot_policy", policy) + monkeypatch.setattr(cli, "update_cve_database", direct) + monkeypatch.setattr(sys, "argv", ["bitprobe", "update-cve-db", "--years", "2"]) + + assert cli.main() == 0 + policy.assert_not_called() + assert direct.call_args.kwargs["years"] == 2 + + +def test_snapshot_only_uses_snapshot_without_direct_nvd(monkeypatch) -> None: + cli = _cli() + policy = mock.Mock(return_value=0) + direct = mock.Mock() + monkeypatch.setattr(cli, "update_with_snapshot_policy", policy) + monkeypatch.setattr(cli, "update_cve_database", direct) + monkeypatch.setattr(sys, "argv", ["bitprobe", "update-cve-db", "--snapshot-only"]) + + assert cli.main() == 0 + policy.assert_called_once_with(snapshot_only=True, verbose=False) + direct.assert_not_called() + + +def test_raw_full_exposes_best_effort_unfiltered_escape_hatch(monkeypatch) -> None: + cli = _cli() + direct = mock.Mock(return_value=1) + monkeypatch.setattr(cli, "update_cve_database", direct) + monkeypatch.setattr(sys, "argv", ["bitprobe", "update-cve-db", "--raw-full"]) + + assert cli.main() == 0 + assert direct.call_args.kwargs["full_sync"] is True + assert direct.call_args.kwargs["raw_full_sync"] is True diff --git a/tests/test_cve_db_bootstrap.py b/tests/test_cve_db_bootstrap.py new file mode 100644 index 0000000..62f9b93 --- /dev/null +++ b/tests/test_cve_db_bootstrap.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import gzip +import hashlib +import json +import sqlite3 +import sys +from pathlib import Path + +import pytest +import requests + + +_BITPROBE = Path(__file__).resolve().parents[1] / "bitprobe" +if str(_BITPROBE) not in sys.path: + sys.path.insert(0, str(_BITPROBE)) + + +def _snapshot_db(path: Path, *, cursor: str = "2026-08-18T00:00:00.000") -> None: + with sqlite3.connect(path) as conn: + conn.execute("CREATE TABLE cve_entries (cve_id TEXT PRIMARY KEY, description TEXT)") + conn.execute("CREATE TABLE cve_products (id INTEGER PRIMARY KEY)") + conn.execute("CREATE TABLE cve_cpes (id INTEGER PRIMARY KEY)") + conn.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)") + conn.execute("INSERT INTO cve_entries VALUES ('CVE-2020-0001', 'old')") + conn.executemany( + "INSERT INTO metadata VALUES (?, ?)", + [ + ("schema_version", "1"), + ("coverage_mode", "full"), + ("coverage_start", None), + ("coverage_end", cursor), + ("nvd_cursor", cursor), + ], + ) + + +def _artifact(tmp_path: Path): + db = tmp_path / "snapshot.sqlite" + gz = tmp_path / "snapshot.sqlite.gz" + _snapshot_db(db) + with db.open("rb") as source, gzip.open(gz, "wb") as target: + target.write(source.read()) + compressed = gz.read_bytes() + manifest = { + "format_version": 1, + "schema_version": 1, + "coverage_mode": "full", + "coverage_start": None, + "coverage_end": "2026-08-18T00:00:00.000", + "built_at": "2026-08-18T00:01:00Z", + "nvd_cursor": "2026-08-18T00:00:00.000", + "cve_count": 1, + "artifact": "cve_db.sqlite.gz", + "sha256_gz": hashlib.sha256(compressed).hexdigest(), + "compressed_size": len(compressed), + "uncompressed_size": db.stat().st_size, + "source_commit": "abc123", + } + return db, gz, manifest + + +def test_validate_manifest_rejects_unsupported_schema(tmp_path: Path) -> None: + from scanner.cve_db_bootstrap import SnapshotValidationError, validate_manifest + + _, _, manifest = _artifact(tmp_path) + manifest["schema_version"] = 99 + with pytest.raises(SnapshotValidationError, match="schema_version"): + validate_manifest(manifest) + + +def test_verify_snapshot_rejects_checksum_mismatch(tmp_path: Path) -> None: + from scanner.cve_db_bootstrap import SnapshotValidationError, verify_snapshot + + _, gz, manifest = _artifact(tmp_path) + manifest["sha256_gz"] = "0" * 64 + with pytest.raises(SnapshotValidationError, match="checksum"): + verify_snapshot(gz, manifest) + + +def test_fetch_manifest_wraps_transport_errors() -> None: + from scanner.cve_db_bootstrap import SnapshotError, fetch_snapshot_manifest + + session = pytest.MonkeyPatch() + client = requests.Session() + session.setattr(client, "get", lambda *args, **kwargs: (_ for _ in ()).throw(requests.Timeout("offline"))) + try: + with pytest.raises(SnapshotError, match="download snapshot manifest"): + fetch_snapshot_manifest(session=client) + finally: + session.undo() + + +def test_download_snapshot_wraps_transport_errors(tmp_path: Path) -> None: + from scanner.cve_db_bootstrap import SnapshotError, download_snapshot + + _, _, manifest = _artifact(tmp_path) + client = requests.Session() + client.get = lambda *args, **kwargs: (_ for _ in ()).throw(requests.ConnectionError("offline")) + with pytest.raises(SnapshotError, match="download CVE snapshot"): + download_snapshot(manifest, tmp_path / "download.gz", session=client) + + +def test_validate_database_rejects_manifest_cursor_mismatch(tmp_path: Path) -> None: + from scanner.cve_db_bootstrap import SnapshotValidationError, validate_snapshot_database + + db, _, manifest = _artifact(tmp_path) + manifest["nvd_cursor"] = "2026-08-17T00:00:00.000" + with pytest.raises(SnapshotValidationError, match="nvd_cursor"): + validate_snapshot_database(db, manifest) + + +def test_atomic_install_preserves_existing_database_on_validation_failure(tmp_path: Path) -> None: + from scanner.cve_db_bootstrap import SnapshotValidationError, install_snapshot_atomically + + destination = tmp_path / "installed.sqlite" + destination.write_bytes(b"existing") + invalid = tmp_path / "invalid.sqlite" + invalid.write_bytes(b"bad") + manifest = {"format_version": 1, "schema_version": 1} + + with pytest.raises(SnapshotValidationError): + install_snapshot_atomically(invalid, manifest, destination=destination) + assert destination.read_bytes() == b"existing" + + +def test_atomic_install_replaces_database_and_returns_cursor(tmp_path: Path) -> None: + from scanner.cve_db_bootstrap import install_snapshot_atomically + + db, _, manifest = _artifact(tmp_path) + destination = tmp_path / "data" / "cve.sqlite" + destination.parent.mkdir() + destination.write_bytes(b"existing") + + cursor = install_snapshot_atomically(db, manifest, destination=destination) + + assert cursor == manifest["nvd_cursor"] + with sqlite3.connect(destination) as conn: + assert conn.execute("SELECT COUNT(*) FROM cve_entries").fetchone()[0] == 1 + + +def test_atomic_install_checkpoints_stale_sidecars_before_replace(tmp_path: Path) -> None: + from scanner.cve_db_bootstrap import install_snapshot_atomically + + db, _, manifest = _artifact(tmp_path) + destination = tmp_path / "data" / "cve.sqlite" + destination.parent.mkdir() + _snapshot_db(destination, cursor="2026-08-17T00:00:00.000") + Path(f"{destination}-wal").touch() + Path(f"{destination}-shm").touch() + + install_snapshot_atomically(db, manifest, destination=destination) + + assert not Path(f"{destination}-wal").exists() + assert not Path(f"{destination}-shm").exists() + with sqlite3.connect(destination) as conn: + assert conn.execute("SELECT COUNT(*) FROM cve_entries").fetchone()[0] == 1 + + +def test_install_local_artifact_used_by_ci(monkeypatch, tmp_path: Path) -> None: + import scanner.cve_db_bootstrap as bootstrap + + _, gz, manifest = _artifact(tmp_path) + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + destination = tmp_path / "data" / "cve.sqlite" + monkeypatch.setattr(bootstrap, "CVE_DB_PATH", str(destination)) + monkeypatch.setattr(bootstrap, "set_state_timestamp", lambda *args: None) + + bootstrap.install_snapshot_artifact(manifest_path, gz) + + with sqlite3.connect(destination) as conn: + assert conn.execute("SELECT COUNT(*) FROM cve_entries").fetchone()[0] == 1 diff --git a/tests/test_cve_metadata.py b/tests/test_cve_metadata.py new file mode 100644 index 0000000..17966a3 --- /dev/null +++ b/tests/test_cve_metadata.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import sqlite3 +import sys +from pathlib import Path + + +_BITPROBE = Path(__file__).resolve().parents[1] / "bitprobe" +if str(_BITPROBE) not in sys.path: + sys.path.insert(0, str(_BITPROBE)) + + +def _isolate(monkeypatch, tmp_path: Path): + import scanner.cve_db_manager as manager + import scanner.update_state as state + + db = tmp_path / "cve_db.sqlite" + monkeypatch.setattr(manager, "CVE_DB_PATH", str(db)) + monkeypatch.setattr(manager, "CVE_META_PATH", str(tmp_path / "cve_meta.json")) + monkeypatch.setattr(manager, "migrate_legacy_cve_database", lambda: False) + monkeypatch.setattr(state, "STATE_DIR", tmp_path / "state") + monkeypatch.setattr(state, "STATE_PATH", tmp_path / "state" / "state.json") + return manager, state, db + + +def test_schema_initializes_coverage_metadata(monkeypatch, tmp_path: Path) -> None: + manager, _, db = _isolate(monkeypatch, tmp_path) + manager.init_cve_database() + + metadata = manager.read_cve_metadata() + assert metadata["schema_version"] == str(manager.CVE_SCHEMA_VERSION) + assert metadata["coverage_mode"] == "windowed" + assert metadata["coverage_start"] is None + assert metadata["coverage_end"] is None + assert metadata["nvd_cursor"] is None + assert db.exists() + + +def test_only_full_coverage_is_bootstrap_complete(monkeypatch, tmp_path: Path) -> None: + manager, _, _ = _isolate(monkeypatch, tmp_path) + manager.init_cve_database() + + assert manager.cve_db_is_complete() is False + manager.write_cve_metadata({"coverage_mode": "full"}) + assert manager.cve_db_is_complete() is True + + +def test_sqlite_cursor_overrides_compatibility_state(monkeypatch, tmp_path: Path) -> None: + manager, state, _ = _isolate(monkeypatch, tmp_path) + manager.init_cve_database() + manager.write_cve_metadata({"nvd_cursor": "2026-08-18T00:00:00.000"}) + state.set_state_timestamp("cve", "last_modified", "2020-01-01T00:00:00.000") + + assert manager.bootstrap_cve_state() == "2026-08-18T00:00:00.000" + assert state.get_state_timestamp("cve", "last_modified") == "2026-08-18T00:00:00.000" + + +def test_mirror_cursor_never_copies_state_into_sqlite(monkeypatch, tmp_path: Path) -> None: + manager, state, _ = _isolate(monkeypatch, tmp_path) + manager.init_cve_database() + state.set_state_timestamp("cve", "last_modified", "2020-01-01T00:00:00.000") + + assert manager.mirror_sqlite_cursor_to_state() is None + assert manager.read_cve_metadata()["nvd_cursor"] is None + + +def test_status_uses_coverage_metadata_not_row_threshold(monkeypatch, tmp_path: Path) -> None: + manager, _, _ = _isolate(monkeypatch, tmp_path) + manager.init_cve_database() + with manager._connect() as conn: + conn.execute( + "INSERT INTO cve_entries (cve_id, description) VALUES (?, ?)", + ("CVE-2026-0001", "test"), + ) + manager.write_cve_metadata( + { + "coverage_mode": "full", + "nvd_cursor": manager._format_nvd_datetime(manager._utcnow()), + } + ) + + assert manager.describe_cve_db_local_status().startswith("ok (1 CVEs loaded)") diff --git a/tests/test_cve_paths.py b/tests/test_cve_paths.py new file mode 100644 index 0000000..810275d --- /dev/null +++ b/tests/test_cve_paths.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import importlib +import sqlite3 +import sys +from pathlib import Path + + +_BITPROBE = Path(__file__).resolve().parents[1] / "bitprobe" +if str(_BITPROBE) not in sys.path: + sys.path.insert(0, str(_BITPROBE)) + + +def _valid_legacy_db(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(path) as conn: + conn.execute("CREATE TABLE cve_entries (cve_id TEXT PRIMARY KEY)") + conn.execute("CREATE TABLE cve_products (id INTEGER PRIMARY KEY)") + conn.execute("CREATE TABLE cve_cpes (id INTEGER PRIMARY KEY)") + conn.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)") + + +def test_cve_paths_honor_data_dir_override(monkeypatch, tmp_path: Path) -> None: + target = tmp_path / "custom-data" + monkeypatch.setenv("BITSENTRY_DATA_DIR", str(target)) + + import scanner.paths as paths + + paths = importlib.reload(paths) + assert Path(paths.CVE_DB_PATH) == target / "cve_db.sqlite" + assert Path(paths.CVE_META_PATH) == target / "cve_meta.json" + + +def test_migration_copies_valid_legacy_database_atomically(tmp_path: Path) -> None: + from scanner.paths import migrate_legacy_cve_database + + legacy = tmp_path / "legacy" / "cve_db.sqlite" + destination = tmp_path / "new" / "cve_db.sqlite" + _valid_legacy_db(legacy) + + assert migrate_legacy_cve_database(legacy, destination) is True + assert legacy.exists() + assert destination.exists() + with sqlite3.connect(destination) as conn: + assert conn.execute("PRAGMA integrity_check").fetchone()[0] == "ok" + + +def test_migration_rejects_invalid_legacy_database(tmp_path: Path) -> None: + from scanner.paths import migrate_legacy_cve_database + + legacy = tmp_path / "legacy.sqlite" + destination = tmp_path / "new" / "cve_db.sqlite" + legacy.write_text("not sqlite", encoding="utf-8") + + assert migrate_legacy_cve_database(legacy, destination) is False + assert not destination.exists() + assert legacy.read_text(encoding="utf-8") == "not sqlite" + + +def test_existing_destination_prevents_legacy_read(tmp_path: Path) -> None: + from scanner.paths import migrate_legacy_cve_database + + legacy = tmp_path / "missing-legacy.sqlite" + destination = tmp_path / "data" / "cve_db.sqlite" + destination.parent.mkdir(parents=True) + destination.write_bytes(b"already installed") + + assert migrate_legacy_cve_database(legacy, destination) is False + assert destination.read_bytes() == b"already installed" + + +def test_legacy_json_adapter_checks_new_sqlite_path() -> None: + import scanner.cve_db as adapter + from scanner.paths import CVE_DB_PATH + + assert adapter.CVE_SQLITE_PATH == CVE_DB_PATH diff --git a/tests/test_cve_resumability.py b/tests/test_cve_resumability.py new file mode 100644 index 0000000..072a437 --- /dev/null +++ b/tests/test_cve_resumability.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from unittest import mock + + +_BITPROBE = Path(__file__).resolve().parents[1] / "bitprobe" +if str(_BITPROBE) not in sys.path: + sys.path.insert(0, str(_BITPROBE)) + + +def _db(monkeypatch, tmp_path: Path): + import scanner.cve_db_manager as manager + + monkeypatch.setattr(manager, "CVE_DB_PATH", str(tmp_path / "cve.sqlite")) + monkeypatch.setattr(manager, "CVE_META_PATH", str(tmp_path / "meta.json")) + monkeypatch.setattr(manager, "migrate_legacy_cve_database", lambda: False) + manager.init_cve_database() + return manager + + +def test_matching_incomplete_window_resumes_next_index(monkeypatch, tmp_path: Path) -> None: + manager = _db(monkeypatch, tmp_path) + with manager._connect() as conn: + assert manager.prepare_sync_window( + conn, mode="publication", window_start="a", window_end="b", results_per_page=2000 + ) == 0 + manager.checkpoint_sync_page(conn, next_start_index=4000, total_expected=9000) + + with manager._connect() as conn: + assert manager.prepare_sync_window( + conn, mode="publication", window_start="a", window_end="b", results_per_page=2000 + ) == 4000 + + +def test_changed_window_invalidates_checkpoint(monkeypatch, tmp_path: Path) -> None: + manager = _db(monkeypatch, tmp_path) + with manager._connect() as conn: + manager.prepare_sync_window( + conn, mode="publication", window_start="a", window_end="b", results_per_page=2000 + ) + manager.checkpoint_sync_page(conn, next_start_index=4000, total_expected=9000) + assert manager.prepare_sync_window( + conn, mode="publication", window_start="b", window_end="c", results_per_page=2000 + ) == 0 + + +def test_completed_window_does_not_resume(monkeypatch, tmp_path: Path) -> None: + manager = _db(monkeypatch, tmp_path) + with manager._connect() as conn: + manager.prepare_sync_window( + conn, mode="modified", window_start="a", window_end="b", results_per_page=2000 + ) + manager.checkpoint_sync_page(conn, next_start_index=2000, total_expected=2000) + manager.complete_sync_window(conn) + assert manager.prepare_sync_window( + conn, mode="modified", window_start="a", window_end="b", results_per_page=2000 + ) == 0 + + +def test_resumed_full_build_preserves_committed_rows(monkeypatch, tmp_path: Path) -> None: + manager = _db(monkeypatch, tmp_path) + target = "2026-08-18T00:00:00.000" + with manager._connect() as conn: + conn.execute( + "INSERT INTO cve_entries (cve_id, description) VALUES (?, ?)", + ("CVE-1999-0001", "already committed"), + ) + manager.write_cve_metadata( + {"coverage_mode": "windowed", "coverage_end": target}, conn=conn + ) + conn.execute( + "INSERT OR REPLACE INTO sync_state " + "(id, mode, window_start, window_end, results_per_page, next_start_index, " + "total_expected, started_at, completed) VALUES (1, ?, ?, ?, 2000, 2000, 4000, ?, 0)", + ( + "full-publication", + "2026-04-21T00:00:00.000", + target, + "2026-08-17T00:00:00.000", + ), + ) + + response = mock.Mock(status_code=200) + response.json.return_value = {"vulnerabilities": [], "totalResults": 0} + monkeypatch.setattr(manager, "_nvd_get", lambda *args, **kwargs: response) + + manager.update_cve_database(full_sync=True, force=True) + + with manager._connect() as conn: + assert conn.execute( + "SELECT COUNT(*) FROM cve_entries WHERE cve_id='CVE-1999-0001'" + ).fetchone()[0] == 1 + assert manager.read_cve_metadata()["coverage_mode"] == "full" diff --git a/tests/test_cve_sync_windows.py b/tests/test_cve_sync_windows.py new file mode 100644 index 0000000..7260d58 --- /dev/null +++ b/tests/test_cve_sync_windows.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import sys +from datetime import datetime, timedelta +from pathlib import Path +from unittest import mock + + +_BITPROBE = Path(__file__).resolve().parents[1] / "bitprobe" +if str(_BITPROBE) not in sys.path: + sys.path.insert(0, str(_BITPROBE)) + + +def test_iter_nvd_windows_splits_long_ranges() -> None: + from scanner.cve_db_manager import iter_nvd_windows + + start = datetime(2020, 1, 1) + end = start + timedelta(days=365) + windows = list(iter_nvd_windows(start, end)) + + assert windows[0][0] == start + assert windows[-1][1] == end + assert all(window_end - window_start <= timedelta(days=119) for window_start, window_end in windows) + assert all(windows[index][1] == windows[index + 1][0] for index in range(len(windows) - 1)) + + +def test_iter_nvd_windows_keeps_exact_boundary_single() -> None: + from scanner.cve_db_manager import iter_nvd_windows + + start = datetime(2026, 1, 1) + end = start + timedelta(days=119) + assert list(iter_nvd_windows(start, end)) == [(start, end)] + + +def test_zero_result_incremental_advances_sqlite_cursor(monkeypatch, tmp_path: Path) -> None: + import scanner.cve_db_manager as manager + import scanner.update_state as state + + monkeypatch.setattr(manager, "CVE_DB_PATH", str(tmp_path / "cve.sqlite")) + monkeypatch.setattr(manager, "CVE_META_PATH", str(tmp_path / "meta.json")) + monkeypatch.setattr(manager, "migrate_legacy_cve_database", lambda: False) + monkeypatch.setattr(state, "STATE_DIR", tmp_path / "state") + monkeypatch.setattr(state, "STATE_PATH", tmp_path / "state" / "state.json") + manager.init_cve_database() + manager.write_cve_metadata({"nvd_cursor": "2026-08-17T00:00:00.000"}) + with manager._connect() as conn: + conn.execute( + "INSERT INTO cve_entries (cve_id, description) VALUES (?, ?)", + ("CVE-2026-0001", "existing"), + ) + + response = mock.Mock(status_code=200) + response.json.return_value = { + "vulnerabilities": [], + "totalResults": 0, + "resultsPerPage": 2000, + } + monkeypatch.setattr(manager, "_nvd_get", lambda *args, **kwargs: response) + + manager.update_cve_database() + + cursor = manager.read_cve_metadata()["nvd_cursor"] + assert cursor is not None + assert cursor > "2026-08-17T00:00:00.000" + assert state.get_state_timestamp("cve", "last_modified") == cursor + + +def test_years_mode_sends_only_nvd_safe_windows(monkeypatch, tmp_path: Path) -> None: + import scanner.cve_db_manager as manager + import scanner.update_state as state + + monkeypatch.setattr(manager, "CVE_DB_PATH", str(tmp_path / "cve.sqlite")) + monkeypatch.setattr(manager, "CVE_META_PATH", str(tmp_path / "meta.json")) + monkeypatch.setattr(manager, "migrate_legacy_cve_database", lambda: False) + monkeypatch.setattr(state, "STATE_DIR", tmp_path / "state") + monkeypatch.setattr(state, "STATE_PATH", tmp_path / "state" / "state.json") + seen = [] + + def response_for(params, *_args, **_kwargs): + seen.append(dict(params)) + response = mock.Mock(status_code=200) + response.json.return_value = {"vulnerabilities": [], "totalResults": 0} + return response + + monkeypatch.setattr(manager, "_nvd_get", response_for) + manager.update_cve_database(years=1, incremental=False) + + assert len(seen) == 4 + for params in seen: + start = datetime.fromisoformat(params["pubStartDate"]) + end = datetime.fromisoformat(params["pubEndDate"]) + assert end - start <= timedelta(days=119) + metadata = manager.read_cve_metadata() + assert metadata["coverage_mode"] == "windowed" + assert metadata["coverage_start"] == seen[0]["pubStartDate"] + assert metadata["coverage_end"] == seen[-1]["pubEndDate"] diff --git a/tests/test_cve_workflow.py b/tests/test_cve_workflow.py new file mode 100644 index 0000000..b684213 --- /dev/null +++ b/tests/test_cve_workflow.py @@ -0,0 +1,24 @@ +from pathlib import Path + + +def test_cve_workflow_has_safe_producer_contract() -> None: + root = Path(__file__).resolve().parents[1] + workflow = (root / ".github/workflows/update-cve-db.yml").read_text(encoding="utf-8") + + assert 'cron: "17 6 * * *"' in workflow + assert "full_rebuild:" in workflow + assert "contents: write" in workflow + assert "group: cve-db-producer" in workflow + assert 'python-version: "3.13"' in workflow + assert "NVD_API_KEY" in workflow + assert workflow.index("Restore previous snapshot") < workflow.index("Update canonical database") + assert "build_cve_snapshot.py" in workflow + assert "update_cve_snapshot_release.sh" in workflow + + +def test_release_script_keeps_database_releases_out_of_latest() -> None: + root = Path(__file__).resolve().parents[1] + script = (root / "scripts/update_cve_snapshot_release.sh").read_text(encoding="utf-8") + assert "cve-db-latest" in script + assert "--latest=false" in script + assert "--clobber" in script diff --git a/tests/test_products_cli.py b/tests/test_products_cli.py index a4842f4..926e54d 100644 --- a/tests/test_products_cli.py +++ b/tests/test_products_cli.py @@ -63,6 +63,29 @@ def test_update_cve_db_passthrough() -> None: assert "7" in cmd +def test_update_cve_db_default_preserves_snapshot_policy() -> None: + completed = mock.Mock(returncode=0) + with ( + mock.patch("subprocess.run", return_value=completed) as run_mock, + mock.patch("sys.argv", ["bitsentry", "update-cve-db"]), + ): + code = bitsentry_main() + assert code == 0 + assert run_mock.call_args.args[0][2:] == ["update-cve-db"] + + +def test_update_cve_db_snapshot_flags_are_forwarded() -> None: + completed = mock.Mock(returncode=0) + for flag in ("--snapshot-only", "--no-snapshot", "--raw-full"): + with ( + mock.patch("subprocess.run", return_value=completed) as run_mock, + mock.patch("sys.argv", ["bitsentry", "update-cve-db", flag]), + ): + code = bitsentry_main() + assert code == 0 + assert flag in run_mock.call_args.args[0] + + def test_update_db_passthrough_to_asn_updater() -> None: completed = mock.Mock(returncode=0) with ( diff --git a/tests/test_update_lock.py b/tests/test_update_lock.py new file mode 100644 index 0000000..d22ee32 --- /dev/null +++ b/tests/test_update_lock.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import sys +from contextlib import nullcontext +from pathlib import Path +from unittest import mock + +import pytest + + +_BITPROBE = Path(__file__).resolve().parents[1] / "bitprobe" +if str(_BITPROBE) not in sys.path: + sys.path.insert(0, str(_BITPROBE)) + + +def test_lock_is_reentrant_and_released_after_exception(tmp_path: Path) -> None: + from scanner.update_lock import bitsentry_update_lock + + lock_path = tmp_path / ".update.lock" + with pytest.raises(RuntimeError): + with bitsentry_update_lock(lock_path=lock_path): + with bitsentry_update_lock(lock_path=lock_path): + raise RuntimeError("boom") + + with bitsentry_update_lock(lock_path=lock_path): + assert lock_path.exists() + + +def test_lock_contention_fails_without_waiting(tmp_path: Path) -> None: + from scanner.update_lock import UpdateLockError, bitsentry_update_lock + + lock_path = tmp_path / ".update.lock" + with bitsentry_update_lock(lock_path=lock_path): + with pytest.raises(UpdateLockError): + with bitsentry_update_lock(lock_path=lock_path, _allow_reentry=False): + pass + + +def test_cve_update_uses_shared_lock(monkeypatch, tmp_path: Path) -> None: + import scanner.cve_db_manager as manager + + monkeypatch.setattr(manager, "CVE_DB_PATH", str(tmp_path / "cve.sqlite")) + monkeypatch.setattr(manager, "CVE_META_PATH", str(tmp_path / "meta.json")) + monkeypatch.setattr(manager, "migrate_legacy_cve_database", lambda: False) + lock = mock.Mock(return_value=nullcontext()) + monkeypatch.setattr(manager, "bitsentry_update_lock", lock) + response = mock.Mock(status_code=200) + response.json.return_value = {"vulnerabilities": [], "totalResults": 0} + monkeypatch.setattr(manager, "_nvd_get", lambda *args, **kwargs: response) + + manager.update_cve_database(incremental=False) + + lock.assert_called_once_with() diff --git a/tests/test_update_notifier_snapshot.py b/tests/test_update_notifier_snapshot.py new file mode 100644 index 0000000..541d9af --- /dev/null +++ b/tests/test_update_notifier_snapshot.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from unittest import mock + + +_BITPROBE = Path(__file__).resolve().parents[1] / "bitprobe" +if str(_BITPROBE) not in sys.path: + sys.path.insert(0, str(_BITPROBE)) + + +def test_scan_time_update_uses_snapshot_policy(monkeypatch) -> None: + import scanner.update_notifier as notifier + + monkeypatch.setattr(notifier, "cve_db_needs_update", lambda: True) + policy = mock.Mock(return_value=5) + monkeypatch.setattr(notifier, "update_with_snapshot_policy", policy) + + notifier.check_and_notify(auto_update=True) + + policy.assert_called_once_with(verbose=False) From 651a7a6e6ff72e1265be1a8b1e552bb43b9ac87f Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 18 Aug 2026 19:18:54 +0000 Subject: [PATCH 3/4] Address CVE distribution review --- .github/workflows/update-cve-db.yml | 12 ++- README.md | 1 + bitprobe/bitprobe.py | 3 + bitprobe/scanner/cve_db_bootstrap.py | 32 +++++-- bitprobe/scanner/cve_db_manager.py | 126 ++++++++++++++++++++------- bitprobe/scanner/paths.py | 3 +- bitprobe/scanner/update_notifier.py | 4 +- scripts/build_cve_snapshot.py | 23 +++-- scripts/install_bitsentry.sh | 2 +- tests/test_build_cve_snapshot.py | 15 ++++ tests/test_cve_bootstrap_policy.py | 36 ++++++++ tests/test_cve_cli.py | 19 ++-- tests/test_cve_db_bootstrap.py | 47 +++++++--- tests/test_cve_metadata.py | 29 ++++++ tests/test_cve_paths.py | 11 +-- tests/test_cve_resumability.py | 34 +++++++- tests/test_cve_sync_windows.py | 29 +++++- tests/test_cve_workflow.py | 32 ++++--- tests/test_products_cli.py | 28 +++--- tests/test_update_lock.py | 3 + 20 files changed, 392 insertions(+), 97 deletions(-) diff --git a/.github/workflows/update-cve-db.yml b/.github/workflows/update-cve-db.yml index dea20ce..8e6f999 100644 --- a/.github/workflows/update-cve-db.yml +++ b/.github/workflows/update-cve-db.yml @@ -21,12 +21,17 @@ concurrency: jobs: sync-and-publish: runs-on: ubuntu-latest + timeout-minutes: 360 env: - BITSENTRY_DATA_DIR: ${{ runner.temp }}/bitsentry-data NVD_API_KEY: ${{ secrets.NVD_API_KEY }} GH_TOKEN: ${{ github.token }} steps: - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Configure data directory + run: echo "BITSENTRY_DATA_DIR=$RUNNER_TEMP/bitsentry-data" >> "$GITHUB_ENV" - uses: actions/setup-python@v5 with: @@ -52,8 +57,11 @@ jobs: fi - name: Update canonical database + env: + FULL_REBUILD: ${{ inputs.full_rebuild }} + SNAPSHOT_RESTORED: ${{ steps.restore.outputs.restored }} run: | - if [[ "${{ inputs.full_rebuild }}" == "true" || "${{ steps.restore.outputs.restored }}" != "true" ]]; then + if [[ "$FULL_REBUILD" == "true" || "$SNAPSHOT_RESTORED" != "true" ]]; then PYTHONPATH=bitprobe python bitprobe/bitprobe.py update-cve-db --full --no-snapshot else PYTHONPATH=bitprobe python bitprobe/bitprobe.py update-cve-db --no-snapshot diff --git a/README.md b/README.md index ea23e5d..91302dc 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ BitSentry is a CLI-first security assessment suite. The public build focuses on - web-focused vulnerability scanning It is built to run cleanly in local shells, CI pipelines, and Docker. +Supported hosts are Linux and macOS. > Use only on systems you own or are explicitly authorized to test. ## Current Product Status diff --git a/bitprobe/bitprobe.py b/bitprobe/bitprobe.py index 2945dcb..7681dcb 100755 --- a/bitprobe/bitprobe.py +++ b/bitprobe/bitprobe.py @@ -264,6 +264,9 @@ def main() -> int: snapshot_only=snapshot_only, verbose=verbose, ) + if snapshot_only: + print("[+] CVE database snapshot installed") + return 0 print(f"[+] CVE database updated with {count} entries") return 0 except Exception as e: diff --git a/bitprobe/scanner/cve_db_bootstrap.py b/bitprobe/scanner/cve_db_bootstrap.py index fe56808..3461185 100644 --- a/bitprobe/scanner/cve_db_bootstrap.py +++ b/bitprobe/scanner/cve_db_bootstrap.py @@ -10,6 +10,7 @@ import shutil import sqlite3 import tempfile +from contextlib import closing from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any @@ -81,6 +82,18 @@ def validate_manifest(manifest: dict[str, Any]) -> dict[str, Any]: raise SnapshotValidationError("invalid sha256_gz") if not isinstance(manifest["nvd_cursor"], str) or not manifest["nvd_cursor"]: raise SnapshotValidationError("invalid nvd_cursor") + artifact = manifest["artifact"] + if ( + not isinstance(artifact, str) + or not artifact + or "\x00" in artifact + or "/" in artifact + or "\\" in artifact + or ":" in artifact + or artifact in {".", ".."} + or Path(artifact).name != artifact + ): + raise SnapshotValidationError("invalid artifact") return manifest @@ -164,7 +177,7 @@ def decompress_snapshot(source: Path, destination: Path, manifest: dict[str, Any def validate_snapshot_database(path: Path, manifest: dict[str, Any]) -> dict[str, str | None]: validate_manifest(manifest) try: - with sqlite3.connect(f"file:{path}?mode=ro", uri=True) as conn: + with closing(sqlite3.connect(f"file:{path}?mode=ro", uri=True)) as conn: if conn.execute("PRAGMA integrity_check").fetchone()[0] != "ok": raise SnapshotValidationError("SQLite integrity_check failed") tables = { @@ -201,7 +214,7 @@ def _checkpoint_existing_database(destination: Path) -> None: raise SnapshotValidationError("SQLite sidecar exists without the main database") try: - with sqlite3.connect(destination, timeout=0) as conn: + with closing(sqlite3.connect(destination, timeout=0)) as conn: conn.execute("PRAGMA busy_timeout=0") busy, _, _ = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() except sqlite3.Error as exc: @@ -244,9 +257,10 @@ def install_snapshot_atomically( ) as temp_file: temp_path = Path(temp_file.name) shutil.copyfile(snapshot_db, temp_path) - with sqlite3.connect(temp_path) as conn: + with closing(sqlite3.connect(temp_path)) as conn: conn.execute("PRAGMA journal_mode=DELETE") conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + conn.commit() validate_snapshot_database(temp_path, manifest) os.replace(temp_path, destination) temp_path = None @@ -285,10 +299,14 @@ def update_with_snapshot_policy( cursor = manager.read_cve_metadata().get("nvd_cursor") if complete else None cursor_too_old = False if cursor: - parsed = datetime.fromisoformat(cursor.replace("Z", "+00:00")) - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - cursor_too_old = datetime.now(timezone.utc) - parsed > timedelta(days=119) + try: + parsed = datetime.fromisoformat(cursor.replace("Z", "+00:00")) + except ValueError: + cursor_too_old = True + else: + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + cursor_too_old = datetime.now(timezone.utc) - parsed > timedelta(days=119) if not complete or cursor_too_old: try: diff --git a/bitprobe/scanner/cve_db_manager.py b/bitprobe/scanner/cve_db_manager.py index 4819d45..03f1955 100644 --- a/bitprobe/scanner/cve_db_manager.py +++ b/bitprobe/scanner/cve_db_manager.py @@ -10,6 +10,7 @@ import os import time import requests +from contextlib import closing from datetime import datetime, timedelta, timezone from typing import List, Dict, Optional, Any from packaging import version @@ -20,8 +21,6 @@ NVD_API_URL = "https://services.nvd.nist.gov/rest/json/cves/2.0" DEFAULT_STALE_DAYS = 7 -# Below this count, local DB is treated as a short publication window bootstrap, not full coverage. -MIN_PRODUCTION_CVE_COUNT = 50_000 # NVD rate limits (https://nvd.nist.gov/developers/start-here): 5 req/30s # without a key (6.0s spacing), 50 req/30s with one (0.6s spacing, +margin). NVD_SLEEP_NO_KEY = 6.0 @@ -36,6 +35,15 @@ "coverage_end", "nvd_cursor", ) +SYNC_PROGRESS_SQL = ( + "UPDATE sync_state SET next_start_index = ?, total_expected = ? WHERE id = 1" +) +_MIRROR_COMPAT_STATE = True + + +def _set_compatibility_cursor(value: str) -> None: + if _MIRROR_COMPAT_STATE: + set_state_timestamp("cve", "last_modified", value) def _utcnow() -> datetime: @@ -196,7 +204,7 @@ def bootstrap_cve_state() -> str | None: cursor.execute("SELECT value FROM metadata WHERE key = 'nvd_cursor'") sqlite_cursor = cursor.fetchone() if sqlite_cursor and sqlite_cursor[0]: - set_state_timestamp("cve", "last_modified", sqlite_cursor[0]) + _set_compatibility_cursor(sqlite_cursor[0]) return sqlite_cursor[0] cursor.execute("SELECT COUNT(*) FROM cve_entries") if cursor.fetchone()[0] == 0: @@ -207,7 +215,7 @@ def bootstrap_cve_state() -> str | None: ) row = cursor.fetchone() if row and row[0]: - set_state_timestamp("cve", "last_modified", row[0]) + _set_compatibility_cursor(row[0]) return row[0] finally: conn.close() @@ -231,7 +239,7 @@ def bootstrap_cve_state() -> str | None: seeded = _format_nvd_datetime(dt) except ValueError: seeded = last_update - set_state_timestamp("cve", "last_modified", seeded) + _set_compatibility_cursor(seeded) return seeded return None @@ -288,7 +296,7 @@ def mirror_sqlite_cursor_to_state() -> str | None: return None cursor = read_cve_metadata().get("nvd_cursor") if cursor: - set_state_timestamp("cve", "last_modified", cursor) + _set_compatibility_cursor(cursor) return cursor @@ -324,10 +332,7 @@ def checkpoint_sync_page( next_start_index: int, total_expected: int, ) -> None: - conn.execute( - "UPDATE sync_state SET next_start_index = ?, total_expected = ? WHERE id = 1", - (next_start_index, total_expected), - ) + conn.execute(SYNC_PROGRESS_SQL, (next_start_index, total_expected)) conn.commit() @@ -359,8 +364,12 @@ def cve_db_needs_update(stale_days: int = DEFAULT_STALE_DAYS) -> bool: cursor.execute("SELECT value FROM metadata WHERE key = 'nvd_cursor'") row = cursor.fetchone() if row and row[0]: - last_dt = datetime.fromisoformat(_normalize_nvd_timestamp(row[0])) - return _utcnow() - last_dt > timedelta(days=stale_days) + try: + last_dt = datetime.fromisoformat(_normalize_nvd_timestamp(row[0])) + except ValueError: + pass + else: + return _utcnow() - last_dt > timedelta(days=stale_days) cursor.execute("SELECT value FROM metadata WHERE key = 'last_updated'") row = cursor.fetchone() if not row or not row[0]: @@ -549,7 +558,7 @@ def _update_cve_database_unlocked( now_dt = _utcnow() if full_sync and not raw_full_sync: metadata = read_cve_metadata() - with _connect() as state_conn: + with closing(_connect()) as state_conn: resume_row = state_conn.execute( "SELECT mode, window_start, window_end, completed " "FROM sync_state WHERE id = 1" @@ -564,7 +573,7 @@ def _update_cve_database_unlocked( build_started = datetime.fromisoformat(metadata["coverage_end"]) else: build_started = now_dt - with _connect() as reset_conn: + with closing(_connect()) as reset_conn: reset_conn.execute("DELETE FROM cve_products") reset_conn.execute("DELETE FROM cve_cpes") reset_conn.execute("DELETE FROM cve_entries") @@ -620,7 +629,7 @@ def _update_cve_database_unlocked( "nvd_cursor": completed, } ) - set_state_timestamp("cve", "last_modified", completed) + _set_compatibility_cursor(completed) return total if incremental and state_last_modified and existing_count and not force: @@ -660,9 +669,10 @@ def _update_cve_database_unlocked( "nvd_cursor": coverage_end, } ) - set_state_timestamp("cve", "last_modified", coverage_end) + _set_compatibility_cursor(coverage_end) return total + use_incremental = False if _window is not None: window_kind, window_start_dt, window_end_dt = _window window_start_text = _format_nvd_datetime(window_start_dt) @@ -685,7 +695,11 @@ def _update_cve_database_unlocked( else: use_incremental = ( - incremental and state_last_modified and not force and not full_sync + incremental + and state_last_modified + and not force + and not full_sync + and not raw_full_sync ) if _window is not None: @@ -936,7 +950,7 @@ def _update_cve_database_unlocked( ) if use_incremental: write_cve_metadata({"nvd_cursor": completed_window_end}) - set_state_timestamp("cve", "last_modified", completed_window_end) + _set_compatibility_cursor(completed_window_end) elif saw_vulnerabilities and not ( _window is not None and _window[0] == "full-publication" ): @@ -949,7 +963,7 @@ def _update_cve_database_unlocked( "coverage_end": params.get("pubEndDate", update_end), } ) - set_state_timestamp("cve", "last_modified", cursor_value) + _set_compatibility_cursor(cursor_value) print( f"[+] CVE database updated: {total_updated} CVEs added/updated " @@ -970,16 +984,67 @@ def update_cve_database( ) -> int: """Run a CVE update while holding the shared mutable-data lock.""" with bitsentry_update_lock(): - return _update_cve_database_unlocked( - days=days, - years=years, - full_sync=full_sync, - raw_full_sync=raw_full_sync, - api_key=api_key, - incremental=incremental, - force=force, - verbose=verbose, + if not full_sync or raw_full_sync: + return _update_cve_database_unlocked( + days=days, + years=years, + full_sync=full_sync, + raw_full_sync=raw_full_sync, + api_key=api_key, + incremental=incremental, + force=force, + verbose=verbose, + ) + + global CVE_DB_PATH, CVE_META_PATH, _MIRROR_COMPAT_STATE + destination = Path(CVE_DB_PATH) + staging = destination.with_name(f".{destination.name}.full-build") + metadata_destination = Path(CVE_META_PATH) + metadata_staging = metadata_destination.with_name( + f".{metadata_destination.name}.full-build" ) + destination.parent.mkdir(parents=True, exist_ok=True) + active_path = CVE_DB_PATH + active_metadata_path = CVE_META_PATH + mirror_state = _MIRROR_COMPAT_STATE + try: + CVE_DB_PATH = str(staging) + CVE_META_PATH = str(metadata_staging) + _MIRROR_COMPAT_STATE = False + count = _update_cve_database_unlocked( + days=days, + years=years, + full_sync=True, + raw_full_sync=False, + api_key=api_key, + incremental=incremental, + force=force, + verbose=verbose, + ) + with closing(_connect()) as conn: + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + conn.execute("PRAGMA journal_mode=DELETE") + conn.commit() + finally: + CVE_DB_PATH = active_path + CVE_META_PATH = active_metadata_path + _MIRROR_COMPAT_STATE = mirror_state + if destination.exists(): + with closing(sqlite3.connect(destination, timeout=0)) as conn: + conn.execute("PRAGMA busy_timeout=0") + busy, _, _ = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + if busy: + raise RuntimeError("current CVE database is busy") + conn.execute("PRAGMA journal_mode=DELETE") + if any(Path(f"{destination}{suffix}").exists() for suffix in ("-wal", "-shm")): + raise RuntimeError("current CVE database still has active WAL state") + os.replace(staging, destination) + if metadata_staging.exists(): + os.replace(metadata_staging, metadata_destination) + cursor = read_cve_metadata().get("nvd_cursor") + if cursor: + set_state_timestamp("cve", "last_modified", cursor) + return count def _extract_cpe_matches_from_node(node: Dict) -> List[Dict]: @@ -1146,10 +1211,7 @@ def _store_cves_batch( """, unique_products) if checkpoint is not None: - cursor.execute( - "UPDATE sync_state SET next_start_index = ?, total_expected = ? WHERE id = 1", - checkpoint, - ) + cursor.execute(SYNC_PROGRESS_SQL, checkpoint) conn.commit() finally: diff --git a/bitprobe/scanner/paths.py b/bitprobe/scanner/paths.py index 07f73f1..742c339 100644 --- a/bitprobe/scanner/paths.py +++ b/bitprobe/scanner/paths.py @@ -6,6 +6,7 @@ import shutil import sqlite3 import tempfile +from contextlib import closing from pathlib import Path # bitprobe/ directory (parent of scanner/) @@ -30,7 +31,7 @@ def _valid_cve_database(path: Path) -> bool: try: - with sqlite3.connect(f"file:{path}?mode=ro", uri=True) as conn: + with closing(sqlite3.connect(f"file:{path}?mode=ro", uri=True)) as conn: if conn.execute("PRAGMA integrity_check").fetchone()[0] != "ok": return False tables = { diff --git a/bitprobe/scanner/update_notifier.py b/bitprobe/scanner/update_notifier.py index 545f448..bc18006 100755 --- a/bitprobe/scanner/update_notifier.py +++ b/bitprobe/scanner/update_notifier.py @@ -12,8 +12,6 @@ META_PATH = CVE_META_PATH REMINDER_DAYS = 14 -# Fast bootstrap on scan startup (not the full NVD corpus) -SCAN_BOOTSTRAP_DAYS = 7 def _load_meta(): @@ -23,7 +21,7 @@ def _load_meta(): return json.load(f) -def check_and_notify(auto_update: bool = True, bootstrap_days: int = SCAN_BOOTSTRAP_DAYS): +def check_and_notify(auto_update: bool = True): """ Runs at scan startup. - Installs a verified full snapshot when local coverage is incomplete diff --git a/scripts/build_cve_snapshot.py b/scripts/build_cve_snapshot.py index 7f2565e..ed24983 100755 --- a/scripts/build_cve_snapshot.py +++ b/scripts/build_cve_snapshot.py @@ -10,6 +10,7 @@ import os import sqlite3 import tempfile +from contextlib import closing from datetime import datetime, timezone from pathlib import Path @@ -22,10 +23,12 @@ "coverage_end", "nvd_cursor", ) +MAX_COMPRESSED_SIZE = 512 * 1024 * 1024 +MAX_UNCOMPRESSED_SIZE = 2 * 1024 * 1024 * 1024 def _inspect_database(path: Path) -> tuple[dict[str, str | None], int]: - with sqlite3.connect(path) as conn: + with closing(sqlite3.connect(path)) as conn: conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") if conn.execute("PRAGMA integrity_check").fetchone()[0] != "ok": raise RuntimeError("CVE database failed integrity_check") @@ -54,6 +57,9 @@ def build_snapshot( source_commit: str, ) -> dict[str, object]: metadata, count = _inspect_database(database) + uncompressed_size = database.stat().st_size + if uncompressed_size > MAX_UNCOMPRESSED_SIZE: + raise RuntimeError("CVE database exceeds the uncompressed size limit") output_dir.mkdir(parents=True, exist_ok=True) artifact = output_dir / "cve_db.sqlite.gz" with database.open("rb") as source, artifact.open("wb") as raw_output: @@ -61,7 +67,14 @@ def build_snapshot( for block in iter(lambda: source.read(1024 * 1024), b""): output.write(block) - compressed = artifact.read_bytes() + compressed_size = artifact.stat().st_size + if compressed_size > MAX_COMPRESSED_SIZE: + artifact.unlink() + raise RuntimeError("CVE snapshot exceeds the compressed size limit") + digest = hashlib.sha256() + with artifact.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) manifest: dict[str, object] = { "format_version": 1, "schema_version": int(metadata["schema_version"] or 0), @@ -72,9 +85,9 @@ def build_snapshot( "nvd_cursor": metadata["nvd_cursor"], "cve_count": count, "artifact": artifact.name, - "sha256_gz": hashlib.sha256(compressed).hexdigest(), - "compressed_size": len(compressed), - "uncompressed_size": database.stat().st_size, + "sha256_gz": digest.hexdigest(), + "compressed_size": compressed_size, + "uncompressed_size": uncompressed_size, "source_commit": source_commit, } manifest_path = output_dir / "manifest.json" diff --git a/scripts/install_bitsentry.sh b/scripts/install_bitsentry.sh index 1f901bf..a6a3ce4 100755 --- a/scripts/install_bitsentry.sh +++ b/scripts/install_bitsentry.sh @@ -70,7 +70,7 @@ print_before_first_scan_notice() { echo -e "${b} 2) export NVD_API_KEY=\"your-nvd-api-key\"${r} ${d}(optional, faster incremental sync)${r}" if [[ "${CVE_NEEDS_BOOTSTRAP}" -eq 1 ]]; then - echo -e "${y} 3) bitsentry update-cve-db${r} ${d}(verified snapshot + catch-up)${r}" + echo -e "${y} 3) bitsentry update-cve-db${r} ${d}(recommended; skipping may delay the first scan)${r}" else echo -e "${b} 3) bitsentry update-cve-db${r} ${d}(CVE loaded — incremental refresh)${r}" fi diff --git a/tests/test_build_cve_snapshot.py b/tests/test_build_cve_snapshot.py index fda5b06..9ff04ab 100644 --- a/tests/test_build_cve_snapshot.py +++ b/tests/test_build_cve_snapshot.py @@ -7,6 +7,8 @@ import sqlite3 from pathlib import Path +import pytest + def _load_builder(): path = Path(__file__).resolve().parents[1] / "scripts" / "build_cve_snapshot.py" @@ -40,9 +42,11 @@ def test_build_snapshot_copies_sqlite_metadata_and_hashes_artifact(tmp_path: Pat builder = _load_builder() db = tmp_path / "cve.sqlite" dist = tmp_path / "dist" + second_dist = tmp_path / "dist-second" _database(db) manifest = builder.build_snapshot(db, dist, source_commit="deadbeef") + second_manifest = builder.build_snapshot(db, second_dist, source_commit="deadbeef") artifact = dist / "cve_db.sqlite.gz" on_disk = json.loads((dist / "manifest.json").read_text(encoding="utf-8")) @@ -51,5 +55,16 @@ def test_build_snapshot_copies_sqlite_metadata_and_hashes_artifact(tmp_path: Pat assert manifest["nvd_cursor"] == "2026-08-18T00:00:00.000" assert manifest["cve_count"] == 1 assert manifest["sha256_gz"] == hashlib.sha256(artifact.read_bytes()).hexdigest() + assert second_manifest["sha256_gz"] == manifest["sha256_gz"] with gzip.open(artifact, "rb") as source: assert source.read() == db.read_bytes() + + +def test_build_snapshot_rejects_oversized_database(monkeypatch, tmp_path: Path) -> None: + builder = _load_builder() + db = tmp_path / "cve.sqlite" + _database(db) + monkeypatch.setattr(builder, "MAX_UNCOMPRESSED_SIZE", 0) + + with pytest.raises(RuntimeError, match="uncompressed size limit"): + builder.build_snapshot(db, tmp_path / "dist", source_commit="deadbeef") diff --git a/tests/test_cve_bootstrap_policy.py b/tests/test_cve_bootstrap_policy.py index 228c0f7..c5b1dc9 100644 --- a/tests/test_cve_bootstrap_policy.py +++ b/tests/test_cve_bootstrap_policy.py @@ -74,3 +74,39 @@ def test_full_database_over_119_days_stale_prefers_snapshot(monkeypatch) -> None assert bootstrap.update_with_snapshot_policy() == 8 install.assert_called_once_with() update.assert_called_once() + + +def test_stale_complete_database_falls_back_to_incremental(monkeypatch) -> None: + import scanner.cve_db_bootstrap as bootstrap + import scanner.cve_db_manager as manager + + monkeypatch.setattr(manager, "cve_db_is_complete", lambda: True) + monkeypatch.setattr( + manager, + "read_cve_metadata", + lambda: {"nvd_cursor": "2020-01-01T00:00:00.000"}, + ) + monkeypatch.setattr( + bootstrap, + "bootstrap_from_snapshot", + mock.Mock(side_effect=bootstrap.SnapshotError("offline")), + ) + update = mock.Mock(return_value=9) + monkeypatch.setattr(manager, "update_cve_database", update) + + assert bootstrap.update_with_snapshot_policy() == 9 + update.assert_called_once_with(days=30, incremental=True, verbose=False) + + +def test_invalid_complete_cursor_prefers_snapshot(monkeypatch) -> None: + import scanner.cve_db_bootstrap as bootstrap + import scanner.cve_db_manager as manager + + monkeypatch.setattr(manager, "cve_db_is_complete", lambda: True) + monkeypatch.setattr(manager, "read_cve_metadata", lambda: {"nvd_cursor": "bad"}) + install = mock.Mock(return_value={}) + monkeypatch.setattr(bootstrap, "bootstrap_from_snapshot", install) + monkeypatch.setattr(manager, "update_cve_database", mock.Mock(return_value=1)) + + assert bootstrap.update_with_snapshot_policy() == 1 + install.assert_called_once_with() diff --git a/tests/test_cve_cli.py b/tests/test_cve_cli.py index 424b801..0de17bd 100644 --- a/tests/test_cve_cli.py +++ b/tests/test_cve_cli.py @@ -2,9 +2,12 @@ import importlib.util import sys +from types import ModuleType from pathlib import Path from unittest import mock +import pytest + ROOT = Path(__file__).resolve().parents[1] BITPROBE = ROOT / "bitprobe" @@ -12,7 +15,7 @@ sys.path.insert(0, str(BITPROBE)) -def _cli(): +def _cli() -> ModuleType: spec = importlib.util.spec_from_file_location("bitprobe_cli_for_test", BITPROBE / "bitprobe.py") module = importlib.util.module_from_spec(spec) assert spec.loader is not None @@ -20,7 +23,7 @@ def _cli(): return module -def test_default_update_uses_snapshot_policy(monkeypatch) -> None: +def test_default_update_uses_snapshot_policy(monkeypatch: pytest.MonkeyPatch) -> None: cli = _cli() policy = mock.Mock(return_value=4) direct = mock.Mock() @@ -33,7 +36,7 @@ def test_default_update_uses_snapshot_policy(monkeypatch) -> None: direct.assert_not_called() -def test_explicit_years_skips_snapshot(monkeypatch) -> None: +def test_explicit_years_skips_snapshot(monkeypatch: pytest.MonkeyPatch) -> None: cli = _cli() policy = mock.Mock() direct = mock.Mock(return_value=2) @@ -46,7 +49,10 @@ def test_explicit_years_skips_snapshot(monkeypatch) -> None: assert direct.call_args.kwargs["years"] == 2 -def test_snapshot_only_uses_snapshot_without_direct_nvd(monkeypatch) -> None: +def test_snapshot_only_uses_snapshot_without_direct_nvd( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: cli = _cli() policy = mock.Mock(return_value=0) direct = mock.Mock() @@ -57,9 +63,12 @@ def test_snapshot_only_uses_snapshot_without_direct_nvd(monkeypatch) -> None: assert cli.main() == 0 policy.assert_called_once_with(snapshot_only=True, verbose=False) direct.assert_not_called() + assert "snapshot installed" in capsys.readouterr().out -def test_raw_full_exposes_best_effort_unfiltered_escape_hatch(monkeypatch) -> None: +def test_raw_full_exposes_best_effort_unfiltered_escape_hatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: cli = _cli() direct = mock.Mock(return_value=1) monkeypatch.setattr(cli, "update_cve_database", direct) diff --git a/tests/test_cve_db_bootstrap.py b/tests/test_cve_db_bootstrap.py index 62f9b93..07e0a3e 100644 --- a/tests/test_cve_db_bootstrap.py +++ b/tests/test_cve_db_bootstrap.py @@ -78,29 +78,52 @@ def test_verify_snapshot_rejects_checksum_mismatch(tmp_path: Path) -> None: verify_snapshot(gz, manifest) -def test_fetch_manifest_wraps_transport_errors() -> None: +def test_fetch_manifest_wraps_transport_errors(monkeypatch: pytest.MonkeyPatch) -> None: from scanner.cve_db_bootstrap import SnapshotError, fetch_snapshot_manifest - session = pytest.MonkeyPatch() client = requests.Session() - session.setattr(client, "get", lambda *args, **kwargs: (_ for _ in ()).throw(requests.Timeout("offline"))) - try: - with pytest.raises(SnapshotError, match="download snapshot manifest"): - fetch_snapshot_manifest(session=client) - finally: - session.undo() - - -def test_download_snapshot_wraps_transport_errors(tmp_path: Path) -> None: + monkeypatch.setattr( + client, + "get", + lambda *args, **kwargs: (_ for _ in ()).throw(requests.Timeout("offline")), + ) + with pytest.raises(SnapshotError, match="download snapshot manifest"): + fetch_snapshot_manifest(session=client) + + +def test_download_snapshot_wraps_transport_errors( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: from scanner.cve_db_bootstrap import SnapshotError, download_snapshot _, _, manifest = _artifact(tmp_path) client = requests.Session() - client.get = lambda *args, **kwargs: (_ for _ in ()).throw(requests.ConnectionError("offline")) + monkeypatch.setattr( + client, + "get", + lambda *args, **kwargs: (_ for _ in ()).throw(requests.ConnectionError("offline")), + ) with pytest.raises(SnapshotError, match="download CVE snapshot"): download_snapshot(manifest, tmp_path / "download.gz", session=client) +@pytest.mark.parametrize( + "artifact", + ["", ".", "..", "../db.gz", "dir/db.gz", "dir\\db.gz", "C:db.gz", "db\x00.gz"], +) +def test_validate_manifest_rejects_unsafe_artifact( + tmp_path: Path, + artifact: str, +) -> None: + from scanner.cve_db_bootstrap import SnapshotValidationError, validate_manifest + + _, _, manifest = _artifact(tmp_path) + manifest["artifact"] = artifact + with pytest.raises(SnapshotValidationError, match="artifact"): + validate_manifest(manifest) + + def test_validate_database_rejects_manifest_cursor_mismatch(tmp_path: Path) -> None: from scanner.cve_db_bootstrap import SnapshotValidationError, validate_snapshot_database diff --git a/tests/test_cve_metadata.py b/tests/test_cve_metadata.py index 17966a3..581a088 100644 --- a/tests/test_cve_metadata.py +++ b/tests/test_cve_metadata.py @@ -4,6 +4,8 @@ import sys from pathlib import Path +import pytest + _BITPROBE = Path(__file__).resolve().parents[1] / "bitprobe" if str(_BITPROBE) not in sys.path: @@ -45,6 +47,14 @@ def test_only_full_coverage_is_bootstrap_complete(monkeypatch, tmp_path: Path) - assert manager.cve_db_is_complete() is True +def test_metadata_rejects_unknown_keys(monkeypatch, tmp_path: Path) -> None: + manager, _, _ = _isolate(monkeypatch, tmp_path) + manager.init_cve_database() + + with pytest.raises(ValueError, match="Unknown CVE metadata keys"): + manager.write_cve_metadata({"unexpected": "value"}) + + def test_sqlite_cursor_overrides_compatibility_state(monkeypatch, tmp_path: Path) -> None: manager, state, _ = _isolate(monkeypatch, tmp_path) manager.init_cve_database() @@ -80,3 +90,22 @@ def test_status_uses_coverage_metadata_not_row_threshold(monkeypatch, tmp_path: ) assert manager.describe_cve_db_local_status().startswith("ok (1 CVEs loaded)") + + +def test_invalid_cursor_uses_last_updated_fallback(monkeypatch, tmp_path: Path) -> None: + manager, _, _ = _isolate(monkeypatch, tmp_path) + manager.init_cve_database() + with manager._connect() as conn: + conn.execute( + "INSERT INTO cve_entries (cve_id, description) VALUES (?, ?)", + ("CVE-2026-0001", "test"), + ) + conn.executemany( + "INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)", + [ + ("nvd_cursor", "invalid"), + ("last_updated", manager._utcnow().isoformat()), + ], + ) + + assert manager.cve_db_needs_update() is False diff --git a/tests/test_cve_paths.py b/tests/test_cve_paths.py index 810275d..e075091 100644 --- a/tests/test_cve_paths.py +++ b/tests/test_cve_paths.py @@ -22,13 +22,14 @@ def _valid_legacy_db(path: Path) -> None: def test_cve_paths_honor_data_dir_override(monkeypatch, tmp_path: Path) -> None: target = tmp_path / "custom-data" - monkeypatch.setenv("BITSENTRY_DATA_DIR", str(target)) - import scanner.paths as paths - paths = importlib.reload(paths) - assert Path(paths.CVE_DB_PATH) == target / "cve_db.sqlite" - assert Path(paths.CVE_META_PATH) == target / "cve_meta.json" + with monkeypatch.context() as patch: + patch.setenv("BITSENTRY_DATA_DIR", str(target)) + paths = importlib.reload(paths) + assert Path(paths.CVE_DB_PATH) == target / "cve_db.sqlite" + assert Path(paths.CVE_META_PATH) == target / "cve_meta.json" + importlib.reload(paths) def test_migration_copies_valid_legacy_database_atomically(tmp_path: Path) -> None: diff --git a/tests/test_cve_resumability.py b/tests/test_cve_resumability.py index 072a437..269b740 100644 --- a/tests/test_cve_resumability.py +++ b/tests/test_cve_resumability.py @@ -1,9 +1,12 @@ from __future__ import annotations import sys +from contextlib import closing from pathlib import Path from unittest import mock +import pytest + _BITPROBE = Path(__file__).resolve().parents[1] / "bitprobe" if str(_BITPROBE) not in sys.path: @@ -12,10 +15,13 @@ def _db(monkeypatch, tmp_path: Path): import scanner.cve_db_manager as manager + import scanner.update_state as state monkeypatch.setattr(manager, "CVE_DB_PATH", str(tmp_path / "cve.sqlite")) monkeypatch.setattr(manager, "CVE_META_PATH", str(tmp_path / "meta.json")) monkeypatch.setattr(manager, "migrate_legacy_cve_database", lambda: False) + monkeypatch.setattr(state, "STATE_DIR", tmp_path / "state") + monkeypatch.setattr(state, "STATE_PATH", tmp_path / "state" / "state.json") manager.init_cve_database() return manager @@ -61,8 +67,10 @@ def test_completed_window_does_not_resume(monkeypatch, tmp_path: Path) -> None: def test_resumed_full_build_preserves_committed_rows(monkeypatch, tmp_path: Path) -> None: manager = _db(monkeypatch, tmp_path) + manager.CVE_DB_PATH = str(tmp_path / ".cve.sqlite.full-build") + manager.init_cve_database() target = "2026-08-18T00:00:00.000" - with manager._connect() as conn: + with closing(manager._connect()) as conn: conn.execute( "INSERT INTO cve_entries (cve_id, description) VALUES (?, ?)", ("CVE-1999-0001", "already committed"), @@ -81,6 +89,9 @@ def test_resumed_full_build_preserves_committed_rows(monkeypatch, tmp_path: Path "2026-08-17T00:00:00.000", ), ) + conn.commit() + + manager.CVE_DB_PATH = str(tmp_path / "cve.sqlite") response = mock.Mock(status_code=200) response.json.return_value = {"vulnerabilities": [], "totalResults": 0} @@ -93,3 +104,24 @@ def test_resumed_full_build_preserves_committed_rows(monkeypatch, tmp_path: Path "SELECT COUNT(*) FROM cve_entries WHERE cve_id='CVE-1999-0001'" ).fetchone()[0] == 1 assert manager.read_cve_metadata()["coverage_mode"] == "full" + + +def test_failed_full_build_preserves_active_database(monkeypatch, tmp_path: Path) -> None: + manager = _db(monkeypatch, tmp_path) + with closing(manager._connect()) as conn: + conn.execute( + "INSERT INTO cve_entries (cve_id, description) VALUES (?, ?)", + ("CVE-2026-0001", "active"), + ) + conn.commit() + monkeypatch.setattr( + manager, + "_update_cve_database_unlocked", + mock.Mock(side_effect=RuntimeError("interrupted")), + ) + + with pytest.raises(RuntimeError, match="interrupted"): + manager.update_cve_database(full_sync=True) + + with closing(manager._connect()) as conn: + assert conn.execute("SELECT description FROM cve_entries").fetchone()[0] == "active" diff --git a/tests/test_cve_sync_windows.py b/tests/test_cve_sync_windows.py index 7260d58..2993dca 100644 --- a/tests/test_cve_sync_windows.py +++ b/tests/test_cve_sync_windows.py @@ -74,6 +74,8 @@ def test_years_mode_sends_only_nvd_safe_windows(monkeypatch, tmp_path: Path) -> monkeypatch.setattr(manager, "migrate_legacy_cve_database", lambda: False) monkeypatch.setattr(state, "STATE_DIR", tmp_path / "state") monkeypatch.setattr(state, "STATE_PATH", tmp_path / "state" / "state.json") + now = datetime(2026, 8, 18) + monkeypatch.setattr(manager, "_utcnow", lambda: now) seen = [] def response_for(params, *_args, **_kwargs): @@ -85,7 +87,8 @@ def response_for(params, *_args, **_kwargs): monkeypatch.setattr(manager, "_nvd_get", response_for) manager.update_cve_database(years=1, incremental=False) - assert len(seen) == 4 + start = now - timedelta(days=365) + assert len(seen) == len(list(manager.iter_nvd_windows(start, now))) for params in seen: start = datetime.fromisoformat(params["pubStartDate"]) end = datetime.fromisoformat(params["pubEndDate"]) @@ -94,3 +97,27 @@ def response_for(params, *_args, **_kwargs): assert metadata["coverage_mode"] == "windowed" assert metadata["coverage_start"] == seen[0]["pubStartDate"] assert metadata["coverage_end"] == seen[-1]["pubEndDate"] + + +def test_raw_full_never_uses_date_filters(monkeypatch, tmp_path: Path) -> None: + import scanner.cve_db_manager as manager + import scanner.update_state as state + + monkeypatch.setattr(manager, "CVE_DB_PATH", str(tmp_path / "cve.sqlite")) + monkeypatch.setattr(manager, "CVE_META_PATH", str(tmp_path / "meta.json")) + monkeypatch.setattr(manager, "migrate_legacy_cve_database", lambda: False) + monkeypatch.setattr(state, "STATE_DIR", tmp_path / "state") + monkeypatch.setattr(state, "STATE_PATH", tmp_path / "state" / "state.json") + seen = [] + + def response_for(params, *_args, **_kwargs): + seen.append(dict(params)) + response = mock.Mock(status_code=200) + response.json.return_value = {"vulnerabilities": [], "totalResults": 0} + return response + + monkeypatch.setattr(manager, "_nvd_get", response_for) + manager.update_cve_database(full_sync=True, raw_full_sync=True, force=False) + + assert len(seen) == 1 + assert not {"lastModStartDate", "lastModEndDate", "pubStartDate", "pubEndDate"} & seen[0].keys() diff --git a/tests/test_cve_workflow.py b/tests/test_cve_workflow.py index b684213..7529b7d 100644 --- a/tests/test_cve_workflow.py +++ b/tests/test_cve_workflow.py @@ -1,19 +1,31 @@ +from __future__ import annotations + from pathlib import Path +import yaml + def test_cve_workflow_has_safe_producer_contract() -> None: root = Path(__file__).resolve().parents[1] - workflow = (root / ".github/workflows/update-cve-db.yml").read_text(encoding="utf-8") + workflow = yaml.safe_load( + (root / ".github/workflows/update-cve-db.yml").read_text(encoding="utf-8") + ) + triggers = workflow.get("on", workflow.get(True)) + job = workflow["jobs"]["sync-and-publish"] + steps = job["steps"] + names = [step.get("name") for step in steps] - assert 'cron: "17 6 * * *"' in workflow - assert "full_rebuild:" in workflow - assert "contents: write" in workflow - assert "group: cve-db-producer" in workflow - assert 'python-version: "3.13"' in workflow - assert "NVD_API_KEY" in workflow - assert workflow.index("Restore previous snapshot") < workflow.index("Update canonical database") - assert "build_cve_snapshot.py" in workflow - assert "update_cve_snapshot_release.sh" in workflow + assert triggers["schedule"][0]["cron"] == "17 6 * * *" + assert "full_rebuild" in triggers["workflow_dispatch"]["inputs"] + assert workflow["permissions"]["contents"] == "write" + assert workflow["concurrency"]["group"] == "cve-db-producer" + assert job["timeout-minutes"] == 360 + assert steps[0]["with"]["persist-credentials"] is False + assert next(step for step in steps if step.get("uses") == "actions/setup-python@v5")["with"]["python-version"] == "3.13" + assert job["env"]["NVD_API_KEY"] == "${{ secrets.NVD_API_KEY }}" + assert names.index("Restore previous snapshot") < names.index("Update canonical database") + assert "build_cve_snapshot.py" in next(step["run"] for step in steps if step.get("name") == "Build snapshot") + assert "update_cve_snapshot_release.sh" in next(step["run"] for step in steps if step.get("name") == "Publish releases") def test_release_script_keeps_database_releases_out_of_latest() -> None: diff --git a/tests/test_products_cli.py b/tests/test_products_cli.py index 926e54d..0f1b664 100644 --- a/tests/test_products_cli.py +++ b/tests/test_products_cli.py @@ -3,6 +3,8 @@ import json from unittest import mock +import pytest + import products from bitsentry import main as bitsentry_main @@ -63,25 +65,27 @@ def test_update_cve_db_passthrough() -> None: assert "7" in cmd -def test_update_cve_db_default_preserves_snapshot_policy() -> None: +def test_update_cve_db_default_preserves_snapshot_policy( + monkeypatch: pytest.MonkeyPatch, +) -> None: completed = mock.Mock(returncode=0) - with ( - mock.patch("subprocess.run", return_value=completed) as run_mock, - mock.patch("sys.argv", ["bitsentry", "update-cve-db"]), - ): - code = bitsentry_main() + run_mock = mock.Mock(return_value=completed) + monkeypatch.setattr("subprocess.run", run_mock) + monkeypatch.setattr("sys.argv", ["bitsentry", "update-cve-db"]) + code = bitsentry_main() assert code == 0 assert run_mock.call_args.args[0][2:] == ["update-cve-db"] -def test_update_cve_db_snapshot_flags_are_forwarded() -> None: +def test_update_cve_db_snapshot_flags_are_forwarded( + monkeypatch: pytest.MonkeyPatch, +) -> None: completed = mock.Mock(returncode=0) for flag in ("--snapshot-only", "--no-snapshot", "--raw-full"): - with ( - mock.patch("subprocess.run", return_value=completed) as run_mock, - mock.patch("sys.argv", ["bitsentry", "update-cve-db", flag]), - ): - code = bitsentry_main() + run_mock = mock.Mock(return_value=completed) + monkeypatch.setattr("subprocess.run", run_mock) + monkeypatch.setattr("sys.argv", ["bitsentry", "update-cve-db", flag]) + code = bitsentry_main() assert code == 0 assert flag in run_mock.call_args.args[0] diff --git a/tests/test_update_lock.py b/tests/test_update_lock.py index d22ee32..e02f102 100644 --- a/tests/test_update_lock.py +++ b/tests/test_update_lock.py @@ -38,10 +38,13 @@ def test_lock_contention_fails_without_waiting(tmp_path: Path) -> None: def test_cve_update_uses_shared_lock(monkeypatch, tmp_path: Path) -> None: import scanner.cve_db_manager as manager + import scanner.update_state as state monkeypatch.setattr(manager, "CVE_DB_PATH", str(tmp_path / "cve.sqlite")) monkeypatch.setattr(manager, "CVE_META_PATH", str(tmp_path / "meta.json")) monkeypatch.setattr(manager, "migrate_legacy_cve_database", lambda: False) + monkeypatch.setattr(state, "STATE_DIR", tmp_path / "state") + monkeypatch.setattr(state, "STATE_PATH", tmp_path / "state" / "state.json") lock = mock.Mock(return_value=nullcontext()) monkeypatch.setattr(manager, "bitsentry_update_lock", lock) response = mock.Mock(status_code=200) From 13ff840b412851c8b04c689309f254bf1500d8e1 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 18 Aug 2026 19:33:38 +0000 Subject: [PATCH 4/4] Strengthen CVE fallback tests --- tests/test_cve_metadata.py | 17 +++++++++++++++-- tests/test_cve_sync_windows.py | 19 ++++++++++++++----- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/tests/test_cve_metadata.py b/tests/test_cve_metadata.py index 581a088..a93977c 100644 --- a/tests/test_cve_metadata.py +++ b/tests/test_cve_metadata.py @@ -2,6 +2,8 @@ import sqlite3 import sys +from contextlib import closing +from datetime import timedelta from pathlib import Path import pytest @@ -95,7 +97,8 @@ def test_status_uses_coverage_metadata_not_row_threshold(monkeypatch, tmp_path: def test_invalid_cursor_uses_last_updated_fallback(monkeypatch, tmp_path: Path) -> None: manager, _, _ = _isolate(monkeypatch, tmp_path) manager.init_cve_database() - with manager._connect() as conn: + now = manager._utcnow() + with closing(manager._connect()) as conn: conn.execute( "INSERT INTO cve_entries (cve_id, description) VALUES (?, ?)", ("CVE-2026-0001", "test"), @@ -104,8 +107,18 @@ def test_invalid_cursor_uses_last_updated_fallback(monkeypatch, tmp_path: Path) "INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)", [ ("nvd_cursor", "invalid"), - ("last_updated", manager._utcnow().isoformat()), + ("last_updated", (now - timedelta(days=30)).isoformat()), ], ) + conn.commit() + + assert manager.cve_db_needs_update() is True + + with closing(manager._connect()) as conn: + conn.execute( + "INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)", + ("last_updated", now.isoformat()), + ) + conn.commit() assert manager.cve_db_needs_update() is False diff --git a/tests/test_cve_sync_windows.py b/tests/test_cve_sync_windows.py index 2993dca..012c843 100644 --- a/tests/test_cve_sync_windows.py +++ b/tests/test_cve_sync_windows.py @@ -88,11 +88,20 @@ def response_for(params, *_args, **_kwargs): manager.update_cve_database(years=1, incremental=False) start = now - timedelta(days=365) - assert len(seen) == len(list(manager.iter_nvd_windows(start, now))) - for params in seen: - start = datetime.fromisoformat(params["pubStartDate"]) - end = datetime.fromisoformat(params["pubEndDate"]) - assert end - start <= timedelta(days=119) + expected_windows = list(manager.iter_nvd_windows(start, now)) + actual_windows = [ + ( + datetime.fromisoformat(params["pubStartDate"]), + datetime.fromisoformat(params["pubEndDate"]), + ) + for params in seen + ] + assert len(seen) == len(expected_windows) + assert actual_windows == expected_windows + assert all( + window_end - window_start <= timedelta(days=119) + for window_start, window_end in actual_windows + ) metadata = manager.read_cve_metadata() assert metadata["coverage_mode"] == "windowed" assert metadata["coverage_start"] == seen[0]["pubStartDate"]