From 57d1066795906a4ae9508de3168e95ab815786ca Mon Sep 17 00:00:00 2001 From: inkantak Date: Wed, 5 Aug 2026 21:02:28 +0530 Subject: [PATCH 1/2] Refactor AST traversal for improved performance and accuracy - Introduced a file indexing mechanism to reduce redundant AST walks across multiple rules, significantly improving scan performance. - Replaced direct descendant traversal with indexed methods for retrieving call expressions and function nodes. - Updated various rules to utilize the new indexing functions, ensuring consistent behavior and performance gains. - Added tests to validate the correctness of the new indexing methods against traditional traversal methods. - Introduced new test fixtures to cover edge cases and ensure robustness of the scanning process. - Enhanced Python AST parsing to correctly identify and handle calls, attributes, and other constructs, improving vulnerability detection. --- .github/workflows/ci.yml | 4 + .github/workflows/codeql.yml | 28 + .github/workflows/dependency-review.yml | 16 + .github/workflows/publish.yml | 38 - .github/workflows/scorecard.yml | 40 + .github/workflows/secureai-scan.yml | 16 +- .secureai-policy.json | 3 +- CHANGELOG.md | 39 +- CITATION.cff | 19 + CLAUDE.md | 19 +- GOVERNANCE.md | 35 + PUBLISHING.md | 385 +--- README.md | 74 +- ROADMAP.md | 17 +- action.yml | 21 +- docs/Architecture.md | 10 +- docs/Contributing.md | 2 +- docs/FAQ.md | 4 +- docs/Performance.md | 48 +- docs/ReleaseAssurance.md | 42 + docs/benchmarks/v0.8.0.json | 59 + docs/index.html | 18 +- docs/llms.txt | 14 + docs/robots.txt | 4 + docs/sitemap.xml | 7 + package-lock.json | 31 +- package.json | 22 +- scripts/regression-scan.js | 108 +- scripts/sync-advisories.js | 220 ++ spike/python-ast-poc/README.md | 51 - spike/python-ast-poc/ai001-poc.ts | 224 -- spike/python-ast-poc/tsconfig.json | 14 - src/scanner/advisories-generated.ts | 1803 +++++++++++++++++ src/scanner/advisories.ts | 30 +- src/scanner/confidence.ts | 13 +- src/scanner/dependency-guard.ts | 161 +- src/scanner/python-ast.ts | 201 ++ src/scanner/python-scanner.ts | 650 +++--- src/scanner/python-source.ts | 14 + src/scanner/rules/excessive-agency.ts | 4 +- .../rules/indirect-prompt-injection.ts | 13 +- src/scanner/rules/llm-before-auth.ts | 6 +- src/scanner/rules/llm-rule-utils.ts | 64 +- .../rules/mcp-dynamic-server-command.ts | 11 +- src/scanner/rules/mcp-dynamic-server-url.ts | 11 +- src/scanner/rules/mcp-tool-desc-injection.ts | 13 +- .../rules/mcp-unvalidated-tool-result.ts | 13 +- .../rules/multiagent-trust-boundary.ts | 13 +- src/scanner/rules/prompt-injection-concat.ts | 18 +- src/scanner/rules/rag-context-injection.ts | 4 +- src/scanner/rules/sensitive-data-to-llm.ts | 4 +- src/scanner/rules/sensitive-prompt-logging.ts | 4 +- src/scanner/rules/system-prompt-leakage.ts | 4 +- src/scanner/rules/unbounded-llm-input.ts | 4 +- src/scanner/rules/unsafe-output-handling.ts | 15 +- .../rules/unvalidated-structured-output.ts | 15 +- .../rules/vec-ingestion-no-namespace.ts | 4 +- .../rules/vec-search-no-access-control.ts | 4 +- src/scanner/rules/vec-unbounded-search.ts | 4 +- src/scanner/rules/vec-user-ingestion.ts | 13 +- src/utils/ast.ts | 86 +- test-fixtures/safe/docstring_example.py | 48 + test-fixtures/safe/vec_chained_filter.py | 15 + test-fixtures/vulnerable/attribute_taint.py | 43 + test/ast-index.test.js | 107 + test/corpus.test.js | 3 + test/dependency-guard.test.js | 58 + test/false-positive-regressions.test.js | 76 + test/python-ast.test.js | 142 ++ test/regression-baseline.json | 40 + test/run-tests.js | 2 + 71 files changed, 4149 insertions(+), 1216 deletions(-) create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/dependency-review.yml delete mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/scorecard.yml create mode 100644 CITATION.cff create mode 100644 GOVERNANCE.md create mode 100644 docs/ReleaseAssurance.md create mode 100644 docs/benchmarks/v0.8.0.json create mode 100644 docs/llms.txt create mode 100644 docs/robots.txt create mode 100644 docs/sitemap.xml create mode 100644 scripts/sync-advisories.js delete mode 100644 spike/python-ast-poc/README.md delete mode 100644 spike/python-ast-poc/ai001-poc.ts delete mode 100644 spike/python-ast-poc/tsconfig.json create mode 100644 src/scanner/advisories-generated.ts create mode 100644 src/scanner/python-ast.ts create mode 100644 src/scanner/python-source.ts create mode 100644 test-fixtures/safe/docstring_example.py create mode 100644 test-fixtures/safe/vec_chained_filter.py create mode 100644 test-fixtures/vulnerable/attribute_taint.py create mode 100644 test/ast-index.test.js create mode 100644 test/python-ast.test.js create mode 100644 test/regression-baseline.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f43b89c..10d3f4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,9 @@ on: push: branches: [main] +permissions: + contents: read + jobs: test: strategy: @@ -34,6 +37,7 @@ jobs: - run: npm ci - run: npm run coverage - uses: actions/upload-artifact@v7 + if: always() with: name: coverage-report path: coverage/ diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..84eb461 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,28 @@ +name: CodeQL + +on: + pull_request: + push: + branches: [main] + schedule: + - cron: "23 4 * * 1" + +permissions: + contents: read + security-events: write + +jobs: + analyze: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: github/codeql-action/init@v4.37.3 + with: + languages: javascript-typescript + - uses: actions/setup-node@v7 + with: + node-version: "22" + cache: npm + - run: npm ci + - run: npm run build + - uses: github/codeql-action/analyze@v4.37.3 \ No newline at end of file diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 0000000..40e97b2 --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,16 @@ +name: Dependency Review + +on: + pull_request: + +permissions: + contents: read + +jobs: + review: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/dependency-review-action@v4 + with: + fail-on-severity: high \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 3d4d7b8..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Publish to npm - -# Requires an NPM_TOKEN secret configured in the repo settings -# (Settings -> Secrets and variables -> Actions) before this will actually -# publish. Documented in PUBLISHING.md, which previously described this -# workflow but never had it committed — publishing was manual until now. - -on: - push: - tags: - - "v*" # Triggers on any tag like v0.2.0, v1.0.0 - -jobs: - publish: - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-node@v7 - with: - node-version: "22" - registry-url: "https://registry.npmjs.org" - - - name: Install dependencies - run: npm ci - - - name: Build - run: npm run build - - - name: Run tests - run: npm test - - - name: Publish to npm - run: npm publish --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 0000000..c7a9a44 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,40 @@ +name: OpenSSF Scorecard + +on: + branch_protection_rule: + schedule: + - cron: "31 5 * * 1" + push: + branches: [main] + +permissions: read-all + +jobs: + scorecard: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + security-events: write + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Run OpenSSF Scorecard + uses: ossf/scorecard-action@v2.4.2 + with: + results_file: scorecard-results.sarif + results_format: sarif + publish_results: true + - name: Upload Scorecard artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: scorecard-results + path: scorecard-results.sarif + retention-days: 5 + - name: Upload Scorecard to code scanning + if: always() + uses: github/codeql-action/upload-sarif@v4.37.3 + with: + sarif_file: scorecard-results.sarif \ No newline at end of file diff --git a/.github/workflows/secureai-scan.yml b/.github/workflows/secureai-scan.yml index d0be357..603cb78 100644 --- a/.github/workflows/secureai-scan.yml +++ b/.github/workflows/secureai-scan.yml @@ -17,19 +17,13 @@ jobs: with: node-version: "20" cache: "npm" - - name: Run SecureAI-Scan (non-blocking) - run: | - npx --yes secureai-scan@latest scan . \ - --baseline secureai-baseline.json \ - --output report.md || true + - run: npm ci + - run: npm run build + - name: Run SecureAI-Scan + run: node dist/index.js scan . --fail-on high --output report.md - name: Upload report artifact + if: always() uses: actions/upload-artifact@v7 with: name: secureai-scan-report path: report.md - - # Optional strict mode: fail CI when High/Critical findings are present. - # - name: Fail on High/Critical findings - # run: | - # npx --yes secureai-scan@latest scan . --severity high --output report.json - # node -e "const fs=require('node:fs'); const r=JSON.parse(fs.readFileSync('report.json','utf-8')); if ((r.summary.bySeverity.critical + r.summary.bySeverity.high) > 0) process.exit(1)" diff --git a/.secureai-policy.json b/.secureai-policy.json index d87dcaa..1e06833 100644 --- a/.secureai-policy.json +++ b/.secureai-policy.json @@ -1,9 +1,10 @@ { "$comment": "SecureAI-Scan policy file — commit this to your repo. See: secureai-scan explain ", + "$skipPathsRationale": "test-fixtures/ is deliberately vulnerable code that the suite requires to fire (see test/corpus.test.js, which scans it directly as its own root); .regression-cache/ holds repos cloned by npm run regression. Neither is this project's source.", "minSeverity": "medium", "minConfidence": 0.45, "failOnSeverity": "high", - "skipPaths": [], + "skipPaths": ["test-fixtures", ".regression-cache"], "blockedRules": [], "onlyRules": [], "requireOutputValidation": true diff --git a/CHANGELOG.md b/CHANGELOG.md index 55087bc..1768861 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,42 @@ # Changelog +## 0.8.0 — 2026-08-05 + +Audit-driven hardening. Every item below came from reviewing the project the way an external evaluator would: claims checked against code, then the gaps closed. + +### Added +- **Public trust controls for a single-maintainer project.** `GOVERNANCE.md` identifies release authority, review expectations, bus-factor limits, and succession behavior. `docs/ReleaseAssurance.md` documents release controls and non-guarantees, while `docs/benchmarks/v0.8.0.json` preserves machine-readable test, coverage, regression, and package evidence. +- **Independent security workflows.** CodeQL, dependency review, and OpenSSF Scorecard now run alongside the existing cross-platform CI. The project's own SecureAI-Scan workflow is blocking at high severity and uploads its report even on failure. +- **Search and evaluation metadata.** The README leads with a pinned install command and measured release evidence; GitHub Pages now exposes canonical metadata, structured application data, `robots.txt`, a sitemap, and `llms.txt` for MCP security, tool-poisoning, prompt-injection, and Agent Skill scanner discovery. `CITATION.cff` gives researchers and ecosystem audits a version-aware citation path. +- **Release integrity is now enforced locally.** `npm run release:check` runs the complete test suite, coverage thresholds, the reviewed real-repository regression gate, and an npm tarball dry run. `prepublishOnly` runs the same gate before a maintainer can publish manually; GitHub Actions never receives npm credentials or publishes packages. +- **The GitHub Action installs an exact scanner version by default.** Its shell inputs are passed through environment variables and Bash arrays instead of expression-expanded command strings, preventing workflow inputs from being reinterpreted as shell syntax. +- **Python is now AST-based.** Every `.py` file is parsed once with `tree-sitter` + `tree-sitter-python`; `src/scanner/python-ast.ts` indexes imports, calls, positional/keyword arguments, assignment targets (identifier, attribute, tuple/list), functions, decorators, scopes, dictionary fields, and strings for all Python rules. The interim lexical `code`/`logical` views and hand-written assignment parser were deleted — structural detection no longer depends on physical-line regex. This closes fake imports/calls in comments and docstrings, multiline calls and keyword arguments, class-handler attribute taint (`self.user_message = request.json[...]`), tuple assignment, and decorated async handlers. Tree-sitter error recovery keeps incomplete files scanable; target code is never imported or executed and no Python interpreter is required. +- **Python AST contract tests** cover node indexing, receiver/call identity, attribute and tuple taint, decorators, multiline keywords, malformed-file recovery, and fake syntax inside comments/docstrings. The existing safe/vulnerable corpus and real-repo regression gate validate the full rule surface. +- **A shared per-file AST index** (`getFileCalls` / `getFileFunctions` / `getCallsWithin` in `src/utils/ast.ts`), replacing the ~20 independent whole-file walks rules used to each perform. `getCallsWithin` slices the pre-order index by compiler span instead of walking a subtree. +- **DEP003 advisory data is now generated from OSV, not hand-typed.** `npm run sync-advisories` ([`scripts/sync-advisories.js`](scripts/sync-advisories.js)) pulls HIGH/CRITICAL advisories for an explicit LLM/MCP/RAG package watchlist and writes `src/scanner/advisories-generated.ts`. The check stays fully offline at scan time — the snapshot is bundled — but it is now refreshable instead of stale-by-construction. **2 → 164 advisories.** Advisories whose affected range can't be parsed into an exact comparison are dropped rather than shipped, since an always-on advisory is a false positive by construction. +- **`npm run regression` is a real gate.** It now writes a structured report per repo, fingerprints every `proven`/`likely` finding as `repo|rule|file` (line-free, so upstream churn isn't noise), and **exits non-zero on anything not in the reviewed baseline** at `test/regression-baseline.json`. `--update-baseline` accepts the current set. Previously this was "run it and read the output with judgment," which is not a gate. +- **Python LLM sinks are resolved through AST constructor bindings.** Variables (and `self.x` attributes) assigned from a known SDK constructor — `gateway = OpenAI()`, `bedrock = boto3.client("bedrock-runtime")` — make invocation-shaped calls on that receiver visible whatever it is named. This is the Python analogue of the TS scanner's import-resolved `resolveLlmSink`. + +### Performance +**A `vercel/ai` scan (5,691 files) went from 217s to 62s — 3.5× — with identical findings.** Driven by a CPU profile rather than guesswork; details and the remaining known gaps are in [`docs/Performance.md`](docs/Performance.md), which previously claimed scan times were "in the multi-second range" and was simply wrong. + +- The profile attributed **~95s of 150s to AST descendant iteration alone**, more than everything else combined — type resolution barely registered. Cause: every rule ran its own `getDescendantsOfKind`/`getDescendants` per file, so each file's AST was walked ~20 times per scan, and nested functions were re-walked once per enclosing scope on top of that. Rules now share one memoized pre-order walk per file. +- `resolveLlmSink` checks the generation-shaped method name and the file's imports *before* consulting the type checker. Since `resolveIdentifierModule` can only ever report a specifier the file itself imports, a file with no LLM SDK import cannot produce a resolved sink — the type checker never needed asking for the vast majority of files. +- `npm test` dropped from 42s to 16s as a side effect. +- `test/ast-index.test.js` asserts the index and the containment slice against ts-morph's own traversal (exact membership *and* document order). The optimization's failure mode is silence, so it is guarded by a correctness test rather than a flaky wall-clock one. + +### Fixed +- **Python DEP003 compared the wrong version.** `readRequirementsCandidates` stripped the comparison operator, so `langchain>=0.1.0` was parsed as an exact pin of `0.1.0` and then tested against advisory ranges as though that were the installed version — producing `proven`-tier findings about a version the repo never declared. Only `==`/`===` are treated as exact pins now; every other specifier keeps its operator and resolves to "unknown". +- **Renaming a Python LLM client silently disabled every rule.** `LLM_CALL_PATTERNS` hardcoded the receiver names `client`, `llm`, `chain`, `model`, `co`, so an app using any other variable name got zero AI-rule coverage with no indication anything was skipped. The legacy name patterns are kept for cross-module clients but every rule relying on them (AI001, AI003, AI004, AI010) is now gated on the file actually importing an LLM SDK — closing the inverse false positive, where a `chain.invoke(...)` in an unrelated ETL file was reported as prompt injection. +- **`test-fixtures/`-style directories weren't recognized as non-production.** `NON_PRODUCTION_SEGMENT` matched suffixed conventions (`ecosystem-tests`) but not prefixed ones, so a real repo laid out with `test-fixtures/`, `example-app/`, or `demo-server/` got undemoted findings — this repo's own self-scan exited 1 with 38 high-severity findings from its own fixtures. Both affix positions are handled now, still per-segment so `attestation`/`protest` are untouched. +- **`isTestFile` in the Python scanner recognized fewer paths than the TypeScript one.** It now delegates to the shared `isTestFilePath` and keeps only the Python-specific additions (`test_*.py`, `*_test.py`, `conftest.py`). + +### Changed +- **CVEs and malicious packages get different ambiguity handling in DEP003.** A documented-malicious package still fires when the version can't be resolved — installing a backdoor is unrecoverable. A CVE now fires at `proven` only when the declared version is an exact pin provably inside the affected range; unpinned-but-possibly-affected drops to `heuristic` (`--paranoid`). Applying the malicious-kind rule to a 162-entry CVE snapshot would have put a critical finding on every repo with `langchain>=0.1.0`. +- Multiple advisories on one package now group into a single finding instead of one per CVE. +- The repo's own [`.secureai-policy.json`](.secureai-policy.json) now skips `test-fixtures/` and `.regression-cache/`, so `secureai-scan scan .` on this repo is a real signal instead of a wall of intentionally-vulnerable fixture hits. It reports clean. +- README: dropped the unprovable "first scanner mapped to all three OWASP frameworks" claim, and corrected the advisory-list and regression-benchmark descriptions to match what the code actually does. + ## 0.7.0 — 2026-08-01 ### Added @@ -26,7 +63,7 @@ Evasion-resistant Agent Skill scanning. In July 2026, [*Cloak and Detonate*](https://arxiv.org/abs/2607.02357) (arXiv:2607.02357) showed that nine published skill scanners could be bypassed by >80% (structural obfuscation) and ≥90% (self-extracting packing) using transformations that preserve the payload exactly; separately, Gecko Security demonstrated an exfiltration payload hidden in a `*.test.ts` file that every public scanner skipped. SecureAI-Scan v0.5.0 was vulnerable to all of these. This release closes each published technique, and was additionally validated against two real-world corpora added to `scripts/regression-scan.js`: the canonical [anthropics/skills](https://github.com/anthropics/skills) repo (18 real skill bundles, zero findings — a pure precision check) and [cisco-ai-defense/skill-scanner](https://github.com/cisco-ai-defense/skill-scanner)'s own labeled eval corpus (20 skills under `evals/`, each with an `_expected.json` verdict and a directory literally named `malicious/` or `safe/`) — a rare case where a real-world repo doubles as a recall check, not just a precision one. Result: 6/6 in-scope malicious fixtures correctly flagged, zero findings on any fixture labeled safe. ### Added -- **Python AST migration spike (`spike/python-ast-poc/`)** — not part of the shipped package (`tree-sitter-python`/`web-tree-sitter` are `devDependencies` only). Confirms `web-tree-sitter` + `tree-sitter-python`'s bundled `.wasm` grammar parses real Python with zero native compilation, and ports enough of AI001 to demonstrate a concrete, real gap in the current regex scanner: `self.user_message = request.json[...]` (any class-based handler — Flask `MethodView`, FastAPI DI classes) is completely invisible to `collectRequestTaintedVars`, which only recognizes bare-identifier assignment targets, even at `--paranoid`. The AST-based POC catches it with no special-casing. See `ROADMAP.md` for the full findings and effort estimate. +- **Python AST migration spike** — established the feasibility of Tree-sitter and demonstrated the class-handler attribute-taint gap (`self.user_message = request.json[...]`). The spike was later superseded and removed when the production AST engine shipped; see the Unreleased section. - **`secureai-scan skill ` / `secureai-scan mcp `** — the pre-install wedge: fetch and scan a single Agent Skill or MCP server *before* trusting it, with no clone, no config, and nothing fetched ever executed. `target` accepts a local path, a full git URL, a GitHub `owner/repo` shorthand, or (for `mcp`) a bare npm package name. npm targets are downloaded with `npm pack` — the tarball only, no `install`, no lifecycle scripts; git targets with `git clone --depth 1`. New module: `src/scanner/fetch-target.ts`. - **MCP server: `scan_untrusted_target` tool** — the same fetch-and-scan capability exposed to Claude itself via the bundled MCP server (`mcp-server/index.js`), so an agent can check a skill or MCP server before recommending or installing it, from inside the conversation. `skills/secureai-scan/SKILL.md` updated to use this flow instead of its previous "clone it first" instructions. - **Deobfuscation layer (`src/scanner/deobfuscate.ts`)** — content checks now match against normalized *variants* of the text rather than one fixed byte sequence: zero-width/bidi stripping, Unicode homoglyph folding, spliced-string-literal joining (`'cu' + 'rl'`), and intra-word line-break joining. Transforms are applied cumulatively, so a payload cloaked with two techniques at once is still recovered. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..a679140 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,19 @@ +cff-version: 1.2.0 +message: "If you use SecureAI-Scan in research or an ecosystem audit, cite the software and the exact version tested." +title: "SecureAI-Scan: evidence-tiered static analysis for LLM, MCP, RAG, and Agent Skill security" +type: software +authors: + - family-names: Kanthed + given-names: Akshay +repository-code: "https://github.com/akanthed/SecureAI-Scan" +url: "https://www.npmjs.com/package/secureai-scan" +license: MIT +abstract: "A local-first static AI security scanner for TypeScript, JavaScript, Python, MCP configurations, and Agent Skill bundles, with source-to-sink evidence, SARIF reporting, and reviewed real-repository regression testing." +keywords: + - AI security + - LLM security + - MCP security + - prompt injection + - Agent Skill security + - RAG poisoning + - static analysis \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 1e7f9d9..0177a06 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,11 +24,13 @@ For any change that touches detection logic (new rule, edited rule, edited share npm run regression ``` -This clones a curated set of real public repos (`scripts/regression-scan.js` — OpenAI/Anthropic/Vercel AI SDKs, official MCP servers and SDK, LlamaIndex) into `.regression-cache/` (gitignored, cached across runs — pass `--fresh` to re-clone, or a repo name to scan just one) and scans each with the built CLI. It is not a pass/fail gate — there's no fixed expected count, since upstream repos change. Read every `proven`/`likely` finding it prints against its source line: +This clones a curated set of real public repos (`scripts/regression-scan.js` — OpenAI/Anthropic/Vercel AI SDKs, official MCP servers and SDK, LlamaIndex) into `.regression-cache/` (gitignored, cached across runs — pass `--fresh` to re-clone, or a repo name to scan just one) and scans each with the built CLI. **It is a gate**: it exits non-zero on any `proven`/`likely` finding whose `repo|rule|file` fingerprint isn't in `test/regression-baseline.json`. Fingerprints omit line numbers so ordinary upstream churn isn't noise. Read every new finding against its source line: -- If it's a real issue in that repo, that's expected — leave it. +- If it's a real issue in that repo, accept it with `npm run regression -- --update-baseline`. - If it isn't, it's a bug in a rule. Fix the rule, then add the offending pattern as a new fixture under `test-fixtures/safe/` (with a comment noting which repo/file it came from) so `npm test` locks in the fix permanently. +Baselining a finding you haven't actually read defeats the whole mechanism — the baseline is a record of human review, not a mute button. + This is how several real false-positive classes were found and fixed in this codebase: `token_endpoint`/`tokenType` fields flagged as leaked secrets (AI002 matched "token" as a bare identifier token with no regard for context), example/demo/`ecosystem-tests`-style directories not being recognized as lower-trust paths (`isTestFilePath` only matched a narrow set of literal `/test/`/`/tests/` segments), `chunks` (an extremely common LLM streaming-response variable name) being treated as unambiguous RAG evidence (AI007), narrative/fiction prompt text containing the bare English words "secret"/"token" being flagged as leaked credentials (AI008 did a raw substring search with no requirement that the match look like an actual credential value), and a Python MCP001 check that flagged *any* `description=` field containing phrases like "system prompt" with `proven` evidence regardless of whether the file had anything to do with MCP at all. Consider that class of bug — a keyword/substring match applied without enough surrounding context, given undeservedly high evidence — the default failure mode to watch for. ### 3. Recall/true-positive validation matters as much as precision — and stay scoped to LLM/MCP/RAG @@ -67,9 +69,9 @@ node dist/index.js bom . ### Three independent scanning surfaces, merged into one output - **TS/JS**: AST-based via `ts-morph` (`src/scanner/project.ts` builds the `Project`; rules in `src/scanner/rules/*.ts` walk the AST). -- **Python**: regex-pattern based, not AST (`src/scanner/python-scanner.ts`). Patterns for LLM SDK calls, request-input taint sources, vector store calls, and exec sinks are matched line-by-line with a small taint-propagation pass. Being regex-based, it's more prone to context-free matches than the AST rules — see the MCP001 false-positive class above. +- **Python**: AST-based via `tree-sitter` + `tree-sitter-python` (`src/scanner/python-ast.ts` builds one indexed tree per file; `src/scanner/python-scanner.ts` runs the rules). Imports, calls, assignments, attributes, tuple/list targets, decorators, functions, scopes, arguments, dictionary fields, and strings must come from AST nodes — never reconstruct structure with raw-line regex. Regex is allowed only to classify a node's text after syntax establishes what the node is (for example, an injection phrase inside an AST-confirmed tool-description string). LLM receivers are bound from SDK constructor assignment nodes, so renamed clients remain visible. Tree-sitter error recovery is deliberate: a malformed file is scanned, not allowed to abort the repository. - **Config/content files scanned off disk directly** (not via the ts-morph `Project`): `src/scanner/mcp-config-scanner.ts` (`.mcp.json`, `claude_desktop_config.json`, `.cursor/mcp.json` → `MCP004`–`MCP006`) and `src/scanner/skill-scanner.ts` (Agent Skill bundles → `SKL001`–`SKL005`, reusing the same invisible-Unicode/injection-phrase/cross-reference checks in `tool-poisoning-checks.ts` that the MCP tool-poisoning rules use). -- **Dependency advisories** (`DEP001`–`DEP003`) run out of `src/scanner/dependency-guard.ts` against a curated offline list in `src/scanner/advisories.ts`. DEP003 (documented malicious/CVE packages) always runs; DEP001/DEP002 (registry lookups, typosquat detection) are opt-in via `--check-dependencies` since they need network access. +- **Dependency advisories** (`DEP001`–`DEP003`) run out of `src/scanner/dependency-guard.ts` against two bundled sources: the hand-curated `src/scanner/advisories.ts` (in-the-wild malicious releases) and the generated `src/scanner/advisories-generated.ts` (HIGH/CRITICAL OSV advisories for an LLM/MCP/RAG watchlist, regenerated with `npm run sync-advisories`). DEP003 always runs offline; DEP001/DEP002 (registry lookups, typosquat detection) are opt-in via `--check-dependencies` since they need network access. `src/scanner/scan.ts` (`scanRepositoryDetailed`) is the entry point that runs all of the above, merges findings, dedupes, and applies `// secureai-ignore RULE_ID: reason` suppression comments. @@ -77,7 +79,9 @@ node dist/index.js bom . Every AST rule lives in `src/scanner/rules/` and exports a `Rule` object (`id`, `title`, `severity`, `run(context)`). New rules must be registered in `src/scanner/rules/index.ts`'s `RULES` array (or the adjacent `CONFIG_RULE_IDS`/`SKILL_RULE_IDS`/`DEPENDENCY_RULE_IDS` for the non-AST scanners) — this is also the source of truth for `AVAILABLE_RULE_IDS`. -Shared helpers: `src/scanner/rules/llm-rule-utils.ts` (resolves whether a call is a real LLM SDK sink via import resolution, extracts prompt message parts *with role* via `getPromptParts` — always prefer this over writing a new ad hoc prompt-part extractor, since a rule-specific reimplementation is exactly what caused the AI007 false-positive class) and `src/utils/ast.ts` (node/line helpers, string-concat detection). +Shared helpers: `src/scanner/rules/llm-rule-utils.ts` (resolves whether a call is a real LLM SDK sink via import resolution, extracts prompt message parts *with role* via `getPromptParts` — always prefer this over writing a new ad hoc prompt-part extractor, since a rule-specific reimplementation is exactly what caused the AI007 false-positive class) and `src/utils/ast.ts` (node/line helpers, string-concat detection, and the shared per-file AST index). + +**Never call `sourceFile.getDescendants()` or `getDescendantsOfKind(...)` from a rule.** Use `getFileCalls(sourceFile)`, `getFileFunctions(sourceFile)`, and `getCallsWithin(node)` from `src/utils/ast.ts`, which serve one memoized pre-order walk per file. Every rule doing its own walk is what made a `vercel/ai` scan take 217s — a CPU profile put ~95s of 150s in descendant iteration alone; routing rules through the index cut the scan to 62s with identical findings. `getCallsWithin` relies on descendants being a contiguous run of the pre-order index, which `test/ast-index.test.js` asserts against ts-morph's own traversal; if that ever breaks, rules silently stop seeing calls. ### Evasion resistance (skill bundles) @@ -129,4 +133,7 @@ Any change to a rule's detection logic must be checked against this corpus, **an ## Package advisories (DEP003) -`src/scanner/advisories.ts` feeds `proven`-tier findings, so the bar for additions is strict (see `CONTRIBUTING.md`): only packages with a public incident report or CVE, with version ranges when documented, and a corresponding case added to `test/dependency-guard.test.js`. Don't add an advisory for an incident where the exact package name/ecosystem isn't publicly documented — a plausible guess is not the same as a citable fact. +Two sources, with deliberately different ambiguity handling — do not collapse them: + +- `src/scanner/advisories.ts` is hand-maintained and holds `kind: "malicious"` incidents (backdoored releases, in-the-wild compromises) that OSV does not carry. These feed `proven`/critical findings and **fire even when the declared version can't be resolved**, because installing a backdoor is unrecoverable. The bar for additions is strict (see `CONTRIBUTING.md`): only packages with a public incident report, with version ranges when documented, and a corresponding case added to `test/dependency-guard.test.js`. Don't add an advisory for an incident where the exact package name/ecosystem isn't publicly documented — a plausible guess is not the same as a citable fact. +- `src/scanner/advisories-generated.ts` is generated by `scripts/sync-advisories.js` from OSV and holds `kind: "vulnerable"` CVEs. These fire at `proven` **only when the declared version is an exact pin provably inside the affected range**. Applying the malicious-kind "fail toward flagging" rule here would produce a critical finding on every repo with `langchain>=0.1.0` — an unactionable false positive at scale. Unpinned-but-possibly-affected surfaces at `heuristic` (`--paranoid`) instead. Never edit the generated file by hand; change the watchlist or severity bar in the script and re-run `npm run sync-advisories`. diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 0000000..5eb8ea8 --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,35 @@ +# Governance + +SecureAI-Scan is currently maintained by Akshay Kanthed (`@akanthed`). This document makes the authority and limitations of a single-maintainer project explicit. + +## Decision authority + +The maintainer is responsible for releases, npm ownership, repository settings, security-response coordination, rule evidence tiers, and regression-baseline changes. Material decisions should be explained in a pull request, issue, changelog entry, or architecture document so they remain reviewable after the decision. + +No contributor, automation account, or AI tool has release authority. Automated contributions receive the same tests and review requirements as human contributions. + +## Contribution review + +Contributions are welcome under [CONTRIBUTING.md](CONTRIBUTING.md). Detection changes must include recall coverage, plausible safe patterns, and the real-repository regression gate required by the project. A reviewed baseline is not a suppression list: each added fingerprint must correspond to a finding read against its source. + +The maintainer may ask external domain experts to review a rule or advisory without granting repository or npm access. Such review is credited only with the reviewer's consent. + +## Release authority + +Only the maintainer publishes the npm package. GitHub Actions never receives npm credentials and never publishes a release. The mandatory local release checks and manual publication procedure are documented in [PUBLISHING.md](PUBLISHING.md). + +## Security reports + +Vulnerabilities in the scanner itself follow [SECURITY.md](SECURITY.md). Detection gaps and false positives use the public issue templates because transparency about scanner coverage benefits users. Private reports are not converted into public issues until disclosure is safe. + +## Continuity + +This project currently has a bus factor of one. It does not claim an enterprise support SLA or guaranteed continuity. + +If the maintainer plans an extended absence or stops maintaining the project, the preferred path is to appoint a contributor with a demonstrated record of technically sound, precision-preserving work. Repository and npm access are transferred separately and minimally. If no suitable successor exists, the repository will be archived with a clear notice rather than presented as actively maintained. + +Organizations requiring contractual support, multiple release approvers, or guaranteed response times should treat SecureAI-Scan as a transparent supplementary control and apply their own vendor-risk process. + +## Changes to governance + +Governance changes are made through a public pull request and recorded in the changelog when they alter release authority, security reporting, or continuity expectations. \ No newline at end of file diff --git a/PUBLISHING.md b/PUBLISHING.md index a829453..ebe3fce 100644 --- a/PUBLISHING.md +++ b/PUBLISHING.md @@ -1,386 +1,73 @@ -# Publishing SecureAI-Scan to npm +# Publishing SecureAI-Scan -Everything you need to publish, update versions, and automate future releases. +Releases are published manually from a maintainer workstation. GitHub Actions builds and tests the project but never receives npm credentials and never publishes packages. ---- +## npm authentication -## One-Time Setup - -### 1. Create an npm Account - -Go to [npmjs.com](https://www.npmjs.com) → Sign up. - -Choose a username that matches or is close to your GitHub username (`akanthed`). - -Enable 2FA immediately after signing up — npm will require it for publishing anyway. - -### 2. Log In on Your Machine +Authenticate directly with npm before publishing: ```bash npm login -``` - -You'll be prompted for username, password, and a one-time 2FA code. -After this, your credentials are stored and you won't need to log in again on this machine. - -Verify it worked: -```bash npm whoami -# should print: akanthed ``` -### 3. Verify the Package Name is Available - -```bash -npm view secureai-scan -``` +Complete the password, browser, and 2FA prompts locally. Never add an npm password, token, or recovery code to GitHub secrets, repository files, terminal logs, or issue comments. -If it returns package info — the name is taken (it's yours if you already published it). -If it returns an error "404 Not Found" — the name is available. +## Release gate ---- - -## Pre-Publish Checklist - -Run through this every time before publishing: +Run the release gate locally: ```bash -# 1. Make sure you're on the main branch with no uncommitted changes -git status - -# 2. Pull latest -git pull origin main - -# 3. Build the TypeScript -npm run build - -# 4. Verify the CLI works -node dist/index.js --version -node dist/index.js --help -node dist/index.js scan . --severity high - -# 5. Run tests -npm test - -# 6. Check what files will be published (should only be dist/, README.md, LICENSE) -npm pack --dry-run -``` - -Expected `npm pack --dry-run` output — you should see ONLY these: -``` -dist/ -dist/index.js -dist/scanner/... -README.md -LICENSE -package.json +npm ci +npm run release:check ``` -If you see `.ts` source files, `.env` files, or test fixtures — add them to `.npmignore`. +The gate must complete all of these: -### Create .npmignore (if it doesn't exist) +1. Build and run every test. +2. Meet the configured coverage thresholds. +3. Pass the reviewed real-repository regression baseline. +4. Show only intended runtime files in `npm pack --dry-run`. -```bash -# .npmignore -src/ -test/ -test-fixtures/ -examples/ -*.cast -*.md -!README.md -.github/ -MARKETING.md -PUBLISHING.md -THREAT_MODEL.md -tsconfig.json -.secureai-policy.json -.secureai-baseline.json -``` +Review every new regression finding against its source. Fix false positives and add permanent safe fixtures. Update `test/regression-baseline.json` only for findings confirmed to be real. ---- +## Create a release -## Publishing — Step by Step - -### First Publish +1. Update `CHANGELOG.md` and set the release date. +2. Set the same version in `package.json` and `package-lock.json`. +3. Run `npm run release:check` from a clean worktree. +4. Commit and merge through the normal reviewed branch workflow. +5. Publish from the authenticated workstation. `prepublishOnly` automatically reruns the release gate: ```bash -# Make sure version in package.json is correct (currently 0.2.0) -cat package.json | grep version - -# Publish to npm (--access public is required for scoped packages, -# harmless for unscoped packages like this one) npm publish --access public ``` -You'll see output like: -``` -npm notice Publishing to https://registry.npmjs.org/ -+ secureai-scan@0.2.0 -``` +6. After npm confirms the publication, create and push an annotated tag matching the package version exactly: -**Verify it's live:** ```bash -# Wait 30 seconds then: -npx --yes secureai-scan@latest --version -# Should print: 0.2.0 +git tag -a v0.8.0 -m "secureai-scan v0.8.0" +git push origin v0.8.0 ``` -Also check: [npmjs.com/package/secureai-scan](https://www.npmjs.com/package/secureai-scan) - ---- - -### Publishing Updates - -Every time you make changes and want to release: - -**Step 1: Bump the version** - -Use npm's version command — it updates `package.json` AND creates a git tag automatically: - -```bash -# For bug fixes (0.2.0 → 0.2.1) -npm version patch - -# For new features that don't break anything (0.2.0 → 0.3.0) -npm version minor - -# For breaking changes (0.2.0 → 1.0.0) -npm version major -``` - -**Step 2: Build** - -```bash -npm run build -``` - -**Step 3: Publish** - -```bash -npm publish -``` - -**Step 4: Push the version tag to GitHub** - -```bash -git push origin main --follow-tags -``` - -This pushes both the commit and the version tag (e.g. `v0.2.1`) to GitHub. - ---- - -## Version Numbering Guide - -Follow semantic versioning (semver). Users depend on this to know if an update is safe. - -| Change type | Example | Command | -|------------|---------|---------| -| Bug fix, no new features | Fixed false positive in AI001 | `npm version patch` | -| New rule added | Added MCP004 | `npm version minor` | -| New command added | Added `threat-model` command | `npm version minor` | -| Removed a rule or changed rule IDs | Renamed VEC001 to VEC100 | `npm version major` | -| Changed CLI flag names | `--only-ai` renamed | `npm version major` | - -**Current planned releases:** -- `0.2.1` — any bug fixes from community feedback -- `0.3.0` — Python support or new rule category -- `1.0.0` — when the rule set is stable and well-tested - ---- +Pushing the tag creates no npm publication job. Keep the tag and manifest version identical so the GitHub release and npm artifact remain traceable. -## Automated Publishing with GitHub Actions +## Verify the public artifact -`.github/workflows/publish.yml` is now committed to the repo — it builds, tests, and publishes on any `v*` tag push. **It won't actually publish anything until the `NPM_TOKEN` secret below is added** (Step 1 and 2 are the remaining setup — Step 3 is done). Until then, publishing stays manual via the steps earlier in this doc. - -### Step 1: Create an npm Access Token - -1. Go to [npmjs.com](https://www.npmjs.com) → Your account → Access Tokens -2. Click "Generate New Token" → Choose "Automation" (works even with 2FA) -3. Copy the token — you'll only see it once - -### Step 2: Add Token to GitHub Secrets - -1. Go to your GitHub repo → Settings → Secrets and Variables → Actions -2. Click "New repository secret" -3. Name: `NPM_TOKEN` -4. Value: paste the token from Step 1 -5. Click "Add secret" - -### Step 3: Create the Publish Workflow - -Create this file in your repo: - -```yaml -# .github/workflows/publish.yml -name: Publish to npm - -on: - push: - tags: - - "v*" # Triggers on any tag like v0.2.0, v1.0.0 - -jobs: - publish: - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: "22" - registry-url: "https://registry.npmjs.org" - - - name: Install dependencies - run: npm ci - - - name: Build - run: npm run build - - - name: Run tests - run: npm test - - - name: Publish to npm - run: npm publish --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} -``` - -**How to use it going forward:** +After `npm publish` succeeds: ```bash -# Make your changes, commit them -git add . -git commit -m "feat: add Python support" - -# Bump version (updates package.json + creates git tag) -npm version minor - -# Push code + tag — GitHub Actions publishes automatically -git push origin main --follow-tags -``` - -GitHub Actions runs the tests, builds, and publishes. You never touch `npm publish` manually again. - ---- - -## npm Page Optimisation - -Your npm package page is at [npmjs.com/package/secureai-scan](https://www.npmjs.com/package/secureai-scan). -It pulls content directly from your repo. To make it look good: - -### Keywords (in package.json — already set, verify these are there) - -```json -"keywords": [ - "security", "ai", "llm", "scanner", "cli", "typescript", - "prompt-injection", "appsec", "mcp", "rag", "vector-database", - "devsecops", "owasp", "langchain", "openai" -] +npm view secureai-scan@0.8.0 version dependencies dist.integrity +npx --yes secureai-scan@0.8.0 --version +npx --yes secureai-scan@0.8.0 scan . ``` -Add `"mcp"`, `"rag"`, `"vector-database"`, `"devsecops"`, `"owasp"`, `"langchain"`, `"openai"` to the keywords array now — they drive npm search results. +Confirm that `tree-sitter` and `tree-sitter-python` are runtime dependencies. Create the matching GitHub release from the tag and include the relevant changelog section. -### Description (in package.json) +## Version policy -Change from: -``` -"Repo-native AI security scanning CLI for LLM-specific risks" -``` - -To: -``` -"Find AI/LLM security vulnerabilities in your code — prompt injection, MCP tool poisoning, RAG data poisoning, agent trust violations. 28 rules mapped to the OWASP LLM/Agentic/MCP Top 10s. Local-first." -``` - -### README Shows on npm Page - -The README.md you have is shown directly on the npm page. The badges, table of contents, and code examples all render. It's already in good shape. - ---- - -## After Publishing — Announce It - -**Immediately after `npm publish` succeeds:** - -1. Update the README badge to show the real npm version: - ```markdown - [![npm version](https://img.shields.io/npm/v/secureai-scan)](https://www.npmjs.com/package/secureai-scan) - ``` - This badge auto-updates — no changes needed. - -2. Post on X/Twitter: - ``` - SecureAI-Scan v0.2.0 is on npm. - - 19 rules covering AI, MCP, and RAG security vulnerabilities. - Runs locally. Free. Zero config. - - npx --yes secureai-scan@latest scan . - - What's new in 0.2.0: MCP rules (tool poisoning, dynamic server URLs), - Vector/RAG rules (data poisoning, unbounded search, cross-tenant leakage), - threat model generation, policy file enforcement. - - [GitHub link] - ``` - -3. Post the Hacker News Show HN (from MARKETING.md) - ---- - -## Troubleshooting - -**"You do not have permission to publish"** -You're not logged in, or the package name is owned by someone else. -Run `npm whoami` — if it returns nothing, run `npm login` again. - -**"Cannot publish over existing version"** -You already published this version number. Run `npm version patch` to bump and try again. - -**"Package name too similar to existing package"** -npm may block names similar to popular packages. If this happens, consider: -`@akanthed/secureai-scan` (scoped package — free, just change the name in package.json) - -**GitHub Actions publish fails with 401** -The NPM_TOKEN secret has expired or was deleted. Generate a new Automation token from npmjs.com and update the GitHub secret. - -**Build succeeds locally but fails in Actions** -Check that your `package.json` `"engines"` field matches the Node version in the workflow (`"node": ">=22"`). - ---- - -## Version History to Document - -Keep a CHANGELOG.md for users who want to know what changed: - -```markdown -# Changelog - -## 0.2.0 — 2026-06-07 -### Added -- 9 new security rules: AI010, AI011, AI012, MCP001, MCP002, MCP003, VEC001, VEC002, VEC003 -- `secureai-scan init` command — first-time setup with policy file and CI workflow -- `secureai-scan threat-model` command — generates THREAT_MODEL.md -- `--only-mcp` and `--only-vec` scan filters -- `--min-confidence` flag — control false-positive sensitivity -- `--policy` flag — enforce .secureai-policy.json in CI -- Policy file exits with code 1 on `failOnSeverity` threshold breach - -### Improved -- Confidence scoring now penalises test files, sanitized contexts, embedding-only calls -- All rules now include detailed `howToFix` guidance -- Terminal output shows confidence-hidden finding count - -## 0.1.6 — 2026-04-28 -- Initial rules: AI001–AI009, AI100 -- Baseline diff mode -- HTML, Markdown, JSON reports -- Dependency scanning -- Prompt risk evaluator -``` +- Patch: false-positive fix or compatible bug fix. +- Minor: new rule, scanner surface, command, or material analysis improvement. +- Major: removed or renamed rule, incompatible CLI flag, report-schema break, or stable `1.0.0` contract. -Create `CHANGELOG.md` in the repo root with this content. +Never use `--ignore-scripts` to bypass `prepublishOnly`. Repair the failing test, coverage threshold, regression result, or package contents instead. \ No newline at end of file diff --git a/README.md b/README.md index 2221984..d4f5464 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,29 @@ # SecureAI-Scan [![npm version](https://img.shields.io/npm/v/secureai-scan)](https://www.npmjs.com/package/secureai-scan) +[![npm downloads](https://img.shields.io/npm/dm/secureai-scan)](https://www.npmjs.com/package/secureai-scan) +[![CI](https://github.com/akanthed/SecureAI-Scan/actions/workflows/ci.yml/badge.svg)](https://github.com/akanthed/SecureAI-Scan/actions/workflows/ci.yml) +[![CodeQL](https://github.com/akanthed/SecureAI-Scan/actions/workflows/codeql.yml/badge.svg)](https://github.com/akanthed/SecureAI-Scan/actions/workflows/codeql.yml) +[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/akanthed/SecureAI-Scan/badge)](https://scorecard.dev/viewer/?uri=github.com/akanthed/SecureAI-Scan) [![license](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) -[![TypeScript](https://img.shields.io/badge/TypeScript-ready-blue.svg)](https://www.typescriptlang.org/) -[![Python](https://img.shields.io/badge/Python-supported-yellow.svg)](https://www.python.org/) [![Node](https://img.shields.io/badge/node-%3E%3D20-brightgreen)](https://nodejs.org) -[![ChatGPT](https://img.shields.io/badge/ChatGPT-GPT%20available-74aa9c?logo=openai&logoColor=white)](https://chatgpt.com/g/g-6a25141758188191a764020c1ab6a226-secureai-scan-ai-security-advisor) [![OWASP](https://img.shields.io/badge/OWASP-LLM%20%C2%B7%20ASI%20%C2%B7%20MCP%20Top%2010-000000)](#rules) **The AI security scanner that proves its findings.** SecureAI-Scan finds LLM, MCP, Agent Skill, and RAG vulnerabilities in **TypeScript, JavaScript, and Python** — and shows you the evidence: the exact source → flow → sink path for every dataflow finding, resolved through real imports, not keyword matching. -It's the first scanner mapped to **all three** OWASP AI security frameworks — the LLM Top 10, the [Top 10 for Agentic Applications (2026)](https://genai.owasp.org/), and the [MCP Top 10](https://owasp.org/www-project-mcp-top-10/) — with a coverage matrix in every threat model showing exactly which risks are checked and which are runtime concerns. +It is mapped to **all three** OWASP AI security frameworks — the LLM Top 10, the [Top 10 for Agentic Applications (2026)](https://genai.owasp.org/), and the [MCP Top 10](https://owasp.org/www-project-mcp-top-10/) — with a coverage matrix in every threat model showing exactly which risks are checked and which are runtime concerns. + +## Get started in 30 seconds + +```bash +npx --yes secureai-scan@0.8.0 scan . +``` + +No account, cloud upload, Python interpreter, or configuration required. TypeScript, JavaScript, Python, MCP configs, and Agent Skill bundles are detected automatically. + +**Measured `0.8.0` release candidate:** 133/133 tests · 88.07% statement coverage · 12,676 files across 9 public repositories · 0 new default-tier fingerprints against the reviewed baseline. [Evidence](docs/benchmarks/v0.8.0.json) · [methodology and limits](docs/ReleaseAssurance.md) ``` ▌ HIGH AI001 Prompt injection via user input @@ -39,6 +50,7 @@ It's the first scanner mapped to **all three** OWASP AI security frameworks — - [Architecture](#architecture) - [MCP server (use it from Claude)](#mcp-server-use-it-from-claude) - [Claude Skill](#claude-skill) +- [Trust and release assurance](#trust-and-release-assurance) - [The precision contract](#the-precision-contract) - [Testing & benchmarking](#testing--benchmarking) - [Roadmap](#roadmap) @@ -48,20 +60,20 @@ It's the first scanner mapped to **all three** OWASP AI security frameworks — - **Evidence tiers, not noise.** Every finding is `proven` (traced dataflow or parsed config fact), `likely` (resolved sink, one heuristic hop), or `heuristic`. **A default scan shows only proven + likely.** Heuristics are opt-in via `--paranoid`. - **Import-resolved detection.** A call is only an "LLM call" if it resolves to a real SDK import (`openai`, `@anthropic-ai/sdk`, `ai`, `@google/genai`, LangChain, Bedrock, …). Your Google Maps client will never be flagged as an LLM again. -- **Precision-gated, and benchmarked against real repos.** The test suite asserts every vulnerable fixture fires *and* every safe fixture stays clean — a false positive on the safe corpus fails the build. Beyond that, every detection change is run against real public repos (OpenAI/Anthropic/Vercel AI SDKs, official MCP servers, LlamaIndex) before shipping. See [Testing & benchmarking](#testing--benchmarking) for the actual before/after numbers. +- **Precision-gated, and benchmarked against real repos.** The test suite asserts every vulnerable fixture fires *and* every safe fixture stays clean — a false positive on the safe corpus fails the build. Beyond that, `npm run regression` scans real public repos (OpenAI/Anthropic/Vercel AI SDKs, official MCP servers, LlamaIndex) against a committed, hand-reviewed baseline and **fails on any new `proven`/`likely` finding**. See [Testing & benchmarking](#testing--benchmarking) for the actual before/after numbers. - **SARIF for GitHub code scanning.** `--output report.sarif` puts findings inline on pull requests and in the Security tab. -- **AI-BOM.** `secureai-scan bom .` inventories every SDK, model ID, vector store, agent framework, and MCP server in your repo — zero-false-positive by construction, mapped to OWASP LLM Top 10 / EU AI Act documentation needs. +- **AI-BOM.** `secureai-scan bom .` builds a syntax-derived inventory of SDKs, model IDs, vector stores, agent frameworks, and MCP servers, mapped to OWASP LLM Top 10 / EU AI Act documentation needs. - **MCP config scanning.** Parses `.mcp.json`, `claude_desktop_config.json`, `.cursor/mcp.json`: unpinned `npx -y` servers, inline secrets, plaintext HTTP transports. - **MCP tool-poisoning detection.** Catches the pattern behind the WhatsApp MCP rug-pull and postmark-mcp backdoor — invisible Unicode, agent-directed injection phrases, and cross-tool shadowing in tool names/descriptions, statically, before you ever run the server. - **MCP command-injection detection.** Flags MCP stdio transport `command`/`args` built from request data — the pattern behind the 2026 MCP STDIO RCE disclosure. - **Agent Skill poisoning detection.** The same invisible-Unicode, injection-phrase, and shadowing checks applied to `SKILL.md` files — Agent Skills load into context wholesale, so a poisoned skill is a poisoned tool description by another name. - **Evasion-resistant skill scanning.** Skill bundles are scanned as *directories*, not just their `SKILL.md`, and every content check runs against deobfuscated variants of the text. This targets the published techniques — homoglyphs, zero-width splitting, payloads staged in `.git/` or `build/`, exfiltration hidden in a `*.test.ts` file — that bypassed **>90% of the nine scanners** surveyed in *Cloak and Detonate* (arXiv:2607.02357). See [Evasion resistance](#evasion-resistance). -- **Known-malicious package advisories, version-aware.** Checks every dependency and every MCP-launched package against a curated advisory list (documented backdoors, critical CVEs) — offline, on every scan, no flag required. Clears a finding once you've actually upgraded past the affected range; stays flagged on any ambiguous or unpinned version, never silently. +- **Known-vulnerable and known-malicious package advisories, version-aware.** Checks every dependency and every MCP-launched package against a bundled advisory snapshot — a hand-curated list of documented in-the-wild backdoors, plus HIGH/CRITICAL OSV advisories for an LLM/MCP/RAG package watchlist, regenerated by [`scripts/sync-advisories.js`](scripts/sync-advisories.js). Runs offline on every scan, no flag required. A CVE only fires when your pinned version is *provably* inside the affected range; a documented-malicious package fires even on an ambiguous range, because installing a backdoor is unrecoverable. - **Local-first.** Nothing leaves your machine. ## How it compares -SecureAI-Scan is not a replacement for a general SAST tool or a container/IaC scanner — run it alongside one, not instead of one. It's the only one of these purpose-built for the LLM/MCP/RAG attack surface with dataflow evidence, not keyword rules. +SecureAI-Scan is not a replacement for a general SAST tool or a container/IaC scanner — run it alongside one, not instead of one. It is purpose-built for the LLM/MCP/RAG attack surface and emphasizes dataflow evidence over flat keyword findings. | | SecureAI-Scan | Semgrep (OSS rules) | Trivy | GitHub Advanced Security | |---|---|---|---|---| @@ -78,14 +90,6 @@ SecureAI-Scan is not a replacement for a general SAST tool or a container/IaC sc If you already run Semgrep or GHAS, keep them — add SecureAI-Scan for the risk surface they don't model at all. -## Get started in 30 seconds - -```bash -npx --yes secureai-scan@latest scan . -``` - -TypeScript, JavaScript, Python, MCP config files, and Agent Skill (`SKILL.md`) files are scanned automatically — one command, no config. - > Prefer to ask questions first? Try the free **[SecureAI-Scan AI Security Advisor on ChatGPT](https://chatgpt.com/g/g-6a25141758188191a764020c1ab6a226-secureai-scan-ai-security-advisor)**. > About to run an MCP server you found on GitHub or Twitter? Paste its tool description into **[MCP X-Ray](https://akanthed.github.io/SecureAI-Scan/)** first — checks it for hidden Unicode, injected instructions, and known-malicious packages in your browser, no install. @@ -185,8 +189,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: akanthed/SecureAI-Scan@main + - uses: akanthed/SecureAI-Scan@v0.8.0 with: + scanner-version: 0.8.0 fail-on: high ``` @@ -253,8 +258,8 @@ Three independent scanning surfaces feed one merged, deduped finding list: └─────────────────────┘ │ │ ┌─────────────────────┐ │ ┌──────────────┐ ┌─────────────────┐ - *.py ───▶ │ Python regex + │───┼───▶ │ scan.ts │───▶ │ evidence filter │ - │ taint-propagation │ │ │ merge/dedupe│ │ → confidence │ + *.py ───▶ │ tree-sitter AST + │───┼───▶ │ scan.ts │───▶ │ evidence filter │ + │ local taint flow │ │ │ merge/dedupe│ │ → confidence │ └─────────────────────┘ │ │ + suppress │ │ → severity │ │ │ (// secure- │ │ → baseline diff │ .mcp.json, ┌─────────────────────┐ │ │ ai-ignore) │ │ → report │ @@ -332,6 +337,16 @@ Validated against two real-world corpora, not just fixtures we wrote ourselves: Honest limitation: the paper's conclusion is that runtime detonation beats static analysis, and that is correct. An adaptive adversary who knows these rules can write a transformation they don't cover. What changes here is the *cost* of evasion — the published, currently-circulating techniques no longer work, and the obfuscation needed to defeat them now itself raises the finding's severity. **Static scanning is a filter, not a security boundary.** Treat an untrusted skill as untrusted code regardless of what any scanner says. +## Trust and release assurance + +- CI runs on Linux, Windows, and macOS across supported Node versions. +- CodeQL, dependency review, OpenSSF Scorecard, Dependabot, and this scanner's own blocking self-scan provide independent checks. +- Every manual npm publication invokes tests, coverage floors, the reviewed real-repository regression gate, and tarball inspection through `prepublishOnly`. +- GitHub Actions receives no npm password or token and cannot publish the package. +- [Release assurance](docs/ReleaseAssurance.md), [single-maintainer governance](GOVERNANCE.md), [security reporting](SECURITY.md), and [versioned benchmark evidence](docs/benchmarks/v0.8.0.json) are public. + +This is a single-maintainer project with no contractual SLA or independent certification. The controls above reduce risk; they do not turn a static scan into proof of security. + ## The precision contract False positives kill scanners. SecureAI-Scan's rule engine follows three hard rules: @@ -355,12 +370,15 @@ npm test **2. Real-world regression benchmark — against public repos we didn't write.** ```bash -npm run regression # scan the full curated repo set -npm run regression -- --fresh # re-clone everything first -npm run regression -- openai-node # scan just one repo by name +npm run regression # scan the full curated repo set +npm run regression -- --fresh # re-clone everything first +npm run regression -- openai-node # scan just one repo by name +npm run regression -- --update-baseline # accept the current findings ``` -[`scripts/regression-scan.js`](scripts/regression-scan.js) clones a curated, diverse set of real public repos (OpenAI/Anthropic/Vercel AI SDKs, the official MCP servers and TypeScript SDK, LlamaIndex, plus [anthropics/skills](https://github.com/anthropics/skills) and [cisco-ai-defense/skill-scanner](https://github.com/cisco-ai-defense/skill-scanner) for skill-bundle coverage — spanning TS and Python, SDK-consumer example code and SDK-author source) and scans each with the built CLI. There's no fixed pass/fail threshold — upstream repos change — so every `proven`/`likely` finding gets read against its source line by hand. Anything that isn't a genuine issue is a rule bug, fixed at the root cause and locked in permanently as a new `test-fixtures/safe/` fixture. +[`scripts/regression-scan.js`](scripts/regression-scan.js) clones a curated, diverse set of real public repos (OpenAI/Anthropic/Vercel AI SDKs, the official MCP servers and TypeScript SDK, LlamaIndex, plus [anthropics/skills](https://github.com/anthropics/skills) and [cisco-ai-defense/skill-scanner](https://github.com/cisco-ai-defense/skill-scanner) for skill-bundle coverage — spanning TS and Python, SDK-consumer example code and SDK-author source) and scans each with the built CLI. + +It **exits non-zero on any `proven`/`likely` finding not already in [`test/regression-baseline.json`](test/regression-baseline.json)** — a hand-reviewed record of findings already read against their source line. Fingerprints are `repo|rule|file`, not line numbers, so ordinary upstream churn doesn't produce noise. A new fingerprint is a claim the scanner has to justify: if it isn't a genuine issue it's a rule bug, fixed at the root cause and locked in as a new `test-fixtures/safe/` fixture. Baselining a finding you haven't read defeats the entire mechanism. **Skill-bundle coverage gets its own line** because `cisco-ai-defense/skill-scanner`'s `evals/` corpus is labeled — each of its 20 fixtures ships an `_expected.json` verdict and sits under a directory literally named `malicious/` or `safe/`, so it doubles as a recall check, not just a precision one: **6/6 in-scope malicious fixtures fire, 0 findings on anything labeled safe**, and 0 findings across all 18 real bundles in `anthropics/skills` and all 14 in `vercel/ai`. (The remaining Cisco categories — SQL injection, path traversal, resource exhaustion, generic `eval()` of a function argument, a payload deliberately split across four files — are either out of the documented LLM/MCP/RAG scope or beyond same-file conjunction analysis; see the [0.6.0 changelog entry](CHANGELOG.md) for the specific reasoning on each.) @@ -374,13 +392,13 @@ Historical before/after from the run that drove the original precision fixes (fi | [modelcontextprotocol/typescript-sdk](https://github.com/modelcontextprotocol/typescript-sdk) | 3 | 0 | `token_endpoint`/`tokenType`-style OAuth metadata fields flagged as leaked secrets | | [run-llama/llama_index](https://github.com/run-llama/llama_index) | 18 | 15 | A Python check flagged any `description=` field containing "system prompt" as `proven` MCP tool poisoning, regardless of context. The remaining 15 are `VEC001` hits on the library's own generic retriever definitions — scanning a vector-DB SDK's own source, not application code, so a filter can't exist to check; an honest, inherent limit, not a bug | -**Current run (v0.6.0, 2026-07-28)** — upstream repos have grown substantially since the table above, so the counts have moved: +**Current run (2026-08-05)** — versioned evidence is recorded in [`docs/benchmarks/v0.8.0.json`](docs/benchmarks/v0.8.0.json): | Repo | Findings | Rules | Status | |------|---------:|-------|--------| | openai-node, anthropic-sdk-typescript, anthropic-sdk-python, modelcontextprotocol/typescript-sdk, modelcontextprotocol/servers | 0 | — | clean | | [anthropics/skills](https://github.com/anthropics/skills) (18 real skill bundles) | 0 | — | clean — pure precision check for SKL001–005 | -| [vercel/ai](https://github.com/vercel/ai) (5,511 files) | 0 | — | **was 40** (AI001, AI003, AI005, AI010, MCP002) before triage — every one hand-reviewed against source and confirmed a false positive, traced to 3 independent root-cause bugs (see below), fixed, and re-confirmed clean on a full re-scan | +| [vercel/ai](https://github.com/vercel/ai) (5,691 files) | 0 | — | **was 40** (AI001, AI003, AI005, AI010, MCP002) before triage — every one hand-reviewed against source and confirmed a false positive, traced to 3 independent root-cause bugs (see below), fixed, and re-confirmed clean on a full re-scan | | [run-llama/llama_index](https://github.com/run-llama/llama_index) | 46 | VEC001 | inherent limit, not a bug — the library's own generic retriever definitions, where no tenant filter can exist to find | | [cisco-ai-defense/skill-scanner](https://github.com/cisco-ai-defense/skill-scanner) | 7 | SKL001, SKL002, SKL005 | **all on fixtures labeled `malicious/`** — 6/6 in-scope, 0 on anything labeled `safe/` | @@ -400,11 +418,13 @@ The two layers above only check that the scanner stays quiet on safe code. `DEP0 node --test test/dependency-guard.test.js ``` -covers: `mcp-remote@0.1.15` (CVE-2025-6514, vulnerable) flagged / `mcp-remote@0.1.16` (patched) clear; `postmark-mcp@1.0.15` (before the backdoor) clear / `postmark-mcp@1.0.20` (after — no legitimate patch exists for a malicious package) still flagged; an unpinned `^0.1.16` range still flagged despite being patchable, since we can't prove what actually resolves. Building this test caught a real gap: `DEP003` used to match advisories by package name only, never actually comparing the declared version against the advisory's affected range — fixed in [`src/scanner/semver.ts`](src/scanner/semver.ts), which clears a finding only when an exact version pin is provably outside the affected range, and fails toward flagging on anything ambiguous. +covers: `mcp-remote@0.1.15` (CVE-2025-6514, vulnerable) flagged / `mcp-remote@0.1.16` (patched) clear; `postmark-mcp@1.0.15` (before the backdoor) clear / `postmark-mcp@1.0.20` (after — no legitimate patch exists for a malicious package) still flagged; `llama-cpp-python==0.2.71` (CVE-2024-34359, from the OSV-generated set) flagged / `==0.2.72` (patched) clear, including under PyPI name normalization (`llama_cpp_python`); and `langchain>=0.1.0`-style unpinned specifiers producing **zero** default-report findings. Building this test caught a real gap: `DEP003` used to match advisories by package name only, never actually comparing the declared version against the advisory's affected range — fixed in [`src/scanner/semver.ts`](src/scanner/semver.ts). + +Ambiguity is resolved differently per advisory kind, deliberately. A **malicious** package fires even when the declared version can't be resolved — installing a backdoor is unrecoverable, so it fails toward flagging. A **CVE** fires at `proven` only when the declared version is an exact pin provably inside the affected range; unpinned-but-possibly-affected drops to `heuristic` (`--paranoid` only). Applying the malicious-kind rule to a 162-entry CVE snapshot would put a critical finding on every repo that declares `langchain>=0.1.0` — unactionable noise at scale. ## Roadmap -See [`ROADMAP.md`](ROADMAP.md) for what's shipped and what's planned — the short version: the Python scanner is regex-based today (a documented, deliberate trade-off, not an oversight — see [`docs/DetectionEngine.md`](docs/DetectionEngine.md)), and moving it to full AST analysis is the largest planned change. +See [`ROADMAP.md`](ROADMAP.md) for what's shipped and what's planned. Both language engines are AST-based: ts-morph for TypeScript/JavaScript and Tree-sitter for Python. Python imports, calls, assignments, decorators, scopes, keyword arguments, dictionary fields, and strings are syntax nodes; target code is never imported or executed, and no Python interpreter is required. The remaining Python gap is bounded cross-function/cross-file taint depth, not parsing. Scan performance and known limits are documented in [`docs/Performance.md`](docs/Performance.md). ## Contributing diff --git a/ROADMAP.md b/ROADMAP.md index 8abf596..e82d334 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,7 +2,7 @@ Where this scanner is going, and why — kept separate from [`CHANGELOG.md`](CHANGELOG.md) (what shipped) and [`MARKETING.md`](MARKETING.md) (how to talk about it). This file is the plan; update it as decisions change rather than letting it drift out of sync with reality. -## Where we stand (2026-07-28) +## Where we stand (2026-08-05) **Shipped in v0.6.0:** evasion-resistant Agent Skill scanning (SKL004/SKL005, deobfuscation-aware SKL001–003), validated against two real-world corpora — [anthropics/skills](https://github.com/anthropics/skills) (18 bundles, 0 findings) and [cisco-ai-defense/skill-scanner](https://github.com/cisco-ai-defense/skill-scanner)'s labeled eval set (6/6 malicious fixtures correctly flagged, 0 false positives). The pre-install wedge (`secureai-scan skill ` / `secureai-scan mcp `) — fetch and scan before you trust, no clone or config required, nothing fetched is ever executed. @@ -16,20 +16,15 @@ This is the exact validation loop the project is built around: real code found r ## The four gaps, and what closes them -### 1. Python is regex-based, not AST-based +### 1. Python AST foundation — shipped -The TS/JS rules resolve real imports and walk a real AST (`ts-morph`). The Python scanner (`python-scanner.ts`) matches patterns line-by-line with a small taint pass — `CLAUDE.md` documents this as the most false-positive-prone surface in the codebase, and it's the majority language for production LLM/agent code. +**Status: complete (2026-08-05).** Python source is parsed synchronously with `tree-sitter` + `tree-sitter-python` in `src/scanner/python-ast.ts`. Imports, calls, arguments, keyword arguments, assignments (identifier, attribute, tuple/list unpacking), decorators, functions, scopes, dictionary fields, and string/docstring nodes all come from the syntax tree. The old lexical `code`/`logical` views and hand-written assignment parser were deleted; no production rule infers Python structure from physical lines. -**Plan:** integrate `tree-sitter-python` (WASM, pure-npm, no external Python interpreter required — keeps `npx secureai-scan` working with zero Python installed). Port rules incrementally behind the existing regex path as a fallback, so the precision gate never regresses mid-migration. Start with AI001 (prompt injection) as the proof of concept, since it has the most-documented false-negative history (see memory: litellm calls, >10-line taint gaps). +The parser is a runtime dependency with prebuilt binaries for macOS/Linux/Windows on x64 and ARM64; no Python interpreter and no execution of target code is required. Tree-sitter error recovery keeps malformed files scanable instead of aborting the repository. -**Status: spiked (2026-07-28), not integrated.** `spike/python-ast-poc/` — not part of the shipped package, `tree-sitter-python`/`web-tree-sitter` are `devDependencies` only. Findings: +This closes the concrete class-handler gap that motivated the migration (`self.user_message = request.json[...]`), fake imports/calls inside comments and docstrings, multiline calls and keyword arguments, tuple assignment, and decorated async handlers. Direct AST contracts in `test/python-ast.test.js`, the vulnerable/safe corpus, and the real-repo regression gate protect both recall and precision. -- **Technically viable exactly as planned.** `web-tree-sitter` + `tree-sitter-python`'s own bundled `.wasm` grammar (not the 51MB `tree-sitter-wasms` multi-language bundle) parses real Python with zero native compilation — confirmed by testing on this machine. `tree-sitter-python`'s own npm entry point pulls in `node-gyp-build`/native bindings; that path was deliberately avoided. -- **Found and closed a real, concrete gap** in the current regex scanner during the spike itself: `collectRequestTaintedVars` in `python-scanner.ts` only recognizes bare-identifier assignment targets, so `self.user_message = request.json[...]` — a common shape in any class-based handler (Flask `MethodView`, FastAPI DI classes, agent/session state) — is invisible to it, confirmed empirically (0 findings, even at `--paranoid`, on a textbook prompt-injection case). The AST-based POC catches it with no special-casing, because an assignment target is either an `identifier` or an `attribute` node either way — the gap only existed because the regex approach had to enumerate LHS shapes by hand and missed one. -- **Recall parity confirmed** on the shape the regex scanner already handles (bare-identifier taint), and **no precision regression** on a safe case (plain function argument, no request source) — 1/1/0 findings across the three test cases, exactly as expected. -- **Effort estimate for a real migration, based on what the POC didn't cover:** the POC is single-function, single-pass, no cross-function propagation, no sanitizer detection (`hasSanitization` in the current scanner), no evidence tiering, and covers only AI001's core shape — not the ~10 other Python rules that share `python-scanner.ts`'s taint infrastructure (AI003, AI004, AI005, AI007, AI010, MCP007–009, DEP checks). A full port is a genuine multi-session rewrite of that shared infrastructure, not an incremental patch — closer to "rebuild the Python surface on a new foundation" than "swap one function's implementation." - -**Recommendation:** proceed. The technical risk is retired — this was the open question, and it's now answered with working code, not a guess. The remaining cost is pure engineering time, and the AI001 case alone (a completely invisible textbook vulnerability) is enough to justify it independent of the strategic case in the rest of this document. Next step, when picked up: expand the POC to a second rule (AI005 or AI007, since they read the same shared taint set) to confirm the "shared infrastructure" assumption before committing to the full rebuild. +**Remaining work:** bounded cross-function and cross-file propagation. The AST establishes correct syntax and scope; it does not automatically make arbitrary interprocedural taint sound. That is tracked below as a separate problem rather than being misrepresented as unfinished parsing. ### 2. No cross-file taint tracking diff --git a/action.yml b/action.yml index 71f30bc..bcc2fb3 100644 --- a/action.yml +++ b/action.yml @@ -6,6 +6,10 @@ branding: color: "purple" inputs: + scanner-version: + description: "Exact secureai-scan npm version to install" + required: false + default: "0.8.0" path: description: "Path to scan" required: false @@ -32,15 +36,22 @@ runs: steps: - name: Install secureai-scan shell: bash - run: npm install -g secureai-scan + env: + SCANNER_VERSION: ${{ inputs.scanner-version }} + run: npm install -g "secureai-scan@$SCANNER_VERSION" - name: Run SecureAI-Scan shell: bash + env: + FAIL_ON: ${{ inputs.fail-on }} + PARANOID: ${{ inputs.paranoid }} + SARIF_FILE: ${{ inputs.sarif-file }} + SCAN_PATH: ${{ inputs.path }} run: | - ARGS="--output ${{ inputs.sarif-file }}" - if [ "${{ inputs.paranoid }}" = "true" ]; then ARGS="$ARGS --paranoid"; fi - if [ -n "${{ inputs.fail-on }}" ]; then ARGS="$ARGS --fail-on ${{ inputs.fail-on }}"; fi - secureai-scan scan "${{ inputs.path }}" $ARGS + args=(--output "$SARIF_FILE") + if [ "$PARANOID" = "true" ]; then args+=(--paranoid); fi + if [ -n "$FAIL_ON" ]; then args+=(--fail-on "$FAIL_ON"); fi + secureai-scan scan "$SCAN_PATH" "${args[@]}" - name: Upload SARIF to GitHub code scanning if: ${{ inputs.upload-sarif == 'true' && always() }} diff --git a/docs/Architecture.md b/docs/Architecture.md index 071ca1d..aba4735 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -1,18 +1,20 @@ # Architecture -SecureAI-Scan runs four independent scanning surfaces and merges their output into one finding list. Each surface exists because the source material is fundamentally different — an AST for TypeScript, text for Python, JSON for MCP config, a whole directory for Agent Skills — and forcing them through one abstraction would weaken all four. +SecureAI-Scan runs four independent scanning surfaces and merges their output into one finding list. Each surface exists because the source material is fundamentally different — language ASTs for TypeScript and Python, JSON for MCP config, a whole directory for Agent Skills — and forcing them through one abstraction would weaken all four. ## The four surfaces ### 1. TypeScript/JavaScript — AST-based (`src/scanner/project.ts`, `src/scanner/rules/*.ts`) -`src/scanner/project.ts` builds one [`ts-morph`](https://ts-morph.com/) `Project` from the target path (`skipAddingFilesFromTsConfig: true`, excluding `node_modules`/`dist`/`build`/`out`/`.next`). Every rule in `src/scanner/rules/` is a `Rule` object (`id`, `title`, `severity`, `run(context)`) that walks this AST independently — rules do not share a traversal pass, each does its own `forEachDescendant`-style walk. +`src/scanner/project.ts` builds one [`ts-morph`](https://ts-morph.com/) `Project` from the target path (`skipAddingFilesFromTsConfig: true`, excluding `node_modules`/`dist`/`build`/`out`/`.next`). Every rule in `src/scanner/rules/` is a `Rule` object (`id`, `title`, `severity`, `run(context)`). Rules share the memoized per-file call/function index in `src/utils/ast.ts`; they do not independently walk every file. The defining property of this surface: **a call is only ever treated as "an LLM call" if it resolves through actual import bindings to a known SDK** (`resolveLlmSink` in `src/scanner/rules/llm-rule-utils.ts` — covers `openai`, `@anthropic-ai/sdk`, `ai`, `@google/genai`, LangChain, Bedrock, and others). A function named `query()` or `chat()` that isn't imported from one of those packages is never flagged, no matter how LLM-shaped its name looks. This is the single biggest lever against false positives in the whole project — see [DetectionEngine.md](DetectionEngine.md) for why. -### 2. Python — regex + taint propagation (`src/scanner/python-scanner.ts`) +### 2. Python — Tree-sitter AST + taint propagation (`src/scanner/python-ast.ts`, `src/scanner/python-scanner.ts`) -Python has no AST pass here — patterns for LLM SDK calls, request-input taint sources, vector-store calls, and exec-style sinks are matched line-by-line, with a small taint-propagation pass connecting a source line to a sink line within the same file. This is a real trade-off, not an oversight: regex matching is inherently more prone to context-free false positives than an AST + import resolution (see the MCP001 false-positive class in [DetectionEngine.md](DetectionEngine.md), which was exactly this). [`ROADMAP.md`](../ROADMAP.md) tracks the plan to move this to full AST analysis; a working spike lives in `spike/python-ast-poc/` (tree-sitter-based, not yet wired into `src/`). +Every Python file is parsed once with `tree-sitter-python`. One indexed tree supplies imports, calls, arguments, keyword arguments, assignment targets, functions, decorators, scopes, dictionary fields, and literal strings to all Python rules. LLM receivers are bound from SDK constructor assignments, and request/LLM-result taint propagates through AST assignment expressions within the enclosing function or module scope. String-content rules still inspect string text, but the fact that a value is a tool description, dictionary field, keyword argument, or docstring comes from its syntax node — not a substring search over physical lines. + +Tree-sitter recovers around syntax errors, so an incomplete file remains scanable. Parsing is static and local: SecureAI-Scan does not need a Python interpreter and never imports or executes target code. ### 3. Config and content files — read directly off disk (`src/scanner/mcp-config-scanner.ts`, `src/scanner/skill-scanner.ts`) diff --git a/docs/Contributing.md b/docs/Contributing.md index 813cc75..ab6c95f 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -22,7 +22,7 @@ node --test test/corpus.test.js # run a single test file directly, after build ## Where things live -- Detection logic: `src/scanner/rules/*.ts` (TS/JS, AST-based), `src/scanner/python-scanner.ts` (Python, regex-based), `src/scanner/mcp-config-scanner.ts` + `src/scanner/skill-scanner.ts` (config/content), `src/scanner/dependency-guard.ts` (advisories). See [Architecture.md](Architecture.md). +- Detection logic: `src/scanner/rules/*.ts` (TS/JS AST), `src/scanner/python-ast.ts` + `src/scanner/python-scanner.ts` (Python AST), `src/scanner/mcp-config-scanner.ts` + `src/scanner/skill-scanner.ts` (config/content), `src/scanner/dependency-guard.ts` (advisories). See [Architecture.md](Architecture.md). - CLI wiring: `src/cli.ts`. Any change here needs a corresponding case in `test/cli.test.js` — it's the only test file that exercises the real built binary via `execFileSync` rather than calling scanner functions directly, so a flag-wiring bug (a dropped parser argument, a flag that silently no-ops) has no other test that would catch it. This has happened once already. - Test corpus: `test-fixtures/vulnerable/` and `test-fixtures/safe/`, enforced by `test/corpus.test.js`. - Documentation you're reading now: `docs/`. Auto-generated/per-project docs (`THREAT_MODEL.md`, the AI-BOM) are separate — see [ThreatModel.md](ThreatModel.md) for the distinction. diff --git a/docs/FAQ.md b/docs/FAQ.md index 63c2226..3c790ff 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -22,7 +22,7 @@ No. Everything runs locally; nothing leaves your machine. The only network calls Yes — that's what `secureai-scan skill ` and `secureai-scan mcp ` are for. They accept a GitHub `owner/repo` shorthand, a full git URL, a local path, or (for `mcp`) a bare npm package name. The target is fetched (npm: `npm pack` only, no `install`, no lifecycle scripts; git: `git clone --depth 1`), scanned, then deleted (`--keep` to inspect it instead). See the README's "Scan before you install" section. **Why does the Python scanner seem weaker than the TS/JS engine?** -It's regex/line-based, not AST-based, which is an honest, documented trade-off — see [Architecture.md](Architecture.md) and [ROADMAP.md](../ROADMAP.md). A tree-sitter-based AST rewrite is planned (spike in `spike/python-ast-poc/`) but not yet shipped. +Both engines are AST-based. Python uses Tree-sitter for imports, calls, assignments, decorators, scopes, arguments, and strings; TypeScript uses ts-morph and has deeper symbol resolution and bounded interprocedural tracing. The remaining difference is semantic depth, not parsing correctness — see [Architecture.md](Architecture.md) and [ROADMAP.md](../ROADMAP.md). **How do I add a new detection rule?** [WritingRules.md](WritingRules.md) is the step-by-step checklist; [RuleDevelopment.md](RuleDevelopment.md) covers the day-to-day loop and how a rule PR gets reviewed. @@ -31,4 +31,4 @@ It's regex/line-based, not AST-based, which is an honest, documented trade-off Yes — SARIF output (`--output report.sarif`) puts findings inline on PRs and in the GitHub Security tab. See the README's GitHub Action example, or run `secureai-scan init` to scaffold a workflow automatically. **What Node/TypeScript/Python versions are supported?** -Node `>=20` (see `package.json` `engines`). TypeScript scanning works on any `.ts`/`.tsx`/`.js`/`.jsx` regardless of the target project's own TS version, since ts-morph parses independently. Python scanning is regex-based and doesn't depend on a specific Python version being installed — SecureAI-Scan never executes your code. +Node `>=20` (see `package.json` `engines`). TypeScript scanning works on any `.ts`/`.tsx`/`.js`/`.jsx` regardless of the target project's own TS version, since ts-morph parses independently. Python is parsed by Tree-sitter and does not require Python to be installed — SecureAI-Scan never executes your code. diff --git a/docs/Performance.md b/docs/Performance.md index 1f7c51a..0aaa8da 100644 --- a/docs/Performance.md +++ b/docs/Performance.md @@ -1,27 +1,45 @@ # Performance -Current state, honestly, as of v0.6.1 — this is a snapshot for contributors thinking about scanning large repos, not a set of benchmarks to hit. +Measured, not estimated. Numbers below come from `node --cpu-prof` runs against the regression corpus (`npm run regression`), largest repo `vercel/ai` at 5,691 files. -## How a scan actually runs +## Where the time actually goes -- `src/scanner/project.ts` builds one `ts-morph` `Project` from the target path per invocation. There is no caching or incremental-scan support: every run parses every file from scratch, including in CI where the same repo is scanned on every PR. -- `src/scanner/scan.ts` runs fully synchronously and serially. Each rule in `RULES` walks the *entire* set of source files independently (`RULES.map(rule => rule.run(context))`) — rules do not share a single traversal, so the cost is roughly O(rules × files × AST nodes), not O(files × AST nodes). -- The four scanning surfaces (TS/JS, Python, MCP config, Agent Skill bundles — see [Architecture.md](Architecture.md)) each do their own file discovery / glob pass independently, rather than sharing one file list from a single walk of the repo. -- `--baseline` diffs *findings* against a saved baseline, not files — it doesn't skip re-scanning unchanged files, it only filters which findings get reported afterward. -- The only asynchronous I/O in the default scan path is `--check-dependencies`'s registry lookups (network calls to npm/PyPI); everything else is synchronous. +The dominant cost in a scan is AST traversal — not type resolution, not I/O, not rule logic. A CPU profile attributed **~95s of a 150s** `vercel/ai` scan to ts-morph descendant iteration alone (`getCompilerDescendantsIterator`, `getCompilerForEachDescendantsIterator`, `getCompilerChildren` and their node-wrapper cache): more than everything else in the profile combined. -None of this is an algorithmic problem (no O(n²) pattern was found scanning the codebase) — it's an absence of caching and sharing, which mainly matters as repo size grows. On the repos in the regression benchmark (`npm run regression` — up to `vercel/ai`'s 5,511 files), scan time in the multi-second range, reported in the terminal header (`v0.6.1 · N files · X.Xs`). +The cause was structural. Each of ~20 rules ran its own `sourceFile.getDescendantsOfKind(CallExpression)` or `sourceFile.getDescendants()` per file, so every file's AST was walked ~20 times per scan — and nested functions were re-walked once per enclosing scope on top of that. -## What's not done yet, and why it's not urgent +Two fixes, both in [`src/utils/ast.ts`](../src/utils/ast.ts): -- **No worker-thread parallelism.** Rules currently run against ts-morph's in-memory `Project`, which isn't trivially thread-safe to share across workers without real design work. Given current scan times are seconds, not minutes, this hasn't been a reported pain point. -- **No incremental scanning (only re-analyze changed files).** Would require tracking file hashes/mtimes and being careful that dataflow rules spanning multiple files still get correctly re-evaluated when an unrelated file in the flow changes. Worth doing once a large-monorepo user reports scan time as an actual blocker — not worth the complexity budget speculatively. -- **No shared file-discovery pass across the four surfaces.** Lower effort than the above two, and the most likely first fix if this area gets prioritized. +- **One walk per file, shared.** `getFileCalls` / `getFileFunctions` build a per-file index in a single `forEachDescendant` pass, memoized in a `WeakMap`. Rules consume the index instead of walking. +- **Containment by binary search.** `getCallsWithin(fn)` slices that index rather than walking the subtree — the index is in pre-order, so a node's descendants are a contiguous run of it. Spans come from `compilerNode.pos`/`.end`, captured during the indexing walk; an earlier version used `getStart()`, which rescans leading trivia on every probe and was *slower* than the walk it replaced. -## What contributors should do today +A third fix, in [`src/scanner/rules/llm-rule-utils.ts`](../src/scanner/rules/llm-rule-utils.ts): `resolveLlmSink` now checks the generation-shaped method name and the file's imports *before* asking the type checker anything. `resolveIdentifierModule` can only ever report a specifier the file itself imports, so a file with no LLM SDK import cannot produce a resolved sink — the type checker never needed consulting for the overwhelming majority of files. -If you're touching the scan pipeline (`scan.ts`, `project.ts`, or a rule that does heavy work per file), the practical check is: run `npm run regression` and compare wall-clock time before/after on the two largest repos in the set (`vercel/ai`, `llama_index`) — there's no formal perf test, so this is a manual sanity check, not a gate. Flag a regression in your PR description if you see one; don't silently absorb it. +### Result + +| | before | after | +|---|---:|---:| +| `vercel/ai` (5,691 files) | 217s | **62s** | +| `npm test` (full suite) | 42s | **16s** | + +Findings are identical before and after across the full regression corpus. This was a cost change, not a behaviour change — and it is verified as such, not assumed. + +## Guarding the optimization + +`getCallsWithin`'s contiguity assumption is the kind that fails silently: if it broke, rules would stop seeing calls and the scanner would simply go quiet, which is the worst failure mode a scanner has. [`test/ast-index.test.js`](../test/ast-index.test.js) therefore asserts the index and the slice against ts-morph's *own* `getDescendantsOfKind` walk — exact membership and document order — over a fixture with nested arrows, class methods, and tagged templates. + +There is no wall-clock perf gate: timing tests are flaky on shared CI runners, and the correctness test above is what actually protects the behaviour. + +## What's still not done + +- **No incremental scanning.** Every run parses every file from scratch, including in CI on every PR. `--baseline` filters *findings* after the fact; it does not skip unchanged files. Real incremental support needs file hashing plus care that cross-file dataflow rules re-evaluate when any file in the flow changes. +- **No worker-thread parallelism.** Rules run against a shared in-memory ts-morph `Project`, which is not trivially shareable across threads. +- **No shared file-discovery pass.** The four scanning surfaces (TS/JS, Python, MCP config, Agent Skill bundles — see [Architecture.md](Architecture.md)) each glob the repo independently. Lowest-effort remaining win. + +## What contributors should do + +If you touch `scan.ts`, `project.ts`, `llm-rule-utils.ts`, or add per-file work to a rule: time `npm run regression` on `vercel/ai` and `llama_index` before and after, and say so in the PR. **Never reintroduce a whole-file `getDescendants*` call in a rule** — use the shared index. That single pattern cost 3.5× scan time. ## Progress feedback -A scan prints a single "Scanning ``..." line to stderr before starting (gated on `isTTY`, so CI/piped logs stay clean), so a multi-second scan on a large repo doesn't look hung. There's no animated progress beyond that — the scan pipeline is synchronous, CPU-bound work with no natural yield point for a spinner, which would need a larger async rework to add. If you're debugging a "the scanner seems frozen" report, check `--debug` output first to confirm it's making progress before assuming a real hang. +A scan prints a single `Scanning ...` line to stderr before starting (gated on `isTTY`, so CI and piped logs stay clean). There's no spinner: the pipeline is synchronous CPU-bound work with no natural yield point. When triaging a "scanner seems frozen" report, check `--debug` first to confirm it's progressing. diff --git a/docs/ReleaseAssurance.md b/docs/ReleaseAssurance.md new file mode 100644 index 0000000..c923432 --- /dev/null +++ b/docs/ReleaseAssurance.md @@ -0,0 +1,42 @@ +# Release Assurance + +SecureAI-Scan is a single-maintainer open-source project. Trust comes from reproducible evidence and constrained release authority, not from a claim of independent certification. + +## Controls applied to every release + +`npm run release:check` performs four gates: + +1. Builds the TypeScript project and runs the complete test suite. +2. Enforces minimum coverage of 80% statements, 80% lines, 80% functions, and 75% branches. +3. Scans the curated real-repository corpus and rejects any new default-tier fingerprint outside the reviewed baseline. +4. Inspects the npm tarball contents before publication. + +`prepublishOnly` invokes the same release gate when the maintainer runs `npm publish`. Target repositories are parsed and read; their code is never imported or executed. + +## Independent platform checks + +Pull requests and `main` run on Linux, Windows, and macOS with supported Node versions. GitHub also runs CodeQL, dependency review, OpenSSF Scorecard, Dependabot, and SecureAI-Scan against its own repository. High and critical self-scan findings block CI. + +These checks are independent implementations, not proof that the package is vulnerability-free. + +## Release authority and credentials + +GitHub Actions does not receive npm credentials and cannot publish the package. The maintainer authenticates directly with npm using account protections and publishes manually after the release gate passes. Release ownership and continuity are documented in [GOVERNANCE.md](../GOVERNANCE.md). + +## Precision evidence + +The fixture corpus tests both directions: expected vulnerable patterns must fire at `proven` or `likely`, and safe fixtures must produce no default-tier findings. The real-repository regression corpus checks behavior on code not written for this scanner. + +Regression baselines are human-reviewed records, not an assertion that every baseline finding is exploitable. Framework source can structurally resemble unsafe application code; those known limits are retained and disclosed rather than hidden. + +Versioned benchmark records live under [`docs/benchmarks/`](benchmarks/). They report the command, corpus, test totals, coverage, new findings, reviewed baseline findings, and known limitations. + +## What this does not guarantee + +- Static analysis cannot establish runtime safety. +- Passing a scan does not prove that an application or skill is secure. +- A default-tier finding can still require application-context review. +- The project has one maintainer and no contractual support SLA. +- The benchmark corpus is curated and cannot represent every framework or coding style. + +Security teams should use SecureAI-Scan alongside general SAST, dependency, secret, runtime, and human review controls. \ No newline at end of file diff --git a/docs/benchmarks/v0.8.0.json b/docs/benchmarks/v0.8.0.json new file mode 100644 index 0000000..a7156b6 --- /dev/null +++ b/docs/benchmarks/v0.8.0.json @@ -0,0 +1,59 @@ +{ + "schema_version": 1, + "product": "secureai-scan", + "version": "0.8.0", + "status": "release-candidate", + "measured_at": "2026-08-05", + "commands": [ + "npm test", + "npm run coverage", + "npm run regression", + "npm pack --dry-run --ignore-scripts" + ], + "tests": { + "total": 133, + "passed": 133, + "failed": 0 + }, + "coverage_percent": { + "statements": 88.07, + "branches": 80.8, + "functions": 84.61, + "lines": 88.07 + }, + "regression": { + "repositories": 9, + "files_scanned": 12676, + "new_default_tier_fingerprints": 0, + "repositories_with_zero_default_tier_findings": [ + "openai/openai-node", + "anthropics/anthropic-sdk-typescript", + "anthropics/anthropic-sdk-python", + "modelcontextprotocol/typescript-sdk", + "modelcontextprotocol/servers", + "vercel/ai", + "anthropics/skills" + ], + "reviewed_baseline_findings": { + "run-llama/llama_index": { + "count": 46, + "rules": ["VEC001"], + "interpretation": "Known framework-source limitation: generic retriever implementations have no authenticated tenant context to inspect." + } + }, + "labeled_malicious_evaluation": { + "repository": "cisco-ai-defense/skill-scanner", + "default_tier_findings": 7, + "in_scope_malicious_cases_detected": "6/6", + "safe_labeled_cases_with_default_tier_findings": 0, + "rules": ["SKL001", "SKL002", "SKL005"] + } + }, + "package": { + "files": 58, + "packed_bytes": 150200, + "unpacked_bytes": 568000 + }, + "methodology": "../ReleaseAssurance.md", + "regression_baseline": "../../test/regression-baseline.json" +} \ No newline at end of file diff --git a/docs/index.html b/docs/index.html index 2d1730d..5af75e0 100644 --- a/docs/index.html +++ b/docs/index.html @@ -5,6 +5,8 @@ MCP X-Ray — see what's hidden before you run it + + @@ -12,6 +14,20 @@ +