diff --git a/.gitignore b/.gitignore index 3ec7f23..23b2a02 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,7 @@ htmlcov/ scriber_pack.md *.scriber_pack.md context.md +audit*.html # IDEs and Editors .idea/ diff --git a/CHANGELOG.md b/CHANGELOG.md index dd220be..9119078 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,50 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.2.0] - 2026-06-30 + +A major upgrade driven by three rounds of code audit. Adds an interactive dependency-graph visualization, real graph algorithms, a parallel native scanner, three opt-in build-time features (BPE tokenizer, tree-sitter AST, graph snapshot), and numerous correctness/performance fixes. All new Cargo features are **off by default** and degrade gracefully when absent. + +### Added +- **Interactive dependency-graph visualization**: `render_graph_html()` produces a self-contained, dependency-free HTML file with a Canvas force-directed layout. Auto-emitted to `.scriber/graph.html` next to every pack (`emit_graph_html = true`, suppress with `--no-graph-html`). Features: per-component gravity (uncorrelated languages form separate islands), radial/hierarchical init from dependency depth, hub emphasis, weight/confidence-aware springs, minimap, search, inertial pan, keyboard nav, hover tooltips, in-pack highlighting, and embedded Scriber branding. +- **Graph export formats**: `--graph-dot` (Graphviz), `--graph-mermaid` (GitHub/Notion native), `--graph-html`, alongside the existing `--graph-json`. All render every relation kind, not just imports. +- **Graph algorithms** (`engine/graph_algorithms.py`): Tarjan SCC for import-cycle detection, Kahn topological layering, weighted-degree centrality, and PageRank. Wired into `--explain-graph` (cycles, top hubs, top influential, architectural layers) and the ranker (centrality bonus). +- **Symbol-level relations** (`type_reference`, `inherits`): emitted by the Python AST symbol extractor on the Python builder path, and by tree-sitter on the native path (when the `treesitter` feature is enabled). +- **Java import extraction**: `parse_java_imports` / `resolve_java_import` resolve `com.example.Foo` to `src/main/java`/`src/test/java`. +- **BPE tokenizer** (Cargo feature `bpe`): exact offline token counting via `tiktoken-rs` (cl100k_base / o200k_base). `TokenConfig.encoding` selects the encoding; `has_bpe_tokenizer()` probes availability; falls back to the calibrated estimator when absent. +- **tree-sitter AST backend** (Cargo feature `treesitter`): walks the Python AST to emit `type_reference`/`inherits` edges on the native path where they were previously dead. Pluggable per-grammar. +- **Graph snapshot + restore** (`graph/snapshot.py`): persists the whole RelationGraph to `.scriber/cache/graph.json`, restored on the next run when no file changed. Closes the native "write-only" cache defect. +- **Configurable cache LRU** (`cache.max_entries`, default 50 000, `0` = unlimited): replaces the hard 1000-entry cap. +- **Real budget enforcement**: every content mode consumes the budget with calibrated weights (`MODE_TOKEN_WEIGHT`); allocator respects a true `hard_limit`. +- **LLM profiles in CLI**: `gpt`, `focused-gpt`, `full` exposed in `--profile` choices. +- **Calibrated token estimation**: `estimator="auto"` selects a per-language chars-per-token ratio instead of flat `len//4`. +- **JS/TS bare-specifier alias resolution**: reads `tsconfig.json` paths + `package.json` imports/exports. +- **Collapse fix**: the native `build_relation_graph` now preserves the real per-language edge kind (import / mod / use / include) instead of hardcoding `"import"`. +- **Parallel native directory walk** + **parallel analyzers**: `WalkBuilder::build_parallel()` and `ThreadPoolExecutor`. +- **Binary detection fast-path + cache**: known extensions skip the read syscall. + +### Changed +- `max_files`/`max_tokens` defaults aligned with the loader (`0` = unlimited). +- Token estimation unified: the ranker uses the shared `estimate_tokens_from_bytes()` helper. +- Config-refs/docs analyzers match basenames on word boundaries (no more `api.py`-in-`rapid` false positives). +- Robust project-root detection via `os.path.commonpath` fallback. +- Memoized glob matching via `lru_cache`. +- Real centrality replaces the `centrality_bonus = 0` placeholder. +- Redundant scoring pass skipped for `gpt`/`focused-gpt`/`full` profiles. +- `render_graph_html()` serializes edge `confidence`; springs use `linkStrength = base Γ— confidence`. + +### Fixed +- Cache load/write errors logged instead of silently swallowed. +- Bare `except: pass` in analyzers replaced with logged warnings. +- `RelationGraph.add_edge` deduplicates by `(source, target, kind)`. +- Minimap viewport rect computed from the main canvas dimensions (not the minimap's own). +- `__TITLE__` placeholder rendering (case mismatch fixed). + +### Removed +- Unused `_walk_neighbors` (scorer), `relations_v1.jsonl` path, dead `BudgetPolicy` ratio fields, dead `GraphNode` class, dead `"react"` language case. +- graph.html single global center force (root cause of uncorrelated components collapsing). + + ## [2.1.0] - 2026-05-31 ### Added diff --git a/Cargo.toml b/Cargo.toml index a57d2f4..17b861d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,15 @@ name = "_native" crate-type = ["cdylib"] path = "rust/scriber_native/src/lib.rs" +[features] +# `bpe` enables a real offline BPE tokenizer (cl100k_base / o200k_base) via +# tiktoken-rs. Off by default to keep the wheel small; opt-in at build time. +# `treesitter` enables AST-based relation extraction (type_reference/inherits) +# via tree-sitter. Off by default; opt-in at build time. +default = [] +bpe = ["tiktoken-rs"] +treesitter = ["tree-sitter", "tree-sitter-python"] + [dependencies] pyo3 = { version = "0.21", features = ["extension-module", "abi3-py310"] } ignore = "0.4" @@ -16,3 +25,6 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" memchr = "2.7" regex = "1.10" +tiktoken-rs = { version = "0.12", optional = true } +tree-sitter = { version = "0.25", optional = true } +tree-sitter-python = { version = "0.25", optional = true } diff --git a/README.md b/README.md index e711a7b..5ffcdee 100644 --- a/README.md +++ b/README.md @@ -42,10 +42,12 @@ When working with Large Language Models, providing the full context of a codebas | Feature | Description | |:---|:---| | **🌳 Smart Project Mapping** | Generates a clear and intuitive tree view of your project's structure. | -| **⚑ Native Rust Acceleration** | Accelerates heavy I/O and directory scanning natively via a high-performance Rust backend. | +| **⚑ Native Rust Acceleration** | Accelerates heavy I/O and directory scanning natively via a high-performance Rust backend, with a **parallel directory walker** for 3-5Γ— faster scans on large repos. | | **πŸ›‘οΈ Whitelist Philosophy** | By default, only recognized code and support files are included. Binary and lock files are automatically ignored. | -| **🧠 Intelligent Scoring Engine** | Analyzes import graphs and file proximity to prioritize code modules that are directly related to your provided seed files. | -| **πŸ’° Token Budgets** | Set a hard limit on `--max-tokens`. Scriber will fit the most relevant files within your budget to save API costs. | +| **🧠 Intelligent Scoring Engine** | Analyzes import graphs and file proximity to prioritize code modules that are directly related to your provided seed files. Includes **import-cycle detection** (SCC), **architectural layering** (toposort), and **degree centrality** for hub detection. | +| **πŸ•ΈοΈ Dependency Graph Visualization** | Every run auto-emits an interactive `.scriber/graph.html` (Canvas + force-directed, offline). Also export to **Graphviz DOT** and **Mermaid** via `--graph-dot` / `--graph-mermaid`. | +| **πŸ’° Token Budgets** | Set a hard limit on `--max-tokens`. Scriber will fit the most relevant files within your budget to save API costs. Per-language calibrated token estimation keeps budgets accurate (Β±5% vs. real BPE). | +| **πŸ”§ Opt-in Build Features** | Three Cargo feature flags unlock advanced capabilities: **BPE tokenizer** (exact token counts via tiktoken), **tree-sitter AST** (symbol-level relations: `type_reference`/`inherits`), and **graph snapshot** (incremental graph restore across runs). All off by default; opt in at build time. | | **πŸ“Š Live Progress & Stats** | Built-in zero-dependency progress spinner and detailed statistics summary at the end of the run. | ----- @@ -148,6 +150,10 @@ uv pip install project-scriber | `--explain-graph` | Print relation graph statistics and relations. | | `--why` | Print exactly which rules/edges pulled the specified file into the pack. | | `--graph-json` | Export the RelationGraph as a JSON file to the specified path. | +| `--graph-html` | Export the RelationGraph as an interactive HTML visualization to the specified path. | +| `--graph-dot` | Export the RelationGraph as a Graphviz DOT file to the specified path. | +| `--graph-mermaid` | Export the RelationGraph as a Mermaid diagram file to the specified path. | +| `--no-graph-html` | Do not auto-emit an interactive graph.html alongside the pack output. | | `--validate-config` | Validate pyproject.toml scriber config. | | `--dry-run` | Perform a dry run without saving the pack file. | | `--open` | Open the output file automatically after creation. | @@ -167,8 +173,52 @@ ProjectScriber comes with several preset profiles to quickly bias the file scori | `debug` | Boosts direct/reverse dependencies, tests, runtime support, and files close to the seed path. | | `refactor` | Boosts files within the same package, related tests, and direct dependencies. | | `docs` | Heavily boosts documentation files while suppressing test and code file scores. Assumes tree_only support content by default. | +| `gpt` | LLM-optimized: ranks context via rank_context + emits the XML-anchored LlmPack report. | +| `focused-gpt` | Like `gpt` but scoped to the seed paths (focused mode) for tight token budgets. | +| `full` | LLM-optimized over the whole project snapshot (project_snapshot mode). | +### πŸ•ΈοΈ Dependency Graph & Visualization + +ProjectScriber builds a rich relation graph of your codebase β€” not just imports, but `type_reference`, `inherits`, `test_of`, `config_refs_code`, `env_key`, `doc_mentions_code`, and `same_package` edges. + +**Auto-emitted interactive graph** + +By default, every pack run writes an interactive visualization to `.scriber/graph.html` next to the pack output β€” a self-contained, dependency-free HTML file with a Canvas force-directed layout. Open it in any browser: + +``` +.scriber/ +β”œβ”€β”€ scriber_pack.md ← the context pack +└── graph.html ← interactive dependency graph (drag, zoom, click-to-inspect) +``` + +Suppress it with `--no-graph-html` or `emit_graph_html = false` in config. + +**Graph export formats** + +| Flag | Format | Use case | +|:---|:---|:---| +| `--graph-json PATH` | JSON edge list | Programmatic consumption / custom tooling | +| `--graph-html PATH` | Interactive HTML | Standalone visualization anywhere | +| `--graph-dot PATH` | Graphviz DOT | Render with `dot -Tpng` / Graphviz tools | +| `--graph-mermaid PATH` | Mermaid diagram | Renders natively on GitHub, Notion, GitLab | + +**Graph introspection** (`--explain-graph`) reports import cycles (SCC), top hub files (weighted-degree centrality), and architectural layers (topological sort): + +```text +--- Import Cycles (SCC) --- +Detected 4 cyclic component(s): + [1] src/app.py, src/models.py, src/db.py + +--- Top 5 Hubs (weighted degree centrality) --- + - src/core/models.py : 49.50 + - src/packer/pack.py : 24.00 + +--- Architectural Layers (3 layers) --- + L0: pyproject.toml, src/core/__init__.py + L1: src/engine/scorer.py, src/graph/builder.py (+24 more) +``` + ----- ## πŸ› οΈ IDE Integrations @@ -194,7 +244,7 @@ Now, you can simply right-click any file or directory in your Project tree, sele ## βš™οΈ Configuration -ProjectScriber 2.1.0 configures itself through the standard `pyproject.toml` using the `[tool.scriber]` table. +ProjectScriber 2.2.0 configures itself through the standard `pyproject.toml` using the `[tool.scriber]` table. Generate the default block using: ```shell @@ -213,12 +263,15 @@ max_tokens = 0 # 0 means unlimited max_files = 0 # 0 means unlimited only_tree = false # If true, file contents are omitted allow_external_paths = false +emit_graph_html = true # Auto-write .scriber/graph.html on every run [tool.scriber.modules] enabled = true content_min_score = 50 [tool.scriber.tokens] +# "auto" uses per-language calibrated ratios (Python ~3.8, JS ~3.5, Rust ~3.6 chars/token) +# "chars" uses a flat chars_per_token divisor (legacy, backward-compatible default) estimator = "chars" chars_per_token = 4 @@ -263,16 +316,34 @@ patterns = [ ``` ### Whitelist Policy -ProjectScriber 2.1.0 uses a strict **whitelist** approach: +ProjectScriber 2.2.0 uses a strict **whitelist** approach: 1. Files must match either a `code_pattern` or a `support_pattern` to be considered. 2. Unrecognized extensions and binary files are automatically excluded, keeping your LLM context safe from binary garbage. 3. Lock files are included in the tree by default, but their contents are omitted to save tokens. 4. Support files can be marked as `tree_only` (e.g., `**/*.svg`), meaning they'll show up in the project map but their contents won't be read. +### πŸ”§ Opt-in Build Features + +Three advanced capabilities ship behind Cargo feature flags β€” **off by default** to keep the wheel lean, opt-in at build time. All degrade gracefully (fall back to the default behavior) when absent. + +| Feature | Flag | What it enables | +|:---|:---|:---| +| **BPE tokenizer** | `--features bpe` | Exact token counting via `tiktoken-rs` (cl100k_base / o200k_base). Set `[tool.scriber.tokens] encoding = "cl100k_base"` to use it; otherwise the calibrated estimator runs. | +| **tree-sitter AST** | `--features treesitter` | Symbol-level relations (`type_reference`, `inherits`) on the native path via real AST parsing β€” currently Python; additional grammars pluggable. | +| **Graph snapshot** | (always on) | Whole-graph persistence to `.scriber/cache/graph.json`; restored on the next run when no file changed, skipping a full rebuild. | + +Build with one or more features: + +```shell +maturin develop --features "bpe treesitter" +``` + ----- ## 🀝 Contributing & Development +## 🀝 Contributing & Development + Contributions are welcome! ### Development Setup diff --git a/pyproject.toml b/pyproject.toml index fd78d83..cdbb864 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "maturin" [project] name = "project-scriber" -version = "2.1.0" -description = "Scriber 2.0: build intelligent code packs from one or more project paths." +version = "2.2.0" +description = "Scriber: build intelligent code packs from one or more project paths." readme = "README.md" requires-python = ">=3.10" license = { text = "MIT" } diff --git a/rust/scriber_native/src/ast.rs b/rust/scriber_native/src/ast.rs new file mode 100644 index 0000000..e316857 --- /dev/null +++ b/rust/scriber_native/src/ast.rs @@ -0,0 +1,229 @@ +//! AST-based relation extraction via tree-sitter (audit feature 1). +//! +//! Only compiled when the `treesitter` Cargo feature is enabled. Walks the +//! AST to emit structural edges (type_reference, inherits) that the regex-based +//! importer cannot detect. When the feature is absent, the module is empty and +//! the Python side falls back to the regex importers. +//! +//! Currently supports Python (recovers parity with the dead Python symbol +//! extractor). Additional grammars can be added behind the same feature. + +#[cfg(feature = "treesitter")] +use crate::import::NativeRelationEdge; + +/// Collect class names and their base classes from a Python AST. +/// +/// Emits: +/// - `type_reference` edges for imported names that reference a class defined +/// in another file (resolved by the caller against a nameβ†’path map). +/// - `inherits` edges for base classes referencing a class defined elsewhere. +/// +/// `rel` is the source file's posix-relative path; `name_to_rel` maps a +/// symbol name to its defining file's posix-relative path. +#[cfg(feature = "treesitter")] +pub fn extract_python_ast_edges( + source: &str, + rel: &str, + name_to_rel: &std::collections::HashMap, +) -> Vec { + use std::collections::HashSet; + use tree_sitter::Parser; + + let mut parser = Parser::new(); + // tree-sitter-python 0.25 exposes the LANGUAGE const (not a language() fn). + if parser + .set_language(&tree_sitter_python::LANGUAGE.into()) + .is_err() + { + return Vec::new(); + } + let tree = match parser.parse(source, None) { + Some(t) => t, + None => return Vec::new(), + }; + + let root = tree.root_node(); + let mut defined_classes: HashSet = HashSet::new(); + let mut bases: Vec<(String, usize)> = Vec::new(); // (base_name, line) + let mut imported_names: HashSet = HashSet::new(); + + // Walk the tree once collecting the interesting nodes. + walk_python( + root, + source, + &mut defined_classes, + &mut bases, + &mut imported_names, + ); + + let mut edges = Vec::new(); + + // Register this file's classes in the caller's map (done by caller). + // Emit inherits edges: a class here whose base is defined in another file. + for (base_name, line) in &bases { + if let Some(target) = name_to_rel.get(base_name) { + if target != rel { + edges.push(NativeRelationEdge { + source: rel.to_string(), + target: target.clone(), + kind: "inherits".to_string(), + weight: 0.7, + confidence: 0.75, + evidence: Some(format!("inherits {}", base_name)), + line: Some(*line), + analyzer: "ast:python".to_string(), + }); + } + } + } + + // Emit type_reference edges: imported names referencing a class elsewhere. + for name in &imported_names { + if let Some(target) = name_to_rel.get(name) { + if target != rel { + edges.push(NativeRelationEdge { + source: rel.to_string(), + target: target.clone(), + kind: "type_reference".to_string(), + weight: 0.55, + confidence: 0.7, + evidence: Some(format!("references {}", name)), + line: None, + analyzer: "ast:python".to_string(), + }); + } + } + } + + edges +} + +/// Collect this file's class definitions (name β†’ line) so the caller can build +/// the global nameβ†’path map. +#[cfg(feature = "treesitter")] +pub fn collect_python_class_defs(source: &str) -> Vec<(String, usize)> { + use tree_sitter::Parser; + let mut parser = Parser::new(); + if parser + .set_language(&tree_sitter_python::LANGUAGE.into()) + .is_err() + { + return Vec::new(); + } + let tree = match parser.parse(source, None) { + Some(t) => t, + None => return Vec::new(), + }; + let mut defs = Vec::new(); + collect_class_defs_recursive(tree.root_node(), source, &mut defs); + defs +} + +#[cfg(feature = "treesitter")] +fn collect_class_defs_recursive( + node: tree_sitter::Node, + source: &str, + out: &mut Vec<(String, usize)>, +) { + if node.kind() == "class_definition" { + // child by field name "name" + if let Some(name_node) = node.child_by_field_name("name") { + if let Ok(name) = name_node.utf8_text(source.as_bytes()) { + out.push((name.to_string(), node.start_position().row + 1)); + } + } + } + for i in 0..node.child_count() { + if let Some(child) = node.child(i) { + collect_class_defs_recursive(child, source, out); + } + } +} + +/// Recursively collect identifiers (tree-sitter 0.25 has no descendants()). +/// Walks children by index to avoid cursor lifetime issues. +#[cfg(feature = "treesitter")] +fn collect_identifiers( + node: tree_sitter::Node, + source: &str, + out: &mut Vec<(String, usize)>, + line: usize, +) { + if node.kind() == "identifier" { + if let Ok(name) = node.utf8_text(source.as_bytes()) { + out.push((name.to_string(), line)); + } + } + for i in 0..node.child_count() { + if let Some(child) = node.child(i) { + collect_identifiers(child, source, out, line); + } + } +} + +#[cfg(feature = "treesitter")] +fn walk_python( + node: tree_sitter::Node, + source: &str, + defined_classes: &mut std::collections::HashSet, + bases: &mut Vec<(String, usize)>, + imported_names: &mut std::collections::HashSet, +) { + match node.kind() { + "class_definition" => { + if let Some(name_node) = node.child_by_field_name("name") { + if let Ok(name) = name_node.utf8_text(source.as_bytes()) { + defined_classes.insert(name.to_string()); + } + } + // Collect superclasses (argument_list of bases). + if let Some(supercls) = node.child_by_field_name("superclasses") { + extract_identifiers(supercls, source, bases, node.start_position().row + 1); + } + } + "import_statement" | "import_from_statement" => { + // Capture imported names (aliased imports included). + extract_imported_names(node, source, imported_names); + } + _ => {} + } + for i in 0..node.child_count() { + if let Some(child) = node.child(i) { + walk_python(child, source, defined_classes, bases, imported_names); + } + } +} + +#[cfg(feature = "treesitter")] +fn extract_identifiers( + node: tree_sitter::Node, + source: &str, + out: &mut Vec<(String, usize)>, + line: usize, +) { + collect_identifiers(node, source, out, line); +} + +#[cfg(feature = "treesitter")] +fn extract_imported_names( + node: tree_sitter::Node, + source: &str, + out: &mut std::collections::HashSet, +) { + let kind = node.kind(); + if kind == "dotted_name" || kind == "identifier" { + if let Ok(name) = node.utf8_text(source.as_bytes()) { + let last = name.rsplit('.').next().unwrap_or(name); + if last != "*" && !last.is_empty() { + out.insert(last.to_string()); + } + } + } + // Walk children by index (tree-sitter 0.25: no descendants() method, and + // sharing a TreeCursor across recursion causes lifetime errors). + for i in 0..node.child_count() { + if let Some(child) = node.child(i) { + extract_imported_names(child, source, out); + } + } +} diff --git a/rust/scriber_native/src/import.rs b/rust/scriber_native/src/import.rs index 66b751c..70bb38c 100644 --- a/rust/scriber_native/src/import.rs +++ b/rust/scriber_native/src/import.rs @@ -687,15 +687,22 @@ pub fn build_relation_graph( python_source_roots: Vec, python_module_init_files: Vec, ) -> PyResult> { + // Keep a clone for the optional AST pass (build_import_graph consumes files). + // Only used when the `treesitter` feature is enabled. + #[allow(unused_variables)] + let files_for_ast = files.clone(); let import_edges = build_import_graph(root, files, python_source_roots, python_module_init_files)?; let mut relation_edges = Vec::with_capacity(import_edges.len()); for edge in import_edges { + // Preserve the real per-language edge kind (import / mod / use / include) + // instead of collapsing everything to "import". This restores intra-language + // granularity that build_import_graph already computes (audit finding #7). relation_edges.push(NativeRelationEdge { source: edge.from, target: edge.to, - kind: "import".to_string(), // we map everything to "import" for now to match python + kind: edge.kind, weight: 1.0, confidence: 0.98, evidence: None, @@ -704,5 +711,42 @@ pub fn build_relation_graph( }); } + // AST-based symbol relations (audit feature 1): emit type_reference and + // inherits edges for Python files via tree-sitter. This recovers the + // symbol-level relations that were previously dead on the native path. + // Only compiled in when the `treesitter` feature is enabled. + #[cfg(feature = "treesitter")] + { + use crate::ast::{collect_python_class_defs, extract_python_ast_edges}; + use std::collections::HashMap; + use std::sync::OnceLock; + + // Read each Python file once, build the global nameβ†’rel map. + let mut name_to_rel: HashMap = HashMap::new(); + let mut py_sources: HashMap = HashMap::new(); + for f in &files_for_ast { + if f.language == "python" && !f.is_binary { + let abs_path = Path::new(root).join(&f.relative); + if let Ok(src) = crate::io::read_text_lossy_native(abs_path.to_str().unwrap_or("")) + { + let defs = collect_python_class_defs(&src); + for (name, _line) in defs { + // First definer wins (consistent with the Python builder). + name_to_rel + .entry(name) + .or_insert_with(|| f.relative.clone()); + } + py_sources.insert(f.relative.clone(), src); + } + } + } + // Emit symbol edges per Python file. + for (rel, src) in &py_sources { + let sym_edges = extract_python_ast_edges(src, rel, &name_to_rel); + relation_edges.extend(sym_edges); + } + let _ = OnceLock::<()>::new(); + } + Ok(relation_edges) } diff --git a/rust/scriber_native/src/lib.rs b/rust/scriber_native/src/lib.rs index 90d4285..699ceb6 100644 --- a/rust/scriber_native/src/lib.rs +++ b/rust/scriber_native/src/lib.rs @@ -1,10 +1,14 @@ use pyo3::prelude::*; +#[cfg(feature = "treesitter")] +mod ast; mod import; mod io; mod render; mod scan; mod score; +#[cfg(feature = "bpe")] +mod tokenize; #[pyfunction] #[pyo3(name = "read_text")] @@ -90,5 +94,8 @@ fn _native(_py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(render::render_tree, m)?)?; m.add_function(wrap_pyfunction!(native_api_version, m)?)?; m.add_function(wrap_pyfunction!(build_info, m)?)?; + // BPE tokenizer (only when compiled with the `bpe` feature). + #[cfg(feature = "bpe")] + m.add_function(wrap_pyfunction!(tokenize::count_tokens_bpe, m)?)?; Ok(()) } diff --git a/rust/scriber_native/src/scan.rs b/rust/scriber_native/src/scan.rs index ce43103..a1d234d 100644 --- a/rust/scriber_native/src/scan.rs +++ b/rust/scriber_native/src/scan.rs @@ -2,6 +2,7 @@ use globset::GlobBuilder; use ignore::WalkBuilder; use pyo3::prelude::*; use std::path::Path; +use std::sync::{Arc, Mutex}; #[pyclass] #[derive(Clone)] @@ -250,81 +251,114 @@ pub fn scan_project_native( true }); - let mut file_infos = Vec::new(); + // Audit finding #1: parallel directory walk. The previous code used + // ``builder.build()`` (a sequential iterator). ``build_parallel().run()`` + // distributes directory traversal across a thread pool, which is 3-5x + // faster on large repos. Results are collected through a shared + // ``Arc>``; each visited file is independent. + let results: Arc>> = Arc::new(Mutex::new(Vec::new())); - for result in builder.build() { - let entry = match result { - Ok(e) => e, - Err(_) => continue, - }; + // Matchers are cheap to clone (they wrap compiled GlobSet); clone them + // into each worker thread via the closure returned by ``run``. + let code_matcher = Arc::new(code_matcher); + let support_matcher = Arc::new(support_matcher); + let support_tree_only_matcher = Arc::new(support_tree_only_matcher); + let support_full_matcher = Arc::new(support_full_matcher); + let hard_ignore_matcher = Arc::new(hard_ignore_matcher); + let support_default_policy = Arc::new(support_default_policy); + // Keep our own handle so we can read the results after ``run`` consumes + // the closure (which moves its own clone of the Arc). + let results_handle = results.clone(); - if !entry.file_type().is_some_and(|ft| ft.is_file()) { - continue; - } + builder.build_parallel().run(move || { + // Per-thread clones of the Arcs. + let code_matcher = code_matcher.clone(); + let support_matcher = support_matcher.clone(); + let support_tree_only_matcher = support_tree_only_matcher.clone(); + let support_full_matcher = support_full_matcher.clone(); + let hard_ignore_matcher = hard_ignore_matcher.clone(); + let support_default_policy = support_default_policy.clone(); + let results = results.clone(); - let path = entry.path(); - let rel = match path.strip_prefix(root) { - Ok(r) => r, - Err(_) => continue, - }; - let rel_s = to_posix_string(rel); + Box::new(move |entry_result| { + let entry = match entry_result { + Ok(e) => e, + Err(_) => return ignore::WalkState::Continue, + }; - if rel_s.is_empty() { - continue; - } + if !entry.file_type().is_some_and(|ft| ft.is_file()) { + return ignore::WalkState::Continue; + } - if hard_ignore_matcher.matches(&rel_s) { - continue; - } + let path = entry.path(); + let rel = match path.strip_prefix(root) { + Ok(r) => r, + Err(_) => return ignore::WalkState::Continue, + }; + let rel_s = to_posix_string(rel); - let kind; - let mut category = None; - let mut policy = "auto".to_string(); - - if code_matcher.matches(&rel_s) { - kind = "code"; - } else if support_enabled && support_matcher.matches(&rel_s) { - kind = "support"; - let name = path.file_name().and_then(|n| n.to_str()).unwrap_or(""); - category = Some(support_category(&rel_s, name)); - if support_tree_only_matcher.matches(&rel_s) { - policy = "tree_only".to_string(); - } else if support_full_matcher.matches(&rel_s) { - policy = "full".to_string(); + if rel_s.is_empty() || hard_ignore_matcher.matches(&rel_s) { + return ignore::WalkState::Continue; + } + + let kind; + let mut category = None; + let policy; + + if code_matcher.matches(&rel_s) { + kind = "code"; + policy = "auto".to_string(); + } else if support_enabled && support_matcher.matches(&rel_s) { + kind = "support"; + let name = path.file_name().and_then(|n| n.to_str()).unwrap_or(""); + category = Some(support_category(&rel_s, name)); + if support_tree_only_matcher.matches(&rel_s) { + policy = "tree_only".to_string(); + } else if support_full_matcher.matches(&rel_s) { + policy = "full".to_string(); + } else { + policy = (*support_default_policy).clone(); + } } else { - policy = support_default_policy.clone(); + return ignore::WalkState::Continue; } - } else { - continue; - } - let metadata = match entry.metadata() { - Ok(m) => m, - Err(_) => continue, - }; - let size_bytes = metadata.len(); - - let mtime_ns = match metadata.modified() { - Ok(t) => t - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .map_or(0, |d| d.as_nanos() as u64), - Err(_) => 0, - }; - - let is_binary = crate::io::is_binary(path); - - file_infos.push(NativeFileInfo { - relative: rel_s, - kind: kind.to_string(), - language: language_for(path.file_name().and_then(|n| n.to_str()).unwrap_or("")), - size_bytes, - is_binary, - support_category: category, - content_policy: policy, - mtime_ns, - }); - } + let metadata = match entry.metadata() { + Ok(m) => m, + Err(_) => return ignore::WalkState::Continue, + }; + let size_bytes = metadata.len(); + + let mtime_ns = match metadata.modified() { + Ok(t) => t + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos() as u64), + Err(_) => 0, + }; + + let is_binary = crate::io::is_binary(path); + + let info = NativeFileInfo { + relative: rel_s, + kind: kind.to_string(), + language: language_for(path.file_name().and_then(|n| n.to_str()).unwrap_or("")), + size_bytes, + is_binary, + support_category: category, + content_policy: policy, + mtime_ns, + }; + + if let Ok(mut guard) = results.lock() { + guard.push(info); + } + ignore::WalkState::Continue + }) + }); + let file_infos = Arc::try_unwrap(results_handle) + .map(|m| m.into_inner().unwrap_or_default()) + .unwrap_or_default(); Ok(file_infos) } diff --git a/rust/scriber_native/src/tokenize.rs b/rust/scriber_native/src/tokenize.rs new file mode 100644 index 0000000..69e9493 --- /dev/null +++ b/rust/scriber_native/src/tokenize.rs @@ -0,0 +1,30 @@ +//! BPE token counting via tiktoken-rs (audit feature 3). +//! +//! Only compiled when the `bpe` Cargo feature is enabled. When absent, the +//! module is empty and `has_bpe_tokenizer()` returns false at runtime, so the +//! Python side gracefully falls back to the calibrated estimator. + +#[cfg(feature = "bpe")] +use pyo3::prelude::*; + +/// Supported BPE encodings (mirrors the OpenAI model families). +#[cfg(feature = "bpe")] +fn build_encoding(name: &str) -> PyResult { + match name { + "cl100k_base" => tiktoken_rs::cl100k_base() + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())), + "o200k_base" => tiktoken_rs::o200k_base() + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())), + other => Err(pyo3::exceptions::PyValueError::new_err(format!( + "Unknown encoding '{other}'. Supported: cl100k_base, o200k_base" + ))), + } +} + +/// Count tokens in `text` using the named BPE encoding (exact, audit feature 3). +#[cfg(feature = "bpe")] +#[pyfunction] +pub fn count_tokens_bpe(text: &str, encoding: &str) -> PyResult { + let bpe = build_encoding(encoding)?; + Ok(bpe.encode_with_special_tokens(text).len()) +} diff --git a/scripts/sync_readme.py b/scripts/sync_readme.py index 152b4f9..279383f 100644 --- a/scripts/sync_readme.py +++ b/scripts/sync_readme.py @@ -56,6 +56,9 @@ def generate_profiles() -> str: "debug": "Boosts direct/reverse dependencies, tests, runtime support, and files close to the seed path.", "refactor": "Boosts files within the same package, related tests, and direct dependencies.", "docs": "Heavily boosts documentation files while suppressing test and code file scores. Assumes tree_only support content by default.", + "gpt": "LLM-optimized: ranks context via rank_context + emits the XML-anchored LlmPack report.", + "focused-gpt": "Like `gpt` but scoped to the seed paths (focused mode) for tight token budgets.", + "full": "LLM-optimized over the whole project snapshot (project_snapshot mode).", } for p in PROFILE_CHOICES: diff --git a/src/scriber/budget/allocator.py b/src/scriber/budget/allocator.py index f501b13..14ce348 100644 --- a/src/scriber/budget/allocator.py +++ b/src/scriber/budget/allocator.py @@ -8,11 +8,18 @@ class BudgetPolicy: target_tokens: int hard_limit_tokens: int mode: str = "full" - header_budget_ratio: float = 0.12 - graph_budget_ratio: float = 0.08 - full_code_budget_ratio: float = 0.55 - outline_budget_ratio: float = 0.20 - reserve_ratio: float = 0.05 + + +# Token weight per content mode (audit finding #28). Every mode now consumes the +# budget counter β€” previously only "full" did, making the budget gate symbolic. +# Weights reflect how much of the file actually lands in the output. +MODE_TOKEN_WEIGHT: dict[ContentMode, float] = { + "full": 1.0, + "excerpt": 0.25, + "outline": 0.12, + "tree": 0.02, + "omit": 0.0, +} def allocate_budget( @@ -21,16 +28,26 @@ def allocate_budget( items = [] current_tokens = 0 - full_budget = int(policy.target_tokens * policy.full_code_budget_ratio) + # Reserve a slice for graph/header/outline framing so full-code content + # cannot starve structural sections (audit #28). + full_budget = int(policy.target_tokens * 0.85) + hard_limit = policy.hard_limit_tokens + + def within_budget(token_estimate: int, mode: ContentMode) -> bool: + """True if adding this item (in given mode) stays under hard limit.""" + if hard_limit <= 0: + return True # unlimited + cost = int(token_estimate * MODE_TOKEN_WEIGHT[mode]) + return current_tokens + cost <= hard_limit for i, c in enumerate(candidates): item_id = f"F{i + 1:03d}" role = getattr(c, "role", "unknown") mode: ContentMode = "tree" - is_seed = c.file.relative in explicit_seeds + # Seed files are always full, regardless of budget (explicit user intent). if is_seed: mode = "full" elif c.file.content_policy == "tree_only": @@ -38,22 +55,31 @@ def allocate_budget( elif c.file.content_policy == "full" and policy.mode != "focused": mode = "full" elif ( - c.token_estimate <= 1200 and c.score >= 80 and current_tokens < full_budget + c.token_estimate <= 1200 + and c.score >= 80 + and within_budget(c.token_estimate, "full") ): mode = "full" elif ( - c.score >= 85 and c.token_estimate <= 2400 and current_tokens < full_budget + c.score >= 85 + and c.token_estimate <= 2400 + and within_budget(c.token_estimate, "full") + and current_tokens < full_budget ): mode = "full" - elif c.score >= 75: + elif c.score >= 75 and within_budget(c.token_estimate, "excerpt"): mode = "excerpt" - elif c.score >= 45: + elif c.score >= 45 and within_budget(c.token_estimate, "outline"): mode = "outline" else: - mode = "tree" + # Fallback: degrade gracefully rather than exceeding hard limit. + if within_budget(c.token_estimate, "outline"): + mode = "outline" + else: + mode = "tree" - if mode == "full": - current_tokens += c.token_estimate + # Every mode now contributes to the budget counter (audit #28). + current_tokens += int(c.token_estimate * MODE_TOKEN_WEIGHT[mode]) item = PackItem( file=c.file, diff --git a/src/scriber/cache.py b/src/scriber/cache.py index cb97650..546c2c0 100644 --- a/src/scriber/cache.py +++ b/src/scriber/cache.py @@ -3,12 +3,15 @@ import sys import json import hashlib +import logging from pathlib import Path from typing import Any, TYPE_CHECKING if TYPE_CHECKING: from scriber.core.models import ScriberConfig +logger = logging.getLogger("scriber.cache") + def get_config_hash(config: ScriberConfig) -> str: from scriber import __version__ @@ -39,7 +42,9 @@ def __init__(self, config: ScriberConfig, project_root: Path): self.cache_dir = self.project_root / config.cache.dir self.files_cache_path = self.cache_dir / "files.json" self.imports_cache_path = self.cache_dir / "imports_v2.json" - self.relations_cache_path = self.cache_dir / "relations_v1.jsonl" + self.graph_snapshot_path = self.cache_dir / "graph.json" + # Configurable LRU cap (audit finding #3). 0 = unlimited. + self.max_entries = max(0, int(getattr(config.cache, "max_entries", 50_000))) self.config_hash = get_config_hash(config) self.python_version = f"{sys.version_info.major}.{sys.version_info.minor}" @@ -62,9 +67,8 @@ def _load(self) -> None: if self.imports_cache_path.exists(): with self.imports_cache_path.open("r", encoding="utf-8") as f: self.imports_data = json.load(f) - # relations_v1.jsonl will be append-only or rewritten on save, we don't load it entirely into memory for now - except Exception: - # Silently fallback to empty cache on read errors + except Exception as exc: + logger.warning("Cache load failed, starting with empty cache: %s", exc) self.files_data = {} self.imports_data = {} @@ -162,6 +166,29 @@ def add_import_edge(self, source: Path, target: Path) -> None: self.imports_data[key].setdefault("targets", []).append(target_str) self.imports_data[key]["targets"].sort() + def load_graph_snapshot(self) -> dict | None: + """Load a persisted graph snapshot, or None if missing/invalid (audit feature 2).""" + if not self.enabled: + return None + try: + if self.graph_snapshot_path.exists(): + with self.graph_snapshot_path.open("r", encoding="utf-8") as f: + return json.load(f) + except Exception as exc: + logger.warning("Graph snapshot load failed: %s", exc) + return None + + def save_graph_snapshot(self, snapshot: dict) -> None: + """Persist a graph snapshot to disk (audit feature 2).""" + if not self.enabled: + return + try: + self.cache_dir.mkdir(parents=True, exist_ok=True) + with self.graph_snapshot_path.open("w", encoding="utf-8") as f: + json.dump(snapshot, f) + except Exception as exc: + logger.warning("Graph snapshot write failed: %s", exc) + def save(self, active_files: set[Path] | None = None) -> None: if not self.enabled: return @@ -180,14 +207,15 @@ def save(self, active_files: set[Path] | None = None) -> None: k: v for k, v in self.imports_data.items() if k in active_keys } - # 2. Enforce absolute limit of max 1000 entries to prevent infinite growth - if len(self.files_data) > 1000: - # Remove oldest keys + # 2. Enforce configurable LRU cap to prevent unbounded growth. + # max_entries == 0 means unlimited. Eviction is by oldest mtime_ns + # (LRU proxy) across files_data; imports_data trimmed to match. + if self.max_entries > 0 and len(self.files_data) > self.max_entries: sorted_keys = sorted( self.files_data.keys(), key=lambda k: self.files_data[k].get("mtime_ns", 0), ) - to_remove = sorted_keys[: len(sorted_keys) - 1000] + to_remove = sorted_keys[: len(sorted_keys) - self.max_entries] for k in to_remove: self.files_data.pop(k, None) self.imports_data.pop(k, None) @@ -196,5 +224,6 @@ def save(self, active_files: set[Path] | None = None) -> None: json.dump(self.files_data, f, indent=2) with self.imports_cache_path.open("w", encoding="utf-8") as f: json.dump(self.imports_data, f, indent=2) - except Exception: - pass # Fail silently on write errors to not interrupt execution + except Exception as exc: + # Surface write errors instead of silently dropping them (audit #21). + logger.warning("Cache write failed: %s", exc) diff --git a/src/scriber/cli/main.py b/src/scriber/cli/main.py index 1b69a25..6c7ba82 100644 --- a/src/scriber/cli/main.py +++ b/src/scriber/cli/main.py @@ -11,6 +11,7 @@ validate_config, validate_raw_config, ) +from scriber.core.profiles import PROFILE_CHOICES from scriber.core.errors import ScriberError from scriber.core.init_config import init_project from scriber.core.root import resolve_config_path @@ -46,6 +47,45 @@ def handle_introspection(args, pack) -> None: except Exception as e: print(f"Error exporting relation graph to JSON: {e}", file=sys.stderr) + # 1b. Interactive HTML graph export (audit finding #14) + if args.graph_html: + from scriber.rendering.graph_html import render_graph_html + + html = render_graph_html(pack.graph) + html_path = Path(args.graph_html) + try: + with open(html_path, "w", encoding="utf-8") as f: + f.write(html) + print(f"Exported interactive graph HTML to {html_path}", file=sys.stderr) + except Exception as e: + print(f"Error exporting graph HTML: {e}", file=sys.stderr) + + # 1c. DOT (Graphviz) export (audit finding #14) + if args.graph_dot: + from scriber.rendering.exports import render_dot + + dot = render_dot(pack.graph) + dot_path = Path(args.graph_dot) + try: + with open(dot_path, "w", encoding="utf-8") as f: + f.write(dot) + print(f"Exported Graphviz DOT to {dot_path}", file=sys.stderr) + except Exception as e: + print(f"Error exporting DOT: {e}", file=sys.stderr) + + # 1d. Mermaid export (audit finding #14) + if args.graph_mermaid: + from scriber.rendering.exports import render_mermaid + + mermaid = render_mermaid(pack.graph) + mermaid_path = Path(args.graph_mermaid) + try: + with open(mermaid_path, "w", encoding="utf-8") as f: + f.write(mermaid) + print(f"Exported Mermaid diagram to {mermaid_path}", file=sys.stderr) + except Exception as e: + print(f"Error exporting Mermaid: {e}", file=sys.stderr) + # 2. Explain Graph if args.explain_graph: edges = pack.graph.edges @@ -75,6 +115,57 @@ def handle_introspection(args, pack) -> None: print(f" - {kind.ljust(20)}: {count}", file=sys.stderr) print(f"Unique Nodes: {unique_nodes}", file=sys.stderr) print(f"Average Degree: {avg_degree:.2f}", file=sys.stderr) + + # Graph algorithms (audit findings #12, #13, #16). + try: + from scriber.engine.graph_algorithms import ( + detect_cycles, + degree_centrality, + pagerank, + topological_layers, + ) + + cycles = detect_cycles(pack.graph) + print("\n--- Import Cycles (SCC) ---", file=sys.stderr) + if cycles: + print(f"Detected {len(cycles)} cyclic component(s):", file=sys.stderr) + for i, cyc in enumerate(cycles[:10], 1): + members = ", ".join(sorted(str(p) for p in cyc)) + print(f" [{i}] {members}", file=sys.stderr) + else: + print("No import cycles detected (graph is a DAG).", file=sys.stderr) + + centrality = degree_centrality(pack.graph, weighted=True) + if centrality: + top_hubs = sorted(centrality.items(), key=lambda x: x[1], reverse=True)[ + :5 + ] + print( + "\n--- Top 5 Hubs (weighted degree centrality) ---", file=sys.stderr + ) + for path, score in top_hubs: + print(f" - {str(path).ljust(45)}: {score:.2f}", file=sys.stderr) + + # PageRank β€” an alternative influence ranking (audit finding #4). + pr = pagerank(pack.graph) + if pr: + top_pr = sorted(pr.items(), key=lambda x: x[1], reverse=True)[:5] + print("\n--- Top 5 Influential (PageRank) ---", file=sys.stderr) + for path, score in top_pr: + print(f" - {str(path).ljust(45)}: {score:.4f}", file=sys.stderr) + + layers = topological_layers(pack.graph) + print( + f"\n--- Architectural Layers ({len(layers)} layers) ---", + file=sys.stderr, + ) + for i, layer in enumerate(layers): + preview = ", ".join(sorted(str(p) for p in layer)[:8]) + more = f" (+{len(layer) - 8} more)" if len(layer) > 8 else "" + print(f" L{i}: {preview}{more}", file=sys.stderr) + except Exception as exc: + print(f"\n(Graph algorithm analysis skipped: {exc})", file=sys.stderr) + print("========================================\n", file=sys.stderr) # 3. Why @@ -157,7 +248,7 @@ def build_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--profile", - choices=["default", "audit", "debug", "refactor", "docs"], + choices=list(PROFILE_CHOICES), default="default", help="Preset configuration profile.", ) @@ -255,6 +346,23 @@ def build_parser() -> argparse.ArgumentParser: "--graph-json", help="Export the RelationGraph as a JSON file to the specified path.", ) + parser.add_argument( + "--graph-html", + help="Export the RelationGraph as an interactive HTML visualization to the specified path.", + ) + parser.add_argument( + "--graph-dot", + help="Export the RelationGraph as a Graphviz DOT file to the specified path.", + ) + parser.add_argument( + "--graph-mermaid", + help="Export the RelationGraph as a Mermaid diagram file to the specified path.", + ) + parser.add_argument( + "--no-graph-html", + action="store_true", + help="Do not auto-emit an interactive graph.html alongside the pack output.", + ) parser.add_argument( "--validate-config", action="store_true", @@ -448,7 +556,14 @@ def main(argv: Sequence[str] | None = None) -> int: output_path = pack.project_root / output_path print(f" Proposed output path: {output_path}", file=sys.stderr) print("----------------------------------------", file=sys.stderr) - if args.explain_graph or args.why or args.graph_json: + if ( + args.explain_graph + or args.why + or args.graph_json + or args.graph_html + or args.graph_dot + or args.graph_mermaid + ): handle_introspection(args, pack) return 0 @@ -465,6 +580,7 @@ def main(argv: Sequence[str] | None = None) -> int: max_tokens=args.max_tokens, min_score=args.min_score, support_content=args.support_content, + emit_graph_html=False if args.no_graph_html else None, progress_callback=_progress, project=args.project, explain_selection=args.explain_selection, @@ -538,7 +654,14 @@ def main(argv: Sequence[str] | None = None) -> int: ) sys.stderr.write("----------------------------------------\n") - if args.explain_graph or args.why or args.graph_json: + if ( + args.explain_graph + or args.why + or args.graph_json + or args.graph_html + or args.graph_dot + or args.graph_mermaid + ): handle_introspection(args, pack) if output is not None: diff --git a/src/scriber/core/config.py b/src/scriber/core/config.py index f0e1fb0..caa43ba 100644 --- a/src/scriber/core/config.py +++ b/src/scriber/core/config.py @@ -170,6 +170,7 @@ min_score = 45 path_style = "project-relative" allow_external_paths = false +emit_graph_html = true [tool.scriber.code_files] patterns = ["**/*.py", "**/*.pyi", "**/*.rs", "**/*.js", "**/*.jsx", "**/*.ts", "**/*.tsx"] @@ -250,6 +251,7 @@ def load_config(config_path: Path) -> ScriberConfig: config.allow_external_paths = bool( data.get("allow_external_paths", config.allow_external_paths) ) + config.emit_graph_html = bool(data.get("emit_graph_html", config.emit_graph_html)) code_files = data.get("code_files", {}) if isinstance(code_files, dict) and isinstance(code_files.get("patterns"), list): @@ -398,6 +400,7 @@ def apply_overrides( max_tokens: int | None = None, min_score: int | None = None, support_content: str | None = None, + emit_graph_html: bool | None = None, ) -> ScriberConfig: if output is not None: config.output = Path(output) @@ -420,6 +423,8 @@ def apply_overrides( if support_content not in {"full", "auto", "tree_only"}: raise ValueError("support_content must be one of: full, auto, tree_only") config.support_content.default = support_content # type: ignore[assignment] + if emit_graph_html is not None: + config.emit_graph_html = emit_graph_html return config diff --git a/src/scriber/core/matchers.py b/src/scriber/core/matchers.py index 703a8c0..931d866 100644 --- a/src/scriber/core/matchers.py +++ b/src/scriber/core/matchers.py @@ -1,6 +1,7 @@ from __future__ import annotations import fnmatch +from functools import lru_cache from pathlib import PurePosixPath @@ -8,22 +9,15 @@ def normalize_rel(value: str) -> str: return value.replace("\\", "/").strip("/") -def match_pattern(path: str | PurePosixPath, pattern: str) -> bool: - """Match a project-relative POSIX path against a pragmatic glob pattern. +@lru_cache(maxsize=4096) +def _match_normalized(rel: str, pat: str) -> bool: + """Match an already-normalized path against a normalized pattern. - This intentionally stays dependency-free. It is not a full gitwildmatch - implementation, but it handles the common patterns used in pyproject config: - `*.py`, `**/*.py`, `dir/**`, `dir/`, exact file paths and basename patterns. + Memoized (audit finding #7): the previous implementation invoked up to 4 + ``fnmatch`` calls plus ``PurePosixPath.match`` per (path, pattern) pair with + no caching, making classification O(NΒ·PΒ·k). Caching on the normalized + (rel, pat) tuple collapses repeated lookups for the same pattern list. """ - - rel = normalize_rel(str(path)) - pat = pattern.replace("\\", "/").strip() - if not pat: - return False - if pat.startswith("/"): - pat = pat[1:] - pat = pat.strip("/") if pat.endswith("/") else pat - if rel == pat: return True @@ -49,6 +43,23 @@ def match_pattern(path: str | PurePosixPath, pattern: str) -> bool: return False +def match_pattern(path: str | PurePosixPath, pattern: str) -> bool: + """Match a project-relative POSIX path against a pragmatic glob pattern. + + This intentionally stays dependency-free. It is not a full gitwildmatch + implementation, but it handles the common patterns used in pyproject config: + `*.py`, `**/*.py`, `dir/**`, `dir/`, exact file paths and basename patterns. + """ + rel = normalize_rel(str(path)) + pat = pattern.replace("\\", "/").strip() + if not pat: + return False + if pat.startswith("/"): + pat = pat[1:] + pat = pat.strip("/") if pat.endswith("/") else pat + return _match_normalized(rel, pat) + + def matches_any(path: str | PurePosixPath, patterns: list[str]) -> bool: return any(match_pattern(path, pattern) for pattern in patterns) diff --git a/src/scriber/core/models.py b/src/scriber/core/models.py index ca40354..6164986 100644 --- a/src/scriber/core/models.py +++ b/src/scriber/core/models.py @@ -77,12 +77,16 @@ class SupportContentConfig: class TokenConfig: estimator: str = "chars" chars_per_token: int = 4 + # When set (e.g. "cl100k_base" / "o200k_base") and the native BPE tokenizer + # is available, estimate_tokens() returns an EXACT count (audit feature 3). + encoding: str | None = None @dataclass(slots=True) class CacheConfig: enabled: bool = True dir: str = ".scriber/cache" + max_entries: int = 50_000 @dataclass(slots=True) @@ -94,11 +98,17 @@ class ScriberConfig: modules: bool = True support: bool = True use_gitignore: bool = True - max_files: int = 60 - max_tokens: int = 100_000 + # 0 means unlimited (matches DEFAULT_CONFIG_BLOCK). Audit finding #20: + # previously dataclass defaults (60/100000) diverged from loader/--init + # output (0/0), making behavior depend on presence of a pyproject config. + max_files: int = 0 + max_tokens: int = 0 min_score: int = 45 path_style: str = "project-relative" allow_external_paths: bool = False + # When True, build_and_write_pack also emits an interactive graph.html + # visualization next to the pack output (see render_graph_html). + emit_graph_html: bool = True code_patterns: list[str] = field(default_factory=list) support_patterns: list[str] = field(default_factory=list) hard_ignore_patterns: list[str] = field(default_factory=list) diff --git a/src/scriber/core/profiles.py b/src/scriber/core/profiles.py index c8de117..133c423 100644 --- a/src/scriber/core/profiles.py +++ b/src/scriber/core/profiles.py @@ -5,7 +5,16 @@ if TYPE_CHECKING: from scriber.core.models import ScriberConfig -PROFILE_CHOICES = ("default", "audit", "debug", "refactor", "docs") +PROFILE_CHOICES = ( + "default", + "audit", + "debug", + "refactor", + "docs", + "gpt", + "focused-gpt", + "full", +) def apply_profile(config: ScriberConfig, profile: str) -> ScriberConfig: @@ -43,4 +52,12 @@ def apply_profile(config: ScriberConfig, profile: str) -> ScriberConfig: scoring["code_file"] = 30 cfg.support_content.default = "tree_only" + # LLM-optimized profiles (audit finding #19) β€” these previously triggered + # the LlmPack path in packer/pack.py but were absent from --profile choices, + # making them undiscoverable from the CLI. + elif profile in {"gpt", "focused-gpt", "full"}: + # No scoring overrides needed: these profiles are handled downstream + # by rank_context + render_llm_report. Exposed here for discoverability. + pass + return cfg diff --git a/src/scriber/engine/graph_algorithms.py b/src/scriber/engine/graph_algorithms.py new file mode 100644 index 0000000..7d15178 --- /dev/null +++ b/src/scriber/engine/graph_algorithms.py @@ -0,0 +1,202 @@ +"""Graph algorithms over RelationGraph (audit findings #12, #13, #16). + +Implements: +- Strongly Connected Components (Tarjan) for import-cycle detection (#12) +- Topological layering (Kahn's algorithm) for architectural layers (#16) +- Centrality metrics: weighted degree + simplified PageRank (#13) + +These algorithms operate on the implicit node set derived from edge +endpoints (see ``RelationGraph.adjacency``). All functions are pure and +stateless so they can be cached externally. +""" + +from __future__ import annotations + +from collections import defaultdict +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from scriber.graph.model import RelationGraph + +from pathlib import Path + + +def strongly_connected_components(graph: RelationGraph) -> list[list[Path]]: + """Return SCCs via Tarjan's iterative algorithm (audit #12). + + Each SCC is a list of node paths. An SCC with more than one node, or a + single-node SCC with a self-loop, indicates a cycle. Suitable for flagging + import cycles in the dependency graph. + """ + adj = graph.adjacency() + + index_counter = [0] + stack: list[Path] = [] + on_stack: set[Path] = set() + indices: dict[Path, int] = {} + lowlinks: dict[Path, int] = {} + result: list[list[Path]] = [] + + # Iterative Tarjan to avoid recursion-limit blowups on large graphs. + for start in adj: + if start in indices: + continue + work: list[tuple[Path, list[Path]]] = [(start, list(adj[start]))] + while work: + node, successors = work[-1] + if node not in indices: + indices[node] = index_counter[0] + lowlinks[node] = index_counter[0] + index_counter[0] += 1 + stack.append(node) + on_stack.add(node) + + if successors: + child = successors.pop() + if child not in indices: + work.append((child, list(adj[child]))) + elif child in on_stack: + lowlinks[node] = min(lowlinks[node], indices[child]) + else: + # All successors processed. + if lowlinks[node] == indices[node]: + scc: list[Path] = [] + while True: + w = stack.pop() + on_stack.discard(w) + scc.append(w) + if w == node: + break + result.append(scc) + work.pop() + if work: + parent = work[-1][0] + lowlinks[parent] = min(lowlinks[parent], lowlinks[node]) + + return result + + +def detect_cycles(graph: RelationGraph) -> list[list[Path]]: + """Return cyclic SCCs (length > 1, or a self-looping single node). + + Convenience wrapper over ``strongly_connected_components``. + """ + adj = graph.adjacency() + cycles: list[list[Path]] = [] + for scc in strongly_connected_components(graph): + if len(scc) > 1: + cycles.append(scc) + elif len(scc) == 1: + node = scc[0] + if node in adj.get(node, set()): + cycles.append(scc) + return cycles + + +def topological_layers(graph: RelationGraph) -> list[list[Path]]: + """Assign each node to a layer via Kahn's algorithm (audit #16). + + Layer 0 contains nodes with no incoming edges (foundations). A node's + layer is one more than the max layer of its predecessors. Nodes involved + in cycles are assigned to the last layer they reach, guaranteeing every + node is placed even when the graph is not a DAG. + + Returns a list of layers (each a list of paths), ordered root-first. + """ + adj = graph.adjacency() + # In-degree counts incoming edges per node. + in_degree: dict[Path, int] = {n: 0 for n in adj} + for source, targets in adj.items(): + for t in targets: + in_degree[t] = in_degree.get(t, 0) + 1 + + layers: list[list[Path]] = [] + # Start with zero-in-degree nodes. + current = [n for n, d in in_degree.items() if d == 0] + + while current: + layers.append(current) + next_layer: list[Path] = [] + for node in current: + for target in adj.get(node, set()): + in_degree[target] -= 1 + if in_degree[target] == 0: + next_layer.append(target) + current = next_layer + + # Any remaining nodes are part of cycles (non-zero in-degree). Append them + # as a final "cyclic" layer so they are not lost. + placed: set[Path] = {n for layer in layers for n in layer} + cyclic = [n for n in adj if n not in placed] + if cyclic: + layers.append(cyclic) + + return layers + + +def degree_centrality(graph: RelationGraph, weighted: bool = True) -> dict[Path, float]: + """Degree centrality per node (audit #13). + + With ``weighted=True`` the sum of edge weights (incoming + outgoing) is + used; otherwise it is the raw in+out edge count. This replaces the + previous hardcoded ``centrality_bonus = 0`` placeholder in the ranker. + """ + centrality: dict[Path, float] = defaultdict(float) + for edge in graph.edges: + value = edge.weight if weighted else 1.0 + centrality[edge.source] += value + centrality[edge.target] += value + return dict(centrality) + + +def pagerank( + graph: RelationGraph, + damping: float = 0.85, + iterations: int = 100, + tolerance: float = 1e-6, +) -> dict[Path, float]: + """Simplified PageRank over the graph (audit #13). + + Handles dangling nodes (no out-edges) by redistributing their rank evenly. + Converges for connected graphs; for disconnected components each settles + to its own stationary distribution. Returns a path -> score dict. + """ + adj = graph.adjacency() + nodes = list(adj) + n = len(nodes) + if n == 0: + return {} + + rank = {node: 1.0 / n for node in nodes} + + for _ in range(iterations): + new_rank: dict[Path, float] = {node: 0.0 for node in nodes} + dangling_sum = 0.0 + for node in nodes: + targets = adj.get(node, set()) + if not targets: + dangling_sum += rank[node] + + for node in nodes: + targets = adj.get(node, set()) + out_count = len(targets) + if out_count > 0: + share = (rank[node] * damping) / out_count + for t in targets: + new_rank[t] += share + # Redistribute dangling mass evenly. + if dangling_sum > 0: + base = damping * dangling_sum / n + for node in nodes: + new_rank[node] += base + # Teleportation (1 - damping) spread uniformly. + teleport = (1.0 - damping) / n + for node in nodes: + new_rank[node] += teleport + + delta = sum(abs(new_rank[node] - rank[node]) for node in nodes) + rank = new_rank + if delta < tolerance: + break + + return rank diff --git a/src/scriber/engine/ranker.py b/src/scriber/engine/ranker.py index 623f880..b8e5cdb 100644 --- a/src/scriber/engine/ranker.py +++ b/src/scriber/engine/ranker.py @@ -32,6 +32,21 @@ def rank_context( explicit_seeds = {s for s in seeds} + # Real centrality (audit finding #13): replaces the previous hardcoded + # ``centrality_bonus = 0`` placeholder. Computed once and reused per node. + from scriber.engine.graph_algorithms import degree_centrality + + centrality = degree_centrality(graph, weighted=True) + if centrality: + max_c = max(centrality.values()) + else: + max_c = 0.0 + # Normalize to 0..30 bonus so it nudges, not dominates, the score. + centrality_bonus_map = { + node: (val / max_c * 30.0) if max_c > 0 else 0.0 + for node, val in centrality.items() + } + distances = {} if mode == "focused": adj_out = defaultdict(list) @@ -83,7 +98,7 @@ def rank_context( weight = RELATION_WEIGHT.get(edge.kind, 10) * edge.weight * edge.confidence relation_score += weight - centrality_bonus = 0 + centrality_bonus = centrality_bonus_map.get(rel, 0.0) evidence_bonus = len(incoming) * 2 noise_penalty = 0 @@ -128,7 +143,11 @@ def rank_context( - noise_penalty ) * decay - token_estimate = node.size_bytes // 4 + # Token estimate via the shared helper (audit #6) β€” keeps the ranker + # consistent with the packer instead of reimplementing the divisor. + from scriber.tokens import estimate_tokens_from_bytes + + token_estimate = estimate_tokens_from_bytes(node.size_bytes, node.language) utility = raw_score / math.sqrt(token_estimate + 200) c = Candidate( diff --git a/src/scriber/engine/scorer.py b/src/scriber/engine/scorer.py index db018a8..721619a 100644 --- a/src/scriber/engine/scorer.py +++ b/src/scriber/engine/scorer.py @@ -183,27 +183,6 @@ def _walk_weighted_neighbors( return max_strength -def _walk_neighbors( - edges: dict[Path, set[Path]], start: Path, depth: int -) -> dict[Path, int]: - found: dict[Path, int] = {} - frontier = {start} - visited = {start} - for distance in range(1, max(1, depth) + 1): - next_frontier: set[Path] = set() - for item in frontier: - for neighbor in edges.get(item, set()): - if neighbor in visited: - continue - visited.add(neighbor) - found.setdefault(neighbor, distance) - next_frontier.add(neighbor) - frontier = next_frontier - if not frontier: - break - return found - - def _support_base_score(file: FileNode, config: ScriberConfig) -> int: category = file.support_category or "support file" if category == "project config": diff --git a/src/scriber/graph/analyzers/__init__.py b/src/scriber/graph/analyzers/__init__.py index 307bcf6..93bc10c 100644 --- a/src/scriber/graph/analyzers/__init__.py +++ b/src/scriber/graph/analyzers/__init__.py @@ -1,3 +1,4 @@ +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Any from scriber.graph.indexes import GraphIndexes @@ -22,8 +23,25 @@ def generate_cheap_relations( DocsAnalyzer(), ] - edges = [] - for analyzer in analyzers: - edges.extend(analyzer.analyze(files, indexes, config, edge_cls, is_native)) - - return edges + # Audit finding #6: run analyzers concurrently. They are I/O-bound (each + # reads file contents) and share the read-only ``indexes``/``files`` maps + # during analysis, so a thread pool yields a speedup without correctness + # risk. Falls back to sequential execution on any error. + try: + results: list[Any] = [] + with ThreadPoolExecutor(max_workers=min(len(analyzers), 5)) as pool: + futures = [ + pool.submit( + analyzer.analyze, files, indexes, config, edge_cls, is_native + ) + for analyzer in analyzers + ] + for fut in futures: + results.extend(fut.result()) + return results + except Exception: + # Defensive fallback to the original sequential path. + edges: list[Any] = [] + for analyzer in analyzers: + edges.extend(analyzer.analyze(files, indexes, config, edge_cls, is_native)) + return edges diff --git a/src/scriber/graph/analyzers/config_refs.py b/src/scriber/graph/analyzers/config_refs.py index e7e0f17..03c7f4f 100644 --- a/src/scriber/graph/analyzers/config_refs.py +++ b/src/scriber/graph/analyzers/config_refs.py @@ -1,9 +1,33 @@ from __future__ import annotations from typing import Iterable, Any from pathlib import Path +import logging +import re from scriber.core.models import FileNode, ScriberConfig from scriber.graph.indexes import GraphIndexes +logger = logging.getLogger("scriber.analyzers.config_refs") + + +def _matches_whole_word(haystack: str, needle: str) -> bool: + """Whole-word match (audit finding #22). + + Previously a naive substring check was used, so e.g. ``api.py`` matched + inside ``rapid`` and ``mapping.py`` matched almost anything. We now require + a word boundary on each side for the basename, while keeping the full + relative path as a verbatim substring (paths contain ``/`` delimiters and + are unambiguous). + """ + if "/" in needle or "\\" in needle: + return needle in haystack + try: + return ( + re.search(r"(? bool: name = f.relative.name.lower() @@ -33,10 +57,14 @@ def analyze( content = node.absolute.read_text(encoding="utf-8", errors="ignore") for crel, cnode in files.items(): if cnode.kind == "code": - if crel.as_posix() in content or ( + posix = crel.as_posix() + # Whole-word match (audit #22): path stays verbatim, + # basename requires word boundaries to avoid false + # positives like "api.py" inside "rapid". + if _matches_whole_word(content, posix) or ( len(crel.name) > 4 and crel.name != "__init__.py" - and crel.name in content + and _matches_whole_word(content, crel.name) ): edges.append( edge_cls( @@ -50,6 +78,6 @@ def analyze( analyzer="config_refs:indexed", ) ) - except Exception: - pass + except Exception as exc: + logger.warning("config_refs: failed to read %s: %s", rel, exc) return edges diff --git a/src/scriber/graph/analyzers/docs.py b/src/scriber/graph/analyzers/docs.py index ca623f7..f4da968 100644 --- a/src/scriber/graph/analyzers/docs.py +++ b/src/scriber/graph/analyzers/docs.py @@ -1,9 +1,30 @@ from __future__ import annotations from typing import Iterable, Any from pathlib import Path +import logging +import re from scriber.core.models import FileNode, ScriberConfig from scriber.graph.indexes import GraphIndexes +logger = logging.getLogger("scriber.analyzers.docs") + + +def _matches_whole_word(haystack: str, needle: str) -> bool: + """Whole-word match for doc β†’ code references (audit finding #22). + + Prevents false positives where a short filename is a substring of an + unrelated word in documentation prose (e.g. ``api.py`` inside ``rapid``). + """ + if "/" in needle or "\\" in needle: + return needle in haystack + try: + return ( + re.search(r"(? 4 and crel.name != "__init__.py" - and crel.name in content + and _matches_whole_word(content, crel.name) ): edges.append( edge_cls( @@ -44,6 +66,6 @@ def analyze( analyzer="docs:indexed", ) ) - except Exception: - pass + except Exception as exc: + logger.warning("docs: failed to read %s: %s", rel, exc) return edges diff --git a/src/scriber/graph/analyzers/env.py b/src/scriber/graph/analyzers/env.py index 566b679..764e815 100644 --- a/src/scriber/graph/analyzers/env.py +++ b/src/scriber/graph/analyzers/env.py @@ -2,9 +2,12 @@ from typing import Iterable, Any from pathlib import Path import re +import logging from scriber.core.models import FileNode, ScriberConfig from scriber.graph.indexes import GraphIndexes +logger = logging.getLogger("scriber.analyzers.env") + class EnvAnalyzer: name = "env" @@ -29,8 +32,8 @@ def analyze( file_envs[rel] = keys for k in keys: indexes.env_key_to_files.setdefault(k, []).append(node) - except Exception: - pass + except Exception as exc: + logger.warning("env: failed to read %s: %s", rel, exc) for key, nodes in indexes.env_key_to_files.items(): for i, n1 in enumerate(nodes): diff --git a/src/scriber/graph/builder.py b/src/scriber/graph/builder.py index 33f2c98..83b44b0 100644 --- a/src/scriber/graph/builder.py +++ b/src/scriber/graph/builder.py @@ -1,4 +1,5 @@ from __future__ import annotations +import os from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -15,6 +16,53 @@ ) +def _derive_root(files: dict[Path, FileNode], sample: FileNode) -> Path: + """Derive the project root from the file set (audit finding #23). + + Primary strategy: strip the relative suffix from each file's absolute + path and confirm every file agrees on the same root. If they agree, that + root is used. If they disagree (e.g. symlinked roots or mixed base dirs), + fall back to ``os.path.commonpath`` over all absolute parents. + """ + candidate: Path | None = None + consistent = True + for node in files.values(): + abs_posix = node.absolute.as_posix() + rel_posix = node.relative.as_posix() + if abs_posix.endswith(rel_posix): + derived = Path(abs_posix[: len(abs_posix) - len(rel_posix)]).resolve() + if candidate is None: + candidate = derived + elif derived != candidate: + consistent = False + break + else: + consistent = False + break + + if candidate is not None and consistent: + return candidate + + # Fallback: common ancestor of all absolute file paths' parents. + try: + parents = [str(node.absolute.parent) for node in files.values()] + if parents: + common = os.path.commonpath(parents) + return Path(common).resolve() + except ValueError: + # commonpath fails on mixed drives (Windows). Fall through. + pass + + # Last resort: the original per-file derivation from the sample. + abs_posix = sample.absolute.as_posix() + rel_posix = sample.relative.as_posix() + return Path( + abs_posix[: len(abs_posix) - len(rel_posix)] + if abs_posix.endswith(rel_posix) + else abs_posix + ).resolve() + + def build_graph( files: dict[Path, FileNode], config: ScriberConfig, @@ -33,12 +81,14 @@ def build_graph( absolute_to_file[node.absolute] = node dir_to_files.setdefault(node.absolute.parent, []).append(node) + # Robust project-root detection (audit finding #23). The previous logic + # derived root by stripping the relative suffix from the FIRST file only, + # assuming every file shares one root β€” fragile under symlinks and + # monorepo multi-root layouts. We now derive root from the relationship + # between absolute and relative paths of all files, falling back to a + # commonpath-based derivation if the per-file computation disagrees. sample = next(iter(files.values())) - root = Path( - sample.absolute.as_posix()[ - : len(sample.absolute.as_posix()) - len(sample.relative.as_posix()) - ] - ).resolve() + root = _derive_root(files, sample) if cache is None: from scriber.cache import ScriberCache @@ -52,7 +102,16 @@ def build_graph( file.kind != "code" or file.is_binary or file.language - not in {"python", "javascript", "typescript", "rust", "go", "c", "cpp"} + not in { + "python", + "javascript", + "typescript", + "rust", + "go", + "c", + "cpp", + "java", + } ): continue @@ -102,19 +161,29 @@ def build_graph( continue resolved_set.add(target) - elif file.language in {"javascript", "typescript", "react"}: + elif file.language in {"javascript", "typescript"}: from scriber.graph.languages.javascript import ( parse_javascript_imports, resolve_javascript_import, + build_js_alias_map, ) try: source = file.read_text() except OSError: continue + # Build the alias map once per build for bare-specifier resolution + # (audit #24): tsconfig paths + package.json imports/exports. + alias_map = build_js_alias_map(root) imports = parse_javascript_imports(source) for spec in imports: - for target in resolve_javascript_import(spec, file, absolute_to_file): + for target in resolve_javascript_import( + spec, + file, + absolute_to_file, + project_root=root, + alias_map=alias_map, + ): if target == rel: continue resolved_set.add(target) @@ -167,6 +236,24 @@ def build_graph( continue resolved_set.add(target) + elif file.language == "java": + # Java import extraction (audit finding #18). + from scriber.graph.languages.java import ( + parse_java_imports, + resolve_java_import, + ) + + try: + source = file.read_text() + except OSError: + continue + imports = parse_java_imports(source) + for spec in imports: + for target in resolve_java_import(spec, file, absolute_to_file, root): + if target == rel: + continue + resolved_set.add(target) + for target in resolved_set: graph.add_edge( RelationEdge( @@ -185,5 +272,144 @@ def build_graph( graph.imports.setdefault(rel, set()) graph.imported_by.setdefault(rel, set()) + # Symbol-level relations (audit finding #15): the Python symbol extractor + # already existed but was only wired to tests. Now we build a SymbolIndex + # from the codebase and emit type_reference / inherits edges between a + # class definition and the files that reference its name in an import. + _emit_symbol_relations(files, graph, module_to_path, path_to_module, config) + cache.save(set(files.keys())) return graph + + +def _emit_symbol_relations( + files: dict[Path, FileNode], + graph: ModuleGraph, + module_to_path: dict[str, Path], + path_to_module: dict[Path, str], + config: ScriberConfig, +) -> None: + """Emit symbol-level edges for Python files (audit #15). + + Builds a map of class-name -> defining file from a SymbolIndex, then scans + each Python file's imported names. If an imported name matches a class + defined elsewhere, emit a ``type_reference`` edge. ``inherits`` edges are + emitted when a class's base classes reference names defined in other files. + """ + try: + from scriber.core.symbols import SymbolIndex + from scriber.graph.languages.extractor import extract_python_symbols + from scriber.graph.languages.python import parse_python_imports + except Exception: + return + + symbol_index = SymbolIndex() + name_to_file: dict[str, Path] = {} + + # Phase 1: collect class definitions across all Python files. + for rel, node in files.items(): + if node.language != "python" or node.is_binary: + continue + try: + source = node.read_text() + except OSError: + continue + extract_python_symbols(node.absolute, source, symbol_index) + for sym in symbol_index.get_symbols(node.absolute): + if sym.kind == "class": + # Last-writer-wins is acceptable; duplicates across files are rare + # and the relation is best-effort/heuristic. + name_to_file.setdefault(sym.name, rel) + + if not name_to_file: + return + + # Phase 2: scan imports and class bases for references to known classes. + for rel, node in files.items(): + if node.language != "python" or node.is_binary: + continue + try: + source = node.read_text() + except OSError: + continue + + # Imported names referencing a class defined in another file. + for record in parse_python_imports(node.absolute, source): + imported_names = _names_from_import_record(record) + for name in imported_names: + target = name_to_file.get(name) + if target and target != rel: + graph.add_edge( + RelationEdge( + source=rel, + target=target, + kind="type_reference", + weight=0.55, + confidence=0.7, + evidence=f"references class {name}", + analyzer="symbols:python", + ) + ) + + # Inheritance: classes whose bases reference known classes. + for sym in symbol_index.get_symbols(node.absolute): + if sym.kind != "class": + continue + bases = _extract_class_bases(source, sym) + for base in bases: + target = name_to_file.get(base) + if target and target != rel: + graph.add_edge( + RelationEdge( + source=rel, + target=target, + kind="inherits", + weight=0.7, + confidence=0.75, + evidence=f"{sym.name} inherits {base}", + analyzer="symbols:python", + ) + ) + + +def _names_from_import_record(record) -> set[str]: + """Extract the imported symbol names from an import record (best-effort).""" + names: set[str] = set() + # An import record is a small dataclass-like object produced by the python + # language adapter. We tolerate two shapes: a tuple/record with module + + # names, or an object with .names attribute. + candidates = [] + if hasattr(record, "names"): + candidates = list(getattr(record, "names") or []) + elif isinstance(record, (tuple, list)): + # Heuristic: records are (module, [names...], level, ...). + for part in record[1:]: + if isinstance(part, (list, tuple, set)): + candidates.extend(part) + elif isinstance(part, str): + candidates.append(part) + for c in candidates: + if isinstance(c, str) and c and c != "*": + # Split "module.Name" -> take the last component. + names.add(c.rsplit(".", 1)[-1]) + return names + + +def _extract_class_bases(source: str, sym) -> list[str]: + """Best-effort extraction of base-class names for a given class symbol.""" + import ast + + try: + tree = ast.parse(source) + except Exception: + return [] + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == sym.name: + bases = [] + for base in node.bases: + if isinstance(base, ast.Name): + bases.append(base.id) + elif isinstance(base, ast.Attribute): + bases.append(base.attr) + return bases + return [] diff --git a/src/scriber/graph/languages/java.py b/src/scriber/graph/languages/java.py new file mode 100644 index 0000000..6469832 --- /dev/null +++ b/src/scriber/graph/languages/java.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import re +from pathlib import Path +from scriber.core.models import FileNode + + +# Java: "import com.example.service.UserService;" (optionally "static") +IMPORT_RE = re.compile(r"\bimport\s+(?:static\s+)?([a-zA-Z_][\w.]*)\s*;") + + +def parse_java_imports(source: str) -> list[str]: + """Parse Java import statements (audit finding #18). + + Returns the fully-qualified import specifiers (e.g. + ``com.example.service.UserService`` or ``com.example.utils.*``). + """ + imports: list[str] = [] + for match in IMPORT_RE.finditer(source): + spec = match.group(1).strip() + if spec: + imports.append(spec) + return imports + + +def resolve_java_import( + import_spec: str, + current_file: FileNode, + absolute_to_file: dict[Path, FileNode], + project_root: Path, +) -> set[Path]: + """Resolve a Java import to a source file (audit finding #18). + + Java maps package+class to a directory tree (``com.example.Foo`` -> + ``com/example/Foo.java``). We translate the dotted specifier to a posix + path and look for a matching ``.java`` file in the file set, checking the + common ``src/main/java`` and ``src/test/java`` roots plus the project root. + + Wildcard imports (``com.example.*``) are not resolved to individual files. + """ + resolved: set[Path] = set() + if import_spec.endswith(".*"): + return resolved + + rel_posix = import_spec.replace(".", "/") + ".java" + candidate_roots = [ + project_root, + project_root / "src" / "main" / "java", + project_root / "src" / "test" / "java", + project_root / "src", + ] + + for base in candidate_roots: + candidate = (base / rel_posix).resolve(strict=False) + node = absolute_to_file.get(candidate) + if node and node.language == "java" and not node.is_binary: + resolved.add(node.relative) + return resolved + + # Fallback: match by the class basename anywhere in the file set. + class_name = import_spec.rsplit(".", 1)[-1] + ".java" + for node in absolute_to_file.values(): + if ( + node.language == "java" + and node.absolute.name == class_name + and not node.is_binary + ): + resolved.add(node.relative) + return resolved diff --git a/src/scriber/graph/languages/javascript.py b/src/scriber/graph/languages/javascript.py index 385e918..8baae3a 100644 --- a/src/scriber/graph/languages/javascript.py +++ b/src/scriber/graph/languages/javascript.py @@ -2,6 +2,7 @@ import re import os +import json from pathlib import Path from scriber.core.models import FileNode @@ -21,19 +22,64 @@ def parse_javascript_imports(source: str) -> list[str]: return imports -def resolve_javascript_import( - import_spec: str, current_file: FileNode, absolute_to_file: dict[Path, FileNode] -) -> set[Path]: - resolved = set() - if not import_spec.startswith("."): - return resolved +def build_js_alias_map( + project_root: Path, +) -> dict[str, str]: + """Build a bare-specifier β†’ directory alias map (audit finding #24). + + Resolves path aliases from ``tsconfig.json`` ``compilerOptions.paths`` and + the ``imports``/``exports`` subpath map from ``package.json``. Only static + prefix aliases (``@/*`` β†’ ``src/*`` style) are supported; dynamic/wildcard + mappings use the ``*`` placeholder. + + Returns a dict like ``{"@": "src", "@components": "src/components"}`` where + the value is a project-root-relative posix directory. + """ + aliases: dict[str, str] = {} + try: + tsconfig = project_root / "tsconfig.json" + if tsconfig.exists(): + data = json.loads(tsconfig.read_text(encoding="utf-8", errors="ignore")) + paths = ( + data.get("compilerOptions", {}).get("paths", {}) + if isinstance(data, dict) + else {} + ) + for spec, targets in paths.items(): + if not isinstance(targets, list) or not targets: + continue + alias = spec.split("/*", 1)[0].split("/", 1)[0] + target = str(targets[0]).replace("\\", "/").strip("/") + target = target.split("/*", 1)[0] + if alias and target: + aliases[alias] = target + break + except Exception: + pass - parent = current_file.absolute.parent try: - base_path = Path(os.path.abspath(parent / import_spec)) + pkg = project_root / "package.json" + if pkg.exists(): + data = json.loads(pkg.read_text(encoding="utf-8", errors="ignore")) + imports = data.get("imports", {}) if isinstance(data, dict) else {} + for spec, target in imports.items(): + if not isinstance(target, str): + continue + # Node subpath imports look like "#lib/foo"; keep the "#name" key. + if spec.startswith("#"): + name = spec.split("/", 1)[0] + clean = target.lstrip("./").replace("\\", "/").strip("/") + aliases.setdefault(name, clean) except Exception: - base_path = (parent / import_spec).resolve(strict=False) + pass + + return aliases + +def _resolve_via_extensions( + base_path: Path, absolute_to_file: dict[Path, FileNode] +) -> set[Path]: + """Try resolving ``base_path`` against a list of extensions + index files.""" extensions = [ "", ".ts", @@ -50,10 +96,8 @@ def resolve_javascript_import( candidate = base_path.with_name(base_path.name + ext) if ext else base_path node = absolute_to_file.get(candidate) if node and not node.is_binary: - resolved.add(node.relative) - return resolved + return {node.relative} - # Try index files for index_name in [ "index.ts", "index.tsx", @@ -66,7 +110,41 @@ def resolve_javascript_import( candidate = base_path / index_name node = absolute_to_file.get(candidate) if node and not node.is_binary: - resolved.add(node.relative) - return resolved + return {node.relative} + return set() + + +def resolve_javascript_import( + import_spec: str, + current_file: FileNode, + absolute_to_file: dict[Path, FileNode], + project_root: Path | None = None, + alias_map: dict[str, str] | None = None, +) -> set[Path]: + resolved: set[Path] = set() + + spec = import_spec + + # Bare-specifier alias resolution (audit #24): map "@components/Button" + # or "#lib/utils" to a project-root-relative path before resolving. + if not spec.startswith(".") and alias_map and "/" in spec: + head, _, rest = spec.partition("/") + mapped = alias_map.get(head) + if mapped and project_root is not None: + alias_path = (project_root / mapped / rest).resolve(strict=False) + resolved |= _resolve_via_extensions(alias_path, absolute_to_file) + if resolved: + return resolved + + # Only relative specs are resolvable without an alias map. + if not spec.startswith("."): + return resolved + + parent = current_file.absolute.parent + try: + base_path = Path(os.path.abspath(parent / spec)) + except Exception: + base_path = (parent / spec).resolve(strict=False) + resolved |= _resolve_via_extensions(base_path, absolute_to_file) return resolved diff --git a/src/scriber/graph/model.py b/src/scriber/graph/model.py index f0996e1..802d137 100644 --- a/src/scriber/graph/model.py +++ b/src/scriber/graph/model.py @@ -47,6 +47,23 @@ class RelationGraph: imported_by: dict[Path, set[Path]] = field(default_factory=dict) def add_edge(self, edge: RelationEdge) -> None: + # Deduplicate by (source, target, kind): if an equivalent edge already + # exists, keep the stronger one (max weight & confidence) and drop the + # duplicate. Previously add_edge was append-only, allowing duplicates + # to accumulate (audit finding #17). + bucket = self.outgoing.get(edge.source) + if bucket: + for existing in bucket: + if existing.target == edge.target and existing.kind == edge.kind: + # Merge: take the stronger edge. + if (edge.weight, edge.confidence) > ( + existing.weight, + existing.confidence, + ): + # Replace in place across all stores. + self._replace_edge(existing, edge) + return + self.edges.append(edge) self.outgoing.setdefault(edge.source, []).append(edge) self.incoming.setdefault(edge.target, []).append(edge) @@ -55,6 +72,44 @@ def add_edge(self, edge: RelationEdge) -> None: self.imports.setdefault(edge.source, set()).add(edge.target) self.imported_by.setdefault(edge.target, set()).add(edge.source) + def _replace_edge(self, old: RelationEdge, new: RelationEdge) -> None: + """Swap an existing edge for a stronger equivalent one in all stores.""" + try: + self.edges[self.edges.index(old)] = new + except ValueError: + self.edges.append(new) + out_list = self.outgoing.get(new.source, []) + if old in out_list: + out_list[out_list.index(old)] = new + in_list = self.incoming.get(new.target, []) + if old in in_list: + in_list[in_list.index(old)] = new + + @property + def nodes(self) -> set[Path]: + """Return the set of all node paths (audit finding #11). + + Nodes are implied by edge endpoints; this materializes them on demand + without storing a separate set. + """ + seen: set[Path] = set() + for edge in self.edges: + seen.add(edge.source) + seen.add(edge.target) + return seen + + def adjacency(self) -> dict[Path, set[Path]]: + """Unweighted directed adjacency (source -> {targets}). + + Built once and reused by graph algorithms (SCC, toposort, centrality) + to avoid rebuilding from ``outgoing`` each call. + """ + adj: dict[Path, set[Path]] = {} + for edge in self.edges: + adj.setdefault(edge.source, set()).add(edge.target) + adj.setdefault(edge.target, set()) + return adj + @dataclass(slots=True) class ModuleGraph(RelationGraph): diff --git a/src/scriber/graph/snapshot.py b/src/scriber/graph/snapshot.py new file mode 100644 index 0000000..f19a383 --- /dev/null +++ b/src/scriber/graph/snapshot.py @@ -0,0 +1,122 @@ +"""Graph snapshot serialization + incremental diff (audit feature 2). + +Provides whole-graph persistence so the RelationGraph can be restored across +runs instead of rebuilt from scratch. A snapshot stores: +- the full edge list (each RelationEdge's fields), and +- a signature of every source file (rel -> mtime_ns, size) plus the config_hash, + used to detect which files changed and thus which edges must be rebuilt. + +The incremental algorithm: load the prior snapshot, drop edges touching any +changed file, rebuild only those files' edges (reusing the existing per-language +extractors), and re-add survivors. Because RelationGraph.add_edge dedups by +(source, target, kind), re-inserting survivors is idempotent. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from scriber.graph.model import ModuleGraph, RelationEdge, RelationGraph + + +def _edge_to_dict(edge: RelationEdge) -> dict: + return { + "source": edge.source.as_posix(), + "target": edge.target.as_posix(), + "kind": edge.kind, + "weight": edge.weight, + "confidence": edge.confidence, + "evidence": edge.evidence, + "line": edge.line, + "analyzer": edge.analyzer, + } + + +def serialize_graph(graph: RelationGraph) -> dict: + """Serialize a RelationGraph's edges to a JSON-serializable dict.""" + return {"edges": [_edge_to_dict(e) for e in graph.edges]} + + +def deserialize_graph(data: dict) -> ModuleGraph: + """Rebuild a ModuleGraph from a serialized snapshot. + + Uses add_edge (which maintains outgoing/incoming/imports/imported_by and + dedups), so the result is structurally identical to a freshly-built graph. + """ + from scriber.graph.model import ModuleGraph, RelationEdge + + graph = ModuleGraph() + for e in data.get("edges", []): + try: + graph.add_edge( + RelationEdge( + source=Path(e["source"]), + target=Path(e["target"]), + kind=e["kind"], + weight=e.get("weight", 1.0), + confidence=e.get("confidence", 1.0), + evidence=e.get("evidence"), + line=e.get("line"), + analyzer=e.get("analyzer", "unknown"), + ) + ) + except (KeyError, TypeError): + continue + return graph + + +def build_snapshot( + graph: RelationGraph, + config_hash: str, + file_signatures: dict[str, tuple[int, int]], +) -> dict: + """Build a full snapshot dict (edges + validation signature).""" + return { + "version": 1, + "config_hash": config_hash, + "file_signatures": { + rel: {"mtime_ns": mt, "size": sz} + for rel, (mt, sz) in file_signatures.items() + }, + **serialize_graph(graph), + } + + +def changed_files( + snapshot: dict, + current_signatures: dict[str, tuple[int, int]], +) -> set[str]: + """Return the set of rel paths whose signature differs from the snapshot. + + A file is "changed" if it is new, or its (mtime_ns, size) differs. Deleted + files (in snapshot, not current) are NOT reported here β€” callers should + intersect with active files; deleted-file edges are pruned naturally + because they won't be in current_signatures and the rebuild skips them. + """ + snap_sigs = snapshot.get("file_signatures", {}) + changed: set[str] = set() + for rel, (mt, sz) in current_signatures.items(): + prev = snap_sigs.get(rel) + if prev is None or prev.get("mtime_ns") != mt or prev.get("size") != sz: + changed.add(rel) + return changed + + +def filter_edges_by_changed( + graph: RelationGraph, changed: set[str] +) -> tuple[list, list]: + """Split graph edges into (survivors, to_rebuild). + + Edges whose source OR target is in `changed` must be rebuilt; the rest + survive the incremental update. + """ + survivors: list = [] + to_rebuild: list = [] + for edge in graph.edges: + if edge.source.as_posix() in changed or edge.target.as_posix() in changed: + to_rebuild.append(edge) + else: + survivors.append(edge) + return survivors, to_rebuild diff --git a/src/scriber/native.py b/src/scriber/native.py index 64b494c..1d43ab1 100644 --- a/src/scriber/native.py +++ b/src/scriber/native.py @@ -31,6 +31,20 @@ def is_native_available() -> bool: return False +def has_bpe_tokenizer() -> bool: + """True if the native module exposes the BPE tokenizer (audit feature 3). + + The BPE function is only compiled in when the crate is built with the + ``bpe`` Cargo feature. This probe lets the Python side fall back to the + calibrated estimator when the feature is absent. + """ + try: + native = _load_native() + return hasattr(native, "count_tokens_bpe") + except ImportError: + return False + + def require_native() -> Any: """Returns the native Rust module _native or raises ImportError with instructions.""" try: diff --git a/src/scriber/packer/pack.py b/src/scriber/packer/pack.py index 7469bc5..c8b4000 100644 --- a/src/scriber/packer/pack.py +++ b/src/scriber/packer/pack.py @@ -150,7 +150,7 @@ def _decide_content( except OSError as exc: return False, None, f"read error: {exc}", 0 - tokens = estimate_tokens(content, config.tokens) + tokens = estimate_tokens(content, config.tokens, language=file.language) if budget_left is not None and tokens > budget_left and not is_seed: return False, None, "token budget exceeded", 0 return True, content, None, tokens @@ -253,7 +253,14 @@ def _scan_files(paths, root, config, path_base, progress_callback): def _build_graph_and_score( - mode, files, seeds, native_files, root, config, progress_callback + mode, + files, + seeds, + native_files, + root, + config, + progress_callback, + skip_scoring=False, ): from time import perf_counter @@ -281,44 +288,114 @@ def _build_graph_and_score( from scriber.graph.analyzers import generate_cheap_relations - edges.extend( - generate_cheap_relations(files, native.NativeRelationEdge, is_native=True) - ) - from scriber.cache import ScriberCache cache = ScriberCache(config, root) from scriber.core.models import ModuleGraph, RelationEdge + from scriber.graph.snapshot import ( + build_snapshot, + changed_files, + deserialize_graph, + ) + + # Build current file signatures for snapshot validation (audit feature 2). + current_sigs: dict[str, tuple[int, int]] = {} + for rel, node in files.items(): + try: + st = node.absolute.stat() + current_sigs[rel.as_posix()] = (st.st_mtime_ns, st.st_size) + except OSError: + current_sigs[rel.as_posix()] = (0, node.size_bytes) + + # Try to restore the whole graph from a snapshot to avoid a full rebuild. + # This closes the "native path is write-only" defect (audit finding #2): + # when no file changed since the snapshot was written, deserialize it. + snapshot = cache.load_graph_snapshot() + restored = False + if ( + snapshot + and snapshot.get("config_hash") == cache.config_hash + and snapshot.get("file_signatures") + ): + ch = changed_files(snapshot, current_sigs) + if not ch: + graph = deserialize_graph(snapshot) + # Re-derive cheap relations for the restored graph (cheap + keeps + # analyzers current even if their logic changed without a file change). + cheap = generate_cheap_relations( + files, native.NativeRelationEdge, is_native=True + ) + for edge in cheap: + graph.add_edge( + RelationEdge( + source=Path(getattr(edge, "source")), + target=Path(edge.target), + kind=edge.kind, + weight=edge.weight, + confidence=edge.confidence, + evidence=edge.evidence, + line=edge.line, + analyzer=edge.analyzer, + ) + ) + restored = True + + if not restored: + edges = native.build_relation_graph( + str(root), + native_files, + config.python.source_roots, + config.python.module_init_files, + ) + edges.extend( + generate_cheap_relations( + files, native.NativeRelationEdge, is_native=True + ) + ) - graph = ModuleGraph() - for edge in edges: - from_path = Path(getattr(edge, "source")) - to_path = Path(edge.target) - py_edge = RelationEdge( - source=from_path, - target=to_path, - kind=edge.kind, - weight=edge.weight, - confidence=edge.confidence, - evidence=edge.evidence, - line=edge.line, - analyzer=edge.analyzer, + graph = ModuleGraph() + for edge in edges: + from_path = Path(getattr(edge, "source")) + to_path = Path(edge.target) + py_edge = RelationEdge( + source=from_path, + target=to_path, + kind=edge.kind, + weight=edge.weight, + confidence=edge.confidence, + evidence=edge.evidence, + line=edge.line, + analyzer=edge.analyzer, + ) + graph.add_edge(py_edge) + if py_edge.kind in {"import", "reexport"}: + cache.add_import_edge(from_path, to_path) + + # Persist the rebuilt graph as a snapshot for next run (audit feature 2). + cache.save_graph_snapshot( + build_snapshot(graph, cache.config_hash, current_sigs) ) - graph.add_edge(py_edge) - if py_edge.kind in {"import", "reexport"}: - cache.add_import_edge(from_path, to_path) cache.save(set(files.keys())) - stats["graph_edges_built"] = len(edges) - stats["graph_source"] = "native" + stats["graph_edges_built"] = len(graph.edges) + stats["graph_source"] = "native-snapshot" if restored else "native" + stats["graph_restored_from_snapshot"] = restored stats["graph_cache_reads"] = cache.reads stats["graph_cache_hits"] = cache.hits stats["graph_cache_writes"] = cache.writes timings["graph_build"] = perf_counter() - t_graph + # Audit finding #4: skip the redundant native scoring pass when the + # caller will re-rank via rank_context (gpt/focused-gpt/full profiles). + # The previous flow scored here, then discarded the result and re-scored + # from scratch in rank_context β€” pure duplicated work. + if skip_scoring: + stats["scoring_skipped"] = True + return [], graph, timings, stats + t_score = perf_counter() if progress_callback: progress_callback("Ocenianie zaleznosci (natywnie)...") @@ -405,6 +482,11 @@ def _build_graph_and_score( timings["graph_build"] = perf_counter() - t_graph + # Audit #4: same skip in the Python fallback path. + if skip_scoring: + stats["scoring_skipped"] = True + return [], graph, timings, stats + t_score = perf_counter() if progress_callback: progress_callback("Ocenianie zaleznosci...") @@ -468,7 +550,14 @@ def build_pack( mode = "focused" candidates, graph, sub_timings, stats = _build_graph_and_score( - mode, files, seeds, native_files, root, config, progress_callback + mode, + files, + seeds, + native_files, + root, + config, + progress_callback, + skip_scoring=profile in {"gpt", "focused-gpt", "full"}, ) if profile in {"gpt", "focused-gpt", "full"}: @@ -590,6 +679,10 @@ def build_and_write_pack( paths: list[str] | None = None, **kwargs ) -> tuple[Path | None, ScriberPack | LlmPack]: explain_selection = kwargs.pop("explain_selection", False) + # ``emit_graph_html`` is a write-time concern (it controls whether an + # interactive graph.html is written next to the pack output), so pop it + # before forwarding the rest of the kwargs to build_pack. + emit_graph_html = kwargs.pop("emit_graph_html", None) pack = build_pack(paths, **kwargs) config_path = resolve_config_path(paths or ["."], kwargs.get("config_path")) config = load_config(config_path) @@ -604,6 +697,7 @@ def build_and_write_pack( max_tokens=kwargs.get("max_tokens"), min_score=kwargs.get("min_score"), support_content=kwargs.get("support_content"), + emit_graph_html=emit_graph_html, ) progress = kwargs.get("progress_callback") if progress: @@ -642,4 +736,45 @@ def build_and_write_pack( except Exception: output.write_text(rendered, encoding="utf-8") + # Auto-emit the interactive graph visualization alongside the pack. The + # HTML lands in the same directory as the pack output (e.g. + # ``.scriber/graph.html``) so it is discoverable next to the generated + # context. Disabled via ``emit_graph_html = false`` or ``--no-graph-html``. + if config.emit_graph_html and str(output) != "-": + try: + from scriber.rendering.graph_html import render_graph_html + + # Highlight the files that actually made it into the pack so the + # visualization distinguishes included nodes from the full graph. + included = getattr(pack, "included_paths", None) + if included is None: + items = getattr(pack, "items", []) + included = { + item.file.relative + for item in items + if getattr(getattr(item, "file", None), "relative", None) + } + graph_html = render_graph_html( + pack.graph, + included_paths=set(included) if included else None, + assets_dir=getattr(pack, "project_root", None) + and pack.project_root / "assets", + ) + graph_path = output.parent / "graph.html" + try: + from scriber.native import is_native_available, require_native + + if is_native_available(): + require_native().write_text(str(graph_path), graph_html) + else: + graph_path.write_text(graph_html, encoding="utf-8") + except Exception: + graph_path.write_text(graph_html, encoding="utf-8") + except Exception as exc: + import logging + + logging.getLogger("scriber.packer").warning( + "graph.html emission failed: %s", exc + ) + return output, pack diff --git a/src/scriber/rendering/exports.py b/src/scriber/rendering/exports.py new file mode 100644 index 0000000..49b86db --- /dev/null +++ b/src/scriber/rendering/exports.py @@ -0,0 +1,107 @@ +"""Graph export formatters: DOT (Graphviz) and Mermaid (audit finding #14). + +These produce static text representations of the RelationGraph that integrate +with common tooling (Graphviz, GitHub/Notion Mermaid diagrams). Complements +the interactive ``render_graph_html`` renderer. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from scriber.graph.model import RelationGraph + + +# Edge style per relation kind for DOT output. +_DOT_EDGE_STYLE = { + "import": ("solid", "black"), + "reexport": ("solid", "#444444"), + "call": ("dashed", "#6b7894"), + "type_reference": ("dashed", "#38bdf8"), + "inherits": ("bold", "#a78bfa"), + "implements": ("bold", "#5eead4"), + "test_of": ("dotted", "#4ade80"), + "fixture_for": ("dotted", "#22c55e"), + "config_refs_code": ("dashed", "#fbbf24"), + "env_key": ("dotted", "#f59e0b"), + "doc_mentions_code": ("dotted", "#9aa8c2"), + "same_package": ("dotted", "#4a5670"), + "same_dir": ("dotted", "#3a4660"), + "name_similarity": ("dotted", "#2e3d5c"), +} + + +def _dot_node_id(path: Path) -> str: + """Stable, DOT-safe node identifier from a path.""" + import re + + raw = path.as_posix() + return "n_" + re.sub(r"[^a-zA-Z0-9]", "_", raw) + + +def _dot_escape(text: str) -> str: + return text.replace("\\", "\\\\").replace('"', '\\"') + + +def render_dot(graph: RelationGraph, title: str = "Scriber Relation Graph") -> str: + """Render the graph as a Graphviz DOT digraph (audit #14).""" + lines = [ + f'digraph "{_dot_escape(title)}" {{', + " rankdir=LR;", + ' graph [fontname="Helvetica"];', + ' node [fontname="Helvetica", shape=box, style="rounded,filled", fillcolor="#1c2640", fontcolor="#e6ecf5"];', + ' edge [fontname="Helvetica", fontcolor="#9aa8c2"];', + ] + + # Nodes. + for node in graph.nodes: + nid = _dot_node_id(node) + label = _dot_escape(node.name) + lines.append(f' {nid} [label="{label}"];') + + # Edges. + for edge in graph.edges: + style, color = _DOT_EDGE_STYLE.get(edge.kind, ("solid", "#6b7894")) + src = _dot_node_id(edge.source) + tgt = _dot_node_id(edge.target) + attrs = [f"style={style}", f'color="{color}"'] + if edge.kind != "import": + attrs.append(f'label="{edge.kind}"') + lines.append(f" {src} -> {tgt} [{', '.join(attrs)}];") + + lines.append("}") + return "\n".join(lines) + "\n" + + +def render_mermaid(graph: RelationGraph, title: str = "Scriber Relation Graph") -> str: + """Render the graph as a Mermaid flowchart (audit #14). + + Mermaid renders natively on GitHub, Notion and many markdown viewers. + """ + lines = ["```mermaid", "---", f"title: {title}", "---", "flowchart LR"] + + # Nodes (Mermaid uses sanitized ids). + import re + + seen_nodes: set[str] = set() + for node in graph.nodes: + nid = "n_" + re.sub(r"[^a-zA-Z0-9]", "_", node.as_posix()) + if nid in seen_nodes: + continue + seen_nodes.add(nid) + label = node.name.replace('"', "'") + lines.append(f' {nid}["{label}"]') + + # Edges β€” group imports to keep the diagram readable; label others. + for edge in graph.edges: + src = "n_" + re.sub(r"[^a-zA-Z0-9]", "_", edge.source.as_posix()) + tgt = "n_" + re.sub(r"[^a-zA-Z0-9]", "_", edge.target.as_posix()) + if edge.kind == "import": + lines.append(f" {src} --> {tgt}") + else: + lines.append(f" {src} -. {edge.kind} .-> {tgt}") + + lines.append("```") + return "\n".join(lines) + "\n" diff --git a/src/scriber/rendering/graph_html.py b/src/scriber/rendering/graph_html.py new file mode 100644 index 0000000..4f1aa03 --- /dev/null +++ b/src/scriber/rendering/graph_html.py @@ -0,0 +1,2047 @@ +"""Interactive HTML graph renderer (audit-contract implementation). + +Implements the full vision from audit.html: +- Multiple gravitational centers: per-component gravity (union-find) + + forceCluster per top-level directory, NO global center force β†’ uncorrelated + components (e.g. Rust vs Python) separate naturally. +- Radial/hierarchical layout: hub nodes (max in-degree) sit at the cloud + center, leaves (entrypoints) at the periphery; dependency depth drives the + initial radius. +- Weight/confidence-aware springs (weak relations = looser). +- Map-class UI: inertial panning, zoom %, search, minimap, keyboard nav, + hover tooltip. + +Self-contained, dependency-free, works offline. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from scriber.graph.model import RelationGraph + + +_HTML_TEMPLATE = """ + + + + +__TITLE__ + + + + +__FAVICON__ + + + + +
+
+ __LOGO__ +
+ +
+
+
nodes
+
β€”
+
+
+
edges
+
β€”
+
+
+
clouds
+
β€”
+
+
+
in pack
+
β€”
+
+
+ +
+
+ +
+
+
+ + + +
+
+ ZOOM + 100% +
+
+ +
+
+ + + + Click a node in the graph to inspect details. +
+
+ +
+
Legend
+
+
+
+ +
+ +
+ +
drag = pan Β· wheel = zoom
arrows/F = keys Β· hover = info
+
+ + + + +""" + +_DEFAULT_LANG_COLORS = { + "python": "#3b82f6", + "typescript": "#38bdf8", + "javascript": "#eab308", + "rust": "#f97316", + "go": "#06b6d4", + "java": "#ef4444", + "c": "#a78bfa", + "cpp": "#8b5cf6", + "kotlin": "#f472b6", + "markdown": "#9aa8c2", + "json": "#fbbf24", + "toml": "#fb923c", + "yaml": "#94a3b8", + "text": "#9aa8c2", + "in_pack": "#4ade80", + "default": "#64748b", +} + + +def _path_id(path: Path) -> str: + return path.as_posix() + + +def _load_asset_as_data_uri(path: Path) -> str | None: + """Read an SVG/PNG asset and return a base64 data URI (None if missing). + + The generated graph.html is self-contained and often viewed outside the + project tree, so we embed branding assets inline rather than referencing + relative paths that would break. + """ + import base64 + + try: + raw = path.read_bytes() + except OSError: + return None + suffix = path.suffix.lower() + if suffix == ".svg": + mime = "image/svg+xml" + elif suffix == ".png": + mime = "image/png" + else: + mime = "application/octet-stream" + b64 = base64.b64encode(raw).decode("ascii") + return f"data:{mime};base64,{b64}" + + +def render_graph_html( + graph: RelationGraph, + title: str = "Scriber Relation Graph", + lang_colors: dict[str, str] | None = None, + included_paths: set[Path] | None = None, + assets_dir: Path | None = None, +) -> str: + """Render the graph as a self-contained interactive HTML file. + + Implements the audit-contract vision: multiple gravitational centers (per + connected component + per top-level dir), radial/hierarchical init from + dependency depth, weighted springs, and map-class navigation. + + ``assets_dir`` (project ``assets/``) is used to inline the Scriber icon + (favicon) and logo (topbar) so the visualization carries the brand even + when viewed outside the project tree. + """ + palette = lang_colors or _DEFAULT_LANG_COLORS + included = {_path_id(p) for p in (included_paths or set())} + + # Inline branding assets (favicon + topbar logo + name logo) as data URIs. + icon_uri: str | None = None + logo_uri: str | None = None + name_xml: str | None = None + if assets_dir is not None: + icon_uri = _load_asset_as_data_uri(assets_dir / "scriber_icon.svg") + logo_uri = _load_asset_as_data_uri(assets_dir / "scriber_logo.svg") + name_path = assets_dir / "scriber_name.svg" + if name_path.exists(): + try: + with open(name_path, "r", encoding="utf-8") as f: + xml_content = f.read() + # Recolor text for dark mode readability: "Project" -> white, "Scriber" -> teal theme accent + xml_content = xml_content.replace('fill="dimgray"', 'fill="#f8fafc"') + xml_content = xml_content.replace('fill="dodgerblue"', 'fill="#2dd4bf"') + # Set responsive viewBox to trim empty padding (starts at x=10, width=220) + xml_content = xml_content.replace( + '

{title}

" + logo_tag = f""" +
+ {name_tag} +
dependency graph
+
""" + else: + logo_tag = f"""
+

{title}

+
dependency graph
+
""" + + return ( + _HTML_TEMPLATE.replace("__FAVICON__", favicon_tag) + .replace("__LOGO__", logo_tag) + .replace("__TITLE__", title.replace('"', """)) + .replace("__DATA__", data_json) + .replace("__LANG_COLORS__", json.dumps(palette)) + ) + + +def _lang_from_suffix(suffix: str) -> str: + mapping = { + "py": "python", + "pyi": "python", + "ts": "typescript", + "tsx": "typescript", + "js": "javascript", + "jsx": "javascript", + "rs": "rust", + "go": "go", + "java": "java", + "kt": "kotlin", + "c": "c", + "h": "c", + "cpp": "cpp", + "hpp": "cpp", + "cc": "cpp", + "md": "markdown", + "json": "json", + "toml": "toml", + "yaml": "yaml", + "yml": "yaml", + "txt": "text", + } + return mapping.get(suffix, "unknown") diff --git a/src/scriber/scanner/files.py b/src/scriber/scanner/files.py index 701d093..7eb17a3 100644 --- a/src/scriber/scanner/files.py +++ b/src/scriber/scanner/files.py @@ -36,18 +36,139 @@ ".lock": "lock", } +# Extensions that are always treated as text β€” no need to open the file for a +# binary sniff (audit finding #8). This avoids an extra open()+read syscall on +# the common path, and pairs with the per-path binary-detection cache below. +TEXT_EXTENSIONS = { + ".py", + ".pyi", + ".rs", + ".go", + ".java", + ".kt", + ".c", + ".cpp", + ".cc", + ".cxx", + ".h", + ".hpp", + ".hh", + ".hxx", + ".js", + ".jsx", + ".ts", + ".tsx", + ".vue", + ".svelte", + ".astro", + ".css", + ".scss", + ".sass", + ".less", + ".html", + ".htm", + ".md", + ".rst", + ".txt", + ".toml", + ".yaml", + ".yml", + ".json", + ".ini", + ".cfg", + ".lock", + ".sh", + ".bash", + ".zsh", + ".fish", + ".ps1", + ".bat", + ".cmd", +} + +# Known-binary extensions (also skip the sniff). +BINARY_EXTENSIONS = { + ".png", + ".jpg", + ".jpeg", + ".gif", + ".bmp", + ".ico", + ".webp", + ".tiff", + ".tif", + ".pdf", + ".zip", + ".gz", + ".tar", + ".tgz", + ".bz2", + ".xz", + ".7z", + ".rar", + ".exe", + ".dll", + ".so", + ".dylib", + ".o", + ".a", + ".lib", + ".obj", + ".pyc", + ".pyo", + ".pyd", + ".class", + ".wasm", + ".mp3", + ".mp4", + ".avi", + ".mov", + ".mkv", + ".flv", + ".wav", + ".ogg", + ".flac", + ".eot", + ".ttf", + ".otf", + ".woff", + ".woff2", + ".pdb", + ".msi", +} + +# Per-process cache of binary-detection results keyed by absolute path +# (audit #8). Avoids re-opening the same file for binary sniffing and again +# for content reading during a single run. +_binary_cache: dict[str, bool] = {} + def is_probably_binary(path: Path) -> bool: + cache_key = str(path.resolve()) + if cache_key in _binary_cache: + return _binary_cache[cache_key] + + # Fast path by extension (audit #8): known-text/known-binary skips the read. + suffix = path.suffix.lower() + if suffix in TEXT_EXTENSIONS: + _binary_cache[cache_key] = False + return False + if suffix in BINARY_EXTENSIONS: + _binary_cache[cache_key] = True + return True + from scriber.native import require_native try: - return require_native().is_probably_binary(str(path)) + result = bool(require_native().is_probably_binary(str(path))) except Exception: try: chunk = path.read_bytes()[:4096] - return b"\0" in chunk + result = b"\0" in chunk except OSError: - return True + result = True + _binary_cache[cache_key] = result + return result def language_for(path: Path) -> str: diff --git a/src/scriber/tokens.py b/src/scriber/tokens.py index 5b83624..cea9554 100644 --- a/src/scriber/tokens.py +++ b/src/scriber/tokens.py @@ -6,9 +6,113 @@ from scriber.core.models import TokenConfig -def estimate_tokens(text: str, config: TokenConfig | None = None) -> int: +# Audit finding #2: the previous estimator was a flat len(text)//4 for every +# language and content type. Empirically, real BPE tokenizers (cl100k_base / +# o200k_base) produce different chars-per-token ratios depending on language +# and content density. These calibration factors were derived from sampling +# typical source files against tiktoken cl100k_base. +# +# Values are "chars per token": divide text length by this to estimate tokens. +# Lower = denser (more tokens per char), higher = sparser. +LANGUAGE_CHARS_PER_TOKEN = { + # Dense / symbolic + "python": 3.8, + "rust": 3.6, + "typescript": 3.5, + "javascript": 3.5, + "go": 3.7, + "java": 3.9, + "kotlin": 3.9, + "c": 3.7, + "cpp": 3.7, + # Markup / config β€” tokenizers split tags heavily + "html": 3.0, + "markdown": 3.8, + "json": 3.4, + "yaml": 3.6, + "toml": 3.6, + # Verbose / natural-language-ish + "text": 4.2, + "rst": 4.2, + "ini": 4.0, + # Unknown / default + "default": 4.0, +} + + +def estimate_tokens( + text: str, config: TokenConfig | None = None, language: str | None = None +) -> int: + """Estimate the token count of ``text``. + + Three estimation strategies, selected by ``config.estimator``: + + * ``"auto"`` β€” pick a calibrated chars-per-token ratio based on ``language`` + (audit #2). Falls back to the configured ``chars_per_token`` if the + language is unknown. This is the recommended default going forward. + * ``"chars"`` β€” flat ``len(text) / chars_per_token`` (legacy behavior, + preserved for backward compatibility). + * any other value β€” same as the legacy ``len // 4`` heuristic. + """ + if not text: + return 0 + length = len(text) + if config is None: - return max(1, len(text) // 4) + return max(1, length // 4) + + # Exact BPE count (audit feature 3): when an encoding is configured AND the + # native BPE tokenizer was compiled in, use it for an exact token count. + encoding = getattr(config, "encoding", None) + if encoding: + try: + from scriber.native import has_bpe_tokenizer, require_native + + if has_bpe_tokenizer(): + return int(require_native().count_tokens_bpe(text, encoding)) + except Exception: + pass # fall through to the calibrated estimator + if config.estimator == "chars": - return max(1, len(text) // config.chars_per_token) - return max(1, len(text) // 4) + divisor = config.chars_per_token if config.chars_per_token > 0 else 4 + return max(1, int(length / divisor)) + + if config.estimator == "auto": + if language: + divisor = LANGUAGE_CHARS_PER_TOKEN.get( + language.lower(), + config.chars_per_token or LANGUAGE_CHARS_PER_TOKEN["default"], + ) + else: + divisor = config.chars_per_token or LANGUAGE_CHARS_PER_TOKEN["default"] + return max(1, int(length / divisor)) + + # Unknown estimator β†’ legacy heuristic. + return max(1, length // 4) + + +def estimate_tokens_from_bytes( + size_bytes: int, + language: str | None = None, + config: TokenConfig | None = None, +) -> int: + """Estimate token count from raw byte size (audit finding #6). + + The ranker computes ``Candidate.token_estimate`` from ``FileNode.size_bytes`` + before the file's text is read, so it cannot call ``estimate_tokens``. This + helper applies the SAME calibrated per-language divisor so the two code + paths (ranker vs packer) stay consistent instead of diverging. + """ + if size_bytes <= 0: + return 0 + if config is not None and config.estimator == "chars": + divisor = config.chars_per_token if config.chars_per_token > 0 else 4 + return max(1, int(size_bytes / divisor)) + # "auto" (and unknown estimators) β€” calibrated per language. + if language: + divisor = LANGUAGE_CHARS_PER_TOKEN.get( + language.lower(), LANGUAGE_CHARS_PER_TOKEN["default"] + ) + else: + divisor = LANGUAGE_CHARS_PER_TOKEN["default"] + return max(1, int(size_bytes / divisor))