add smart git

QUICKSTART.md

@@ -43,6 +43,8 @@ dope scan docs --branch main
 
 _Expected:_ Lists documentation files.
 
+When you run this command, a documentation term index file (`doc-terms.json`) will be created in the state directory. This index helps the application match relevant terms between code and docs, improving the relevance of suggestions for future commands.
+
 ## Verify Setup
 
 To describe the code structure:
@@ -80,5 +82,7 @@ dope scope create --project-size medium --output scope.yml --branch <branch-name
 dope scope apply --scope-file scope.yml --branch <branch-name>
 ```
 
+For `dope suggest` and `dope apply`, the commands utilize the generated `doc-terms.json` index and intelligent file pre-filtering, focusing suggestions and updates on high-priority documentation changes where code and documentation terms align.
+
 - Read `CONTRIBUTING.md` to learn how to contribute
 - See `CHANGELOG.md` for the latest changes

README.md

@@ -1,4 +1,4 @@
-# ![Alt](resources/banner.png)
+![Alt](resources/banner.png)
 
 # Getting Started with dope
 
@@ -59,6 +59,8 @@ dope status
 dope apply
 ```
 
+> Scanning operations (`dope scan docs` and `dope scan code`) now generate a documentation term index file named `doc-terms.json` in the configured state directory. This file is automatically used in later commands (`dope suggest`, `dope apply`) to boost the relevance of suggestions and updates based on documentation-term matching.
+
 > You can inspect and update your configuration at any time using `dope config show`, `dope config validate`, and `dope config set <key> <value>`.
 
 ### Command Reference
@@ -74,17 +76,17 @@ dope config validate          # Validate configuration
 dope config set KEY VALUE     # Update a single setting
 
 # Scanning Commands
-dope scan docs                # Scan documentation files
-dope scan code -b <branch>    # Scan code changes against branch
+dope scan docs [--branch <branch>]   # Scan documentation files, build a `doc-terms.json` index in the state directory, and classify files for later filtering.
+dope scan code [--branch <branch>]   # Scan code files with intelligent pre-filtering (classification and change-magnitude scoring) and use the `doc-terms.json` index to boost relevance of code-to-doc mappings.
 
 # Documentation Workflow
-dope suggest -b <branch>      # Generate documentation suggestions
-dope apply -b <branch>        # Apply suggested changes
-dope status                   # Show current processing status
+dope suggest -b <branch>             # Generate documentation suggestions
+dope apply -b <branch>               # Apply suggested changes
+dope status                          # Show current processing status
 
 # Documentation Structure
-dope scope create             # Create documentation scope
-dope scope apply              # Apply documentation scope
+dope scope create                    # Create documentation scope
+dope scope apply                     # Apply documentation scope
 ```
 
 ## Key Features
@@ -94,6 +96,8 @@ dope scope apply              # Apply documentation scope
 - **Smart Suggestions**: Generate human-readable summaries and documentation update suggestions
 - **Automated Updates**: Apply AI-generated suggestions directly to documentation files
 - **Status Tracking**: Monitor scan progress and pending suggestions with `dope status`
+- **Intelligent file pre-filtering**: Files are automatically classified (SKIP, NORMAL, HIGH) and quantified by change magnitude to skip trivial changes and prioritize critical files (e.g., README, config, entry points) before invoking LLM processing.
+- **Documentation term indexing**: A `doc-terms.json` index is built during scanning to match code changes to related documentation terms, improving the focus and quality of subsequent suggestions and applies.
 
 ### Configuration
 - **Quick Setup**: Get started with just 2-3 questions using `dope config init`

dope/cli/scan.py

@@ -11,7 +11,11 @@
 from dope.core.progress import track
 from dope.core.usage import UsageTracker
 from dope.core.utils import require_config
-from dope.models.constants import DESCRIBE_CODE_STATE_FILENAME, DESCRIBE_DOCS_STATE_FILENAME
+from dope.models.constants import (
+    DESCRIBE_CODE_STATE_FILENAME,
+    DESCRIBE_DOCS_STATE_FILENAME,
+    DOC_TERM_INDEX_FILENAME,
+)
 from dope.services.describer.describer_base import CodeDescriberService, DescriberService
 
 app = typer.Typer(help="Scan documentation and code for changes")
@@ -35,6 +39,7 @@ def docs(
         ),
         state_filepath=get_state_path(settings, DESCRIBE_DOCS_STATE_FILENAME),
         usage_tracker=tracker,
+        doc_term_index_path=get_state_path(settings, DOC_TERM_INDEX_FILENAME),
     )
     doc_state = doc_scanner.scan()
     try:
@@ -63,6 +68,7 @@ def code(
         GitConsumer(repo_root, branch),
         state_filepath=get_state_path(settings, DESCRIBE_CODE_STATE_FILENAME),
         usage_tracker=tracker,
+        doc_term_index_path=get_state_path(settings, DOC_TERM_INDEX_FILENAME),
     )
     code_state = code_scanner.scan()
     try:

dope/cli/suggest.py

@@ -14,6 +14,7 @@
 from dope.models.constants import (
     DESCRIBE_CODE_STATE_FILENAME,
     DESCRIBE_DOCS_STATE_FILENAME,
+    DOC_TERM_INDEX_FILENAME,
     SUGGESTION_STATE_FILENAME,
 )
 from dope.models.domain.scope_template import ScopeTemplate
@@ -44,6 +45,7 @@ def suggest(
         GitConsumer(Path("."), branch),
         state_filepath=get_state_path(settings, DESCRIBE_CODE_STATE_FILENAME),
         usage_tracker=tracker,
+        doc_term_index_path=get_state_path(settings, DOC_TERM_INDEX_FILENAME),
     )
     doc_scanner = DescriberService(
         DocConsumer(
@@ -53,6 +55,7 @@ def suggest(
         ),
         state_filepath=get_state_path(settings, DESCRIBE_DOCS_STATE_FILENAME),
         usage_tracker=tracker,
+        doc_term_index_path=get_state_path(settings, DOC_TERM_INDEX_FILENAME),
     )
     doc_state = doc_scanner.get_state()
     code_state = code_scanner.get_state()

dope/consumers/git_consumer.py

@@ -1,11 +1,59 @@
+import fnmatch
+from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Literal
+from typing import TYPE_CHECKING, Literal
 
 from git import Repo
 
 from dope.consumers.base import BaseConsumer
 from dope.models.domain.doc import CodeMetadata
 
+if TYPE_CHECKING:
+    pass
+
+# File patterns for automatic classification
+TRIVIAL_FILE_PATTERNS = {
+    "test": ["*_test.py", "*.spec.ts", "*.spec.js", "tests/*", "test_*.py", "**/test/**"],
+    "lock": ["*.lock", "*.sum", "package-lock.json", "poetry.lock", "Cargo.lock", "yarn.lock"],
+    "vendor": ["vendor/*", "node_modules/*", "dist/*", "build/*", ".venv/*", "venv/*"],
+    "generated": ["*_pb2.py", "*.pb.go", "*_generated.py", "*.generated.ts"],
+    "minified": ["*.min.js", "*.min.css", "*.bundle.js"],
+}
+
+DOC_CRITICAL_PATTERNS = {
+    "readme": ["README.md", "readme.md", "README.rst"],
+    "api_docs": ["docs/api/*", "api/*"],
+    "entry_points": ["__init__.py", "index.ts", "index.js", "lib.rs", "main.py", "main.go"],
+    "config": ["*.config.js", "*.config.ts", "pyproject.toml", "setup.py", "Cargo.toml"],
+}
+
+
+@dataclass
+class FileClassification:
+    """Classification of a file based on path analysis."""
+
+    classification: Literal["SKIP", "HIGH", "NORMAL"]
+    reason: str
+    matched_pattern: str | None = None
+
+
+@dataclass
+class ChangeMagnitude:
+    """Magnitude of changes in a file."""
+
+    lines_added: int
+    lines_deleted: int
+    total_lines: int
+    is_rename: bool
+    score: float  # 0.0 to 1.0, higher = more significant
+    rename_similarity: int | None = None
+    related_docs: list[str] = field(default_factory=list)  # Docs mentioning terms from this change
+
+    def __post_init__(self):
+        """Initialize mutable default."""
+        if self.related_docs is None:
+            self.related_docs = []
+
 
 class GitConsumer(BaseConsumer):
     """Git consumer."""
@@ -58,9 +106,49 @@ def _get_all_files(self):
         file_list = self.repo.git.ls_files().splitlines()
         return [Path(path) for path in file_list]
 
-    def get_content(self, file_path) -> bytes:
-        """Return diff content of changed file as bytes."""
-        diff = self.repo.git.diff(self.base_branch, f"--unified={5}", "--", str(file_path))
+    def get_content(self, file_path, normalize_whitespace: bool = False) -> bytes:
+        """Return diff content of changed file as bytes.
+
+        Args:
+            file_path: Path to the file to get diff for.
+            normalize_whitespace: If True, ignore whitespace changes in diff.
+
+        Returns:
+            Diff content as bytes.
+        """
+        args = [self.base_branch, f"--unified={5}"]
+
+        if normalize_whitespace:
+            args.extend(["-w", "-b", "--ignore-blank-lines"])
+
+        args.extend(["--", str(file_path)])
+        diff = self.repo.git.diff(*args)
+        return diff.encode("utf-8")
+
+    def get_normalized_diff(self, file_path) -> bytes:
+        """Get whitespace-normalized diff for better comparison.
+
+        This diff ignores:
+        - Whitespace changes (-w)
+        - Blank line changes (-b)
+        - Uses histogram algorithm for better diffs
+
+        Args:
+            file_path: Path to the file to get diff for.
+
+        Returns:
+            Normalized diff content as bytes.
+        """
+        diff = self.repo.git.diff(
+            self.base_branch,
+            "-w",  # ignore whitespace
+            "-b",  # ignore blank lines
+            "--ignore-blank-lines",
+            "--diff-algorithm=histogram",
+            f"--unified={5}",
+            "--",
+            str(file_path),
+        )
         return diff.encode("utf-8")
 
     def get_full_content(self, file_path):
@@ -106,3 +194,132 @@ def get_metadata(self, branch_name: str | None = None) -> CodeMetadata:
             tags=tags,
             lines_of_code=self._get_lines_of_code(),
         )
+
+    def classify_file_by_path(self, file_path: Path) -> FileClassification:
+        """Fast path-based classification before any LLM processing.
+
+        Classifies files into three categories:
+        - SKIP: Trivial files that don't need documentation (tests, locks, vendor)
+        - HIGH: Critical files that likely need documentation (README, entry points)
+        - NORMAL: Regular files that may need documentation
+
+        Args:
+            file_path: Path to classify.
+
+        Returns:
+            FileClassification with classification and reasoning.
+        """
+        path_str = str(file_path).lower()
+
+        # Check for trivial files to skip
+        for category, patterns in TRIVIAL_FILE_PATTERNS.items():
+            for pattern in patterns:
+                if fnmatch.fnmatch(path_str, pattern.lower()):
+                    return FileClassification(
+                        classification="SKIP",
+                        reason=f"Trivial file type: {category}",
+                        matched_pattern=pattern,
+                    )
+
+        # Check for critical files to prioritize
+        for category, patterns in DOC_CRITICAL_PATTERNS.items():
+            for pattern in patterns:
+                if fnmatch.fnmatch(path_str, pattern.lower()):
+                    return FileClassification(
+                        classification="HIGH",
+                        reason=f"Critical file type: {category}",
+                        matched_pattern=pattern,
+                    )
+
+        # Default to normal priority
+        return FileClassification(
+            classification="NORMAL", reason="Regular file requiring standard analysis"
+        )
+
+    def get_change_magnitude(self, file_path: Path) -> ChangeMagnitude:
+        """Calculate the magnitude of changes in a file.
+
+        Analyzes:
+        - Lines added/deleted
+        - Whether file was renamed
+        - Rename similarity percentage
+        - Overall significance score
+
+        Args:
+            file_path: Path to analyze.
+
+        Returns:
+            ChangeMagnitude with detailed change metrics.
+        """
+        # Get diff with rename detection
+        diff_output = self.repo.git.diff(
+            self.base_branch,
+            "-M90%",  # Detect renames with 90% similarity threshold
+            "--numstat",  # Get line counts
+            "--",
+            str(file_path),
+        )
+
+        # Parse numstat output: "added\tdeleted\tfilename"
+        lines_added = 0
+        lines_deleted = 0
+        is_rename = False
+        rename_similarity = None
+
+        if diff_output:
+            lines = diff_output.strip().split("\n")
+            if lines:
+                parts = lines[0].split("\t")
+                if len(parts) >= 2:
+                    # Handle binary files (marked as '-')
+                    added_str = parts[0]
+                    deleted_str = parts[1]
+
+                    lines_added = 0 if added_str == "-" else int(added_str)
+                    lines_deleted = 0 if deleted_str == "-" else int(deleted_str)
+
+        # Check for rename/move
+        rename_output = self.repo.git.diff(
+            self.base_branch, "-M90%", "--summary", "--", str(file_path)
+        )
+
+        if "rename" in rename_output.lower():
+            is_rename = True
+            # Try to extract similarity percentage
+            import re
+
+            match = re.search(r"(\d+)%", rename_output)
+            if match:
+                rename_similarity = int(match.group(1))
+
+        # Calculate significance score (0.0 to 1.0)
+        total_lines = lines_added + lines_deleted
+
+        # Base score on change volume
+        if total_lines == 0:
+            score = 0.0
+        elif total_lines < 5:
+            score = 0.2
+        elif total_lines < 20:
+            score = 0.4
+        elif total_lines < 50:
+            score = 0.6
+        elif total_lines < 100:
+            score = 0.8
+        else:
+            score = 1.0
+
+        # Reduce score for renames (mostly trivial)
+        if is_rename and rename_similarity and rename_similarity > 95:
+            score *= 0.3  # Pure rename with minimal changes
+        elif is_rename:
+            score *= 0.6  # Rename with some changes
+
+        return ChangeMagnitude(
+            lines_added=lines_added,
+            lines_deleted=lines_deleted,
+            total_lines=total_lines,
+            is_rename=is_rename,
+            rename_similarity=rename_similarity,
+            score=score,
+        )