From c7d70719526ba75c91991bb409f68d5e0c8cc973 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 19:44:18 +0000 Subject: [PATCH] feat: make local_files chunking opt-in instead of default Files were always split at 2000 chars with 200 overlap. Now each file becomes one memory regardless of size unless chunk_size is explicitly passed in the connector config. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0186k3gs6jRSfTtaCjjY9Xj7 --- CHANGELOG.md | 6 +++++ server/connectors/local_files.py | 40 +++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d3ada5..d14421f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +- `local_files` connector no longer chunks by default. Each file becomes one + memory regardless of size; pass `chunk_size` in the connector config to opt + back into splitting large files (with `overlap`, default 200). + ## 2.28.0 ### Launch remediation: all four Gate 0A P0 findings closed diff --git a/server/connectors/local_files.py b/server/connectors/local_files.py index 1c65846..918ff50 100644 --- a/server/connectors/local_files.py +++ b/server/connectors/local_files.py @@ -56,7 +56,6 @@ ".kt": ["kotlin", "code"], } -DEFAULT_CHUNK_SIZE: int = 2000 DEFAULT_OVERLAP: int = 200 @@ -66,13 +65,16 @@ class LocalFilesConnector(BaseConnector): name: str = "local_files" description: str = ( "Import local files (markdown, code, JSON, etc.) into memories. " - "Recursively scans a directory, filters by extension, and chunks large files." + "Recursively scans a directory, filters by extension. Each file becomes " + "one memory unless chunk_size is set, in which case large files are split." ) def __init__(self) -> None: self._root: Path | None = None self._extensions: set[str] = DEFAULT_EXTENSIONS - self._chunk_size: int = DEFAULT_CHUNK_SIZE + # None = no chunking: every file becomes exactly one memory, + # however large. Set chunk_size in config to opt into splitting. + self._chunk_size: int | None = None self._overlap: int = DEFAULT_OVERLAP self._exclude_dirs: set[str] = { ".git", "__pycache__", "node_modules", ".venv", "venv", @@ -89,8 +91,11 @@ async def connect(self, config: dict) -> bool: Config keys: directory (str): Root directory to scan. extensions (list[str], optional): File extensions to include. - chunk_size (int, optional): Max chars per chunk. Default 2000. + chunk_size (int, optional): Max chars per chunk. Unset by default, + meaning files are never split. Set this to opt into chunking + large files. overlap (int, optional): Overlap between chunks. Default 200. + Only used when chunk_size is set. exclude_dirs (list[str], optional): Directories to skip. """ directory = config.get("directory", ".") @@ -103,14 +108,15 @@ async def connect(self, config: dict) -> bool: exts = config["extensions"] self._extensions = {e if e.startswith(".") else f".{e}" for e in exts} - if "chunk_size" in config: + if "chunk_size" in config and config["chunk_size"] is not None: self._chunk_size = int(config["chunk_size"]) if "overlap" in config: self._overlap = int(config["overlap"]) - if self._chunk_size <= 0: - raise ValueError("chunk_size must be greater than zero") - if self._overlap < 0 or self._overlap >= self._chunk_size: - raise ValueError("overlap must satisfy 0 <= overlap < chunk_size") + if self._chunk_size is not None: + if self._chunk_size <= 0: + raise ValueError("chunk_size must be greater than zero") + if self._overlap < 0 or self._overlap >= self._chunk_size: + raise ValueError("overlap must satisfy 0 <= overlap < chunk_size") if "exclude_dirs" in config: self._exclude_dirs = set(config["exclude_dirs"]) @@ -192,8 +198,9 @@ def _process_file(self, fpath: Path) -> list[dict]: if ext == ".json": return self._process_json_file(raw, tags, metadata) - # Chunk if large - if len(raw) <= self._chunk_size: + # Chunk only if the caller opted in via chunk_size; otherwise the + # whole file becomes one memory, however large. + if self._chunk_size is None or len(raw) <= self._chunk_size: return [{ "content": raw.strip(), "tags": tags, @@ -211,8 +218,11 @@ def _process_json_file( try: data = json.loads(raw) except json.JSONDecodeError: + content = raw.strip() + if self._chunk_size is not None: + content = content[: self._chunk_size] return [{ - "content": raw.strip()[:self._chunk_size], + "content": content, "tags": tags + ["json-parse-error"], "metadata": metadata, }] @@ -229,8 +239,12 @@ def _process_json_file( else: text = str(data) + content = text.strip() + if self._chunk_size is not None: + content = content[: self._chunk_size * 2] + return [{ - "content": text.strip()[:self._chunk_size * 2], + "content": content, "tags": tags, "metadata": {**metadata, "json_keys": list(data.keys()) if isinstance(data, dict) else None}, }]