diff --git a/CHANGELOG.md b/CHANGELOG.md index ad4860419..554bb7981 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.9.10 (2026-07-08) + +- Fix: TS/JS member calls on a builtin-typed receiver no longer collapse onto a same-named user symbol (#1726). `_resolve_typescript_member_calls` matched a receiver's type to a definition by casefolded label, so `x: Date; x.getTime()` bound the caller to a user `class DATE`/`const DATE` in another file — inventing hundreds of phantom `references` edges and a false god node. Builtin-global receiver types (`Date`, `Promise`, `Map`, ...) are now skipped, mirroring the cross-file call guard; genuine user types are unaffected. +- Fix: never bind a cross-file `calls` edge to a definition in a different language family (#1718, thanks @edinaldoof). Name-only matching resolved a TSX callback passed by name to a same-named Kotlin method (and a Python call to a Kotlin fun) — phantom edges the spec forbids. Candidates are now filtered by interop family (JVM, native C-family, JS/TS module graph, ...); unknown families stay permissive. +- Fix: an ambiguous legacy-stem alias in `build_merge` no longer silently merges two unrelated files (#1713, thanks @mallyskies). The `#1504` old-stem alias (`ping.h`/`ping.php` → bare `ping`) resolved by hash-order, riding a dangling edge onto an arbitrary same-named file. Aliases are now committed only when exactly one file claims them; a salted `.h`/`.cpp` file node is recognized as its own claimant so a genuine collision stays ambiguous (and dropped) instead of picking a wrong winner. +- Fix: inline base-class stubs are tagged with `origin_file` (#1707, thanks @mallyskies). Five inheritance handlers built cross-file base-class stubs without `origin_file`, so same-named bases across files collapsed onto one shared stub that could then merge with an unrelated real class (218 wrong `inherits` edges observed). They now route through `ensure_named_node`, which sets the tag. +- Fix: Java enum constants are extracted as nodes with `case_of` edges to their enum (#1719, thanks @ivanzhl). Closes the Java half of #1700; `affected ErrorCode` / "where is ErrorCode.X used" now works for Java. +- Fix: `graphify` rebuilds recover from a deleted hook working directory instead of crashing (#1703, thanks @FranciscoJSBarragan). A detached git hook can inherit a CWD that no longer exists; the rebuild now recovers via `GRAPHIFY_REPO_ROOT` or fails cleanly instead of raising `FileNotFoundError`. +- Feat: the semantic cache is checkpointed per chunk so an interrupted extraction resumes instead of restarting (#1715, thanks @A-Levin). Each completed chunk is unioned into the cache immediately (opt out with `GRAPHIFY_NO_INCREMENTAL_CACHE`); the final write still overwrites authoritatively. +- Docs: `SECURITY.md` no longer claims stdio-only now that an opt-in `--transport http` (binds `127.0.0.1` by default) exists (#1714, thanks @Thizeidler); added tests for `GRAPHIFY_MAX_GRAPH_BYTES` parsing and corrected its unit docstring to binary MiB/GiB (#1722, thanks @Cekaru). + ## 0.9.9 (2026-07-07) - Fix: `graphify explain` resolves an exactly-typed punctuated label symmetrically against `norm_label` (#1704). The search term tokenized on `\w+` ("blockStream.ts" -> "blockstream ts", space where the '.' was) while a node's stored `norm_label` keeps punctuation ("blockstream.ts"). The verbatim case was already rescued by the tokenized-label tier, but that broke if a node's `label` and `norm_label` diverged; a punctuation-preserving `norm_query` is now matched against `norm_label` across the exact/prefix/substring tiers (and fed to the trigram prefilter), so it is robust by construction. diff --git a/graphify/__main__.py b/graphify/__main__.py index 285d19c1b..708c45819 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -23,30 +23,114 @@ # same override (#1423). from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +# Install/uninstall subsystem moved to graphify/install.py; re-exported here so +# `from graphify.__main__ import ` keeps working unchanged. +from graphify.install import ( # noqa: E402,F401 + dispatch_install_cli, + _agents_install, + _agents_platform_install, + _agents_platform_uninstall, + _agents_uninstall, + _always_on, + _amp_install, + _amp_legacy_cleanup, + _amp_uninstall, + _antigravity_finalize, + _antigravity_install, + _antigravity_uninstall, + _canonical_platform, + _claude_pretooluse_hooks, + _copy_skill_file, + _cursor_install, + _cursor_uninstall, + _devin_rules_install, + _devin_rules_uninstall, + _gemini_hook, + _install_claude_hook, + _install_codebuddy_hook, + _install_codex_hook, + _install_gemini_hook, + _install_kilo_plugin, + _install_opencode_plugin, + _install_skill_references, + _kilo_config_path, + _kilo_config_write_path, + _kilo_install, + _kilo_uninstall, + _kilo_uninstall_global, + _kiro_install, + _kiro_uninstall, + _load_json_like, + _packaged_skill_refs_dir, + _platform_skill_destination, + _print_banner, + _print_install_usage, + _print_project_git_add_hint, + _project_install, + _project_scope_root, + _project_uninstall, + _project_uninstall_all, + _refresh_all_version_stamps, + _remove_claude_skill_registration, + _remove_skill_file, + _replace_or_append_section, + _resolve_graphify_exe, + _skill_registration, + _strip_graphify_hook, + _strip_graphify_md_section, + _strip_json_comments, + _uninstall_claude_hook, + _uninstall_codebuddy_hook, + _uninstall_codex_hook, + _uninstall_gemini_hook, + _uninstall_kilo_plugin, + _uninstall_opencode_plugin, + claude_install, + claude_uninstall, + codebuddy_install, + codebuddy_uninstall, + gemini_install, + gemini_uninstall, + install, + uninstall_all, + vscode_install, + vscode_uninstall, + _PLATFORM_ALIASES, + _CLAUDE_MD_MARKER, + _CODEBUDDY_MD_MARKER, + _AGENTS_MD_MARKER, + _GEMINI_MD_MARKER, + _VSCODE_INSTRUCTIONS_MARKER, + _ANTIGRAVITY_RULES_PATH, + _ANTIGRAVITY_WORKFLOW_PATH, + _ANTIGRAVITY_WORKFLOW, + _CURSOR_RULE_PATH, + _CURSOR_RULE, + _DEVIN_RULES_PATH, + _DEVIN_RULES, + _KILO_PLUGIN_JS, + _KILO_PLUGIN_PATH, + _KILO_CONFIG_JSON_PATH, + _KILO_CONFIG_JSONC_PATH, + _OPENCODE_PLUGIN_JS, + _OPENCODE_PLUGIN_PATH, + _OPENCODE_CONFIG_PATH, + _PLATFORM_CONFIG, +) +from graphify.cli import ( # noqa: E402,F401 + dispatch_command, + _StageTimer, + _clone_repo, + _default_graph_path, + _enforce_graph_size_cap_or_exit, + _run_hook_guard, + _SEARCH_NUDGE, + _READ_NUDGE, + _HOOK_SOURCE_EXTS, + _GEMINI_NUDGE_TEXT, +) + -@functools.lru_cache(maxsize=None) -def _always_on(basename: str) -> str: - """Read a packaged always-on instruction block from graphify/always_on/. - - The six always-on blocks (CLAUDE.md / AGENTS.md / GEMINI.md / VS Code - Copilot instructions / Antigravity rules / Kiro steering) live as committed - markdown next to this module, generated by tools/skillgen from a single - human-edited fragment and guarded against drift by ``skillgen --check``. The - installer injects them verbatim via ``_replace_or_append_section``, so the - bytes here must match the former triple-quoted constant exactly — the - always-on-roundtrip validator proves that. - """ - path = Path(__file__).parent / "always_on" / f"{basename}.md" - try: - return path.read_text(encoding="utf-8") - except OSError as exc: - # Defer to use-time so a missing/corrupt packaged block can't crash module - # import (which would brick every CLI command, not just install). Reached - # only by an install/integration path that actually needs this block. - raise RuntimeError( - f"graphify install is incomplete: missing always-on block '{basename}' " - f"at {path}. Reinstall graphifyy (e.g. `uv tool install --reinstall graphifyy`)." - ) from exc _ALWAYS_ON_ALIASES = { @@ -70,53 +154,10 @@ def __getattr__(name: str) -> str: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") -def _default_graph_path() -> str: - return str(Path(_GRAPHIFY_OUT) / "graph.json") -class _StageTimer: - """Print per-stage wall-clock timings to stderr when --timing is set (#1490). - Monotonic (perf_counter), diagnostic-only: emits ``[graphify timing] : - N.Ns`` after each stage and a final total. Off by default, so normal output is - byte-identical and machine-read stdout is untouched. - """ - def __init__(self, enabled: bool) -> None: - import time as _time - self._now = _time.perf_counter - self.enabled = enabled - self.start = self._now() - self._last = self.start - - def mark(self, stage: str) -> None: - now = self._now() - if self.enabled: - print(f"[graphify timing] {stage}: {now - self._last:.1f}s", file=sys.stderr) - self._last = now - - def total(self) -> None: - if self.enabled: - print(f"[graphify timing] total: {self._now() - self.start:.1f}s", file=sys.stderr) - - -def _enforce_graph_size_cap_or_exit(gp: Path) -> None: - """Reject oversized graph files before parsing (CLI exit-on-fail flavor). - - Delegates to ``graphify.security.check_graph_file_size_cap`` and turns the - raised ``ValueError`` into a CLI-style ``error: ...`` message + exit 1. - Use this from ``__main__.py`` subcommands that already use the ``print + - sys.exit(1)`` idiom. Library/MCP/loader callers (``serve._load_graph``, - ``build``, ``benchmark``, ``tree_html``, ``callflow_html``, ``prs``, - ``global_graph``, ``watch``, ``export``) call the security helper directly - and let the ``ValueError`` propagate. - """ - from graphify.security import check_graph_file_size_cap - try: - check_graph_file_size_cap(gp) - except ValueError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) def _check_skill_version(skill_dst: Path) -> None: @@ -185,254 +226,22 @@ def _version_tuple(version: str) -> tuple[int, ...]: return tuple(parts) -def _refresh_all_version_stamps() -> None: - """After a successful install, update .graphify_version in all other known skill dirs. - Prevents stale-version warnings from platforms that were installed previously - but not explicitly re-installed during this upgrade. - """ - for name in _PLATFORM_CONFIG: - skill_dst = _platform_skill_destination(name) - vf = skill_dst.parent / ".graphify_version" - if skill_dst.exists(): - vf.write_text(__version__, encoding="utf-8") - - -def _platform_skill_destination(platform_name: str, *, project: bool = False, project_dir: Path | None = None) -> Path: - """Return the skill destination for a platform and scope.""" - if platform_name == "gemini": - if project: - return (project_dir or Path(".")) / ".gemini" / "skills" / "graphify" / "SKILL.md" - if platform.system() == "Windows": - return Path.home() / ".agents" / "skills" / "graphify" / "SKILL.md" - return Path.home() / ".gemini" / "skills" / "graphify" / "SKILL.md" - - if platform_name == "opencode": - if project: - return (project_dir or Path(".")) / ".opencode" / "skills" / "graphify" / "SKILL.md" - return Path.home() / ".config" / "opencode" / "skills" / "graphify" / "SKILL.md" - - if platform_name == "hermes": - if project: - return (project_dir or Path(".")) / ".hermes" / "skills" / "graphify" / "SKILL.md" - # On Windows, Hermes scans %LOCALAPPDATA%\hermes\skills, not ~/.hermes (#1403). - if platform.system() == "Windows": - local_appdata = Path(os.environ.get("LOCALAPPDATA") or (Path.home() / "AppData" / "Local")) - return local_appdata / "hermes" / "skills" / "graphify" / "SKILL.md" - return Path.home() / ".hermes" / "skills" / "graphify" / "SKILL.md" - - if platform_name == "devin": - if project: - return (project_dir or Path(".")) / ".devin" / "skills" / "graphify" / "SKILL.md" - return Path.home() / ".config" / "devin" / "skills" / "graphify" / "SKILL.md" - - if platform_name == "amp": - if project: - return (project_dir or Path(".")) / ".agents" / "skills" / "graphify" / "SKILL.md" - return Path.home() / ".config" / "agents" / "skills" / "graphify" / "SKILL.md" - - if platform_name == "agents": - # The generic Agent-Skills target: project ./.agents/skills, global the - # spec's user-global ~/.agents/skills (read by `npx skills` and compliant - # frameworks), NOT amp's ~/.config/agents/skills. - if project: - return (project_dir or Path(".")) / ".agents" / "skills" / "graphify" / "SKILL.md" - return Path.home() / ".agents" / "skills" / "graphify" / "SKILL.md" - - if platform_name in ("antigravity", "antigravity-windows"): - if project: - return (project_dir or Path(".")) / ".agents" / "skills" / "graphify" / "SKILL.md" - # Global Antigravity skill dir (all workspaces): ~/.gemini/config/skills/ - return Path.home() / ".gemini" / "config" / "skills" / "graphify" / "SKILL.md" - - cfg = _PLATFORM_CONFIG[platform_name] - if project: - return (project_dir or Path(".")) / cfg["skill_dst"] - - if platform_name in ("claude", "windows") and os.environ.get("CLAUDE_CONFIG_DIR"): - return Path(os.environ["CLAUDE_CONFIG_DIR"]) / "skills" / "graphify" / "SKILL.md" - return Path.home() / cfg["skill_dst"] - - -def _packaged_skill_refs_dir(platform_name: str) -> Path | None: - """Return the packaged references source dir for a progressive platform, else None. - - A platform opts into progressive disclosure by setting ``skill_refs`` in its - ``_PLATFORM_CONFIG`` entry. The value names a bundle under - ``graphify/skills//references/``. Reuse keys (e.g. trae-cn) point at - their twin's bundle. - - ``gemini`` has no ``_PLATFORM_CONFIG`` entry: it installs claude's - ``skill.md`` body verbatim (see ``_copy_skill_file``). Since that body is the - lean progressive core that links to ``references/``, gemini needs claude's - references/ sidecar too, or its SKILL.md ships with dead pointers. So gemini - resolves to the claude bundle rather than opting out. - - Bundles ship one platform-group at a time. A host whose bundle directory - ``graphify/skills//`` is not in this build has not gone progressive - yet, so this returns None and the host installs today's monolithic SKILL.md - with no references/ sidecar. Only when the bundle directory IS present does - this return the references path; if that directory then lacks its - ``references/`` subdir, ``_copy_skill_file`` hard-fails (a malformed bundle, - the empty-sidecar regression the wheel-content test also guards). - """ - if platform_name == "gemini": - bundle = "claude" - else: - bundle = _PLATFORM_CONFIG[platform_name].get("skill_refs") - if not bundle: - return None - bundle_dir = Path(__file__).parent / "skills" / bundle - if not bundle_dir.is_dir(): - return None - return bundle_dir / "references" - - -def _install_skill_references(skill_dst: Path, refs_src: Path) -> None: - """Atomically install a packaged references/ sidecar next to SKILL.md. - - Stages the packaged dir into ``references.tmp`` (copytree), drops any stale - ``references/`` already on disk, then ``os.replace``-renames the staged dir - into place. The rename is atomic on the same filesystem, so an interrupted - install never leaves a half-written references/ visible to the agent. - """ - refs_dst = skill_dst.parent / "references" - refs_staged = skill_dst.parent / "references.tmp" - if refs_staged.exists(): - shutil.rmtree(refs_staged) - try: - shutil.copytree(refs_src, refs_staged) - if refs_dst.exists(): - shutil.rmtree(refs_dst) - os.replace(refs_staged, refs_dst) - except Exception: - if refs_staged.exists(): - shutil.rmtree(refs_staged, ignore_errors=True) - raise - - -def _copy_skill_file(platform_name: str, *, project: bool = False, project_dir: Path | None = None) -> Path: - """Copy a packaged skill file and write its version stamp. - - For progressive platforms (those with ``skill_refs`` set), the packaged - ``references/`` sidecar is installed alongside SKILL.md and the single - ``.graphify_version`` stamp covers both. For monolith platforms (no - ``skill_refs``), any orphan ``references/`` left by a prior progressive - install is removed so the on-disk layout matches the package. - """ - skill_file = "skill.md" if platform_name == "gemini" else _PLATFORM_CONFIG[platform_name]["skill_file"] - skill_src = Path(__file__).parent / skill_file - if not skill_src.exists(): - print(f"error: {skill_file} not found in package - reinstall graphify", file=sys.stderr) - sys.exit(1) - - refs_src = _packaged_skill_refs_dir(platform_name) - if refs_src is not None and not refs_src.exists(): - # Progressive platform declared a references bundle that is missing from - # the package. Fail loud rather than silently shipping an empty sidecar. - print( - f"error: references for '{platform_name}' not found in package " - f"({refs_src}) - reinstall graphify", - file=sys.stderr, - ) - sys.exit(1) - - skill_dst = _platform_skill_destination(platform_name, project=project, project_dir=project_dir) - skill_dst.parent.mkdir(parents=True, exist_ok=True) - - # Install the references/ sidecar (or clear an orphan one) BEFORE writing - # SKILL.md, so SKILL.md is the last artifact laid down. An install that is - # interrupted partway then leaves no SKILL.md rather than a SKILL.md that - # points at an absent references/ dir. - if refs_src is not None: - _install_skill_references(skill_dst, refs_src) - print(f" references -> {skill_dst.parent / 'references'}") - else: - # Monolith (or progressive-with-no-refs): clear any orphan references/. - orphan_refs = skill_dst.parent / "references" - if orphan_refs.exists(): - shutil.rmtree(orphan_refs) - - # SKILL.md last (crash-safety), via an atomic temp + rename. - tmp_dst = skill_dst.with_suffix(skill_dst.suffix + ".tmp") - try: - shutil.copy(skill_src, tmp_dst) - os.replace(tmp_dst, skill_dst) - except Exception: - try: - tmp_dst.unlink(missing_ok=True) - except OSError: - pass - raise - - (skill_dst.parent / ".graphify_version").write_text(__version__, encoding="utf-8") - print(f" skill installed -> {skill_dst}") - return skill_dst - - -def _remove_skill_file(platform_name: str, *, project: bool = False, project_dir: Path | None = None) -> bool: - """Remove a platform skill file and its version stamp without touching other scopes.""" - skill_dst = _platform_skill_destination(platform_name, project=project, project_dir=project_dir) - removed = False - if skill_dst.exists(): - skill_dst.unlink() - print(f" skill removed -> {skill_dst}") - removed = True - version_file = skill_dst.parent / ".graphify_version" - if version_file.exists(): - version_file.unlink() - removed = True - refs_dir = skill_dst.parent / "references" - if refs_dir.exists(): - shutil.rmtree(refs_dir) - removed = True - for d in (skill_dst.parent, skill_dst.parent.parent, skill_dst.parent.parent.parent): - try: - d.rmdir() - except OSError: - break - return removed - - -def _project_scope_root(path: Path, project_dir: Path) -> Path: - """Return the top-level project artifact for a project-scoped skill path.""" - try: - rel = path.relative_to(project_dir) - except ValueError: - return path - return project_dir / rel.parts[0] if rel.parts else path -def _remove_claude_skill_registration(project_dir: Path) -> None: - """Remove the project-scoped Claude skill registration file/section.""" - claude_md = project_dir / ".claude" / "CLAUDE.md" - if not claude_md.exists(): - return - content = claude_md.read_text(encoding="utf-8") - if "# graphify" not in content: - return - cleaned = re.sub(r"\n*# graphify\n.*?(?=\n# |\Z)", "", content, flags=re.DOTALL).rstrip() - if cleaned: - claude_md.write_text(cleaned + "\n", encoding="utf-8") - print(f" CLAUDE.md -> graphify skill registration removed from {claude_md}") - else: - claude_md.unlink() - print(f" CLAUDE.md -> deleted {claude_md}") - - -def _print_project_git_add_hint(paths: list[Path]) -> None: - unique: list[str] = [] - for path in paths: - text = path.as_posix().rstrip("/") - if path.exists() and path.is_dir(): - text += "/" - if text not in unique: - unique.append(text) - if not unique: - return - print() - print("Project-scoped install. Add to version control:") - print(f" git add {' '.join(unique)}") + + + + + + + + + + + + + # PreToolUse nudge payloads, emitted verbatim by the shell-agnostic # `graphify hook-guard` subcommand (see _run_hook_guard). The previous hooks @@ -444,395 +253,27 @@ def _print_project_git_add_hint(paths: list[Path]) -> None: # additionalContext on PreToolUse (Codex Desktop does not — that path stays a # no-op via `hook-check`). Compact separators keep the payload byte-for-byte the # same JSON the old `echo` emitted. -_SEARCH_NUDGE = json.dumps({ - "hookSpecificOutput": { - "hookEventName": "PreToolUse", - "additionalContext": ( - 'MANDATORY: graphify-out/graph.json exists. You MUST run ' - '`graphify query ""` before grepping raw files. Only grep ' - 'after graphify has oriented you, or to modify/debug specific lines.' - ), - } -}, ensure_ascii=False, separators=(",", ":")) + "\n" - -_READ_NUDGE = json.dumps({ - "hookSpecificOutput": { - "hookEventName": "PreToolUse", - "additionalContext": ( - 'MANDATORY: graphify-out/graph.json exists. You MUST run graphify ' - 'before reading source files. Use: `graphify query ""` ' - '(scoped subgraph), `graphify explain ""`, or ' - '`graphify path "" ""`. Only read raw files after graphify has ' - 'oriented you, or to modify/debug specific lines. This rule applies to ' - 'subagents too — include it in every subagent prompt involving code ' - 'exploration.' - ), - } -}, ensure_ascii=False, separators=(",", ":")) + "\n" + # Source/doc extensions the Read|Glob guard nudges on (verbatim from the old hook). # The trailing-extension test (real final path segment, then its last '.') means # '.json' never false-matches '.js', and framework files like '.astro' are kept. -_HOOK_SOURCE_EXTS = ( - '.py', '.js', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.go', - '.rs', '.java', '.rb', '.c', '.h', '.cpp', '.hpp', '.cc', '.cs', '.kt', - '.swift', '.php', '.scala', '.lua', '.sh', '.md', '.rst', '.txt', '.mdx', -) - -def _claude_pretooluse_hooks() -> "list[dict]": - """graphify's Claude/Codebuddy PreToolUse hooks, resolved at install time. - - The command invokes `graphify hook-guard ` via the absolute exe - path (`_resolve_graphify_exe`), so it parses under sh, cmd.exe and PowerShell - alike — this is the #522 fix, and mirrors the codex hook. Matchers stay "Bash" - and "Read|Glob" and the command always contains "graphify", so the existing - install/uninstall filters find and replace both old bash hooks and these. - """ - exe = _resolve_graphify_exe() - if " " in exe and not exe.startswith('"'): - exe = f'"{exe}"' - return [ - {"matcher": "Bash", - "hooks": [{"type": "command", "command": f"{exe} hook-guard search"}]}, - {"matcher": "Read|Glob", - "hooks": [{"type": "command", "command": f"{exe} hook-guard read"}]}, - ] - -def _skill_registration(skill_path: str = "~/.claude/skills/graphify/SKILL.md") -> str: - return ( - "\n# graphify\n" - f"- **graphify** (`{skill_path}`) " - "- any input to knowledge graph. Trigger: `/graphify`\n" - "When the user types `/graphify`, use the installed graphify skill " - "or instructions before doing anything else.\n" - ) - - -_PLATFORM_CONFIG: dict[str, dict] = { - "claude": { - "skill_file": "skill.md", - "skill_dst": Path(".claude") / "skills" / "graphify" / "SKILL.md", - "claude_md": True, - "skill_refs": "claude", - }, - "codex": { - "skill_file": "skill-codex.md", - "skill_dst": Path(".codex") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "codex", - }, - "opencode": { - "skill_file": "skill-opencode.md", - "skill_dst": Path(".config") / "opencode" / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "opencode", - }, - "kilo": { - "skill_file": "skill-kilo.md", - "skill_dst": Path(".config") / "kilo" / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "kilo", - }, - "aider": { - # Monolith: aider ships the full SKILL.md inline, no references/ sidecar. - "skill_file": "skill-aider.md", - "skill_dst": Path(".aider") / "graphify" / "SKILL.md", - "claude_md": False, - }, - "copilot": { - "skill_file": "skill-copilot.md", - "skill_dst": Path(".copilot") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "copilot", - }, - "claw": { - "skill_file": "skill-claw.md", - "skill_dst": Path(".openclaw") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "claw", - }, - "droid": { - "skill_file": "skill-droid.md", - "skill_dst": Path(".factory") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "droid", - }, - "trae": { - "skill_file": "skill-trae.md", - "skill_dst": Path(".trae") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "trae", - }, - "trae-cn": { - # Reuses trae's split bundle (same skill body + references). - "skill_file": "skill-trae.md", - "skill_dst": Path(".trae-cn") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "trae", - }, - "hermes": { - # Reuses claw's split bundle. - "skill_file": "skill-claw.md", - "skill_dst": Path(".hermes") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "claw", - }, - "kiro": { - "skill_file": "skill-kiro.md", - "skill_dst": Path(".kiro") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "kiro", - }, - "pi": { - "skill_file": "skill-pi.md", - "skill_dst": Path(".pi") / "agent" / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "pi", - }, - "codebuddy": { - # Reuses claude's split bundle (shares skill.md). - "skill_file": "skill.md", - "skill_dst": Path(".codebuddy") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "claude", - }, - "antigravity": { - # Rides claude's split bundle (shares skill.md). - "skill_file": "skill.md", - "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "claude", - }, - "antigravity-windows": { - # Rides windows' split bundle. - "skill_file": "skill-windows.md", - "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "windows", - }, - "windows": { - "skill_file": "skill-windows.md", - "skill_dst": Path(".claude") / "skills" / "graphify" / "SKILL.md", - "claude_md": True, - "skill_refs": "windows", - }, - "kimi": { - # Reuses claude's split bundle (shares skill.md). - "skill_file": "skill.md", - "skill_dst": Path(".kimi") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "claude", - }, - "amp": { - # Amp searches .agents/skills (project) and ~/.config/agents/skills (user), - # not .amp/skills. The user-scope path is set in _platform_skill_destination. - "skill_file": "skill-amp.md", - "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "amp", - }, - "agents": { - # The generic cross-framework Agent-Skills target. Global: ~/.agents/skills - # (the spec's user-global location, read by `npx skills` and compliant - # frameworks); project: ./.agents/skills. The CLI accepts `skills` as an - # alias (see _canonical_platform). Ships its own rendered bundle. - "skill_file": "skill-agents.md", - "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "agents", - }, - "devin": { - # Monolith: devin ships the full SKILL.md inline, no references/ sidecar. - "skill_file": "skill-devin.md", - # User scope: ~/.config/devin/skills/graphify/SKILL.md - # Project scope: .devin/skills/graphify/SKILL.md (overridden in _platform_skill_destination) - "skill_dst": Path(".config") / "devin" / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - }, -} -# CLI-only platform aliases, resolved to a real _PLATFORM_CONFIG key before -# dispatch. `skills` is the friendly alias for the generic `agents` platform -# (the Agent-Skills ecosystem calls them "skills"). -_PLATFORM_ALIASES: dict[str, str] = {"skills": "agents"} -def _canonical_platform(platform_name: str) -> str: - """Resolve a CLI platform alias to its real _PLATFORM_CONFIG key.""" - return _PLATFORM_ALIASES.get(platform_name, platform_name) -def _replace_or_append_section(content: str, marker: str, new_section: str) -> str: - """Idempotently update or append a graphify-owned section in shared files. - If no line is exactly ``marker`` (the heading, at column 0), append - ``new_section`` to the end (with a blank-line separator if there's existing - content). - If a real ``marker`` heading exists, replace the existing section in place. - The section runs from that heading to the line before the next H2 heading - (``## `` at line start), or to EOF if no later H2 exists. This lets older - installs receive the updated copy without users having to uninstall and - reinstall (issue #580). - The heading is matched only when a line *is* exactly ``marker`` (after - stripping surrounding whitespace), never as a substring. Matching ``## - graphify`` inside a bullet or an inline reference used to anchor the replace - on that mention and delete every line from there to the next heading, - silently destroying hand-curated content (#1688). When several exact - headings exist, the last one is used, since graphify's section is appended. - """ - lines = content.split("\n") - starts = [i for i, line in enumerate(lines) if line.strip() == marker] - if not starts: - if content.strip(): - return content.rstrip() + "\n\n" + new_section.lstrip() - return new_section.lstrip() - - start = starts[-1] - end = len(lines) - for j in range(start + 1, len(lines)): - if lines[j].startswith("## "): - end = j - break - - head = "\n".join(lines[:start]).rstrip() - tail = "\n".join(lines[end:]).lstrip() - section = new_section.strip() - - parts: list[str] = [] - if head: - parts.append(head) - parts.append(section) - if tail: - parts.append(tail) - out = "\n\n".join(parts) - if not out.endswith("\n"): - out += "\n" - return out - - -def _print_banner() -> None: - """Amber brain banner on graphify install. TTY-only, never raises.""" - if not sys.stdout.isatty(): - return - try: - if sys.platform == "win32": - import ctypes - ctypes.windll.kernel32.SetConsoleMode( - ctypes.windll.kernel32.GetStdHandle(-11), 7 - ) - A = "\033[38;5;214m" - D = "\033[38;5;130m" - R = "\033[0m" - print(f"""{A} - ╭──◉──╮ ╭──◉──╮ - ╱ ◉ ◉ ╲ ╱ ◉ ◉ ╲ -│ ◉─◉─◉ ◉ ◉─◉─◉ │ -│ ◉ ◉ │ ◉ ◉ │ -│ ◉─◉─◉ ◉ ◉─◉─◉ │ - ╲ ◉ ◉ ╱ ╲ ◉ ◉ ╱ - ╰──◉──╯ ╰──◉──╯ - ◉ - - █▀▀ █▀█ ▄▀█ █▀█ █ █ █ █▀▀ █▄█ - █▄█ █▀▄ █▀█ █▀▀ █▀█ █ █▀ █{D} {__version__}{R} -""") - except Exception: - pass - - -def install(platform: str = "claude", *, project: bool = False, project_dir: Path | None = None) -> None: - _print_banner() - platform = _canonical_platform(platform) - if platform == "gemini": - gemini_install(project_dir=project_dir, project=project) - return - if platform == "cursor": - _cursor_install(Path(".")) - return - # On Windows, antigravity needs the PowerShell skill, not the bash one - if platform == "antigravity" and sys.platform == "win32": - platform = "antigravity-windows" - if platform not in _PLATFORM_CONFIG: - print( - f"error: unknown platform '{platform}'. Choose from: {', '.join(_PLATFORM_CONFIG)}, gemini, cursor", - file=sys.stderr, - ) - sys.exit(1) - cfg = _PLATFORM_CONFIG[platform] - project_dir = project_dir or Path(".") - skill_dst = _copy_skill_file(platform, project=project, project_dir=project_dir) - if platform == "kilo": - # Kilo Code also supports a native /graphify command file. - command_src = Path(__file__).parent / "command-kilo.md" - if not command_src.exists(): - print( - f"error: command-kilo.md not found in package - reinstall graphify", - file=sys.stderr, - ) - sys.exit(1) - command_dst = Path.home() / ".config" / "kilo" / "command" / "graphify.md" - command_dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy(command_src, command_dst) - print(f" command installed -> {command_dst}") - - if cfg["claude_md"]: - # Register in the matching Claude Code scope. - claude_md = (project_dir / ".claude" / "CLAUDE.md") if project else Path.home() / ".claude" / "CLAUDE.md" - registration = _skill_registration(".claude/skills/graphify/SKILL.md" if project else "~/.claude/skills/graphify/SKILL.md") - if claude_md.exists(): - content = claude_md.read_text(encoding="utf-8") - if "graphify" in content: - print(f" CLAUDE.md -> already registered (no change)") - else: - claude_md.write_text(content.rstrip() + registration, encoding="utf-8") - print(f" CLAUDE.md -> skill registered in {claude_md}") - else: - claude_md.parent.mkdir(parents=True, exist_ok=True) - claude_md.write_text(registration.lstrip(), encoding="utf-8") - print(f" CLAUDE.md -> created at {claude_md}") - - if platform == "codebuddy": - # Register in ~/.codebuddy/CODEBUDDY.md (CodeBuddy only) - codebuddy_md = Path.home() / ".codebuddy" / "CODEBUDDY.md" - registration = _skill_registration("~/.codebuddy/skills/graphify/SKILL.md") - if codebuddy_md.exists(): - content = codebuddy_md.read_text(encoding="utf-8") - if "graphify" in content: - print(f" CODEBUDDY.md -> already registered (no change)") - else: - codebuddy_md.write_text(content.rstrip() + registration, encoding="utf-8") - print(f" CODEBUDDY.md -> skill registered in {codebuddy_md}") - else: - codebuddy_md.parent.mkdir(parents=True, exist_ok=True) - codebuddy_md.write_text(registration.lstrip(), encoding="utf-8") - print(f" CODEBUDDY.md -> created at {codebuddy_md}") - if platform == "opencode": - _install_opencode_plugin(project_dir if project else Path(".")) - # Refresh version stamps in all other previously-installed skill dirs so - # stale-version warnings don't fire for platforms not explicitly re-installed. - if project: - _print_project_git_add_hint([_project_scope_root(skill_dst, project_dir)]) - else: - _refresh_all_version_stamps() - print() - print("Done. Open your AI coding assistant and type:") - print() - print(" /graphify .") - print() -def _print_install_usage() -> None: - platforms = ", ".join([*_PLATFORM_CONFIG, "gemini", "cursor"]) - print("Usage: graphify install [--project] [--platform P|P]") - print(f"Platforms: {platforms}") # The always-on instruction blocks are packaged markdown under graphify/always_on/, @@ -840,17 +281,13 @@ def _print_install_usage() -> None: # load keeps the install-string / issue-#580 contract byte-for-byte while letting # a human edit one fragment instead of a triple-quoted literal here. -_CLAUDE_MD_MARKER = "## graphify" -_CODEBUDDY_MD_MARKER = "## graphify" # AGENTS.md section for Codex, OpenCode, and OpenClaw. # All three platforms read AGENTS.md in the project root for persistent instructions. -_AGENTS_MD_MARKER = "## graphify" -_GEMINI_MD_MARKER = "## graphify" # Gemini CLI BeforeTool hook nudge text. The hook always returns # {"decision":"allow"} (never blocks a tool) and appends this as additionalContext @@ -859,737 +296,74 @@ def _print_install_usage() -> None: # `python`/`py` or absent on Windows) and embedded backticks + escaped quotes that # Windows PowerShell mangles (#522 follow-up); the subcommand form has no such # dependency and parses under every shell. -_GEMINI_NUDGE_TEXT = ( - 'graphify: knowledge graph at graphify-out/. For focused questions, run ' - '`graphify query ""` (scoped subgraph, usually much smaller than ' - 'GRAPH_REPORT.md) instead of grepping raw files. Read GRAPH_REPORT.md only ' - 'for broad architecture context.' -) -def _gemini_hook() -> dict: - """Gemini CLI BeforeTool hook, resolved to a shell-agnostic `graphify` call.""" - exe = _resolve_graphify_exe() - if " " in exe and not exe.startswith('"'): - exe = f'"{exe}"' - return { - "matcher": "read_file|list_directory", - "hooks": [{"type": "command", "command": f"{exe} hook-guard gemini"}], - } -def gemini_install(project_dir: Path | None = None, *, project: bool = False) -> None: - """Copy skill file, write GEMINI.md section, and install BeforeTool hook.""" - project_dir = project_dir or Path(".") - skill_dst = _copy_skill_file("gemini", project=project, project_dir=project_dir) - target = project_dir / "GEMINI.md" - if target.exists(): - content = target.read_text(encoding="utf-8") - new_content = _replace_or_append_section( - content, _GEMINI_MD_MARKER, _always_on("gemini-md") - ) - else: - new_content = _always_on("gemini-md") - - if target.exists() and new_content == target.read_text(encoding="utf-8"): - print(f"graphify already configured in {target.resolve()} (no change)") - else: - target.write_text(new_content, encoding="utf-8") - print(f"graphify section written to {target.resolve()}") - - # Always re-install the Gemini hook so an older payload (e.g. pre-issue-#580 - # wording) is replaced on upgrade. - _install_gemini_hook(project_dir) - if project: - _print_project_git_add_hint([_project_scope_root(skill_dst, project_dir), project_dir / "GEMINI.md", project_dir / ".gemini"]) - print() - print("Gemini CLI will now check the knowledge graph before answering") - print("codebase questions and rebuild it after code changes.") - - -def _install_gemini_hook(project_dir: Path) -> None: - settings_path = project_dir / ".gemini" / "settings.json" - settings_path.parent.mkdir(parents=True, exist_ok=True) - try: - settings = ( - json.loads(settings_path.read_text(encoding="utf-8")) - if settings_path.exists() - else {} - ) - except json.JSONDecodeError: - settings = {} - before_tool = settings.setdefault("hooks", {}).setdefault("BeforeTool", []) - settings["hooks"]["BeforeTool"] = [ - h for h in before_tool if "graphify" not in str(h) - ] - settings["hooks"]["BeforeTool"].append(_gemini_hook()) - settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") - print(" .gemini/settings.json -> BeforeTool hook registered") - - -def _uninstall_gemini_hook(project_dir: Path) -> None: - settings_path = project_dir / ".gemini" / "settings.json" - if not settings_path.exists(): - return - try: - settings = json.loads(settings_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return - before_tool = settings.get("hooks", {}).get("BeforeTool", []) - filtered = [h for h in before_tool if "graphify" not in str(h)] - if len(filtered) == len(before_tool): - return - settings["hooks"]["BeforeTool"] = filtered - settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") - print(" .gemini/settings.json -> BeforeTool hook removed") -def gemini_uninstall(project_dir: Path | None = None, *, project: bool = False) -> None: - """Remove the graphify section from GEMINI.md, uninstall hook, and remove skill file.""" - project_dir = project_dir or Path(".") - _remove_skill_file("gemini", project=project, project_dir=project_dir) - target = project_dir / "GEMINI.md" - if not target.exists(): - print("No GEMINI.md found in current directory - nothing to do") - return - content = target.read_text(encoding="utf-8") - if _GEMINI_MD_MARKER not in content: - print("graphify section not found in GEMINI.md - nothing to do") - return - cleaned = re.sub( - r"\n*## graphify\n.*?(?=\n## |\Z)", "", content, flags=re.DOTALL - ).rstrip() - if cleaned: - target.write_text(cleaned + "\n", encoding="utf-8") - print(f"graphify section removed from {target.resolve()}") - else: - target.unlink() - print(f"GEMINI.md was empty after removal - deleted {target.resolve()}") - _uninstall_gemini_hook(project_dir) - - -_VSCODE_INSTRUCTIONS_MARKER = "## graphify" - - -def vscode_install(project_dir: Path | None = None) -> None: - """Install graphify skill for VS Code Copilot Chat + write .github/copilot-instructions.md.""" - skill_src = Path(__file__).parent / "skill-vscode.md" - refs_bundle = "vscode" - if not skill_src.exists(): - skill_src = Path(__file__).parent / "skill-copilot.md" - refs_bundle = "copilot" - skill_dst = Path.home() / ".copilot" / "skills" / "graphify" / "SKILL.md" - skill_dst.parent.mkdir(parents=True, exist_ok=True) - tmp_dst = skill_dst.with_suffix(skill_dst.suffix + ".tmp") - try: - shutil.copy(skill_src, tmp_dst) - os.replace(tmp_dst, skill_dst) - except Exception: - try: - tmp_dst.unlink(missing_ok=True) - except OSError: - pass - raise - # Progressive-capable: install the packaged references/ sidecar when present. - refs_src = Path(__file__).parent / "skills" / refs_bundle / "references" - if refs_src.exists(): - _install_skill_references(skill_dst, refs_src) - print(f" references -> {skill_dst.parent / 'references'}") - else: - orphan_refs = skill_dst.parent / "references" - if orphan_refs.exists(): - shutil.rmtree(orphan_refs) - (skill_dst.parent / ".graphify_version").write_text(__version__, encoding="utf-8") - print(f" skill installed -> {skill_dst}") - - instructions = (project_dir or Path(".")) / ".github" / "copilot-instructions.md" - instructions.parent.mkdir(parents=True, exist_ok=True) - if instructions.exists(): - content = instructions.read_text(encoding="utf-8") - new_content = _replace_or_append_section( - content, _VSCODE_INSTRUCTIONS_MARKER, _always_on("vscode-instructions") - ) - if new_content == content: - print(f" {instructions} -> already configured (no change)") - else: - instructions.write_text(new_content, encoding="utf-8") - print(f" {instructions} -> graphify section {'updated' if _VSCODE_INSTRUCTIONS_MARKER in content else 'added'}") - else: - instructions.write_text(_always_on("vscode-instructions"), encoding="utf-8") - print(f" {instructions} -> created") - - print() - print( - "VS Code Copilot Chat configured. Type /graphify in the chat panel to build the graph." - ) - print("Note: for GitHub Copilot CLI (terminal), use: graphify copilot install") - - -def vscode_uninstall(project_dir: Path | None = None) -> None: - """Remove graphify VS Code Copilot Chat skill and .github/copilot-instructions.md section.""" - skill_dst = Path.home() / ".copilot" / "skills" / "graphify" / "SKILL.md" - if skill_dst.exists(): - skill_dst.unlink() - print(f" skill removed -> {skill_dst}") - version_file = skill_dst.parent / ".graphify_version" - if version_file.exists(): - version_file.unlink() - refs_dir = skill_dst.parent / "references" - if refs_dir.exists(): - shutil.rmtree(refs_dir) - for d in ( - skill_dst.parent, - skill_dst.parent.parent, - skill_dst.parent.parent.parent, - ): - try: - d.rmdir() - except OSError: - break - - instructions = (project_dir or Path(".")) / ".github" / "copilot-instructions.md" - if not instructions.exists(): - return - content = instructions.read_text(encoding="utf-8") - if _VSCODE_INSTRUCTIONS_MARKER not in content: - return - cleaned = re.sub( - r"\n*## graphify\n.*?(?=\n## |\Z)", "", content, flags=re.DOTALL - ).rstrip() - if cleaned: - instructions.write_text(cleaned + "\n", encoding="utf-8") - print(f" graphify section removed from {instructions}") - else: - instructions.unlink() - print(f" {instructions} -> deleted (was empty after removal)") -_ANTIGRAVITY_RULES_PATH = Path(".agents") / "rules" / "graphify.md" -_ANTIGRAVITY_WORKFLOW_PATH = Path(".agents") / "workflows" / "graphify.md" -_ANTIGRAVITY_WORKFLOW = """\ ---- -name: graphify -description: Turn any folder of files into a navigable knowledge graph ---- -# Workflow: graphify -Follow the graphify skill installed at ~/.gemini/config/skills/graphify/SKILL.md to run the full pipeline. -If no path argument is given, use `.` (current directory). -""" + + + + _KIRO_STEERING_MARKER = "graphify: A knowledge graph of this project" -def _kiro_install(project_dir: Path) -> None: - """Write graphify skill + steering file for Kiro IDE/CLI.""" - project_dir = project_dir or Path(".") - - # Skill file + references/ sidecar + .graphify_version stamp via the shared - # progressive-disclosure helper. Previously this used a bare write_text that - # bypassed _copy_skill_file, so the references/ dir and version stamp were - # never written even though kiro declares skill_refs: "kiro" (#1142). - _copy_skill_file("kiro", project=True, project_dir=project_dir) - - # Steering file → .kiro/steering/graphify.md (always-on) - steering_dir = project_dir / ".kiro" / "steering" - steering_dir.mkdir(parents=True, exist_ok=True) - steering_dst = steering_dir / "graphify.md" - if steering_dst.exists() and steering_dst.read_text(encoding="utf-8") == _always_on("kiro-steering"): - print(f" .kiro/steering/graphify.md -> already configured (no change)") - else: - # File is wholly graphify-owned. Overwrite on upgrade so older - # report-first wording does not silently linger (issue #580). - action = "updated" if steering_dst.exists() else "written" - steering_dst.write_text(_always_on("kiro-steering"), encoding="utf-8") - print(f" .kiro/steering/graphify.md -> always-on steering {action}") - - print() - print("Kiro will now read the knowledge graph before every conversation.") - print("Use /graphify to build or update the graph.") - - -def _kiro_uninstall(project_dir: Path) -> None: - """Remove graphify skill + steering file for Kiro.""" - project_dir = project_dir or Path(".") - removed = [] - - # Skill + .graphify_version + references/ sidecar + empty-dir walk. - skill_dst = _platform_skill_destination("kiro", project=True, project_dir=project_dir) - if _remove_skill_file("kiro", project=True, project_dir=project_dir): - removed.append(str(skill_dst.relative_to(project_dir))) - - steering_dst = project_dir / ".kiro" / "steering" / "graphify.md" - if steering_dst.exists(): - steering_dst.unlink() - removed.append(str(steering_dst.relative_to(project_dir))) - - print("Removed: " + (", ".join(removed) if removed else "nothing to remove")) - - -def _antigravity_finalize(skill_dst: Path, project_dir: Path) -> None: - """Write Antigravity's always-on layer next to an installed skill. - - Injects the native tool-discovery YAML frontmatter into *skill_dst*, then - writes ``.agents/rules/graphify.md`` and ``.agents/workflows/graphify.md`` - under *project_dir*. Shared by the global ``antigravity install`` and the - project-scoped ``install --project --platform antigravity`` paths, so both lay - down the rules/workflows that the uninstall path already expects to remove. - """ - # Inject YAML frontmatter for native Antigravity tool discovery. - if skill_dst.exists(): - content = skill_dst.read_text(encoding="utf-8") - if not content.startswith("---\n"): - frontmatter = "---\nname: graphify-manager\ndescription: Rebuild the code graph or perform manual CLI queries when MCP server is offline.\n---\n\n" - skill_dst.write_text(frontmatter + content, encoding="utf-8") - - # .agents/rules/graphify.md - rules_path = project_dir / _ANTIGRAVITY_RULES_PATH - rules_path.parent.mkdir(parents=True, exist_ok=True) - if rules_path.exists(): - existing = rules_path.read_text(encoding="utf-8") - if _always_on("antigravity-rules").strip() != existing.strip(): - rules_path.write_text(_always_on("antigravity-rules"), encoding="utf-8") - print(f"graphify rule updated at {rules_path.resolve()}") - else: - print(f"graphify rule already configured at {rules_path.resolve()} (no change)") - else: - rules_path.write_text(_always_on("antigravity-rules"), encoding="utf-8") - print(f"graphify rule written to {rules_path.resolve()}") - - # .agents/workflows/graphify.md - wf_path = project_dir / _ANTIGRAVITY_WORKFLOW_PATH - wf_path.parent.mkdir(parents=True, exist_ok=True) - if wf_path.exists(): - existing = wf_path.read_text(encoding="utf-8") - if _ANTIGRAVITY_WORKFLOW.strip() != existing.strip(): - wf_path.write_text(_ANTIGRAVITY_WORKFLOW, encoding="utf-8") - print(f"graphify workflow updated at {wf_path.resolve()}") - else: - print(f"graphify workflow already configured at {wf_path.resolve()} (no change)") - else: - wf_path.write_text(_ANTIGRAVITY_WORKFLOW, encoding="utf-8") - print(f"graphify workflow written to {wf_path.resolve()}") - - -def _antigravity_install(project_dir: Path) -> None: - """Install graphify for Google Antigravity (global skill + .agents/rules + .agents/workflows).""" - # Copy the skill to ~/.gemini/config/skills/graphify/SKILL.md (global), then - # lay down the always-on rules/workflows under the project dir. - install(platform="antigravity") - _antigravity_finalize(_platform_skill_destination("antigravity"), project_dir) - - print() - print("Antigravity will now check the knowledge graph before answering") - print("codebase questions. Run /graphify first to build the graph.") - print() - print( - "To enable full MCP architecture navigation, add this to ~/.gemini/antigravity/mcp_config.json:" - ) - print(' "graphify": {') - print(' "command": "uv",') - print( - ' "args": ["run", "--with", "graphifyy", "--with", "mcp", "-m", "graphify.serve", "${workspace.path}/graphify-out/graph.json"]' - ) - print(" }") - - -def _antigravity_uninstall(project_dir: Path, *, project: bool = False) -> None: - """Remove graphify Antigravity rules, workflow, and skill files.""" - # Remove rules file - rules_path = project_dir / _ANTIGRAVITY_RULES_PATH - if rules_path.exists(): - rules_path.unlink() - print(f"graphify rule removed from {rules_path.resolve()}") - else: - print("No graphify Antigravity rule found - nothing to do") - - # Remove workflow file - wf_path = project_dir / _ANTIGRAVITY_WORKFLOW_PATH - if wf_path.exists(): - wf_path.unlink() - print(f"graphify workflow removed from {wf_path.resolve()}") - - # Remove skill file - skill_dst = _platform_skill_destination("antigravity", project=project, project_dir=project_dir) - if skill_dst.exists(): - skill_dst.unlink() - print(f"graphify skill removed from {skill_dst}") - version_file = skill_dst.parent / ".graphify_version" - if version_file.exists(): - version_file.unlink() - refs_dir = skill_dst.parent / "references" - if refs_dir.exists(): - shutil.rmtree(refs_dir) - for d in ( - skill_dst.parent, - skill_dst.parent.parent, - skill_dst.parent.parent.parent, - ): - try: - d.rmdir() - except OSError: - break - - -_CURSOR_RULE_PATH = Path(".cursor") / "rules" / "graphify.mdc" -_CURSOR_RULE = """\ ---- -description: graphify knowledge graph context -alwaysApply: true ---- - -This project has a graphify knowledge graph at graphify-out/. - -**MANDATORY: Before using Read, Grep, Glob, or Bash to explore the codebase, you MUST run graphify first:** -- `graphify query ""` — scoped subgraph for any codebase or architecture question -- `graphify path "" ""` — dependency path between two symbols -- `graphify explain ""` — all nodes related to a concept - -This applies to YOU and to every subagent you spawn. Include this rule explicitly in every subagent prompt that involves code exploration. Do not skip graphify because files are "already known" or because you are executing a plan — the graph surfaces cross-file dependencies and INFERRED edges that grep and Read cannot find. - -Only use Read/Grep/Glob directly when: -1. graphify has already oriented you and you need to modify or debug specific lines -2. `graphify-out/graph.json` does not exist yet - -- If `graphify-out/wiki/index.md` exists, navigate it instead of reading raw files -- Read `graphify-out/GRAPH_REPORT.md` only for broad architecture review when query/path/explain do not surface enough context -- After modifying code files, run `graphify update .` to keep the graph current (AST-only, no API cost) -""" - - -def _cursor_install(project_dir: Path) -> None: - """Write .cursor/rules/graphify.mdc with alwaysApply: true.""" - rule_path = (project_dir or Path(".")) / _CURSOR_RULE_PATH - rule_path.parent.mkdir(parents=True, exist_ok=True) - if rule_path.exists() and rule_path.read_text(encoding="utf-8") == _CURSOR_RULE: - print(f"graphify rule at {rule_path} already configured (no change)") - return - # File is wholly graphify-owned. Overwrite on upgrade so older - # report-first wording does not silently linger (issue #580). - action = "updated" if rule_path.exists() else "written" - rule_path.write_text(_CURSOR_RULE, encoding="utf-8") - print(f"graphify rule {action} at {rule_path.resolve()}") - print() - print("Cursor will now always include the knowledge graph context.") - print("Run /graphify . first to build the graph if you haven't already.") - - -def _cursor_uninstall(project_dir: Path) -> None: - """Remove .cursor/rules/graphify.mdc.""" - rule_path = (project_dir or Path(".")) / _CURSOR_RULE_PATH - if not rule_path.exists(): - print("No graphify Cursor rule found - nothing to do") - return - rule_path.unlink() - print(f"graphify Cursor rule removed from {rule_path.resolve()}") -# Devin CLI — .windsurf/rules/graphify.md (always-on context) -# Devin reads .windsurf/rules/*.md files the same way Windsurf IDE does. -_DEVIN_RULES_PATH = Path(".windsurf") / "rules" / "graphify.md" -_DEVIN_RULES = """\ -## graphify -This project has a graphify knowledge graph at graphify-out/. -Rules: -- For codebase or architecture questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. -- If graphify-out/wiki/index.md exists, navigate it instead of reading raw files -- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context -- After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) -""" -def _devin_rules_install(project_dir: Path) -> None: - """Write .windsurf/rules/graphify.md for always-on Devin context.""" - rules_path = (project_dir or Path(".")) / _DEVIN_RULES_PATH - rules_path.parent.mkdir(parents=True, exist_ok=True) - if rules_path.exists() and rules_path.read_text(encoding="utf-8") == _DEVIN_RULES: - print(f" {rules_path} -> already configured (no change)") - return - action = "updated" if rules_path.exists() else "written" - rules_path.write_text(_DEVIN_RULES, encoding="utf-8") - print(f" rules {action} -> {rules_path}") -def _devin_rules_uninstall(project_dir: Path) -> None: - """Remove .windsurf/rules/graphify.md.""" - rules_path = (project_dir or Path(".")) / _DEVIN_RULES_PATH - if not rules_path.exists(): - return - rules_path.unlink() - print(f" rules removed -> {rules_path}") - - -_KILO_PLUGIN_JS = """\ -// graphify Kilo plugin -// Injects a knowledge graph reminder before bash tool calls when the graph exists. -import { existsSync } from "fs"; -import { join } from "path"; - -export const GraphifyPlugin = async ({ directory }) => { - let reminded = false; - - return { - "tool.execute.before": async (input, output) => { - if (reminded) return; - if (!existsSync(join(directory, "graphify-out", "graph.json"))) return; - - if (input.tool === "bash") { - // Separate with ';' not '&&' — Windows PowerShell 5.1 rejects '&&' as a - // statement separator ("not a valid statement separator"), which broke - // the first bash command in every OpenCode session on Windows (#1646). - // ';' works in PowerShell 5.1, Bash, and POSIX shells alike. - output.args.command = - 'echo "[graphify] Knowledge graph available. Read graphify-out/GRAPH_REPORT.md for god nodes and architecture context before searching files." ; ' + - output.args.command; - reminded = true; - } - }, - }; -}; -""" - -_KILO_PLUGIN_PATH = Path(".kilo") / "plugins" / "graphify.js" -_KILO_CONFIG_JSON_PATH = Path(".kilo") / "kilo.json" -_KILO_CONFIG_JSONC_PATH = Path(".kilo") / "kilo.jsonc" - - -def _strip_json_comments(raw: str) -> str: - """Remove JSONC-style comments while leaving string content intact.""" - result: list[str] = [] - in_string = False - escaped = False - line_comment = False - block_comment = False - i = 0 - - while i < len(raw): - ch = raw[i] - nxt = raw[i + 1] if i + 1 < len(raw) else "" - - if line_comment: - if ch == "\n": - line_comment = False - result.append(ch) - i += 1 - continue - - if block_comment: - if ch == "*" and nxt == "/": - block_comment = False - i += 2 - else: - i += 1 - continue - - if in_string: - result.append(ch) - if escaped: - escaped = False - elif ch == "\\": - escaped = True - elif ch == '"': - in_string = False - i += 1 - continue - - if ch == "/" and nxt == "/": - line_comment = True - i += 2 - continue - if ch == "/" and nxt == "*": - block_comment = True - i += 2 - continue - - result.append(ch) - if ch == '"': - in_string = True - i += 1 - - return re.sub(r",(\s*[}\]])", r"\1", "".join(result)) - - -def _load_json_like(config_file: Path) -> dict: - if not config_file.exists(): - return {} - try: - raw = config_file.read_text(encoding="utf-8") - if config_file.suffix == ".jsonc": - raw = _strip_json_comments(raw) - loaded = json.loads(raw) - except (OSError, json.JSONDecodeError): - return {} - return loaded if isinstance(loaded, dict) else {} - - -def _kilo_config_path(project_dir: Path) -> Path: - kilo_dir = (project_dir or Path(".")) / ".kilo" - json_path = kilo_dir / _KILO_CONFIG_JSON_PATH.name - if json_path.exists(): - return json_path - jsonc_path = kilo_dir / _KILO_CONFIG_JSONC_PATH.name - if jsonc_path.exists(): - return jsonc_path - return json_path - - -def _kilo_config_write_path(project_dir: Path) -> Path: - """Write automated Kilo edits to kilo.json so existing JSONC stays untouched.""" - kilo_dir = (project_dir or Path(".")) / ".kilo" - return kilo_dir / _KILO_CONFIG_JSON_PATH.name - - -def _install_kilo_plugin(project_dir: Path) -> None: - """Write graphify.js plugin and register it without rewriting user JSONC.""" - plugin_file = project_dir / _KILO_PLUGIN_PATH - plugin_file.parent.mkdir(parents=True, exist_ok=True) - plugin_file.write_text(_KILO_PLUGIN_JS, encoding="utf-8") - print(f" {_KILO_PLUGIN_PATH} -> tool.execute.before hook written") - - config_file = _kilo_config_path(project_dir) - write_config_file = _kilo_config_write_path(project_dir) - write_config_file.parent.mkdir(parents=True, exist_ok=True) - config = _load_json_like(config_file) - plugins = config.get("plugin") - if not isinstance(plugins, list): - plugins = [] - config["plugin"] = plugins - entry = plugin_file.resolve().as_uri() - if entry not in plugins: - plugins.append(entry) - write_config_file.write_text(json.dumps(config, indent=2), encoding="utf-8") - print(f" {write_config_file.relative_to(project_dir)} -> plugin registered") - else: - print( - f" {config_file.relative_to(project_dir)} -> plugin already registered (no change)" - ) -def _uninstall_kilo_plugin(project_dir: Path) -> None: - """Remove graphify.js plugin and deregister it without rewriting user JSONC.""" - plugin_file = project_dir / _KILO_PLUGIN_PATH - if plugin_file.exists(): - plugin_file.unlink() - print(f" {_KILO_PLUGIN_PATH} -> removed") - config_file = _kilo_config_path(project_dir) - if not config_file.exists(): - return - write_config_file = _kilo_config_write_path(project_dir) - config = _load_json_like(config_file) - plugins = config.get("plugin", []) - if not isinstance(plugins, list): - plugins = [] - entry = plugin_file.resolve().as_uri() - if entry in plugins: - config["plugin"] = [plugin for plugin in plugins if plugin != entry] - if not config["plugin"]: - config.pop("plugin") - write_config_file.parent.mkdir(parents=True, exist_ok=True) - write_config_file.write_text(json.dumps(config, indent=2), encoding="utf-8") - print( - f" {write_config_file.relative_to(project_dir)} -> plugin deregistered" - ) -# OpenCode tool.execute.before plugin — fires before every tool call. -# Injects a graph reminder into bash command output when graph.json exists. -_OPENCODE_PLUGIN_JS = """\ -// graphify OpenCode plugin -// Injects a knowledge graph reminder before bash tool calls when the graph exists. -// -// IMPORTANT: keep the reminder string free of backticks and $(...) constructs. -// The hook prepends `echo "" && ` to the user's bash command; -// backticks inside the double-quoted echo trigger bash command substitution, -// which both corrupts tool output and silently executes the very graphify -// command we are only suggesting. Plain words render fine in opencode's TUI. -import { existsSync } from "fs"; -import { join } from "path"; - -export const GraphifyPlugin = async ({ directory }) => { - let reminded = false; - - return { - "tool.execute.before": async (input, output) => { - if (reminded) return; - if (!existsSync(join(directory, "graphify-out", "graph.json"))) return; - - if (input.tool === "bash") { - // ';' not '&&' — Windows PowerShell 5.1 rejects '&&' as a statement - // separator, breaking the first bash command of the session (#1646). - output.args.command = - 'echo "[graphify] knowledge graph at graphify-out/. For focused questions, run graphify query with your question (scoped subgraph, usually much smaller than GRAPH_REPORT.md) instead of grepping raw files. Read GRAPH_REPORT.md only for broad architecture context." ; ' + - output.args.command; - reminded = true; - } - }, - }; -}; -""" - -_OPENCODE_PLUGIN_PATH = Path(".opencode") / "plugins" / "graphify.js" -_OPENCODE_CONFIG_PATH = Path(".opencode") / "opencode.json" - - -def _install_opencode_plugin(project_dir: Path) -> None: - """Write graphify.js plugin and register it in opencode.json.""" - plugin_file = project_dir / _OPENCODE_PLUGIN_PATH - plugin_file.parent.mkdir(parents=True, exist_ok=True) - plugin_file.write_text(_OPENCODE_PLUGIN_JS, encoding="utf-8") - print(f" {_OPENCODE_PLUGIN_PATH} -> tool.execute.before hook written") - - config_file = project_dir / _OPENCODE_CONFIG_PATH - if config_file.exists(): - try: - config = json.loads(config_file.read_text(encoding="utf-8")) - except json.JSONDecodeError: - config = {} - else: - config = {} - - plugins = config.setdefault("plugin", []) - entry = _OPENCODE_PLUGIN_PATH.as_posix() - if entry not in plugins: - plugins.append(entry) - config_file.write_text(json.dumps(config, indent=2), encoding="utf-8") - print(f" {_OPENCODE_CONFIG_PATH} -> plugin registered") - else: - print(f" {_OPENCODE_CONFIG_PATH} -> plugin already registered (no change)") - - -def _uninstall_opencode_plugin(project_dir: Path) -> None: - """Remove graphify.js plugin and deregister from opencode.json.""" - plugin_file = project_dir / _OPENCODE_PLUGIN_PATH - if plugin_file.exists(): - plugin_file.unlink() - print(f" {_OPENCODE_PLUGIN_PATH} -> removed") - - config_file = project_dir / _OPENCODE_CONFIG_PATH - if not config_file.exists(): - return - try: - config = json.loads(config_file.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return - plugins = config.get("plugin", []) - entry = _OPENCODE_PLUGIN_PATH.as_posix() - if entry in plugins: - plugins.remove(entry) - if not plugins: - config.pop("plugin") - config_file.write_text(json.dumps(config, indent=2), encoding="utf-8") - print(f" {_OPENCODE_CONFIG_PATH} -> plugin deregistered") + + + + + + + + + + + + + + + + + + + + + + + + + + + + + _CODEX_HOOK = { @@ -1613,700 +387,61 @@ def _uninstall_opencode_plugin(project_dir: Path) -> None: } -def _resolve_graphify_exe() -> str: - """Return the absolute path to the graphify executable. - Falls back to bare 'graphify' if resolution fails. Using an absolute path - ensures the hook works in environments where the venv Scripts/ directory is - not on PATH (e.g. VS Code Codex extension on Windows). - """ - import shutil - found = shutil.which("graphify") - if found: - return found - # Derive from sys.executable: same Scripts/ (Windows) or bin/ (Unix) dir - scripts_dir = Path(sys.executable).parent - for name in ("graphify.exe", "graphify"): - candidate = scripts_dir / name - if candidate.exists(): - return str(candidate) - return "graphify" - - -def _run_hook_guard(kind: str) -> None: - """Shell-agnostic PreToolUse guard (#522). - - Reads the tool-call JSON from stdin and, when a knowledge graph exists in the - current output dir, prints a nudge (`additionalContext`) telling the agent to - use graphify instead of grepping/reading raw files. Replaces the old inline - bash hooks that failed to parse on Windows. Always fails open: any error, or a - non-matching tool call, prints nothing and the caller exits 0, so a legitimate - tool call is never blocked. Detection mirrors the previous hooks exactly. - """ - from graphify.paths import out_path, GRAPHIFY_OUT_NAME - # Gemini's BeforeTool hook takes no stdin and must ALWAYS return a decision so - # the tool is never blocked; the graph nudge is appended only when a graph - # exists. Handled before the stdin read below (which the search/read guards need). - if kind == "gemini": - payload = {"decision": "allow"} - try: - if out_path("graph.json").is_file(): - payload["additionalContext"] = _GEMINI_NUDGE_TEXT - except Exception: - pass - sys.stdout.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":"))) - return - try: - d = json.loads(sys.stdin.buffer.read().decode("utf-8", "replace")) - except Exception: - return - if not isinstance(d, dict): - return - t = d.get("tool_input", d) - if not isinstance(t, dict): - return - try: - if kind == "search": - cmd_str = str(t.get("command", "") or "") - # Same set the old `case` matched: *grep*, *ripgrep*, and rg/find/fd/ - # ack/ag as a token (name followed by a space). - if any(tok in cmd_str for tok in ("grep", "ripgrep", "rg ", "find ", "fd ", "ack ", "ag ")) \ - and out_path("graph.json").is_file(): - sys.stdout.write(_SEARCH_NUDGE) - elif kind == "read": - vals = [str(t.get("file_path") or ""), str(t.get("pattern") or ""), str(t.get("path") or "")] - j = " ".join(vals).lower().replace("\\", "/") - tails = [ - "." + seg.rsplit(".", 1)[-1] - for v in vals if v - for seg in [v.lower().replace("\\", "/").rsplit("/", 1)[-1]] - if "." in seg - ] - under_out = "graphify-out/" in j or (GRAPHIFY_OUT_NAME.lower() + "/") in j - if not under_out and any(tl in _HOOK_SOURCE_EXTS for tl in tails) \ - and out_path("graph.json").is_file(): - sys.stdout.write(_READ_NUDGE) - except Exception: - pass - - -def _install_codex_hook(project_dir: Path) -> None: - """Add graphify PreToolUse hook to .codex/hooks.json.""" - hooks_path = project_dir / ".codex" / "hooks.json" - hooks_path.parent.mkdir(parents=True, exist_ok=True) - - if hooks_path.exists(): - try: - existing = json.loads(hooks_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - existing = {} - else: - existing = {} - - graphify_exe = _resolve_graphify_exe() - hook_entry = { - "hooks": { - "PreToolUse": [ - { - "matcher": "Bash", - "hooks": [{"type": "command", "command": f"{graphify_exe} hook-check"}], - } - ] - } - } - pre_tool = existing.setdefault("hooks", {}).setdefault("PreToolUse", []) - existing["hooks"]["PreToolUse"] = [h for h in pre_tool if "graphify" not in str(h)] - existing["hooks"]["PreToolUse"].extend(hook_entry["hooks"]["PreToolUse"]) - hooks_path.write_text(json.dumps(existing, indent=2), encoding="utf-8") - print(f" .codex/hooks.json -> PreToolUse hook registered ({graphify_exe} hook-check)") -def _uninstall_codex_hook(project_dir: Path) -> None: - """Remove graphify PreToolUse hook from .codex/hooks.json.""" - hooks_path = project_dir / ".codex" / "hooks.json" - if not hooks_path.exists(): - return - try: - existing = json.loads(hooks_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return - pre_tool = existing.get("hooks", {}).get("PreToolUse", []) - filtered = [h for h in pre_tool if "graphify" not in str(h)] - existing["hooks"]["PreToolUse"] = filtered - hooks_path.write_text(json.dumps(existing, indent=2), encoding="utf-8") - print(f" .codex/hooks.json -> PreToolUse hook removed") -def _agents_install(project_dir: Path, platform: str) -> None: - """Write the graphify section to the local AGENTS.md for always-on platforms.""" - target = (project_dir or Path(".")) / "AGENTS.md" - if target.exists(): - content = target.read_text(encoding="utf-8") - new_content = _replace_or_append_section( - content, _AGENTS_MD_MARKER, _always_on("agents-md") - ) - else: - new_content = _always_on("agents-md") - - if target.exists() and new_content == target.read_text(encoding="utf-8"): - print(f"graphify already configured in {target.resolve()} (no change)") - else: - target.write_text(new_content, encoding="utf-8") - print(f"graphify section written to {target.resolve()}") - - if platform == "codex": - _install_codex_hook(project_dir or Path(".")) - elif platform == "opencode": - _install_opencode_plugin(project_dir or Path(".")) - elif platform == "kilo": - _install_kilo_plugin(project_dir or Path(".")) - - print() - print( - f"{platform.capitalize()} will now check the knowledge graph before answering" - ) - print("codebase questions and rebuild it after code changes.") - if platform not in ("codex", "opencode", "kilo"): - print() - print("Note: unlike Claude Code, there is no PreToolUse hook equivalent for") - print( - f"{platform.capitalize()} — the AGENTS.md rules are the always-on mechanism." - ) -def _amp_legacy_cleanup() -> None: - """Best-effort removal of the pre-fix ~/.amp/skills/graphify install dir. - Older graphify versions wrote the Amp skill to ~/.amp/skills, which Amp does - not search. Clean it up on install so a stale, never-loaded copy does not - linger. Failures are ignored (the new path is what matters). - """ - legacy = Path.home() / ".amp" / "skills" / "graphify" - if legacy.exists(): - shutil.rmtree(legacy, ignore_errors=True) - if not legacy.exists(): - print(f" legacy removed -> {legacy}") -def _amp_install(project_dir: Path | None = None) -> None: - """User-scope Amp install: skill into ~/.config/agents/skills + AGENTS.md.""" - _amp_legacy_cleanup() - _copy_skill_file("amp") - _agents_install(project_dir or Path("."), "amp") -def _amp_uninstall(project_dir: Path | None = None) -> None: - """User-scope Amp uninstall: remove the skill and the AGENTS.md section.""" - removed = _remove_skill_file("amp") - if removed: - print("skill removed") - _agents_uninstall(project_dir or Path("."), platform="amp") -def _agents_platform_install(project_dir: Path | None = None) -> None: - """`graphify agents install`: skill into ~/.agents/skills + AGENTS.md. - The amp-twin of the generic Agent-Skills target. Mirrors _amp_install but - lands the skill at the spec's user-global ~/.agents/skills (set in - _platform_skill_destination). Wiring AGENTS.md keeps it honest with the - rendered hooks reference, which points at `graphify agents install`. The bare - `graphify install --platform agents` path stays skill-only (via install()), - exactly as amp's `--platform amp` does. - """ - _copy_skill_file("agents") - _agents_install(project_dir or Path("."), "agents") - - -def _agents_platform_uninstall(project_dir: Path | None = None) -> None: - """`graphify agents uninstall`: remove the skill and the AGENTS.md section.""" - removed = _remove_skill_file("agents") - if removed: - print("skill removed") - _agents_uninstall(project_dir or Path("."), platform="agents") - - -def _project_install(platform_name: str, project_dir: Path | None = None) -> None: - """Install platform skill/config files in the current project.""" - project_dir = project_dir or Path(".") - platform_name = _canonical_platform(platform_name) - if platform_name in ("claude", "windows"): - install(platform=platform_name, project=True, project_dir=project_dir) - claude_install(project_dir) - _print_project_git_add_hint([project_dir / ".claude", project_dir / "CLAUDE.md"]) - elif platform_name == "gemini": - gemini_install(project_dir, project=True) - elif platform_name == "cursor": - _cursor_install(project_dir) - _print_project_git_add_hint([project_dir / ".cursor"]) - elif platform_name == "kiro": - _kiro_install(project_dir) - _print_project_git_add_hint([project_dir / ".kiro"]) - elif platform_name in ("aider", "amp", "codex", "opencode", "claw", "droid", "trae", "trae-cn", "hermes"): - skill_dst = _copy_skill_file(platform_name, project=True, project_dir=project_dir) - _agents_install(project_dir, platform_name) - hint_paths = [_project_scope_root(skill_dst, project_dir), project_dir / "AGENTS.md"] - if platform_name == "opencode": - hint_paths.append(project_dir / ".opencode") - elif platform_name == "codex": - hint_paths.append(project_dir / ".codex") - _print_project_git_add_hint(hint_paths) - elif platform_name == "devin": - skill_dst = _copy_skill_file("devin", project=True, project_dir=project_dir) - _devin_rules_install(project_dir) - _print_project_git_add_hint([_project_scope_root(skill_dst, project_dir), project_dir / ".windsurf"]) - elif platform_name == "antigravity": - # Project-scoped: skill in .agents/skills/ PLUS the .agents/rules + - # .agents/workflows always-on layer (previously this path wrote only the - # skill, leaving the rules/workflows the uninstall path removes unset). - skill_dst = _copy_skill_file("antigravity", project=True, project_dir=project_dir) - _antigravity_finalize(skill_dst, project_dir) - _print_project_git_add_hint([_project_scope_root(skill_dst, project_dir), project_dir / ".agents"]) - elif platform_name in ("copilot", "pi", "kimi", "agents"): - # Skill-only project install: drop SKILL.md (+ references) at the scope - # root. `agents` -> ./.agents/skills/graphify/SKILL.md. - skill_dst = _copy_skill_file(platform_name, project=True, project_dir=project_dir) - _print_project_git_add_hint([_project_scope_root(skill_dst, project_dir)]) - else: - install(platform=platform_name, project=True, project_dir=project_dir) - - -def _project_uninstall(platform_name: str, project_dir: Path | None = None) -> None: - """Remove project-scoped platform skill/config files only.""" - project_dir = project_dir or Path(".") - platform_name = _canonical_platform(platform_name) - if platform_name in ("claude", "windows"): - _remove_skill_file(platform_name, project=True, project_dir=project_dir) - _remove_claude_skill_registration(project_dir) - claude_uninstall(project_dir, project=True) - elif platform_name == "gemini": - gemini_uninstall(project_dir, project=True) - elif platform_name == "cursor": - _cursor_uninstall(project_dir) - elif platform_name == "kiro": - _kiro_uninstall(project_dir) - elif platform_name in ("aider", "amp", "codex", "opencode", "claw", "droid", "trae", "trae-cn", "hermes"): - _remove_skill_file(platform_name, project=True, project_dir=project_dir) - _agents_uninstall(project_dir, platform=platform_name) - if platform_name == "codex": - _uninstall_codex_hook(project_dir) - elif platform_name == "antigravity": - _antigravity_uninstall(project_dir, project=True) - elif platform_name == "devin": - removed = _remove_skill_file("devin", project=True, project_dir=project_dir) - _devin_rules_uninstall(project_dir) - if not removed: - print("nothing to remove") - elif platform_name in ("copilot", "pi", "kimi", "agents"): - removed = _remove_skill_file(platform_name, project=True, project_dir=project_dir) - if not removed: - print("nothing to remove") - elif platform_name == "codebuddy": - codebuddy_uninstall(project_dir) - else: - _remove_skill_file(platform_name, project=True, project_dir=project_dir) - - -def _project_uninstall_all(project_dir: Path | None = None) -> None: - """Remove project-scoped install files without touching user-scope installs.""" - project_dir = project_dir or Path(".") - print("Uninstalling project-scoped graphify files...\n") - for platform_name in _PLATFORM_CONFIG: - _project_uninstall(platform_name, project_dir) - for platform_name in ("gemini", "cursor"): - _project_uninstall(platform_name, project_dir) - print("\nDone.") - - -def _agents_uninstall(project_dir: Path, platform: str = "") -> None: - """Remove the graphify section from the local AGENTS.md.""" - target = (project_dir or Path(".")) / "AGENTS.md" - - if not target.exists(): - print("No AGENTS.md found in current directory - nothing to do") - if platform == "opencode": - _uninstall_opencode_plugin(project_dir or Path(".")) - elif platform == "kilo": - _uninstall_kilo_plugin(project_dir or Path(".")) - return - content = target.read_text(encoding="utf-8") - if _AGENTS_MD_MARKER not in content: - print("graphify section not found in AGENTS.md - nothing to do") - if platform == "opencode": - _uninstall_opencode_plugin(project_dir or Path(".")) - elif platform == "kilo": - _uninstall_kilo_plugin(project_dir or Path(".")) - return - cleaned = re.sub( - r"\n*## graphify\n.*?(?=\n## |\Z)", - "", - content, - flags=re.DOTALL, - ).rstrip() - if cleaned: - target.write_text(cleaned + "\n", encoding="utf-8") - print(f"graphify section removed from {target.resolve()}") - else: - target.unlink() - print(f"AGENTS.md was empty after removal - deleted {target.resolve()}") - - if platform == "opencode": - _uninstall_opencode_plugin(project_dir or Path(".")) - elif platform == "kilo": - _uninstall_kilo_plugin(project_dir or Path(".")) - - -def _kilo_uninstall_global() -> list[str]: - removed = [] - command_dst = Path.home() / ".config" / "kilo" / "command" / "graphify.md" - if command_dst.exists(): - command_dst.unlink() - removed.append(f"command removed: {command_dst}") - try: - command_dst.parent.rmdir() - except OSError: - pass - skill_dst = Path.home() / _PLATFORM_CONFIG["kilo"]["skill_dst"] - if skill_dst.exists(): - skill_dst.unlink() - removed.append(f"skill removed: {skill_dst}") - version_file = skill_dst.parent / ".graphify_version" - if version_file.exists(): - version_file.unlink() - for d in ( - skill_dst.parent, - skill_dst.parent.parent, - skill_dst.parent.parent.parent, - ): - try: - d.rmdir() - except OSError: - break - - return removed - - -def _kilo_install(project_dir: Path) -> None: - """Install native Kilo skill + command globally and always-on project wiring locally.""" - install(platform="kilo") - _agents_install(project_dir or Path("."), "kilo") - - -def _kilo_uninstall(project_dir: Path) -> None: - """Remove Kilo always-on project wiring and global skill/command files.""" - _agents_uninstall(project_dir or Path("."), platform="kilo") - removed = _kilo_uninstall_global() - print("; ".join(removed) if removed else "nothing to remove") - - -def claude_install(project_dir: Path | None = None) -> None: - """Write the graphify section to the local CLAUDE.md.""" - target = (project_dir or Path(".")) / "CLAUDE.md" - - if target.exists(): - content = target.read_text(encoding="utf-8") - new_content = _replace_or_append_section( - content, _CLAUDE_MD_MARKER, _always_on("claude-md") - ) - else: - new_content = _always_on("claude-md") - - if target.exists() and new_content == target.read_text(encoding="utf-8"): - print(f"graphify already configured in {target.resolve()} (no change)") - else: - target.write_text(new_content, encoding="utf-8") - print(f"graphify section written to {target.resolve()}") - - # Always re-install the Claude Code PreToolUse hook so an old hook - # payload (e.g. pre-issue-#580 wording) is replaced on upgrade. - _install_claude_hook(project_dir or Path(".")) - - print() - print("Claude Code will now check the knowledge graph before answering") - print("codebase questions and rebuild it after code changes.") - - -def _install_claude_hook(project_dir: Path) -> None: - """Add graphify PreToolUse hook to .claude/settings.json.""" - settings_path = project_dir / ".claude" / "settings.json" - settings_path.parent.mkdir(parents=True, exist_ok=True) - - if settings_path.exists(): - try: - settings = json.loads(settings_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - settings = {} - else: - settings = {} - - hooks = settings.setdefault("hooks", {}) - pre_tool = hooks.setdefault("PreToolUse", []) - - hooks["PreToolUse"] = [h for h in pre_tool if not (h.get("matcher") in ("Glob|Grep", "Bash", "Read|Glob") and "graphify" in str(h))] - hooks["PreToolUse"].extend(_claude_pretooluse_hooks()) - settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") - print(f" .claude/settings.json -> PreToolUse hooks registered (Bash search + Read/Glob)") - - -def _uninstall_claude_hook(project_dir: Path) -> None: - """Remove graphify PreToolUse hook from .claude/settings.json.""" - settings_path = project_dir / ".claude" / "settings.json" - if not settings_path.exists(): - return - try: - settings = json.loads(settings_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return - pre_tool = settings.get("hooks", {}).get("PreToolUse", []) - filtered = [h for h in pre_tool if not (h.get("matcher") in ("Glob|Grep", "Bash", "Read|Glob") and "graphify" in str(h))] - if len(filtered) == len(pre_tool): - return - settings["hooks"]["PreToolUse"] = filtered - settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") - print(f" .claude/settings.json -> PreToolUse hook removed") - - -def uninstall_all(project_dir: Path | None = None, purge: bool = False) -> None: - """Remove graphify from every platform detected in the current project.""" - pd = project_dir or Path(".") - print("Uninstalling graphify from all detected platforms...\n") - - # Skill-file / config-section uninstallers - claude_uninstall(pd) - codebuddy_uninstall(pd) - gemini_uninstall(pd) - vscode_uninstall(pd) - _cursor_uninstall(pd) - _kiro_uninstall(pd) - _antigravity_uninstall(pd) - # AGENTS.md covers: codex, aider, opencode, claw, droid, trae, trae-cn, hermes, copilot - _agents_uninstall(pd) - # Amp also drops a user-scope skill at ~/.config/agents/skills, which the - # AGENTS.md cleanup above does not touch. - _remove_skill_file("amp") - # The generic agents platform's user-scope skill lives at ~/.agents/skills, - # which neither the AGENTS.md cleanup nor amp's removal reaches. - _remove_skill_file("agents") - _uninstall_opencode_plugin(pd) - _uninstall_codex_hook(pd) - - # Git hook - try: - from graphify.hooks import uninstall as hook_uninstall - result = hook_uninstall(pd) - if result: - print(result) - except Exception: - pass - - if purge: - import shutil as _shutil - out = pd / _GRAPHIFY_OUT - if out.exists(): - _shutil.rmtree(out) - print(f"\n {_GRAPHIFY_OUT}/ -> deleted (--purge)") - else: - print(f"\n {_GRAPHIFY_OUT}/ -> not found (nothing to purge)") - print("\nDone. Run 'pip uninstall graphifyy' to remove the package itself.") -def claude_uninstall(project_dir: Path | None = None, *, project: bool = False) -> None: - """Remove the graphify skill tree (SKILL.md + references/) and the CLAUDE.md section. - Mirrors gemini_uninstall: the bare `graphify uninstall` and `graphify claude - uninstall` must remove the installed skill, not just strip CLAUDE.md, or the - progressive-disclosure tree (SKILL.md + references/) is orphaned (#1121). - """ - project_dir = project_dir or Path(".") - _remove_skill_file("claude", project=project, project_dir=project_dir) - target = project_dir / "CLAUDE.md" - if not target.exists(): - print("No CLAUDE.md found in current directory - nothing to do") - return - content = target.read_text(encoding="utf-8") - if _CLAUDE_MD_MARKER not in content: - print("graphify section not found in CLAUDE.md - nothing to do") - return - # Remove the ## graphify section: from the marker to the next ## heading or EOF - cleaned = re.sub( - r"\n*## graphify\n.*?(?=\n## |\Z)", - "", - content, - flags=re.DOTALL, - ).rstrip() - if cleaned: - target.write_text(cleaned + "\n", encoding="utf-8") - print(f"graphify section removed from {target.resolve()}") - else: - target.unlink() - print(f"CLAUDE.md was empty after removal - deleted {target.resolve()}") - - _uninstall_claude_hook(project_dir or Path(".")) - - -def codebuddy_install(project_dir: Path | None = None) -> None: - """Install the graphify skill and CODEBUDDY.md section for CodeBuddy.""" - _copy_skill_file("codebuddy", project=bool(project_dir), project_dir=project_dir) - target = (project_dir or Path(".")) / "CODEBUDDY.md" - - if target.exists(): - content = target.read_text(encoding="utf-8") - new_content = _replace_or_append_section( - content, _CODEBUDDY_MD_MARKER, _always_on("claude-md") - ) - else: - new_content = _always_on("claude-md") - if target.exists() and new_content == target.read_text(encoding="utf-8"): - print(f"graphify already configured in {target.resolve()} (no change)") - else: - target.write_text(new_content, encoding="utf-8") - print(f"graphify section written to {target.resolve()}") - # Also write CodeBuddy PreToolUse hook to .codebuddy/settings.json - _install_codebuddy_hook(project_dir or Path(".")) - print() - print("CodeBuddy will now check the knowledge graph before answering") - print("codebase questions and rebuild it after code changes.") -def _install_codebuddy_hook(project_dir: Path) -> None: - """Add graphify PreToolUse hook to .codebuddy/settings.json.""" - settings_path = project_dir / ".codebuddy" / "settings.json" - settings_path.parent.mkdir(parents=True, exist_ok=True) - if settings_path.exists(): - try: - settings = json.loads(settings_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - settings = {} - else: - settings = {} - hooks = settings.setdefault("hooks", {}) - pre_tool = hooks.setdefault("PreToolUse", []) - hooks["PreToolUse"] = [h for h in pre_tool if not (h.get("matcher") in ("Glob|Grep", "Bash", "Read|Glob") and "graphify" in str(h))] - hooks["PreToolUse"].extend(_claude_pretooluse_hooks()) - settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") - print(f" .codebuddy/settings.json -> PreToolUse hooks registered") -def _uninstall_codebuddy_hook(project_dir: Path) -> None: - """Remove graphify PreToolUse hook from .codebuddy/settings.json.""" - settings_path = project_dir / ".codebuddy" / "settings.json" - if not settings_path.exists(): - return - try: - settings = json.loads(settings_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return - pre_tool = settings.get("hooks", {}).get("PreToolUse", []) - filtered = [h for h in pre_tool if not (h.get("matcher") in ("Glob|Grep", "Bash", "Read|Glob") and "graphify" in str(h))] - if len(filtered) == len(pre_tool): - return - settings["hooks"]["PreToolUse"] = filtered - settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") - print(f" .codebuddy/settings.json -> PreToolUse hook removed") -def codebuddy_uninstall(project_dir: Path | None = None, *, project: bool = False) -> None: - """Remove the graphify skill tree (SKILL.md + references/) and the CODEBUDDY.md section.""" - project_dir = project_dir or Path(".") - _remove_skill_file("codebuddy", project=project, project_dir=project_dir) - target = project_dir / "CODEBUDDY.md" - if not target.exists(): - print("No CODEBUDDY.md found in current directory - nothing to do") - return - content = target.read_text(encoding="utf-8") - if _CODEBUDDY_MD_MARKER not in content: - print("graphify section not found in CODEBUDDY.md - nothing to do") - return - # Remove the ## graphify section: from the marker to the next ## heading or EOF - cleaned = re.sub( - r"\n*## graphify\n.*?(?=\n## |\Z)", - "", - content, - flags=re.DOTALL, - ).rstrip() - if cleaned: - target.write_text(cleaned + "\n", encoding="utf-8") - print(f"graphify section removed from {target.resolve()}") - else: - target.unlink() - print(f"CODEBUDDY.md was empty after removal - deleted {target.resolve()}") - - _uninstall_codebuddy_hook(project_dir or Path(".")) - -def _clone_repo( - url: str, branch: str | None = None, out_dir: Path | None = None -) -> Path: - """Clone a GitHub repo to a local cache dir and return the path. - - Clones into ~/.graphify/repos// by default so repeated - runs on the same URL reuse the existing clone (git pull instead of clone). - """ - import subprocess as _sp - import re as _re - - # Normalise URL — strip trailing .git if present - url = url.rstrip("/") - if not url.endswith(".git"): - git_url = url + ".git" - else: - git_url = url - url = url[:-4] - - # Extract owner/repo from URL - m = _re.search(r"github\.com[:/]([^/]+)/([^/]+?)(?:\.git)?$", url) - if not m: - print(f"error: not a recognised GitHub URL: {url}", file=sys.stderr) - sys.exit(1) - owner, repo = m.group(1), m.group(2) - - if out_dir: - dest = out_dir - else: - dest = Path.home() / ".graphify" / "repos" / owner / repo - - if branch and branch.startswith("-"): - print(f"error: invalid branch name: {branch!r}", file=sys.stderr) - sys.exit(1) - - if dest.exists(): - print(f"Repo already cloned at {dest} - pulling latest...", flush=True) - cmd = ["git", "-C", str(dest), "pull"] - if branch: - cmd += ["origin", "--", branch] - result = _sp.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"warning: git pull failed:\n{result.stderr}", file=sys.stderr) - else: - dest.parent.mkdir(parents=True, exist_ok=True) - print(f"Cloning {url} -> {dest} ...", flush=True) - cmd = ["git", "clone", "--depth", "1"] - if branch: - cmd += ["--branch", branch] - cmd += ["--", git_url, str(dest)] - result = _sp.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"error: git clone failed:\n{result.stderr}", file=sys.stderr) - sys.exit(1) - - print(f"Ready at: {dest}", flush=True) - return dest + + + + + + + + + + + + + def main() -> None: @@ -2433,6 +568,7 @@ def main() -> None: print(" --out DIR output dir (default: ); writes /graphify-out/") print(" --google-workspace export .gdoc/.gsheet/.gslides shortcuts via gws before extraction") print(" --no-cluster skip clustering, write raw extraction only") + print(" --code-only index code (local AST, no API key) and skip doc/paper/image files") print(" --postgres DSN extract schema from a live PostgreSQL database") print(" maps tables, views, functions + FK relationships;") print(" column-level detail is not represented in the graph") @@ -2528,2772 +664,9 @@ def main() -> None: print(f"Run 'graphify --help' for full usage.") return - if cmd == "install": - # Default to windows platform on Windows, claude elsewhere - default_platform = "windows" if platform.system() == "Windows" else "claude" - selected_platform: str | None = None - project_scope = False - args = sys.argv[2:] - i = 0 - while i < len(args): - arg = args[i] - if arg in ("-h", "--help"): - _print_install_usage() - return - if arg == "--project": - project_scope = True - i += 1 - elif arg.startswith("--platform="): - candidate = arg.split("=", 1)[1] - if selected_platform and selected_platform != candidate: - print("error: specify install platform only once", file=sys.stderr) - sys.exit(1) - selected_platform = candidate - i += 1 - elif arg == "--platform": - if i + 1 >= len(args): - print("error: --platform requires a value", file=sys.stderr) - sys.exit(1) - candidate = args[i + 1] - if selected_platform and selected_platform != candidate: - print("error: specify install platform only once", file=sys.stderr) - sys.exit(1) - selected_platform = candidate - i += 2 - elif arg.startswith("-"): - print(f"error: unknown install option '{arg}'", file=sys.stderr) - sys.exit(1) - else: - if selected_platform and selected_platform != arg: - print("error: specify install platform only once", file=sys.stderr) - sys.exit(1) - selected_platform = arg - i += 1 - chosen_platform = selected_platform or default_platform - if project_scope: - _project_install(chosen_platform, Path(".")) - else: - install(platform=chosen_platform) - elif cmd == "uninstall": - args = sys.argv[2:] - purge = "--purge" in args - project_scope = "--project" in args - selected_platform = None - i = 0 - while i < len(args): - arg = args[i] - if arg in ("--purge", "--project"): - i += 1 - elif arg.startswith("--platform="): - selected_platform = arg.split("=", 1)[1] - i += 1 - elif arg == "--platform": - if i + 1 >= len(args): - print("error: --platform requires a value", file=sys.stderr) - sys.exit(1) - selected_platform = args[i + 1] - i += 2 - elif arg.startswith("-"): - print(f"error: unknown uninstall option '{arg}'", file=sys.stderr) - sys.exit(1) - else: - selected_platform = arg - i += 1 - if project_scope: - if selected_platform: - _project_uninstall(selected_platform, Path(".")) - else: - _project_uninstall_all(Path(".")) - else: - uninstall_all(purge=purge) - elif cmd == "claude": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("claude", Path(".")) - else: - claude_install() - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("claude", Path(".")) - else: - claude_uninstall() - else: - print("Usage: graphify claude [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "codebuddy": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - codebuddy_install() - elif subcmd == "uninstall": - codebuddy_uninstall() - else: - print("Usage: graphify codebuddy [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "gemini": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - gemini_install(project=("--project" in sys.argv[3:])) - elif subcmd == "uninstall": - gemini_uninstall(project=("--project" in sys.argv[3:])) - else: - print("Usage: graphify gemini [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "cursor": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - _cursor_install(Path(".")) - elif subcmd == "uninstall": - _cursor_uninstall(Path(".")) - else: - print("Usage: graphify cursor [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "vscode": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - vscode_install() - elif subcmd == "uninstall": - vscode_uninstall() - else: - print("Usage: graphify vscode [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "copilot": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("copilot", Path(".")) - else: - install(platform="copilot") - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("copilot", Path(".")) - else: - removed = _remove_skill_file("copilot") - print("skill removed" if removed else "nothing to remove") - else: - print("Usage: graphify copilot [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "kilo": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - _kilo_install(Path(".")) - elif subcmd == "uninstall": - _kilo_uninstall(Path(".")) - else: - print("Usage: graphify kilo [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "kiro": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - _kiro_install(Path(".")) - elif subcmd == "uninstall": - _kiro_uninstall(Path(".")) - else: - print("Usage: graphify kiro [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "devin": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("devin", Path(".")) - else: - install(platform="devin") - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("devin", Path(".")) - else: - removed = _remove_skill_file("devin") - print("skill removed" if removed else "nothing to remove") - else: - print("Usage: graphify devin [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "pi": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("pi", Path(".")) - else: - install("pi") - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("pi", Path(".")) - else: - _remove_skill_file("pi") - else: - print("Usage: graphify pi [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "amp": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("amp", Path(".")) - else: - _amp_install(Path(".")) - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("amp", Path(".")) - else: - _amp_uninstall(Path(".")) - else: - print("Usage: graphify amp [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd in ("agents", "skills"): - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("agents", Path(".")) - else: - _agents_platform_install(Path(".")) - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("agents", Path(".")) - else: - _agents_platform_uninstall(Path(".")) - else: - print(f"Usage: graphify {cmd} [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd in ("aider", "codex", "opencode", "claw", "droid", "trae", "trae-cn", "hermes"): - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install(cmd, Path(".")) - else: - _agents_install(Path("."), cmd) - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall(cmd, Path(".")) - else: - _agents_uninstall(Path("."), platform=cmd) - if cmd == "codex": - _uninstall_codex_hook(Path(".")) - else: - print(f"Usage: graphify {cmd} [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "antigravity": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("antigravity", Path(".")) - else: - _antigravity_install(Path(".")) - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("antigravity", Path(".")) - else: - _antigravity_uninstall(Path(".")) - else: - print("Usage: graphify antigravity [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "provider": - from graphify.llm import _custom_providers_path, BACKENDS - import json as _json - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - global_path = _custom_providers_path(global_=True) - - if subcmd == "list": - global_path.parent.mkdir(parents=True, exist_ok=True) - existing: dict = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if not existing: - print("No custom providers registered.") - else: - for name in existing: - print(f" {name} ({existing[name].get('base_url', '')})") - - elif subcmd == "show": - name = sys.argv[3] if len(sys.argv) > 3 else "" - if not name: - print("Usage: graphify provider show ", file=sys.stderr) - sys.exit(1) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if name not in existing: - print(f"Provider '{name}' not found.", file=sys.stderr) - sys.exit(1) - print(_json.dumps({name: existing[name]}, indent=2)) - - elif subcmd == "add": - args = sys.argv[3:] - name = args[0] if args and not args[0].startswith("-") else "" - if not name: - print("Usage: graphify provider add --base-url URL --default-model MODEL --env-key KEY", file=sys.stderr) - sys.exit(1) - if name in BACKENDS: - print(f"Error: '{name}' is a built-in provider and cannot be overridden.", file=sys.stderr) - sys.exit(1) - base_url = "" - default_model = "" - env_key = "" - pricing_input = 0.0 - pricing_output = 0.0 - i = 1 - while i < len(args): - a = args[i] - if a == "--base-url" and i + 1 < len(args): - base_url = args[i + 1]; i += 2 - elif a.startswith("--base-url="): - base_url = a.split("=", 1)[1]; i += 1 - elif a == "--default-model" and i + 1 < len(args): - default_model = args[i + 1]; i += 2 - elif a.startswith("--default-model="): - default_model = a.split("=", 1)[1]; i += 1 - elif a == "--env-key" and i + 1 < len(args): - env_key = args[i + 1]; i += 2 - elif a.startswith("--env-key="): - env_key = a.split("=", 1)[1]; i += 1 - elif a == "--pricing-input" and i + 1 < len(args): - pricing_input = float(args[i + 1]); i += 2 - elif a == "--pricing-output" and i + 1 < len(args): - pricing_output = float(args[i + 1]); i += 2 - else: - i += 1 - if not base_url or not default_model or not env_key: - print("Error: --base-url, --default-model, and --env-key are required.", file=sys.stderr) - sys.exit(1) - from graphify.llm import provider_base_url_ok - if not provider_base_url_ok(base_url, name): - print(f"Error: refusing to add provider with unsafe base_url {base_url!r}.", file=sys.stderr) - sys.exit(1) - global_path.parent.mkdir(parents=True, exist_ok=True) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - existing[name] = { - "base_url": base_url, - "default_model": default_model, - "env_key": env_key, - "pricing": {"input": pricing_input, "output": pricing_output}, - "temperature": 0, - } - global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") - print(f"Provider '{name}' added. Use with: graphify extract . --backend {name}") - - elif subcmd == "remove": - name = sys.argv[3] if len(sys.argv) > 3 else "" - if not name: - print("Usage: graphify provider remove ", file=sys.stderr) - sys.exit(1) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if name not in existing: - print(f"Provider '{name}' not found.", file=sys.stderr) - sys.exit(1) - del existing[name] - global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") - print(f"Provider '{name}' removed.") - - else: - print("Usage: graphify provider [add|list|show|remove]", file=sys.stderr) - if subcmd: - sys.exit(1) - elif cmd == "prs": - from graphify.prs import cmd_prs - cmd_prs(sys.argv[2:]) - elif cmd == "hook": - from graphify.hooks import ( - install as hook_install, - uninstall as hook_uninstall, - status as hook_status, - ) - - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - print(hook_install(Path("."))) - elif subcmd == "uninstall": - print(hook_uninstall(Path("."))) - elif subcmd == "status": - print(hook_status(Path("."))) - else: - print("Usage: graphify hook [install|uninstall|status]", file=sys.stderr) - sys.exit(1) - elif cmd == "query": - if len(sys.argv) < 3: - print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) - sys.exit(1) - from graphify.serve import _query_graph_text - from graphify.security import sanitize_label - from networkx.readwrite import json_graph - from graphify import querylog - - question = sys.argv[2] - use_dfs = "--dfs" in sys.argv - budget = 2000 - graph_path = _default_graph_path() - context_filters: list[str] = [] - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--budget" and i + 1 < len(args): - try: - budget = int(args[i + 1]) - except ValueError: - print(f"error: --budget must be an integer", file=sys.stderr) - sys.exit(1) - i += 2 - elif args[i].startswith("--budget="): - try: - budget = int(args[i].split("=", 1)[1]) - except ValueError: - print(f"error: --budget must be an integer", file=sys.stderr) - sys.exit(1) - i += 1 - elif args[i] == "--context" and i + 1 < len(args): - context_filters.append(args[i + 1]) - i += 2 - elif args[i].startswith("--context="): - context_filters.append(args[i].split("=", 1)[1]) - i += 1 - elif args[i] == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - i += 2 - else: - i += 1 - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print(f"error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - try: - import json as _json - import networkx as _nx - - _raw = _json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - try: - from graphify.build import graph_has_legacy_ids as _legacy - if _legacy(_raw.get("nodes", [])): - print( - "[graphify] note: this graph uses the pre-#1504 node-ID scheme; " - "rebuild with `graphify extract --force` to get path-qualified IDs " - "(fixes same-name-file collisions).", - file=sys.stderr, - ) - except Exception: - pass - except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) - sys.exit(1) - import time as _time - _t0 = _time.perf_counter() - _mode = "dfs" if use_dfs else "bfs" - _result = _query_graph_text( - G, - question, - mode=_mode, - depth=2, - token_budget=budget, - context_filters=context_filters, - ) - querylog.log_query( - kind="query", - question=question, - corpus=str(gp), - result=_result, - mode=_mode, - depth=2, - token_budget=budget, - duration_ms=(_time.perf_counter() - _t0) * 1000, - ) - print(_result) - elif cmd == "affected": - if len(sys.argv) < 3: - print("Usage: graphify affected \"\" [--relation R] [--depth N] [--graph path]", file=sys.stderr) - sys.exit(1) - from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected, load_graph - query = sys.argv[2] - graph_path = _default_graph_path() - depth = 2 - relations: list[str] = [] - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - i += 2 - elif args[i].startswith("--graph="): - graph_path = args[i].split("=", 1)[1] - i += 1 - elif args[i] == "--depth" and i + 1 < len(args): - try: - depth = int(args[i + 1]) - except ValueError: - print("error: --depth must be an integer", file=sys.stderr) - sys.exit(1) - i += 2 - elif args[i].startswith("--depth="): - try: - depth = int(args[i].split("=", 1)[1]) - except ValueError: - print("error: --depth must be an integer", file=sys.stderr) - sys.exit(1) - i += 1 - elif args[i] == "--relation" and i + 1 < len(args): - relations.append(args[i + 1]) - i += 2 - elif args[i].startswith("--relation="): - relations.append(args[i].split("=", 1)[1]) - i += 1 - else: - i += 1 - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print("error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) - try: - graph = load_graph(gp) - except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) - sys.exit(1) - print( - format_affected( - graph, - query, - relations=relations or DEFAULT_AFFECTED_RELATIONS, - depth=depth, - ) - ) - elif cmd == "save-result": - # graphify save-result --question Q --answer A [--type T] [--nodes N1 N2 ...] - # [--outcome useful|dead_end|corrected] [--correction TEXT] - import argparse as _ap - - p = _ap.ArgumentParser(prog="graphify save-result") - p.add_argument("--question", required=True) - p.add_argument("--answer", default=None) - p.add_argument("--answer-file", dest="answer_file", default=None) - p.add_argument("--type", dest="query_type", default="query") - p.add_argument("--nodes", nargs="*", default=[]) - p.add_argument("--outcome", choices=("useful", "dead_end", "corrected"), default=None) - p.add_argument("--correction", default=None) - p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) - opts = p.parse_args(sys.argv[2:]) - if opts.answer_file: - opts.answer = Path(opts.answer_file).read_text(encoding="utf-8").strip() - elif not opts.answer: - p.error("--answer or --answer-file is required") - from graphify.ingest import save_query_result as _sqr - - out = _sqr( - question=opts.question, - answer=opts.answer, - memory_dir=Path(opts.memory_dir), - query_type=opts.query_type, - source_nodes=opts.nodes or None, - outcome=opts.outcome, - correction=opts.correction, - ) - print(f"Saved to {out}") - elif cmd == "reflect": - import argparse as _ap - - p = _ap.ArgumentParser(prog="graphify reflect") - p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) - p.add_argument( - "--out", - default=str(Path(_GRAPHIFY_OUT) / "reflections" / "LESSONS.md"), - ) - p.add_argument("--graph", default=None) - p.add_argument("--analysis", default=None) - p.add_argument("--labels", default=None) - p.add_argument("--half-life-days", type=float, default=30.0, - help="signal weight halves every N days (default 30)") - p.add_argument("--min-corroboration", type=int, default=2, - help="distinct useful results to promote a node to preferred (default 2)") - p.add_argument("--if-stale", action="store_true", - help="skip when LESSONS.md is already newer than every input " - "(e.g. the git hook just refreshed it)") - opts = p.parse_args(sys.argv[2:]) - from graphify.reflect import reflect as _reflect, lessons_fresh as _lessons_fresh - - graph_arg = opts.graph - if graph_arg is None: - default_graph = Path(_GRAPHIFY_OUT) / "graph.json" - if default_graph.exists(): - graph_arg = str(default_graph) - - _gp = Path(graph_arg) if graph_arg else None - _analysis_path = None - _labels_path = None - if _gp is not None: - _analysis_path = Path(opts.analysis) if opts.analysis else ( - _gp.parent / ".graphify_analysis.json") - _labels_path = Path(opts.labels) if opts.labels else ( - _gp.parent / ".graphify_labels.json") - - if opts.if_stale and _lessons_fresh( - Path(opts.out), Path(opts.memory_dir), _gp, _analysis_path, _labels_path - ): - print(f"Lessons already up to date -> {opts.out} (skipped; omit --if-stale to force)") - else: - out_path, agg = _reflect( - memory_dir=Path(opts.memory_dir), - out_path=Path(opts.out), - graph_path=_gp, - analysis_path=_analysis_path, - labels_path=_labels_path, - half_life_days=opts.half_life_days, - min_corroboration=opts.min_corroboration, - ) - c = agg["counts"] - print( - f"Reflected {agg['total']} memories " - f"({c['useful']} useful, {c['dead_end']} dead ends, " - f"{c['corrected']} corrected) -> {out_path}" - ) - elif cmd == "path": - if len(sys.argv) < 4: - print( - 'Usage: graphify path "" "" [--graph path]', - file=sys.stderr, - ) - sys.exit(1) - from graphify.serve import _score_nodes - from networkx.readwrite import json_graph - import networkx as _nx - - source_label = sys.argv[2] - target_label = sys.argv[3] - graph_path = _default_graph_path() - args = sys.argv[4:] - for i, a in enumerate(args): - if a == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee direction. - _raw = {**_raw, "directed": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - src_scored = _score_nodes(G, [t.lower() for t in source_label.split()]) - tgt_scored = _score_nodes(G, [t.lower() for t in target_label.split()]) - if not src_scored: - print(f"No node matching '{source_label}' found.", file=sys.stderr) - sys.exit(1) - if not tgt_scored: - print(f"No node matching '{target_label}' found.", file=sys.stderr) - sys.exit(1) - src_nid, tgt_nid = src_scored[0][1], tgt_scored[0][1] - # Ambiguity guard: when both queries resolve to the same node, the - # shortest path is trivially zero hops, which is almost never what the - # caller wanted (see bug #828). - if src_nid == tgt_nid: - print( - f"'{source_label}' and '{target_label}' both resolved to the same " - f"node '{src_nid}'. Use a more specific label or the exact node ID.", - file=sys.stderr, - ) - sys.exit(1) - for _name, _scored in (("source", src_scored), ("target", tgt_scored)): - if len(_scored) >= 2: - _top, _runner = _scored[0][0], _scored[1][0] - if _top > 0 and (_top - _runner) / _top < 0.10: - print( - f"warning: {_name} match was ambiguous " - f"(top score {_top:g}, runner-up {_runner:g})", - file=sys.stderr, - ) - try: - path_nodes = _nx.shortest_path(G.to_undirected(as_view=True), src_nid, tgt_nid) - except (_nx.NetworkXNoPath, _nx.NodeNotFound): - print(f"No path found between '{source_label}' and '{target_label}'.") - sys.exit(0) - hops = len(path_nodes) - 1 - segments = [] - from graphify.build import edge_data - for i in range(len(path_nodes) - 1): - u, v = path_nodes[i], path_nodes[i + 1] - # Check which direction the stored edge points. - if G.has_edge(u, v): - edata = edge_data(G, u, v) - forward = True - else: - edata = edge_data(G, v, u) - forward = False - rel = edata.get("relation", "") - conf = edata.get("confidence", "") - conf_str = f" [{conf}]" if conf else "" - if i == 0: - segments.append(G.nodes[u].get("label", u)) - if forward: - segments.append(f"--{rel}{conf_str}--> {G.nodes[v].get('label', v)}") - else: - segments.append(f"<--{rel}{conf_str}-- {G.nodes[v].get('label', v)}") - print(f"Shortest path ({hops} hops):\n " + " ".join(segments)) - from graphify import querylog - querylog.log_query( - kind="path", - question=f"{sys.argv[2]} -> {sys.argv[3]}", - corpus=str(gp), - nodes_returned=hops, - ) - - elif cmd == "explain": - if len(sys.argv) < 3: - print('Usage: graphify explain "" [--graph path]', file=sys.stderr) - sys.exit(1) - from graphify.serve import _find_node - from networkx.readwrite import json_graph - - label = sys.argv[2] - graph_path = _default_graph_path() - args = sys.argv[3:] - for i, a in enumerate(args): - if a == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee direction. - _raw = {**_raw, "directed": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - matches = _find_node(G, label) - if not matches: - print(f"No node matching '{label}' found.") - sys.exit(0) - nid = matches[0] - d = G.nodes[nid] - print(f"Node: {d.get('label', nid)}") - print(f" ID: {nid}") - print( - f" Source: {d.get('source_file', '')} {d.get('source_location', '')}".rstrip() - ) - print(f" Type: {d.get('file_type', '')}") - print(f" Community: {d.get('community_name') or d.get('community', '')}") - # Work-memory overlay: a derived experiential hint from `graphify reflect`, - # merged in display-only from the .graphify_learning.json sidecar next to - # graph.json. No line when the node has no overlay entry. - try: - from graphify.reflect import load_learning_overlay as _llo - from graphify.security import sanitize_label as _sl - _overlay = _llo(gp) - _entry = _overlay.get(str(nid)) - if _entry: - _status = _sl(str(_entry.get("status", ""))) - if _status == "contested": - _line = (f" Lesson: contested (useful {_entry.get('uses', 0)} / " - f"dead-end {_entry.get('neg', 0)})") - elif _status == "preferred": - _line = (f" Lesson: preferred source (start here) — " - f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") - else: - _line = (f" Lesson: {_status or 'tentative'} — " - f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") - if _entry.get("stale"): - _line += " [code changed since — re-verify]" - print(_line) - except Exception: - pass - print(f" Degree: {G.degree(nid)}") - from graphify.build import edge_data - connections: list[tuple[str, str, dict]] = [] # (direction, neighbor_id, edge_data) - for nb in G.successors(nid): - connections.append(("out", nb, edge_data(G, nid, nb))) - for nb in G.predecessors(nid): - connections.append(("in", nb, edge_data(G, nb, nid))) - if connections: - print(f"\nConnections ({len(connections)}):") - connections.sort(key=lambda c: G.degree(c[1]), reverse=True) - for direction, nb, edata in connections[:20]: - rel = edata.get("relation", "") - conf = edata.get("confidence", "") - arrow = "-->" if direction == "out" else "<--" - print(f" {arrow} {G.nodes[nb].get('label', nb)} [{rel}] [{conf}]") - if len(connections) > 20: - print(f" ... and {len(connections) - 20} more") - from graphify import querylog - querylog.log_query( - kind="explain", - question=sys.argv[2], - corpus=str(gp), - nodes_returned=len(connections), - ) - - elif cmd == "diagnose": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd != "multigraph": - print( - "Usage: graphify diagnose multigraph " - "[--graph path] [--json] [--max-examples N] " - "[--directed] [--undirected] [--extract-path path]", - file=sys.stderr, - ) - sys.exit(1) - - graph_path = Path(_default_graph_path()) - max_examples = 5 - directed: bool | None = None - direction_flag: str | None = None - json_output = False - extract_path: Path | None = None - - i = 3 - while i < len(sys.argv): - arg = sys.argv[i] - if arg == "--graph": - i += 1 - if i >= len(sys.argv): - print("error: --graph requires a path", file=sys.stderr) - sys.exit(1) - graph_path = Path(sys.argv[i]) - elif arg == "--json": - json_output = True - elif arg == "--max-examples": - i += 1 - if i >= len(sys.argv): - print("error: --max-examples requires an integer", file=sys.stderr) - sys.exit(1) - try: - max_examples = int(sys.argv[i]) - except ValueError: - print("error: --max-examples requires an integer", file=sys.stderr) - sys.exit(1) - if max_examples < 0: - print("error: --max-examples must be >= 0", file=sys.stderr) - sys.exit(1) - elif arg == "--directed": - if direction_flag == "undirected": - print( - "error: --directed and --undirected are mutually exclusive", - file=sys.stderr, - ) - sys.exit(1) - direction_flag = "directed" - directed = True - elif arg == "--undirected": - if direction_flag == "directed": - print( - "error: --directed and --undirected are mutually exclusive", - file=sys.stderr, - ) - sys.exit(1) - direction_flag = "undirected" - directed = False - elif arg == "--extract-path": - i += 1 - if i >= len(sys.argv): - print("error: --extract-path requires a path", file=sys.stderr) - sys.exit(1) - extract_path = Path(sys.argv[i]) - else: - print(f"error: unknown diagnose option {arg}", file=sys.stderr) - sys.exit(1) - i += 1 - - from graphify.diagnostics import ( - diagnose_file, - format_diagnostic_json, - format_diagnostic_report, - ) - - try: - summary = diagnose_file( - graph_path, - directed=directed, - root=Path(".").resolve(), - max_examples=max_examples, - extract_path=extract_path, - ) - except Exception as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - - if json_output: - print(json.dumps(format_diagnostic_json(summary), indent=2)) - else: - print(format_diagnostic_report(summary)) - - elif cmd == "add": - if len(sys.argv) < 3: - print( - "Usage: graphify add [--author Name] [--contributor Name] [--dir ./raw]", - file=sys.stderr, - ) - sys.exit(1) - from graphify.ingest import ingest as _ingest - - url = sys.argv[2] - author: str | None = None - contributor: str | None = None - target_dir = Path("raw") - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--author" and i + 1 < len(args): - author = args[i + 1] - i += 2 - elif args[i] == "--contributor" and i + 1 < len(args): - contributor = args[i + 1] - i += 2 - elif args[i] == "--dir" and i + 1 < len(args): - target_dir = Path(args[i + 1]) - i += 2 - else: - i += 1 - try: - saved = _ingest(url, target_dir, author=author, contributor=contributor) - print(f"Saved to {saved}") - print("Run /graphify --update in your AI assistant to update the graph.") - except Exception as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - - elif cmd == "watch": - watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") - if not watch_path.exists(): - print(f"error: path not found: {watch_path}", file=sys.stderr) - sys.exit(1) - from graphify.watch import watch as _watch - - try: - _watch(watch_path) - except ImportError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - - elif cmd in ("cluster-only", "label"): - # `label` is `cluster-only` that always (re)generates community names with - # the configured backend, even when a .graphify_labels.json already exists. - force_relabel = cmd == "label" - # Mirror the tree/export arg-parsing pattern: walk argv so flags and - # the optional positional path can appear in any order (#724). - no_viz = "--no-viz" in sys.argv - no_label = "--no-label" in sys.argv - missing_only = "--missing-only" in sys.argv - co_timing = "--timing" in sys.argv - _backend_arg = next((a for a in sys.argv if a.startswith("--backend=")), None) - label_backend = _backend_arg.split("=", 1)[1] if _backend_arg else None - _model_arg = next((a for a in sys.argv if a.startswith("--model=")), None) - label_model = _model_arg.split("=", 1)[1] if _model_arg else None - _min_cs_arg = next((a for a in sys.argv if a.startswith("--min-community-size=")), None) - min_community_size = int(_min_cs_arg.split("=")[1]) if _min_cs_arg else 3 - args = sys.argv[2:] - watch_path: Path | None = None - graph_override: Path | None = None - co_resolution: float = 1.0 - co_exclude_hubs: float | None = None - label_max_concurrency: int = 4 - label_batch_size: int = 100 - i_arg = 0 - while i_arg < len(args): - a = args[i_arg] - if a == "--graph" and i_arg + 1 < len(args): - graph_override = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--backend" and i_arg + 1 < len(args): - label_backend = args[i_arg + 1]; i_arg += 2 - elif a.startswith("--backend="): - label_backend = a.split("=", 1)[1]; i_arg += 1 - elif a == "--model" and i_arg + 1 < len(args): - label_model = args[i_arg + 1]; i_arg += 2 - elif a.startswith("--model="): - label_model = a.split("=", 1)[1]; i_arg += 1 - elif a == "--resolution" and i_arg + 1 < len(args): - co_resolution = float(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--resolution="): - co_resolution = float(a.split("=", 1)[1]); i_arg += 1 - elif a == "--exclude-hubs" and i_arg + 1 < len(args): - co_exclude_hubs = float(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--exclude-hubs="): - co_exclude_hubs = float(a.split("=", 1)[1]); i_arg += 1 - elif a == "--max-concurrency" and i_arg + 1 < len(args): - label_max_concurrency = int(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--max-concurrency="): - label_max_concurrency = int(a.split("=", 1)[1]); i_arg += 1 - elif a == "--batch-size" and i_arg + 1 < len(args): - label_batch_size = int(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--batch-size="): - label_batch_size = int(a.split("=", 1)[1]); i_arg += 1 - elif a in ("--no-viz", "--missing-only") or a.startswith("--min-community-size="): - i_arg += 1 - elif a.startswith("--"): - i_arg += 1 - elif watch_path is None: - watch_path = Path(a); i_arg += 1 - else: - i_arg += 1 - if watch_path is None: - watch_path = Path(".") - graph_json = graph_override if graph_override is not None else watch_path / _GRAPHIFY_OUT / "graph.json" - if not graph_json.exists(): - print( - f"error: no graph found at {graph_json} — run /graphify first", - file=sys.stderr, - ) - sys.exit(1) - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json - from graphify.cluster import cluster, score_all, remap_communities_to_previous - from graphify.analyze import ( - god_nodes, - surprising_connections, - suggest_questions, - ) - from graphify.report import generate - from graphify.export import to_json, to_html - - stages = _StageTimer(co_timing) - print("Loading existing graph...") - # Solution 3 (#1019): don't hard-exit on an oversized graph.json here. - # Core outputs (graph.json + GRAPH_REPORT.md) still get written; the - # graph.html render below falls back to the community-aggregation view - # (node_limit=5000) when over the cap. - from graphify.security import check_graph_file_size_cap as _check_cap - _over_cap = False - try: - _check_cap(graph_json) - except ValueError: - _over_cap = True - try: - _over_cap_bytes = graph_json.stat().st_size - except OSError: - _over_cap_bytes = -1 - print( - f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " - f"falling back to community-aggregation view (node_limit=5000)", - file=sys.stderr, - ) - _raw = json.loads(graph_json.read_text(encoding="utf-8")) - _directed = bool(_raw.get("directed", False)) - G = build_from_json(_raw, directed=_directed) - print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") - stages.mark("load") - print("Re-clustering...") - communities = cluster(G, resolution=co_resolution, exclude_hubs_percentile=co_exclude_hubs) - # Mirror the watch/update path (#822): map new cids to prior ones by - # node-overlap so the existing .graphify_labels.json keeps attaching - # to the same conceptual community after re-clustering. Without this, - # labels follow raw cid index and become misaligned whenever the - # graph has changed between labeling and cluster-only (#1027). - previous_node_community = { - n["id"]: n["community"] - for n in _raw.get("nodes", []) - if n.get("community") is not None and n.get("id") is not None - } - if previous_node_community: - communities = remap_communities_to_previous(communities, previous_node_community) - stages.mark("cluster") - cohesion = score_all(G, communities) - gods = god_nodes(G) - surprises = surprising_connections(G, communities) - stages.mark("analyze") - out = watch_path / _GRAPHIFY_OUT - out.mkdir(parents=True, exist_ok=True) - labels_path = out / ".graphify_labels.json" - existing_labels: dict[int, str] = {} - if labels_path.exists(): - try: - existing_labels = { - int(k): v - for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items() - if isinstance(v, str) - } - except Exception: - existing_labels = {} - # Accumulate token usage from the labeling LLM calls so cluster-only mode - # reports real cost instead of a hardcoded zero (#1694). Stays {0, 0} on - # the reuse / no-label paths, which make no LLM calls. - label_token_usage = {"input": 0, "output": 0} - if labels_path.exists() and not force_relabel: - # Reuse saved labels, but don't blindly trust them: the graph may have - # been re-scoped/re-clustered since labeling, in which case a cid now - # covers a DIFFERENT community and its old (LLM) name is wrong (#label-stale). - # Validate each community against the membership signature saved beside the - # labels; any community that changed (or has no saved label) is renamed by - # its current hub — deterministic and correct-by-construction — and the user - # is told to `graphify label` for fresh LLM names. Unchanged communities keep - # their saved label. When no signature sidecar exists (labels predate this), - # fall back to hub-filling only the communities missing a label. - from graphify.cluster import community_member_sigs, label_communities_by_hub - sig_path = labels_path.parent / (labels_path.name + ".sig") - saved_sigs: dict[int, str] = {} - if sig_path.exists(): - try: - saved_sigs = { - int(k): v for k, v in - json.loads(sig_path.read_text(encoding="utf-8")).items() - if isinstance(v, str) - } - except Exception: - saved_sigs = {} - cur_sigs = community_member_sigs(communities) - count_mismatch = len(existing_labels) != len(communities) - labels = {} - hub_labels: dict[int, str] | None = None - changed = 0 - for cid in communities: - have_label = cid in existing_labels - if saved_sigs: - # Precise: the membership signature tells us if this exact - # community changed since it was labeled. - fresh = have_label and saved_sigs.get(cid) == cur_sigs.get(cid) - else: - # No signature sidecar (labels predate it). A differing community - # COUNT means the labels describe a different clustering, so a cid's - # old label can't be trusted; equal count is the best "same" signal. - fresh = have_label and not count_mismatch - if fresh: - labels[cid] = existing_labels[cid] - else: - if hub_labels is None: - hub_labels = label_communities_by_hub(G, communities) - labels[cid] = hub_labels[cid] - if have_label: - changed += 1 - if changed: - print( - f"[graphify] community set changed since labeling " - f"({len(existing_labels)} saved labels, {len(communities)} communities now; " - f"renamed {changed} community(ies) by their hub). " - f"Run `graphify label` to refresh names with the LLM.", - file=sys.stderr, - ) - elif no_label and not force_relabel: - labels = {cid: f"Community {cid}" for cid in communities} - else: - # No labels file yet (or `graphify label` forced a refresh). When run - # standalone there is no orchestrating agent to do skill.md Step 5, so - # auto-name communities rather than leave "Community N" (#1097). - from graphify.cluster import label_communities_by_hub - from graphify.llm import generate_community_labels - print("Labeling communities...") - # Deterministic, LLM-free base labels: name each community after its - # highest-degree hub, so the report is readable even with no backend - # (previously bare "Community N"). A configured LLM backend overrides these - # with richer names below; its no-backend placeholder fallback does NOT. - hub_labels = label_communities_by_hub(G, communities) - label_communities_input = communities - labels = dict(hub_labels) - if missing_only: - labels = { - cid: existing_labels.get(cid, hub_labels[cid]) - for cid in communities - } - label_communities_input = { - cid: members - for cid, members in communities.items() - if cid not in existing_labels or existing_labels.get(cid) == f"Community {cid}" - } - generated_labels, _ = generate_community_labels( - G, label_communities_input, backend=label_backend, model=label_model, gods=gods, - max_concurrency=label_max_concurrency, batch_size=label_batch_size, - usage_out=label_token_usage, - ) - # Only let the LLM OVERRIDE where it produced a real name — its no-backend - # fallback returns "Community {cid}" placeholders, which must not clobber - # the deterministic hub labels. - labels.update({ - cid: v for cid, v in generated_labels.items() - if v and v != f"Community {cid}" - }) - stages.mark("label") - questions = suggest_questions(G, communities, labels) - tokens = label_token_usage - from graphify.export import _git_head as _gh - _commit = _gh() - from graphify.report import load_learning_for_report as _llfr - report = generate(G, communities, cohesion, labels, gods, surprises, - {"warning": "cluster-only mode — file stats not available"}, - tokens, str(watch_path), suggested_questions=questions, - min_community_size=min_community_size, built_at_commit=_commit, - learning=_llfr(out / "graph.json")) - (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") - stages.mark("report") - from graphify.export import backup_if_protected as _backup - _backup(out) - analysis = { - "communities": {str(k): v for k, v in communities.items()}, - "cohesion": {str(k): v for k, v in cohesion.items()}, - "gods": gods, - "surprises": surprises, - "questions": questions, - } - (out / ".graphify_analysis.json").write_text( - json.dumps(analysis, indent=2, ensure_ascii=False), - encoding="utf-8", - ) - to_json(G, communities, str(out / "graph.json"), community_labels=labels) - labels_path.write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8") - # Membership signatures beside the labels so a later cluster-only can detect - # which communities changed and avoid reusing a stale label (see reuse above). - from graphify.cluster import community_member_sigs as _cms - (labels_path.parent / (labels_path.name + ".sig")).write_text( - json.dumps({str(k): v for k, v in _cms(communities).items()}), encoding="utf-8") - - # Mirror watch.py pattern: gate to_html so core outputs (graph.json + - # GRAPH_REPORT.md) always land. Honor --no-viz explicitly; otherwise - # fall back to ValueError handling so an oversized graph doesn't crash - # the CLI mid-write and leave a stale graph.html on disk. - html_target = out / "graph.html" - if no_viz: - if html_target.exists(): - html_target.unlink() - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated (--no-viz; graph.html removed).") - else: - try: - # Over-cap fallback (#1019): force the community-aggregation - # path so an oversized graph still renders a usable graph.html. - _node_limit = 5000 if _over_cap else None - to_html(G, communities, str(html_target), community_labels=labels or None, - node_limit=_node_limit) - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md, graph.json and graph.html updated.") - except ValueError as viz_err: - if html_target.exists(): - html_target.unlink() - print(f"Skipped graph.html: {viz_err}") - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated.") - - elif cmd == "update": - force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") - no_cluster = False - args = sys.argv[2:] - watch_arg: str | None = None - for a in args: - if a == "--force": - force = True - continue - if a == "--no-cluster": - no_cluster = True - continue - if a.startswith("-"): - print(f"error: unknown update option: {a}", file=sys.stderr) - sys.exit(2) - if watch_arg is not None: - print("error: update accepts at most one path argument", file=sys.stderr) - sys.exit(2) - watch_arg = a - - if watch_arg is not None: - watch_path = Path(watch_arg) - else: - # Try to recover the scan root saved by the last full build - saved = Path(_GRAPHIFY_OUT) / ".graphify_root" - if saved.exists(): - watch_path = Path(saved.read_text(encoding="utf-8").strip()) - else: - watch_path = Path(".") - if not watch_path.exists(): - print(f"error: path not found: {watch_path}", file=sys.stderr) - sys.exit(1) - from graphify.watch import _rebuild_code - - print(f"Re-extracting code files in {watch_path} (no LLM needed)...") - # Interactive CLI: block on the per-repo lock rather than skip, so the - # user sees their explicit `graphify update` complete instead of - # exiting silently when a hook-driven rebuild happens to be running. - ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) - if ok: - print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") - if not ( - os.environ.get("GEMINI_API_KEY") - or os.environ.get("GOOGLE_API_KEY") - or os.environ.get("MOONSHOT_API_KEY") - or os.environ.get("DEEPSEEK_API_KEY") - or os.environ.get("GRAPHIFY_NO_TIPS") - ): - print("Tip: set GEMINI_API_KEY or GOOGLE_API_KEY to use Gemini for semantic extraction.") - else: - print( - "Nothing to update or rebuild failed — check output above.", - file=sys.stderr, - ) - sys.exit(1) - - elif cmd == "hook-check": - # Codex Desktop rejects hookSpecificOutput.additionalContext on PreToolUse. - # Keep this as a cross-platform no-op so installed hooks never break Bash - # tool calls. Graph guidance reaches the agent via AGENTS.md / skill instead. - sys.exit(0) - elif cmd == "hook-guard": - # Shell-agnostic Claude/Codebuddy PreToolUse guard (#522). Replaces the old - # inline-bash hooks that failed on Windows. Prints an additionalContext nudge - # toward graphify when a graph exists; always exits 0 (never blocks a tool). - _run_hook_guard(sys.argv[2] if len(sys.argv) > 2 else "") - sys.exit(0) - elif cmd == "check-update": - if len(sys.argv) < 3: - print("Usage: graphify check-update ", file=sys.stderr) - sys.exit(1) - from graphify.watch import check_update - - check_update(Path(sys.argv[2]).resolve()) - sys.exit(0) - elif cmd == "tree": - # Emit a D3 v7 collapsible-tree HTML view of graph.json: - # expand-all / collapse-all / reset-view buttons, multi-line - # wrapText labels with separately-coloured name + count, - # depth-based palette, click-to-toggle subtree, hover inspector - # showing top-K outbound edges per symbol. - from typing import Optional as _Opt - from graphify.tree_html import write_tree_html, DEFAULT_MAX_CHILDREN - graph_path = Path(_GRAPHIFY_OUT) / "graph.json" - output_path: "_Opt[Path]" = None - root: "_Opt[str]" = None - max_children = DEFAULT_MAX_CHILDREN - top_k_edges = 0 - project_label: "_Opt[str]" = None - args = sys.argv[2:] - i_arg = 0 - while i_arg < len(args): - a = args[i_arg] - if a == "--graph" and i_arg + 1 < len(args): - graph_path = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--output" and i_arg + 1 < len(args): - output_path = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--root" and i_arg + 1 < len(args): - root = args[i_arg + 1]; i_arg += 2 - elif a == "--max-children" and i_arg + 1 < len(args): - max_children = int(args[i_arg + 1]); i_arg += 2 - elif a == "--top-k-edges" and i_arg + 1 < len(args): - top_k_edges = int(args[i_arg + 1]); i_arg += 2 - elif a == "--label" and i_arg + 1 < len(args): - project_label = args[i_arg + 1]; i_arg += 2 - elif a in ("-h", "--help"): - print("Usage: graphify tree [--graph PATH] [--output HTML]") - print(" --graph PATH path to graph.json (default graphify-out/graph.json)") - print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") - print(" --root PATH filesystem root (default: longest common dir of all source_files)") - print(" --max-children N cap visible children per node (default 200)") - print(" --top-k-edges N pre-compute top-K outbound edges per symbol (default 12)") - print(" --label NAME project label shown in the page header") - return - else: - i_arg += 1 - if not graph_path.is_file(): - print(f"error: graph.json not found at {graph_path}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(graph_path) - if output_path is None: - output_path = graph_path.parent / "GRAPH_TREE.html" - out = write_tree_html( - graph_path=graph_path, output_path=output_path, - root=root, max_children=max_children, - top_k_edges=top_k_edges, project_label=project_label, - ) - size_kb = out.stat().st_size / 1024 - print(f"wrote {out} ({size_kb:.1f} KB)") - print(f"open with: xdg-open {out} (or file://{out.resolve()})") - sys.exit(0) - - elif cmd == "merge-driver": - # git merge driver for graph.json — takes (base, current, other) and writes - # the union of current+other nodes/edges back to current. Exits 1 on - # corrupt input so git surfaces the conflict instead of silently - # accepting a poisoned merge (see F-005). - # Usage: graphify merge-driver %O %A %B (set in .git/config merge driver) - if len(sys.argv) < 5: - print("Usage: graphify merge-driver ", file=sys.stderr) - sys.exit(1) - _base_path, _current_path, _other_path = sys.argv[2], sys.argv[3], sys.argv[4] - # Hard caps so a malicious or corrupted graph.json cannot exhaust memory - # at parse time. 50 MB / 100k nodes are well above any realistic graph - # (typical graphs are <5 MB / <50k nodes); anything larger should fail - # the merge so a human can investigate. - _MERGE_MAX_BYTES = 50 * 1024 * 1024 - _MERGE_MAX_NODES = 100_000 - import networkx as _nx - from networkx.readwrite import json_graph as _jg - def _load_graph(p: str): - path_obj = Path(p) - try: - size = path_obj.stat().st_size - except OSError as exc: - raise RuntimeError(f"cannot stat {p}: {exc}") from exc - if size > _MERGE_MAX_BYTES: - raise RuntimeError( - f"graph.json {p} is {size} bytes, exceeds {_MERGE_MAX_BYTES}-byte cap" - ) - data = json.loads(path_obj.read_text(encoding="utf-8")) - try: - return _jg.node_link_graph(data, edges="links"), data - except TypeError: - return _jg.node_link_graph(data), data - try: - G_cur, _ = _load_graph(_current_path) - G_oth, _ = _load_graph(_other_path) - except Exception as exc: - print(f"[graphify merge-driver] error loading graphs: {exc}", file=sys.stderr) - sys.exit(1) # surface the conflict so git doesn't accept a corrupt merge - merged = _nx.compose(G_cur, G_oth) - if merged.number_of_nodes() > _MERGE_MAX_NODES: - print( - f"[graphify merge-driver] merged graph has {merged.number_of_nodes()} nodes, " - f"exceeds {_MERGE_MAX_NODES}-node cap; aborting merge.", - file=sys.stderr, - ) - sys.exit(1) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) - Path(_current_path).write_text(json.dumps(out_data, indent=2), encoding="utf-8") - sys.exit(0) - - elif cmd == "merge-graphs": - # graphify merge-graphs graph1.json graph2.json ... --out merged.json - args = sys.argv[2:] - graph_paths: list[Path] = [] - out_path = Path(_GRAPHIFY_OUT) / "merged-graph.json" - i = 0 - while i < len(args): - if args[i] == "--out" and i + 1 < len(args): - out_path = Path(args[i + 1]) - i += 2 - else: - graph_paths.append(Path(args[i])) - i += 1 - if len(graph_paths) < 2: - print( - "Usage: graphify merge-graphs [...] [--out merged.json]", - file=sys.stderr, - ) - sys.exit(1) - import networkx as _nx - from networkx.readwrite import json_graph as _jg - from graphify.build import prefix_graph_for_global as _prefix - graphs = [] - for gp in graph_paths: - if not gp.exists(): - print(f"error: not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - data = json.loads(gp.read_text(encoding="utf-8")) - # Normalize edges/links key before loading — graphify writes "links" - # via node_link_data but older runs may have used "edges" (#738). - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - G = _jg.node_link_graph(data, edges="links") - except TypeError: - G = _jg.node_link_graph(data) - graphs.append(G) - # nx.compose requires all graphs to be the same type. When input graphs - # come from different sources (e.g. an AST-only run vs a full LLM run) one - # may be a MultiGraph and another a Graph. Normalise everything to Graph - # (the graphify default) by converting MultiGraphs with nx.Graph(). - def _to_simple(g: "_nx.Graph") -> "_nx.Graph": - # nx.compose requires every graph to be the same type. Inputs may - # disagree on BOTH axes — directed vs undirected, and multi vs simple - # — because per-repo graph.json files are written by different extract - # paths at different times. Normalise everything to a plain undirected - # Graph (the merged cross-repo view is undirected anyway), which covers - # DiGraph / MultiGraph / MultiDiGraph. Without this a directed input - # crashed compose with "All graphs must be directed or undirected" (#1606). - if type(g) is not _nx.Graph: - return _nx.Graph(g) - return g - merged = _nx.Graph() - for G, gp in zip(graphs, graph_paths): - repo_tag = gp.parent.parent.name # graphify-out/../ → repo dir name - prefixed = _to_simple(_prefix(G, repo_tag)) - merged = _nx.compose(merged, prefixed) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(out_data, indent=2), encoding="utf-8") - print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges") - print(f"Written to: {out_path}") - - elif cmd == "clone": - if len(sys.argv) < 3: - print( - "Usage: graphify clone [--branch ] [--out ]", - file=sys.stderr, - ) - sys.exit(1) - url = sys.argv[2] - branch: str | None = None - out_dir: Path | None = None - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--branch" and i + 1 < len(args): - branch = args[i + 1] - i += 2 - elif args[i] == "--out" and i + 1 < len(args): - out_dir = Path(args[i + 1]) - i += 2 - else: - i += 1 - local_path = _clone_repo(url, branch=branch, out_dir=out_dir) - print(local_path) - - elif cmd == "export": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd not in ("html", "callflow-html", "obsidian", "wiki", "svg", "graphml", "neo4j", "falkordb"): - print("Usage: graphify export ", file=sys.stderr) - print(" html [--graph PATH] [--labels PATH] [--node-limit N] [--no-viz]", file=sys.stderr) - print(" callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH] [--report PATH] [--sections PATH] [--output HTML]", file=sys.stderr) - print(" [--lang auto|zh-CN|en] [--max-sections N] [--diagram-scale N]", file=sys.stderr) - print(" obsidian [--graph PATH] [--labels PATH] [--dir PATH]", file=sys.stderr) - print(" wiki [--graph PATH] [--labels PATH]", file=sys.stderr) - print(" svg [--graph PATH] [--labels PATH]", file=sys.stderr) - print(" graphml [--graph PATH]", file=sys.stderr) - print(" neo4j [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) - print(" (or set NEO4J_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) - print(" falkordb [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) - print(" (or set FALKORDB_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) - sys.exit(1) - - # Parse shared args - args = sys.argv[3:] - graph_path = Path(_GRAPHIFY_OUT) / "graph.json" - graph_path_explicit = False - labels_path = Path(_GRAPHIFY_OUT) / ".graphify_labels.json" - labels_path_explicit = False - report_path = Path(_GRAPHIFY_OUT) / "GRAPH_REPORT.md" - report_path_explicit = False - sections_path: Path | None = None - callflow_output: Path | None = None - callflow_lang = "auto" - callflow_max_sections = 15 - callflow_diagram_scale = 1.0 - callflow_max_diagram_nodes = 18 - callflow_max_diagram_edges = 24 - analysis_path = Path(_GRAPHIFY_OUT) / ".graphify_analysis.json" - node_limit = 5000 - no_viz = False - obsidian_dir = Path(_GRAPHIFY_OUT) / "obsidian" - # Shared push-connection settings for the graph-database sinks (neo4j, - # falkordb), parsed from the generic --push/--user/--password flags below. - push_uri: str | None = None - push_user = "neo4j" # Neo4j default user; FalkorDB auth is optional and ignores it - # F-031: prefer an env var so the password never appears on argv (visible - # in `ps` output / shell history). The explicit --password flag still - # overrides it. Each sink reads its own var: FALKORDB_PASSWORD for falkordb, - # NEO4J_PASSWORD otherwise. - push_password: str | None = ( - os.environ.get("FALKORDB_PASSWORD") if subcmd == "falkordb" - else os.environ.get("NEO4J_PASSWORD") - ) or None - i = 0 - while i < len(args): - a = args[i] - if a == "--graph" and i + 1 < len(args): - graph_path = Path(args[i + 1]) - graph_path_explicit = True - i += 2 - elif a == "--labels" and i + 1 < len(args): - labels_path = Path(args[i + 1]) - labels_path_explicit = True - i += 2 - elif a == "--report" and i + 1 < len(args): - report_path = Path(args[i + 1]) - report_path_explicit = True - i += 2 - elif a == "--sections" and i + 1 < len(args): - sections_path = Path(args[i + 1]); i += 2 - elif a == "--output" and i + 1 < len(args): - callflow_output = Path(args[i + 1]).expanduser() - if not callflow_output.is_absolute(): - callflow_output = Path.cwd() / callflow_output - i += 2 - elif a == "--lang" and i + 1 < len(args): - callflow_lang = args[i + 1]; i += 2 - elif a == "--max-sections" and i + 1 < len(args): - callflow_max_sections = int(args[i + 1]); i += 2 - elif a == "--diagram-scale" and i + 1 < len(args): - callflow_diagram_scale = float(args[i + 1]); i += 2 - elif a == "--max-diagram-nodes" and i + 1 < len(args): - callflow_max_diagram_nodes = int(args[i + 1]); i += 2 - elif a == "--max-diagram-edges" and i + 1 < len(args): - callflow_max_diagram_edges = int(args[i + 1]); i += 2 - elif a in ("-h", "--help") and subcmd == "callflow-html": - print("Usage: graphify export callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH]") - print(" --report PATH path to GRAPH_REPORT.md") - print(" --sections PATH JSON section definitions") - print(" --output HTML output path (default graphify-out/-callflow.html)") - print(" --lang LANG auto, zh-CN, en, etc. (default auto)") - print(" --max-sections N maximum auto-derived sections (default 15)") - print(" --diagram-scale N Mermaid diagram scale (default 1.0)") - print(" --max-diagram-nodes N representative nodes per section (default 18)") - print(" --max-diagram-edges N representative edges per section (default 24)") - sys.exit(0) - elif a == "--node-limit" and i + 1 < len(args): - node_limit = int(args[i + 1]); i += 2 - elif a == "--no-viz": - no_viz = True; i += 1 - elif a == "--dir" and i + 1 < len(args): - obsidian_dir = Path(args[i + 1]); i += 2 - elif a == "--push" and i + 1 < len(args): - push_uri = args[i + 1]; i += 2 - elif a == "--user" and i + 1 < len(args): - push_user = args[i + 1]; i += 2 - elif a == "--password" and i + 1 < len(args): - push_password = args[i + 1]; i += 2 - elif subcmd == "callflow-html" and not a.startswith("-") and not graph_path_explicit: - candidate = Path(a) - if candidate.name == "graph.json" or candidate.suffix.lower() == ".json": - graph_path = candidate - elif (candidate / "graph.json").exists(): - graph_path = candidate / "graph.json" - else: - graph_path = candidate / _GRAPHIFY_OUT / "graph.json" - graph_path_explicit = True - i += 1 - else: - i += 1 - - graph_path = graph_path.expanduser() - if graph_path_explicit: - graph_out_dir = graph_path.parent - if not labels_path_explicit: - labels_path = graph_out_dir / ".graphify_labels.json" - if not report_path_explicit: - report_path = graph_out_dir / "GRAPH_REPORT.md" - labels_path = labels_path.expanduser() - report_path = report_path.expanduser() - - if not graph_path.exists(): - print(f"error: graph not found: {graph_path}. Run /graphify first.", file=sys.stderr) - sys.exit(1) - - if subcmd == "callflow-html": - from graphify.callflow_html import write_callflow_html as _write_callflow_html - out = _write_callflow_html( - graph=graph_path, - report=report_path, - labels=labels_path, - sections=sections_path, - output=callflow_output, - lang=callflow_lang, - max_sections=callflow_max_sections, - diagram_scale=callflow_diagram_scale, - max_diagram_nodes=callflow_max_diagram_nodes, - max_diagram_edges=callflow_max_diagram_edges, - verbose=True, - ) - print(f"callflow HTML written - open in any browser: {out}") - sys.exit(0) - - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json as _bfj - from graphify.security import check_graph_file_size_cap as _check_cap - - # Solution 3 (#1019): for the HTML view, an oversized graph.json should - # not be a hard error. Detect the over-cap condition here and fall back - # to the community-aggregation view (node_limit=5000) below instead of - # exiting 1. All other subcommands keep the hard cap. - _over_cap = False - try: - _check_cap(graph_path) - except ValueError as _cap_err: - if subcmd == "html": - _over_cap = True - try: - _over_cap_bytes = graph_path.stat().st_size - except OSError: - _over_cap_bytes = -1 - print( - f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " - f"falling back to community-aggregation view (node_limit=5000)", - file=sys.stderr, - ) - else: - print(f"error: {_cap_err}", file=sys.stderr) - sys.exit(1) - _raw = json.loads(graph_path.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - try: - G = _jg.node_link_graph(_raw, edges="links") - except TypeError: - G = _jg.node_link_graph(_raw) - - # Load optional analysis/labels - communities: dict[int, list[str]] = {} - if analysis_path.exists(): - _an = json.loads(analysis_path.read_text(encoding="utf-8")) - communities = {int(k): v for k, v in _an.get("communities", {}).items()} - cohesion: dict[int, float] = {int(k): v for k, v in _an.get("cohesion", {}).items()} - gods_data = _an.get("gods", []) - else: - cohesion = {} - gods_data = [] - - # Fallback: graph.json carries the per-node community as a node attribute - # (`to_json` writes it on every node). The analysis sidecar is the - # canonical source — but the post-commit / watch rebuild path doesn't - # regenerate it, and `extract` may have its temp files cleaned up. When - # that happens, `graphify export html` previously bailed with - # "Single community - aggregated view not useful." even though the - # per-node attribute had the right data all along. Reconstruct from - # the graph itself so downstream subcommands (html, obsidian, wiki, - # svg, graphml, neo4j) don't silently produce a degraded artifact. - if not communities: - reconstructed: dict[int, list[str]] = {} - for node_id, data in G.nodes(data=True): - cid_raw = data.get("community") - if cid_raw is None: - continue - try: - cid = int(cid_raw) - except (TypeError, ValueError): - continue - reconstructed.setdefault(cid, []).append(str(node_id)) - if reconstructed: - communities = reconstructed - - labels: dict[int, str] = {} - if labels_path.exists(): - labels = {int(k): v for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items()} - - out_dir = graph_path.parent - - if subcmd == "html": - from graphify.export import to_html as _to_html - if no_viz: - html_target = out_dir / "graph.html" - if html_target.exists(): - html_target.unlink() - print("--no-viz: skipped graph.html") - else: - # Over-cap fallback (#1019): force the community-aggregation - # path so the oversized graph still renders a usable artifact. - _effective_node_limit = 5000 if _over_cap else node_limit - _to_html(G, communities, str(out_dir / "graph.html"), - community_labels=labels or None, node_limit=_effective_node_limit) - if G.number_of_nodes() <= _effective_node_limit: - print(f"graph.html written - open in any browser, no server needed") - if _over_cap: - sys.exit(0) - - elif subcmd == "obsidian": - from graphify.export import to_obsidian as _to_obsidian, to_canvas as _to_canvas - n = _to_obsidian(G, communities, str(obsidian_dir), - community_labels=labels or None, cohesion=cohesion or None) - print(f"Obsidian vault: {n} notes in {obsidian_dir}/") - _to_canvas(G, communities, str(obsidian_dir / "graph.canvas"), - community_labels=labels or None) - print(f"Canvas: {obsidian_dir}/graph.canvas") - print(f"Open {obsidian_dir}/ as a vault in Obsidian.") - - elif subcmd == "wiki": - from graphify.wiki import to_wiki as _to_wiki - from graphify.analyze import god_nodes as _god_nodes - if not communities: - print( - "error: .graphify_analysis.json is missing or empty — refusing to export wiki to prevent data loss.\n" - "Run `graphify extract .` (or `graphify cluster-only .`) to regenerate community data first.", - file=sys.stderr, - ) - sys.exit(1) - if not gods_data: - gods_data = _god_nodes(G) - n = _to_wiki(G, communities, str(out_dir / "wiki"), - community_labels=labels or None, cohesion=cohesion or None, - god_nodes_data=gods_data) - print(f"Wiki: {n} articles written to {out_dir}/wiki/") - print(f" {out_dir}/wiki/index.md -> agent entry point") - - elif subcmd == "svg": - from graphify.export import to_svg as _to_svg - _to_svg(G, communities, str(out_dir / "graph.svg"), - community_labels=labels or None) - print(f"graph.svg written - embeds in Obsidian, Notion, GitHub READMEs") - - elif subcmd == "graphml": - from graphify.export import to_graphml as _to_graphml - _to_graphml(G, communities, str(out_dir / "graph.graphml")) - print(f"graph.graphml written - open in Gephi, yEd, or any GraphML tool") - - elif subcmd == "neo4j": - if push_uri: - from graphify.export import push_to_neo4j as _push - if push_password is None: - print("error: --password required for --push", file=sys.stderr) - sys.exit(1) - result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) - print(f"Pushed to Neo4j: {result['nodes']} nodes, {result['edges']} edges") - else: - from graphify.export import to_cypher as _to_cypher - _to_cypher(G, str(out_dir / "cypher.txt")) - print(f"cypher.txt written - import with: cypher-shell < {out_dir}/cypher.txt") - - elif subcmd == "falkordb": - if push_uri: - from graphify.export import push_to_falkordb as _push - result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) - print(f"Pushed to FalkorDB: {result['nodes']} nodes, {result['edges']} edges") - else: - from graphify.export import to_cypher as _to_cypher - _to_cypher(G, str(out_dir / "cypher.txt")) - print(f"cypher.txt written ({out_dir}/cypher.txt) - statements are OpenCypher. " - f"FalkorDB's GRAPH.QUERY runs one statement at a time (no bulk script " - f"import), so load a graph with: graphify export falkordb --push " - f"falkordb://localhost:6379") - - elif cmd == "benchmark": - from graphify.benchmark import run_benchmark, print_benchmark - - graph_path = sys.argv[2] if len(sys.argv) > 2 else _default_graph_path() - _enforce_graph_size_cap_or_exit(Path(graph_path)) - # Try to load corpus_words from detect output - corpus_words = None - detect_path = Path(".graphify_detect.json") - if detect_path.exists(): - try: - detect_data = json.loads(detect_path.read_text(encoding="utf-8")) - corpus_words = detect_data.get("total_words") - except Exception: - pass - result = run_benchmark(graph_path, corpus_words=corpus_words) - print_benchmark(result) - - elif cmd == "global": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - from graphify.global_graph import ( - global_add as _global_add, - global_remove as _global_remove, - global_list as _global_list, - global_path as _global_path, - ) - if subcmd == "add": - # graphify global add [--as ] - args = sys.argv[3:] - source = None - tag = None - i = 0 - while i < len(args): - if args[i] == "--as" and i + 1 < len(args): - tag = args[i + 1]; i += 2 - elif not source: - source = Path(args[i]); i += 1 - else: - i += 1 - if not source: - print("Usage: graphify global add [--as ]", file=sys.stderr) - sys.exit(1) - tag = tag or source.parent.parent.name - try: - result = _global_add(source, tag) - if result["skipped"]: - print(f"'{tag}' unchanged since last add - global graph not modified.") - else: - print(f"Added '{tag}' to global graph: +{result['nodes_added']} nodes, " - f"-{result['nodes_removed']} pruned. Global: {_global_path()}") - except Exception as exc: - print(f"error: {exc}", file=sys.stderr); sys.exit(1) - elif subcmd == "remove": - tag = sys.argv[3] if len(sys.argv) > 3 else "" - if not tag: - print("Usage: graphify global remove ", file=sys.stderr); sys.exit(1) - try: - removed = _global_remove(tag) - print(f"Removed '{tag}' from global graph ({removed} nodes pruned).") - except KeyError as exc: - print(f"error: {exc}", file=sys.stderr); sys.exit(1) - elif subcmd == "list": - repos = _global_list() - if not repos: - print("Global graph is empty. Use 'graphify global add' to add a project.") - else: - print(f"Global graph: {_global_path()}") - for tag, info in repos.items(): - print(f" {tag}: {info.get('node_count', '?')} nodes, added {info.get('added_at', '?')[:10]}") - elif subcmd == "path": - print(_global_path()) - else: - print("Usage: graphify global [add|remove|list|path]", file=sys.stderr); sys.exit(1) - - elif cmd == "extract": - # Headless full-pipeline extraction for CI / scripts (#698). - # Runs detect -> AST extraction on code -> semantic LLM extraction on - # docs/papers/images -> merge -> build -> cluster -> write outputs. - # Unlike the skill.md path (which runs through Claude Code subagents), - # this calls extract_corpus_parallel directly using whichever backend - # has an API key set. - if len(sys.argv) < 3: - print( - "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " - "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] " - "[--max-workers N] [--token-budget N] [--max-concurrency N] " - "[--api-timeout S] [--postgres DSN] [--cargo] [--timing]", - file=sys.stderr, - ) - sys.exit(1) - - has_path = True - if sys.argv[2].startswith("-"): - has_path = False - target = Path(".").resolve() - else: - target = Path(sys.argv[2]).resolve() - if not target.exists(): - print(f"error: path not found: {target}", file=sys.stderr) - sys.exit(1) - - backend: str | None = None - model: str | None = None - extract_mode: str | None = None - out_dir: Path | None = None - cli_postgres_dsn: str | None = None - cli_cargo: bool = False - no_cluster = False - dedup_llm = False - google_workspace = False - global_merge = False - global_repo_tag: str | None = None - # Performance/tuning knobs (issue #792). None means "use library default". - cli_max_workers: int | None = None - cli_token_budget: int | None = None - cli_max_concurrency: int | None = None - cli_api_timeout: float | None = None - # Clustering tuning knobs - cli_resolution: float = 1.0 - cli_exclude_hubs: float | None = None - cli_excludes: list[str] = [] - cli_timing: bool = False - - def _parse_int(name: str, raw: str) -> int: - try: - v = int(raw) - except ValueError: - print(f"error: {name} must be a positive integer (got {raw!r})", file=sys.stderr) - sys.exit(2) - if v <= 0: - print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) - sys.exit(2) - return v - - def _parse_float(name: str, raw: str) -> float: - try: - v = float(raw) - except ValueError: - print(f"error: {name} must be a positive number (got {raw!r})", file=sys.stderr) - sys.exit(2) - if v <= 0: - print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) - sys.exit(2) - return v - - args = sys.argv[3:] if has_path else sys.argv[2:] - i = 0 - while i < len(args): - a = args[i] - if a == "--backend" and i + 1 < len(args): - backend = args[i + 1]; i += 2 - elif a.startswith("--backend="): - backend = a.split("=", 1)[1]; i += 1 - elif a == "--model" and i + 1 < len(args): - model = args[i + 1]; i += 2 - elif a.startswith("--model="): - model = a.split("=", 1)[1]; i += 1 - elif a == "--mode" and i + 1 < len(args): - extract_mode = args[i + 1]; i += 2 - elif a.startswith("--mode="): - extract_mode = a.split("=", 1)[1]; i += 1 - elif a == "--out" and i + 1 < len(args): - out_dir = Path(args[i + 1]); i += 2 - elif a.startswith("--out="): - out_dir = Path(a.split("=", 1)[1]); i += 1 - elif a == "--no-cluster": - no_cluster = True; i += 1 - elif a == "--dedup-llm": - dedup_llm = True; i += 1 - elif a == "--google-workspace": - google_workspace = True; i += 1 - elif a == "--global": - global_merge = True; i += 1 - elif a == "--as" and i + 1 < len(args): - global_repo_tag = args[i + 1]; i += 2 - elif a == "--max-workers" and i + 1 < len(args): - cli_max_workers = _parse_int("--max-workers", args[i + 1]); i += 2 - elif a.startswith("--max-workers="): - cli_max_workers = _parse_int("--max-workers", a.split("=", 1)[1]); i += 1 - elif a == "--token-budget" and i + 1 < len(args): - cli_token_budget = _parse_int("--token-budget", args[i + 1]); i += 2 - elif a.startswith("--token-budget="): - cli_token_budget = _parse_int("--token-budget", a.split("=", 1)[1]); i += 1 - elif a == "--max-concurrency" and i + 1 < len(args): - cli_max_concurrency = _parse_int("--max-concurrency", args[i + 1]); i += 2 - elif a.startswith("--max-concurrency="): - cli_max_concurrency = _parse_int("--max-concurrency", a.split("=", 1)[1]); i += 1 - elif a == "--api-timeout" and i + 1 < len(args): - cli_api_timeout = _parse_float("--api-timeout", args[i + 1]); i += 2 - elif a.startswith("--api-timeout="): - cli_api_timeout = _parse_float("--api-timeout", a.split("=", 1)[1]); i += 1 - elif a == "--resolution" and i + 1 < len(args): - cli_resolution = _parse_float("--resolution", args[i + 1]); i += 2 - elif a.startswith("--resolution="): - cli_resolution = _parse_float("--resolution", a.split("=", 1)[1]); i += 1 - elif a == "--exclude-hubs" and i + 1 < len(args): - cli_exclude_hubs = float(args[i + 1]); i += 2 - elif a.startswith("--exclude-hubs="): - cli_exclude_hubs = float(a.split("=", 1)[1]); i += 1 - elif a == "--exclude" and i + 1 < len(args): - cli_excludes.append(args[i + 1]); i += 2 - elif a.startswith("--exclude="): - cli_excludes.append(a.split("=", 1)[1]); i += 1 - elif a == "--postgres" and i + 1 < len(args): - cli_postgres_dsn = args[i + 1]; i += 2 - elif a.startswith("--postgres="): - cli_postgres_dsn = a.split("=", 1)[1]; i += 1 - elif a == "--cargo": - cli_cargo = True - i += 1 - elif a == "--timing": - cli_timing = True; i += 1 - else: - i += 1 - - if not has_path and cli_postgres_dsn is None: - print("error: must specify a path to scan or a --postgres DSN", file=sys.stderr) - sys.exit(1) - - _VALID_MODES = {"deep"} - if extract_mode is not None and extract_mode not in _VALID_MODES: - print( - f"error: unknown --mode '{extract_mode}'. " - f"Available: {', '.join(sorted(_VALID_MODES))}", - file=sys.stderr, - ) - sys.exit(2) - deep_mode = extract_mode == "deep" - if deep_mode: - print("[graphify extract] deep mode enabled: richer semantic extraction") - - # CLI flag wins over env var. Setting GRAPHIFY_API_TIMEOUT here so - # _call_openai_compat picks it up without needing a new kwarg path. - if cli_api_timeout is not None: - os.environ["GRAPHIFY_API_TIMEOUT"] = str(cli_api_timeout) - if cli_max_workers is not None: - os.environ["GRAPHIFY_MAX_WORKERS"] = str(cli_max_workers) - - # Resolve output dir. The user-facing contract is "/graphify-out/" - # so a fresh checkout writes graphify-out/ at the project root, matching - # the skill.md pipeline. - out_root = (out_dir.resolve() if out_dir else target) - graphify_out = out_root / _GRAPHIFY_OUT - graphify_out.mkdir(parents=True, exist_ok=True) - - stages = _StageTimer(cli_timing) - - from graphify.detect import ( - detect as _detect, - detect_incremental as _detect_incremental, - save_manifest as _save_manifest, - ) - manifest_path = graphify_out / "manifest.json" - existing_graph_path = graphify_out / "graph.json" - incremental_mode = manifest_path.exists() and existing_graph_path.exists() if has_path else False - - if not has_path: - code_files = [] - doc_files = [] - paper_files = [] - image_files = [] - deleted_files = [] - unchanged_total = 0 - files_by_type = {} - elif incremental_mode: - print(f"[graphify extract] incremental scan of {target}") - detection = _detect_incremental( - target, - manifest_path=str(manifest_path), - google_workspace=google_workspace or None, - extra_excludes=cli_excludes or None, - ) - files_by_type = detection.get("files", {}) - new_by_type = detection.get("new_files", {}) - code_files = [Path(p) for p in new_by_type.get("code", [])] - doc_files = [Path(p) for p in new_by_type.get("document", [])] - paper_files = [Path(p) for p in new_by_type.get("paper", [])] - image_files = [Path(p) for p in new_by_type.get("image", [])] - deleted_files = list(detection.get("deleted_files", [])) - unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) - else: - print(f"[graphify extract] scanning {target}") - detection = _detect(target, google_workspace=google_workspace or None, extra_excludes=cli_excludes or None) - files_by_type = detection.get("files", {}) - code_files = [Path(p) for p in files_by_type.get("code", [])] - doc_files = [Path(p) for p in files_by_type.get("document", [])] - paper_files = [Path(p) for p in files_by_type.get("paper", [])] - image_files = [Path(p) for p in files_by_type.get("image", [])] - deleted_files = [] - unchanged_total = 0 - - semantic_files = doc_files + paper_files + image_files - if incremental_mode: - print( - f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " - f"{len(paper_files)} papers, {len(image_files)} images changed; " - f"{unchanged_total} unchanged; {len(deleted_files)} deleted" - ) - else: - print( - f"[graphify extract] found {len(code_files)} code, " - f"{len(doc_files)} docs, {len(paper_files)} papers, " - f"{len(image_files)} images" - ) - # Surface files that were seen but not classified (extensionless non-shebang - # project files like Dockerfile/Makefile, or unsupported extensions), so they - # are no longer invisible in graphify's own output (#1692). - _unclassified = detection.get("unclassified", []) if isinstance(detection, dict) else [] - if _unclassified: - _names = ", ".join(sorted({Path(p).name for p in _unclassified})[:6]) - _more = f" (+{len(_unclassified) - 6} more)" if len(_unclassified) > 6 else "" - print( - f"[graphify extract] {len(_unclassified)} file(s) not classified " - f"(no supported extension or shebang), skipped: {_names}{_more}" - ) - stages.mark("detect") - - # Resolve the LLM backend only now that we know whether the corpus - # needs one. A code-only corpus is pure local AST and must not require - # an API key; the key is enforced below only when there's LLM work. - from graphify.llm import ( - BACKENDS as _BACKENDS, - detect_backend as _detect_backend, - estimate_cost as _estimate_cost, - extract_corpus_parallel as _extract_corpus_parallel, - _format_backend_env_keys, - _get_backend_api_key, - ) - needs_llm = bool(semantic_files) or dedup_llm - if backend is None and needs_llm: - backend = _detect_backend() - if backend is not None and backend not in _BACKENDS: - print( - f"error: unknown backend '{backend}'. " - f"Available: {', '.join(sorted(_BACKENDS))}", - file=sys.stderr, - ) - sys.exit(1) - if needs_llm: - if backend is None: - reasons = [] - if semantic_files: - reasons.append( - f"{len(semantic_files)} doc/paper/image file(s) need semantic extraction" - ) - if dedup_llm: - reasons.append("--dedup-llm was passed") - print( - "error: no LLM API key found (" + "; ".join(reasons) + "). " - "Set GEMINI_API_KEY or GOOGLE_API_KEY (gemini), MOONSHOT_API_KEY " - "(kimi), ANTHROPIC_API_KEY (claude), OPENAI_API_KEY (openai), " - "DEEPSEEK_API_KEY (deepseek), or pass --backend. A code-only " - "corpus needs no key.", - file=sys.stderr, - ) - sys.exit(1) - if backend == "ollama": - from graphify.llm import _validate_ollama_base_url - _oll_url = os.environ.get("OLLAMA_BASE_URL", _BACKENDS["ollama"].get("base_url", "")) - try: - _validate_ollama_base_url(_oll_url, warn=False) - except ValueError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(2) - if not _get_backend_api_key(backend): - allow_no_key = False - if backend == "ollama": - from urllib.parse import urlparse - ollama_url = os.environ.get( - "OLLAMA_BASE_URL", - _BACKENDS["ollama"].get("base_url", ""), - ) - try: - host = (urlparse(ollama_url).hostname or "").lower() - except Exception: - host = "" - allow_no_key = ( - host in ("localhost", "127.0.0.1", "::1") - or host.startswith("127.") - ) - elif backend == "bedrock": - allow_no_key = bool( - os.environ.get("AWS_PROFILE") - or os.environ.get("AWS_REGION") - or os.environ.get("AWS_DEFAULT_REGION") - or os.environ.get("AWS_ACCESS_KEY_ID") - ) - elif backend == "claude-cli": - import shutil as _shutil - allow_no_key = _shutil.which("claude") is not None - if not allow_no_key: - print( - "error: backend 'claude-cli' requires the `claude` CLI on $PATH " - "(install Claude Code and run `claude` once to authenticate).", - file=sys.stderr, - ) - sys.exit(1) - if not allow_no_key: - print( - f"error: backend '{backend}' requires {_format_backend_env_keys(backend)} to be set.", - file=sys.stderr, - ) - sys.exit(1) - - # AST extraction on code files. Empty code list (docs-only corpus) is - # the issue #698 case — skip cleanly instead of crashing inside extract(). - ast_result: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} - if code_files: - from graphify.extract import extract as _ast_extract - # Anchor the cache at the output root, not the scanned project: - # with --out, a /graphify-out/cache/ would leak a - # graphify-out/ dir into a project that asked for external output. - ast_kwargs: dict = {"cache_root": out_root} - if cli_max_workers is not None: - ast_kwargs["max_workers"] = cli_max_workers - print(f"[graphify extract] AST extraction on {len(code_files)} code files...") - try: - ast_result = _ast_extract(code_files, **ast_kwargs) - except Exception as exc: - print(f"[graphify extract] AST extraction failed: {exc}", file=sys.stderr) - ast_result = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} - stages.mark("AST extract") - - # Semantic extraction on docs/papers/images. Check cache first. - from graphify.cache import ( - check_semantic_cache as _check_semantic_cache, - prune_semantic_cache as _prune_semantic_cache, - save_semantic_cache as _save_semantic_cache, - ) - sem_result: dict = { - "nodes": [], "edges": [], "hyperedges": [], - "input_tokens": 0, "output_tokens": 0, - } - sem_cache_hits = 0 - sem_cache_misses = 0 - if semantic_files: - sem_paths_str = [str(p) for p in semantic_files] - cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( - _check_semantic_cache(sem_paths_str, root=out_root) - ) - sem_cache_hits = len(semantic_files) - len(uncached_paths) - sem_cache_misses = len(uncached_paths) - sem_result["nodes"].extend(cached_nodes) - sem_result["edges"].extend(cached_edges) - sem_result["hyperedges"].extend(cached_hyperedges) - if sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} hit / {sem_cache_misses} miss") - - if uncached_paths: - print(f"[graphify extract] semantic extraction on {len(uncached_paths)} files via {backend}...") - corpus_kwargs: dict = { - "backend": backend, - "model": model, - "root": target, - } - if deep_mode: - corpus_kwargs["deep_mode"] = True - if cli_token_budget is not None: - corpus_kwargs["token_budget"] = cli_token_budget - if cli_max_concurrency is not None: - corpus_kwargs["max_concurrency"] = cli_max_concurrency - - # Minimal progress callback so the CLI is no longer silent - # during long local-inference runs (issue #792 addendum). - # Also track per-chunk success so we can fail loudly when - # every chunk errors (e.g. missing backend SDK package). - _chunk_stats = {"total": 0, "succeeded": 0} - def _progress(idx: int, total: int, _result: dict) -> None: - _chunk_stats["total"] = total - _chunk_stats["succeeded"] += 1 - print( - f"[graphify extract] chunk {idx + 1}/{total} done", - flush=True, - ) - corpus_kwargs["on_chunk_done"] = _progress - - try: - fresh = _extract_corpus_parallel( - [Path(p) for p in uncached_paths], - **corpus_kwargs, - ) - except ImportError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - except Exception as exc: - print( - f"[graphify extract] semantic extraction failed: {exc}", - file=sys.stderr, - ) - fresh = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - - # on_chunk_done only fires after a chunk succeeds. If fresh - # semantic extraction was requested and no chunks completed, - # fail instead of writing an AST-only graph with exit 0. - if uncached_paths and _chunk_stats["succeeded"] == 0: - print( - f"[graphify extract] error: all semantic chunks failed " - f"for backend '{backend}' ({len(uncached_paths)} uncached files) - " - f"see per-chunk errors above. If you see 'requires the X package', " - f"run `pip install X` and retry.", - file=sys.stderr, - ) - sys.exit(1) - try: - _save_semantic_cache( - fresh.get("nodes", []), - fresh.get("edges", []), - fresh.get("hyperedges", []), - root=out_root, - ) - except Exception as exc: - print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) - sem_result["nodes"].extend(fresh.get("nodes", [])) - sem_result["edges"].extend(fresh.get("edges", [])) - sem_result["hyperedges"].extend(fresh.get("hyperedges", [])) - sem_result["input_tokens"] += fresh.get("input_tokens", 0) - sem_result["output_tokens"] += fresh.get("output_tokens", 0) - - # Prune orphaned semantic cache entries. The semantic cache is - # content-hash-keyed and unversioned, so it is never swept by the AST - # version-cleanup: every content change or file deletion leaves a - # permanent orphan that accumulates unbounded (#1527). Sweep it against - # the FULL live document set (``files_by_type`` — present in both the - # incremental and full branches), NOT the incremental ``semantic_files`` - # changed-subset, which would delete every unchanged doc's valid entry. - # Best-effort: a prune failure must never break extraction. - try: - from graphify.cache import file_hash as _file_hash - _live_hashes: set[str] = set() - for _kind in ("document", "paper", "image"): - for _fp in files_by_type.get(_kind, []): - _abs = Path(_fp) - if not _abs.is_absolute(): - _abs = Path(out_root) / _abs - if not _abs.is_file(): - continue # deleted/missing — leave out so its entry is pruned - try: - _live_hashes.add(_file_hash(_abs, out_root)) - except OSError: - pass - _prune_semantic_cache(out_root, _live_hashes) - except Exception as exc: - print(f"[graphify extract] warning: could not prune semantic cache: {exc}", file=sys.stderr) - stages.mark("semantic extract") - - pg_result: dict = {"nodes": [], "edges": []} - if cli_postgres_dsn is not None: - from graphify.pg_introspect import introspect_postgres - print(f"[graphify extract] introspecting PostgreSQL schema...") - try: - pg_result = introspect_postgres(cli_postgres_dsn) - except (ConnectionError, ImportError) as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, " - f"{len(pg_result['edges'])} edges") - - cargo_result: dict = {"nodes": [], "edges": []} - if cli_cargo: - from graphify.cargo_introspect import introspect_cargo - print("[graphify extract] introspecting Cargo workspace...") - try: - cargo_result = introspect_cargo(target) - except (ConnectionError, ImportError, OSError) as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - print(f"[graphify extract] Cargo: {len(cargo_result['nodes'])} nodes, " - f"{len(cargo_result['edges'])} edges") - - # Merge AST + semantic + pg_result + cargo_result. Order matters for deduplication: passing AST - # first means semantic node attributes win on collision (richer labels - # for symbols also referenced in docs). Hyperedges only come from the - # semantic side. - merged: dict = { - "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), - "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), - "hyperedges": list(sem_result.get("hyperedges", [])), - "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), - "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), - } - - graph_json_path = graphify_out / "graph.json" - analysis_path = graphify_out / ".graphify_analysis.json" - - # Build a manifest-safe files dict: only stamp semantic_hash for files - # that actually produced output (cache hit or fresh extraction). Files - # whose chunk failed have no source_file entry in sem_result — leaving - # their semantic_hash empty so detect_incremental re-queues them (#933). - _sem_extracted: set[str] = { - n.get("source_file", "") for n in sem_result.get("nodes", []) - } | { - e.get("source_file", "") for e in sem_result.get("edges", []) - } - _sem_extracted.discard("") - _sem_types = {"document", "paper", "image"} - _manifest_files = { - ftype: [f for f in flist if ftype not in _sem_types or f in _sem_extracted] - for ftype, flist in files_by_type.items() - } - - if no_cluster: - # --no-cluster: dump the raw merged extraction as graph.json. - # No NetworkX, no community detection, no analysis sidecar. - # Dedupe nodes (by id) and parallel edges so the raw output matches the - # clustered path (whose DiGraph collapses both) and stays deterministic - # across modes (#1317; node dedup also collapses shared Swift module - # anchors emitted per importing file, #1327). - from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes - from graphify.export import backup_if_protected as _backup - if ( - incremental_mode - and not code_files - and not semantic_files - and not deleted_files - and not pg_result.get("nodes") - and not pg_result.get("edges") - and not cargo_result.get("nodes") - and not cargo_result.get("edges") - ): - print( - "[graphify extract] no incremental changes detected " - "(--no-cluster); outputs left untouched." - ) - try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - stages.total() - sys.exit(0) - - merged["nodes"] = _dedupe_nodes(merged["nodes"]) - merged["edges"] = _dedupe_edges(merged["edges"]) - # Backfill source_file from endpoint nodes — this raw path bypasses - # build_from_json's backfill, and semantic edges sometimes omit it (#1279). - _node_sf = {n.get("id"): n.get("source_file") for n in merged["nodes"]} - for _e in merged["edges"]: - if not _e.get("source_file"): - _e["source_file"] = ( - _node_sf.get(_e.get("source")) or _node_sf.get(_e.get("target")) or "" - ) - _backup(graphify_out) - graph_json_path.write_text( - json.dumps(merged, indent=2), encoding="utf-8" - ) - stages.mark("write") - cost = _estimate_cost( - backend, merged["input_tokens"], merged["output_tokens"] - ) - print( - f"[graphify extract] wrote {graph_json_path} — " - f"{len(merged['nodes'])} nodes, {len(merged['edges'])} edges " - f"(no clustering)" - ) - if merged["input_tokens"] or merged["output_tokens"]: - print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost: ${cost:.4f}" - ) - try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - if global_merge: - from graphify.global_graph import global_add as _global_add - _tag = global_repo_tag or target.name - try: - result = _global_add(graphify_out / "graph.json", _tag) - if result["skipped"]: - print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") - else: - print(f"[graphify global] '{_tag}' merged into global graph " - f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") - except Exception as exc: - print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) - stages.total() - sys.exit(0) - - # Build graph + cluster + score + write. - from graphify.build import ( - build as _build, - build_from_json as _build_from_json, - build_merge as _build_merge, - ) - from graphify.cluster import cluster as _cluster, score_all as _score_all - from graphify.export import to_json as _to_json - from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising - dedup_backend = backend if dedup_llm else None - if incremental_mode: - G = _build_merge( - [merged], - graph_path=existing_graph_path, - prune_sources=deleted_files or None, - dedup=True, - dedup_llm_backend=dedup_backend, - root=target, - ) - else: - G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target) - stages.mark("build") - if G.number_of_nodes() == 0: - print( - "[graphify extract] graph is empty — extraction produced no nodes. " - "Possible causes: all files skipped, binary-only corpus, or LLM " - "returned no edges.", - file=sys.stderr, - ) - sys.exit(1) - - communities = _cluster(G, resolution=cli_resolution, exclude_hubs_percentile=cli_exclude_hubs) - stages.mark("cluster") - cohesion = _score_all(G, communities) - try: - gods = _god_nodes(G) - except Exception: - gods = [] - try: - surprises = _surprising(G, communities) - except Exception: - surprises = [] - stages.mark("analyze") - - from graphify.export import backup_if_protected as _backup - _backup(graphify_out) - _to_json(G, communities, str(graph_json_path), force=True) - stages.mark("export") - if merged.get("output_tokens", 0) > 0: - (graphify_out / ".graphify_semantic_marker").write_text( - json.dumps({"output_tokens": merged["output_tokens"]}), encoding="utf-8" - ) - if global_merge: - from graphify.global_graph import global_add as _global_add - _tag = global_repo_tag or target.name - try: - result = _global_add(graphify_out / "graph.json", _tag) - if result["skipped"]: - print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") - else: - print(f"[graphify global] '{_tag}' merged into global graph " - f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") - except Exception as exc: - print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) - analysis = { - "communities": {str(k): v for k, v in communities.items()}, - "cohesion": {str(k): v for k, v in cohesion.items()}, - "gods": gods, - "surprises": surprises, - "tokens": { - "input": merged["input_tokens"], - "output": merged["output_tokens"], - }, - } - analysis_path.write_text(json.dumps(analysis, indent=2), encoding="utf-8") - try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - - cost = _estimate_cost(backend, merged["input_tokens"], merged["output_tokens"]) - print( - f"[graphify extract] wrote {graph_json_path}: " - f"{G.number_of_nodes()} nodes, {G.number_of_edges()} edges, " - f"{len(communities)} communities" - ) - print(f"[graphify extract] wrote {analysis_path}") - if incremental_mode: - print( - f"[graphify extract] incremental summary: " - f"{sem_cache_hits + unchanged_total} files cached/unchanged, " - f"{len(code_files) + sem_cache_misses} re-extracted, " - f"{len(deleted_files)} deleted" - ) - elif sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") - if merged["input_tokens"] or merged["output_tokens"]: - print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost (~{backend}): ${cost:.4f}" - ) - # extract intentionally stops at graph.json + analysis; the report and - # community labels are produced by `cluster-only` (or an agent's Step 5). - # Point standalone users at it so communities get named (#1097). - print( - "[graphify extract] next: run " - f"`graphify cluster-only {graphify_out.parent}` " - "to generate GRAPH_REPORT.md and name communities" - ) - stages.total() - - elif cmd == "cache-check": - # graphify cache-check [--root ] - # Reads file paths (one per line) from , checks semantic cache. - # Writes: - # graphify-out/.graphify_cached.json — already-cached nodes/edges/hyperedges - # graphify-out/.graphify_uncached.txt — paths that need extraction - # Stdout: "Cache: N hit, M miss" - from graphify.cache import check_semantic_cache - if len(sys.argv) < 3: - print("Usage: graphify cache-check [--root ]", file=sys.stderr) - sys.exit(1) - files_from = Path(sys.argv[2]) - root = Path(".") - i = 3 - while i < len(sys.argv): - if sys.argv[i] == "--root" and i + 1 < len(sys.argv): - root = Path(sys.argv[i + 1]) - i += 2 - else: - i += 1 - files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()] - cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(files, root) - out = root / _GRAPHIFY_OUT - out.mkdir(parents=True, exist_ok=True) - if cached_nodes or cached_edges or cached_hyperedges: - (out / ".graphify_cached.json").write_text( - json.dumps({"nodes": cached_nodes, "edges": cached_edges, "hyperedges": cached_hyperedges}, - ensure_ascii=False), - encoding="utf-8", - ) - (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8") - print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss") - - elif cmd == "merge-chunks": - # graphify merge-chunks --out - # Concatenates .graphify_chunk_*.json files written by semantic subagents. - # Deduplicates nodes by id (first writer wins). Sums token counts. - import glob as _glob - if len(sys.argv) < 3: - print("Usage: graphify merge-chunks --out ", file=sys.stderr) - sys.exit(1) - out_path: Path | None = None - chunk_args: list[str] = [] - i = 2 - while i < len(sys.argv): - if sys.argv[i] == "--out" and i + 1 < len(sys.argv): - out_path = Path(sys.argv[i + 1]) - i += 2 - else: - chunk_args.append(sys.argv[i]) - i += 1 - if not out_path: - print("error: --out required", file=sys.stderr) - sys.exit(1) - chunk_files: list[str] = [] - for arg in chunk_args: - expanded = _glob.glob(arg) - chunk_files.extend(sorted(expanded) if expanded else [arg]) - merged: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - seen_ids: set[str] = set() - for cf in chunk_files: - try: - chunk = json.loads(Path(cf).read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - print(f"[graphify merge-chunks] warning: skipping {cf}: {exc}", file=sys.stderr) - continue - for n in chunk.get("nodes", []): - if n.get("id") not in seen_ids: - seen_ids.add(n["id"]) - merged["nodes"].append(n) - merged["edges"].extend(chunk.get("edges", [])) - merged["hyperedges"].extend(chunk.get("hyperedges", [])) - merged["input_tokens"] += chunk.get("input_tokens", 0) - merged["output_tokens"] += chunk.get("output_tokens", 0) - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(merged, ensure_ascii=False), encoding="utf-8") - print( - f"Merged {len(chunk_files)} chunks: {len(merged['nodes'])} nodes, {len(merged['edges'])} edges, " - f"{merged['input_tokens']:,} in / {merged['output_tokens']:,} out tokens" - ) - - elif cmd == "merge-semantic": - # graphify merge-semantic --cached --new --out - # Merges cached semantic results with freshly-extracted chunk results. - # Deduplicates nodes by id (cached entries take priority over new ones). - if len(sys.argv) < 3: - print("Usage: graphify merge-semantic --cached --new --out ", file=sys.stderr) - sys.exit(1) - cached_path: Path | None = None - new_path: Path | None = None - out_path2: Path | None = None - i = 2 - while i < len(sys.argv): - if sys.argv[i] == "--cached" and i + 1 < len(sys.argv): - cached_path = Path(sys.argv[i + 1]); i += 2 - elif sys.argv[i] == "--new" and i + 1 < len(sys.argv): - new_path = Path(sys.argv[i + 1]); i += 2 - elif sys.argv[i] == "--out" and i + 1 < len(sys.argv): - out_path2 = Path(sys.argv[i + 1]); i += 2 - else: - i += 1 - if not out_path2: - print("error: --out required", file=sys.stderr) - sys.exit(1) - empty: dict = {"nodes": [], "edges": [], "hyperedges": []} - cached_data = json.loads(cached_path.read_text(encoding="utf-8")) if cached_path and cached_path.exists() else empty - new_data = json.loads(new_path.read_text(encoding="utf-8")) if new_path and new_path.exists() else empty - seen_ids2: set[str] = set() - all_nodes: list[dict] = [] - for n in cached_data.get("nodes", []) + new_data.get("nodes", []): - if n.get("id") not in seen_ids2: - seen_ids2.add(n["id"]) - all_nodes.append(n) - merged2 = { - "nodes": all_nodes, - "edges": cached_data.get("edges", []) + new_data.get("edges", []), - "hyperedges": cached_data.get("hyperedges", []) + new_data.get("hyperedges", []), - } - out_path2.parent.mkdir(parents=True, exist_ok=True) - out_path2.write_text(json.dumps(merged2, ensure_ascii=False), encoding="utf-8") - print(f"Merged: {len(merged2['nodes'])} nodes, {len(merged2['edges'])} edges") - - elif Path(cmd).exists() or cmd in (".", "..") or cmd.startswith(("./", "../", "/", "~")): - # User ran `graphify ` directly — treat as `graphify extract `. - # Common when following the PowerShell note in README (`graphify .`) or - # copy-pasting skill invocations without the leading slash. - sys.argv.insert(2, sys.argv[1]) - sys.argv[1] = "extract" - main() - else: - print(f"error: unknown command '{cmd}'", file=sys.stderr) - print("Run 'graphify --help' for usage.", file=sys.stderr) - sys.exit(1) + if dispatch_install_cli(cmd): + return + dispatch_command(cmd) if __name__ == "__main__": diff --git a/graphify/build.py b/graphify/build.py index b26ddcb61..f0676a8d5 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -956,6 +956,34 @@ def prefix_graph_for_global(G: nx.Graph, repo_tag: str) -> nx.Graph: return H +def distinct_repo_tags(graph_paths: "list[Path]") -> "list[str]": + """Return a unique, human-meaningful repo tag per input graph for merge-graphs. + + The naive tag (the ``graphify-out`` parent dir name) is NOT unique across + inputs: ``src/graphify-out`` and ``frontend/src/graphify-out`` both yield + ``src``. Prefixing both node sets with ``src::`` then makes same-stem nodes + (a backend ``src/app.js`` and a frontend ``App.jsx``, both bare ``app``) + collide, so ``nx.compose`` silently merges two unrelated entities and invents + cross-runtime edges (#1729). Colliding tags are widened with their own parent + dir (``frontend_src``), then an index suffix guarantees uniqueness so no two + graphs ever share a prefix. + """ + repo_dirs = [p.parent.parent for p in graph_paths] # graphify-out/.. → repo dir + tags = [d.name or "repo" for d in repo_dirs] + if len(set(tags)) != len(tags): + widened: list[str] = [] + for d in repo_dirs: + parent = d.parent.name + widened.append(f"{parent}_{d.name}" if parent and d.name else (d.name or "repo")) + tags = widened + seen: dict[str, int] = {} + unique: list[str] = [] + for t in tags: + seen[t] = seen.get(t, 0) + 1 + unique.append(t if seen[t] == 1 else f"{t}-{seen[t]}") + return unique + + def prune_repo_from_graph(G: nx.Graph, repo_tag: str) -> int: """Remove all nodes tagged with repo_tag from G in-place. Returns count removed.""" to_remove = [n for n, d in G.nodes(data=True) if d.get("repo") == repo_tag] diff --git a/graphify/cli.py b/graphify/cli.py new file mode 100644 index 000000000..540a43510 --- /dev/null +++ b/graphify/cli.py @@ -0,0 +1,2753 @@ +"""graphify command dispatch — every non-install subcommand. + +Extracted verbatim from __main__.main(); __main__ now calls dispatch_command(cmd) +after the install/platform dispatch. Kept out of __main__ to shrink the CLI entry +module. The path-redirect (`graphify ` -> extract) re-enters via a lazy +import of main to avoid a cli<->__main__ import cycle. +""" +from __future__ import annotations +import json +import os +import sys +from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from pathlib import Path + + +_SEARCH_NUDGE = json.dumps({ + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "additionalContext": ( + 'MANDATORY: graphify-out/graph.json exists. You MUST run ' + '`graphify query ""` before grepping raw files. Only grep ' + 'after graphify has oriented you, or to modify/debug specific lines.' + ), + } +}, ensure_ascii=False, separators=(",", ":")) + "\n" +_READ_NUDGE = json.dumps({ + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "additionalContext": ( + 'MANDATORY: graphify-out/graph.json exists. You MUST run graphify ' + 'before reading source files. Use: `graphify query ""` ' + '(scoped subgraph), `graphify explain ""`, or ' + '`graphify path "" ""`. Only read raw files after graphify has ' + 'oriented you, or to modify/debug specific lines. This rule applies to ' + 'subagents too — include it in every subagent prompt involving code ' + 'exploration.' + ), + } +}, ensure_ascii=False, separators=(",", ":")) + "\n" +_HOOK_SOURCE_EXTS = ( + '.py', '.js', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.go', + '.rs', '.java', '.rb', '.c', '.h', '.cpp', '.hpp', '.cc', '.cs', '.kt', + '.swift', '.php', '.scala', '.lua', '.sh', '.md', '.rst', '.txt', '.mdx', +) +_GEMINI_NUDGE_TEXT = ( + 'graphify: knowledge graph at graphify-out/. For focused questions, run ' + '`graphify query ""` (scoped subgraph, usually much smaller than ' + 'GRAPH_REPORT.md) instead of grepping raw files. Read GRAPH_REPORT.md only ' + 'for broad architecture context.' +) + + +def _default_graph_path() -> str: + return str(Path(_GRAPHIFY_OUT) / "graph.json") +class _StageTimer: + """Print per-stage wall-clock timings to stderr when --timing is set (#1490). + + Monotonic (perf_counter), diagnostic-only: emits ``[graphify timing] : + N.Ns`` after each stage and a final total. Off by default, so normal output is + byte-identical and machine-read stdout is untouched. + """ + + def __init__(self, enabled: bool) -> None: + import time as _time + self._now = _time.perf_counter + self.enabled = enabled + self.start = self._now() + self._last = self.start + + def mark(self, stage: str) -> None: + now = self._now() + if self.enabled: + print(f"[graphify timing] {stage}: {now - self._last:.1f}s", file=sys.stderr) + self._last = now + + def total(self) -> None: + if self.enabled: + print(f"[graphify timing] total: {self._now() - self.start:.1f}s", file=sys.stderr) +def _enforce_graph_size_cap_or_exit(gp: Path) -> None: + """Reject oversized graph files before parsing (CLI exit-on-fail flavor). + + Delegates to ``graphify.security.check_graph_file_size_cap`` and turns the + raised ``ValueError`` into a CLI-style ``error: ...`` message + exit 1. + Use this from ``__main__.py`` subcommands that already use the ``print + + sys.exit(1)`` idiom. Library/MCP/loader callers (``serve._load_graph``, + ``build``, ``benchmark``, ``tree_html``, ``callflow_html``, ``prs``, + ``global_graph``, ``watch``, ``export``) call the security helper directly + and let the ``ValueError`` propagate. + """ + from graphify.security import check_graph_file_size_cap + try: + check_graph_file_size_cap(gp) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) +def _run_hook_guard(kind: str) -> None: + """Shell-agnostic PreToolUse guard (#522). + + Reads the tool-call JSON from stdin and, when a knowledge graph exists in the + current output dir, prints a nudge (`additionalContext`) telling the agent to + use graphify instead of grepping/reading raw files. Replaces the old inline + bash hooks that failed to parse on Windows. Always fails open: any error, or a + non-matching tool call, prints nothing and the caller exits 0, so a legitimate + tool call is never blocked. Detection mirrors the previous hooks exactly. + """ + from graphify.paths import out_path, GRAPHIFY_OUT_NAME + # Gemini's BeforeTool hook takes no stdin and must ALWAYS return a decision so + # the tool is never blocked; the graph nudge is appended only when a graph + # exists. Handled before the stdin read below (which the search/read guards need). + if kind == "gemini": + payload = {"decision": "allow"} + try: + if out_path("graph.json").is_file(): + payload["additionalContext"] = _GEMINI_NUDGE_TEXT + except Exception: + pass + sys.stdout.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":"))) + return + try: + d = json.loads(sys.stdin.buffer.read().decode("utf-8", "replace")) + except Exception: + return + if not isinstance(d, dict): + return + t = d.get("tool_input", d) + if not isinstance(t, dict): + return + try: + if kind == "search": + cmd_str = str(t.get("command", "") or "") + # Same set the old `case` matched: *grep*, *ripgrep*, and rg/find/fd/ + # ack/ag as a token (name followed by a space). + if any(tok in cmd_str for tok in ("grep", "ripgrep", "rg ", "find ", "fd ", "ack ", "ag ")) \ + and out_path("graph.json").is_file(): + sys.stdout.write(_SEARCH_NUDGE) + elif kind == "read": + vals = [str(t.get("file_path") or ""), str(t.get("pattern") or ""), str(t.get("path") or "")] + j = " ".join(vals).lower().replace("\\", "/") + tails = [ + "." + seg.rsplit(".", 1)[-1] + for v in vals if v + for seg in [v.lower().replace("\\", "/").rsplit("/", 1)[-1]] + if "." in seg + ] + under_out = "graphify-out/" in j or (GRAPHIFY_OUT_NAME.lower() + "/") in j + if not under_out and any(tl in _HOOK_SOURCE_EXTS for tl in tails) \ + and out_path("graph.json").is_file(): + sys.stdout.write(_READ_NUDGE) + except Exception: + pass +def _clone_repo( + url: str, branch: str | None = None, out_dir: Path | None = None +) -> Path: + """Clone a GitHub repo to a local cache dir and return the path. + + Clones into ~/.graphify/repos// by default so repeated + runs on the same URL reuse the existing clone (git pull instead of clone). + """ + import subprocess as _sp + import re as _re + + # Normalise URL — strip trailing .git if present + url = url.rstrip("/") + if not url.endswith(".git"): + git_url = url + ".git" + else: + git_url = url + url = url[:-4] + + # Extract owner/repo from URL + m = _re.search(r"github\.com[:/]([^/]+)/([^/]+?)(?:\.git)?$", url) + if not m: + print(f"error: not a recognised GitHub URL: {url}", file=sys.stderr) + sys.exit(1) + owner, repo = m.group(1), m.group(2) + + if out_dir: + dest = out_dir + else: + dest = Path.home() / ".graphify" / "repos" / owner / repo + + if branch and branch.startswith("-"): + print(f"error: invalid branch name: {branch!r}", file=sys.stderr) + sys.exit(1) + + if dest.exists(): + print(f"Repo already cloned at {dest} - pulling latest...", flush=True) + cmd = ["git", "-C", str(dest), "pull"] + if branch: + cmd += ["origin", "--", branch] + result = _sp.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"warning: git pull failed:\n{result.stderr}", file=sys.stderr) + else: + dest.parent.mkdir(parents=True, exist_ok=True) + print(f"Cloning {url} -> {dest} ...", flush=True) + cmd = ["git", "clone", "--depth", "1"] + if branch: + cmd += ["--branch", branch] + cmd += ["--", git_url, str(dest)] + result = _sp.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"error: git clone failed:\n{result.stderr}", file=sys.stderr) + sys.exit(1) + + print(f"Ready at: {dest}", flush=True) + return dest + + +def _reenter_main() -> None: + from graphify.__main__ import main + main() + + +def dispatch_command(cmd: str) -> None: + if cmd == "provider": + from graphify.llm import _custom_providers_path, BACKENDS + import json as _json + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + global_path = _custom_providers_path(global_=True) + + if subcmd == "list": + global_path.parent.mkdir(parents=True, exist_ok=True) + existing: dict = {} + if global_path.is_file(): + try: + existing = _json.loads(global_path.read_text(encoding="utf-8")) + except Exception: + pass + if not existing: + print("No custom providers registered.") + else: + for name in existing: + print(f" {name} ({existing[name].get('base_url', '')})") + + elif subcmd == "show": + name = sys.argv[3] if len(sys.argv) > 3 else "" + if not name: + print("Usage: graphify provider show ", file=sys.stderr) + sys.exit(1) + existing = {} + if global_path.is_file(): + try: + existing = _json.loads(global_path.read_text(encoding="utf-8")) + except Exception: + pass + if name not in existing: + print(f"Provider '{name}' not found.", file=sys.stderr) + sys.exit(1) + print(_json.dumps({name: existing[name]}, indent=2)) + + elif subcmd == "add": + args = sys.argv[3:] + name = args[0] if args and not args[0].startswith("-") else "" + if not name: + print("Usage: graphify provider add --base-url URL --default-model MODEL --env-key KEY", file=sys.stderr) + sys.exit(1) + if name in BACKENDS: + print(f"Error: '{name}' is a built-in provider and cannot be overridden.", file=sys.stderr) + sys.exit(1) + base_url = "" + default_model = "" + env_key = "" + pricing_input = 0.0 + pricing_output = 0.0 + i = 1 + while i < len(args): + a = args[i] + if a == "--base-url" and i + 1 < len(args): + base_url = args[i + 1]; i += 2 + elif a.startswith("--base-url="): + base_url = a.split("=", 1)[1]; i += 1 + elif a == "--default-model" and i + 1 < len(args): + default_model = args[i + 1]; i += 2 + elif a.startswith("--default-model="): + default_model = a.split("=", 1)[1]; i += 1 + elif a == "--env-key" and i + 1 < len(args): + env_key = args[i + 1]; i += 2 + elif a.startswith("--env-key="): + env_key = a.split("=", 1)[1]; i += 1 + elif a == "--pricing-input" and i + 1 < len(args): + pricing_input = float(args[i + 1]); i += 2 + elif a == "--pricing-output" and i + 1 < len(args): + pricing_output = float(args[i + 1]); i += 2 + else: + i += 1 + if not base_url or not default_model or not env_key: + print("Error: --base-url, --default-model, and --env-key are required.", file=sys.stderr) + sys.exit(1) + from graphify.llm import provider_base_url_ok + if not provider_base_url_ok(base_url, name): + print(f"Error: refusing to add provider with unsafe base_url {base_url!r}.", file=sys.stderr) + sys.exit(1) + global_path.parent.mkdir(parents=True, exist_ok=True) + existing = {} + if global_path.is_file(): + try: + existing = _json.loads(global_path.read_text(encoding="utf-8")) + except Exception: + pass + existing[name] = { + "base_url": base_url, + "default_model": default_model, + "env_key": env_key, + "pricing": {"input": pricing_input, "output": pricing_output}, + "temperature": 0, + } + global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") + print(f"Provider '{name}' added. Use with: graphify extract . --backend {name}") + + elif subcmd == "remove": + name = sys.argv[3] if len(sys.argv) > 3 else "" + if not name: + print("Usage: graphify provider remove ", file=sys.stderr) + sys.exit(1) + existing = {} + if global_path.is_file(): + try: + existing = _json.loads(global_path.read_text(encoding="utf-8")) + except Exception: + pass + if name not in existing: + print(f"Provider '{name}' not found.", file=sys.stderr) + sys.exit(1) + del existing[name] + global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") + print(f"Provider '{name}' removed.") + + else: + print("Usage: graphify provider [add|list|show|remove]", file=sys.stderr) + if subcmd: + sys.exit(1) + elif cmd == "prs": + from graphify.prs import cmd_prs + cmd_prs(sys.argv[2:]) + elif cmd == "hook": + from graphify.hooks import ( + install as hook_install, + uninstall as hook_uninstall, + status as hook_status, + ) + + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd == "install": + print(hook_install(Path("."))) + elif subcmd == "uninstall": + print(hook_uninstall(Path("."))) + elif subcmd == "status": + print(hook_status(Path("."))) + else: + print("Usage: graphify hook [install|uninstall|status]", file=sys.stderr) + sys.exit(1) + elif cmd == "query": + if len(sys.argv) < 3: + print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) + sys.exit(1) + from graphify.serve import _query_graph_text + from graphify.security import sanitize_label + from networkx.readwrite import json_graph + from graphify import querylog + + question = sys.argv[2] + use_dfs = "--dfs" in sys.argv + budget = 2000 + graph_path = _default_graph_path() + context_filters: list[str] = [] + args = sys.argv[3:] + i = 0 + while i < len(args): + if args[i] == "--budget" and i + 1 < len(args): + try: + budget = int(args[i + 1]) + except ValueError: + print(f"error: --budget must be an integer", file=sys.stderr) + sys.exit(1) + i += 2 + elif args[i].startswith("--budget="): + try: + budget = int(args[i].split("=", 1)[1]) + except ValueError: + print(f"error: --budget must be an integer", file=sys.stderr) + sys.exit(1) + i += 1 + elif args[i] == "--context" and i + 1 < len(args): + context_filters.append(args[i + 1]) + i += 2 + elif args[i].startswith("--context="): + context_filters.append(args[i].split("=", 1)[1]) + i += 1 + elif args[i] == "--graph" and i + 1 < len(args): + graph_path = args[i + 1] + i += 2 + else: + i += 1 + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + if not gp.suffix == ".json": + print(f"error: graph file must be a .json file", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) + try: + import json as _json + import networkx as _nx + + _raw = _json.loads(gp.read_text(encoding="utf-8")) + if "links" not in _raw and "edges" in _raw: + _raw = dict(_raw, links=_raw["edges"]) + try: + G = json_graph.node_link_graph(_raw, edges="links") + except TypeError: + G = json_graph.node_link_graph(_raw) + try: + from graphify.build import graph_has_legacy_ids as _legacy + if _legacy(_raw.get("nodes", [])): + print( + "[graphify] note: this graph uses the pre-#1504 node-ID scheme; " + "rebuild with `graphify extract --force` to get path-qualified IDs " + "(fixes same-name-file collisions).", + file=sys.stderr, + ) + except Exception: + pass + except Exception as exc: + print(f"error: could not load graph: {exc}", file=sys.stderr) + sys.exit(1) + import time as _time + _t0 = _time.perf_counter() + _mode = "dfs" if use_dfs else "bfs" + _result = _query_graph_text( + G, + question, + mode=_mode, + depth=2, + token_budget=budget, + context_filters=context_filters, + ) + querylog.log_query( + kind="query", + question=question, + corpus=str(gp), + result=_result, + mode=_mode, + depth=2, + token_budget=budget, + duration_ms=(_time.perf_counter() - _t0) * 1000, + ) + print(_result) + elif cmd == "affected": + if len(sys.argv) < 3: + print("Usage: graphify affected \"\" [--relation R] [--depth N] [--graph path]", file=sys.stderr) + sys.exit(1) + from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected, load_graph + query = sys.argv[2] + graph_path = _default_graph_path() + depth = 2 + relations: list[str] = [] + args = sys.argv[3:] + i = 0 + while i < len(args): + if args[i] == "--graph" and i + 1 < len(args): + graph_path = args[i + 1] + i += 2 + elif args[i].startswith("--graph="): + graph_path = args[i].split("=", 1)[1] + i += 1 + elif args[i] == "--depth" and i + 1 < len(args): + try: + depth = int(args[i + 1]) + except ValueError: + print("error: --depth must be an integer", file=sys.stderr) + sys.exit(1) + i += 2 + elif args[i].startswith("--depth="): + try: + depth = int(args[i].split("=", 1)[1]) + except ValueError: + print("error: --depth must be an integer", file=sys.stderr) + sys.exit(1) + i += 1 + elif args[i] == "--relation" and i + 1 < len(args): + relations.append(args[i + 1]) + i += 2 + elif args[i].startswith("--relation="): + relations.append(args[i].split("=", 1)[1]) + i += 1 + else: + i += 1 + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + if not gp.suffix == ".json": + print("error: graph file must be a .json file", file=sys.stderr) + sys.exit(1) + try: + graph = load_graph(gp) + except Exception as exc: + print(f"error: could not load graph: {exc}", file=sys.stderr) + sys.exit(1) + print( + format_affected( + graph, + query, + relations=relations or DEFAULT_AFFECTED_RELATIONS, + depth=depth, + ) + ) + elif cmd == "save-result": + # graphify save-result --question Q --answer A [--type T] [--nodes N1 N2 ...] + # [--outcome useful|dead_end|corrected] [--correction TEXT] + import argparse as _ap + + p = _ap.ArgumentParser(prog="graphify save-result") + p.add_argument("--question", required=True) + p.add_argument("--answer", default=None) + p.add_argument("--answer-file", dest="answer_file", default=None) + p.add_argument("--type", dest="query_type", default="query") + p.add_argument("--nodes", nargs="*", default=[]) + p.add_argument("--outcome", choices=("useful", "dead_end", "corrected"), default=None) + p.add_argument("--correction", default=None) + p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) + opts = p.parse_args(sys.argv[2:]) + if opts.answer_file: + opts.answer = Path(opts.answer_file).read_text(encoding="utf-8").strip() + elif not opts.answer: + p.error("--answer or --answer-file is required") + from graphify.ingest import save_query_result as _sqr + + out = _sqr( + question=opts.question, + answer=opts.answer, + memory_dir=Path(opts.memory_dir), + query_type=opts.query_type, + source_nodes=opts.nodes or None, + outcome=opts.outcome, + correction=opts.correction, + ) + print(f"Saved to {out}") + elif cmd == "reflect": + import argparse as _ap + + p = _ap.ArgumentParser(prog="graphify reflect") + p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) + p.add_argument( + "--out", + default=str(Path(_GRAPHIFY_OUT) / "reflections" / "LESSONS.md"), + ) + p.add_argument("--graph", default=None) + p.add_argument("--analysis", default=None) + p.add_argument("--labels", default=None) + p.add_argument("--half-life-days", type=float, default=30.0, + help="signal weight halves every N days (default 30)") + p.add_argument("--min-corroboration", type=int, default=2, + help="distinct useful results to promote a node to preferred (default 2)") + p.add_argument("--if-stale", action="store_true", + help="skip when LESSONS.md is already newer than every input " + "(e.g. the git hook just refreshed it)") + opts = p.parse_args(sys.argv[2:]) + from graphify.reflect import reflect as _reflect, lessons_fresh as _lessons_fresh + + graph_arg = opts.graph + if graph_arg is None: + default_graph = Path(_GRAPHIFY_OUT) / "graph.json" + if default_graph.exists(): + graph_arg = str(default_graph) + + _gp = Path(graph_arg) if graph_arg else None + _analysis_path = None + _labels_path = None + if _gp is not None: + _analysis_path = Path(opts.analysis) if opts.analysis else ( + _gp.parent / ".graphify_analysis.json") + _labels_path = Path(opts.labels) if opts.labels else ( + _gp.parent / ".graphify_labels.json") + + if opts.if_stale and _lessons_fresh( + Path(opts.out), Path(opts.memory_dir), _gp, _analysis_path, _labels_path + ): + print(f"Lessons already up to date -> {opts.out} (skipped; omit --if-stale to force)") + else: + out_path, agg = _reflect( + memory_dir=Path(opts.memory_dir), + out_path=Path(opts.out), + graph_path=_gp, + analysis_path=_analysis_path, + labels_path=_labels_path, + half_life_days=opts.half_life_days, + min_corroboration=opts.min_corroboration, + ) + c = agg["counts"] + print( + f"Reflected {agg['total']} memories " + f"({c['useful']} useful, {c['dead_end']} dead ends, " + f"{c['corrected']} corrected) -> {out_path}" + ) + elif cmd == "path": + if len(sys.argv) < 4: + print( + 'Usage: graphify path "" "" [--graph path]', + file=sys.stderr, + ) + sys.exit(1) + from graphify.serve import _score_nodes + from networkx.readwrite import json_graph + import networkx as _nx + + source_label = sys.argv[2] + target_label = sys.argv[3] + graph_path = _default_graph_path() + args = sys.argv[4:] + for i, a in enumerate(args): + if a == "--graph" and i + 1 < len(args): + graph_path = args[i + 1] + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) + _raw = json.loads(gp.read_text(encoding="utf-8")) + if "links" not in _raw and "edges" in _raw: + _raw = dict(_raw, links=_raw["edges"]) + # Force directed so the renderer can recover stored caller→callee direction. + _raw = {**_raw, "directed": True} + try: + G = json_graph.node_link_graph(_raw, edges="links") + except TypeError: + G = json_graph.node_link_graph(_raw) + src_scored = _score_nodes(G, [t.lower() for t in source_label.split()]) + tgt_scored = _score_nodes(G, [t.lower() for t in target_label.split()]) + if not src_scored: + print(f"No node matching '{source_label}' found.", file=sys.stderr) + sys.exit(1) + if not tgt_scored: + print(f"No node matching '{target_label}' found.", file=sys.stderr) + sys.exit(1) + src_nid, tgt_nid = src_scored[0][1], tgt_scored[0][1] + # Ambiguity guard: when both queries resolve to the same node, the + # shortest path is trivially zero hops, which is almost never what the + # caller wanted (see bug #828). + if src_nid == tgt_nid: + print( + f"'{source_label}' and '{target_label}' both resolved to the same " + f"node '{src_nid}'. Use a more specific label or the exact node ID.", + file=sys.stderr, + ) + sys.exit(1) + for _name, _scored in (("source", src_scored), ("target", tgt_scored)): + if len(_scored) >= 2: + _top, _runner = _scored[0][0], _scored[1][0] + if _top > 0 and (_top - _runner) / _top < 0.10: + print( + f"warning: {_name} match was ambiguous " + f"(top score {_top:g}, runner-up {_runner:g})", + file=sys.stderr, + ) + try: + path_nodes = _nx.shortest_path(G.to_undirected(as_view=True), src_nid, tgt_nid) + except (_nx.NetworkXNoPath, _nx.NodeNotFound): + print(f"No path found between '{source_label}' and '{target_label}'.") + sys.exit(0) + hops = len(path_nodes) - 1 + segments = [] + from graphify.build import edge_data + for i in range(len(path_nodes) - 1): + u, v = path_nodes[i], path_nodes[i + 1] + # Check which direction the stored edge points. + if G.has_edge(u, v): + edata = edge_data(G, u, v) + forward = True + else: + edata = edge_data(G, v, u) + forward = False + rel = edata.get("relation", "") + conf = edata.get("confidence", "") + conf_str = f" [{conf}]" if conf else "" + if i == 0: + segments.append(G.nodes[u].get("label", u)) + if forward: + segments.append(f"--{rel}{conf_str}--> {G.nodes[v].get('label', v)}") + else: + segments.append(f"<--{rel}{conf_str}-- {G.nodes[v].get('label', v)}") + print(f"Shortest path ({hops} hops):\n " + " ".join(segments)) + from graphify import querylog + querylog.log_query( + kind="path", + question=f"{sys.argv[2]} -> {sys.argv[3]}", + corpus=str(gp), + nodes_returned=hops, + ) + + elif cmd == "explain": + if len(sys.argv) < 3: + print('Usage: graphify explain "" [--graph path]', file=sys.stderr) + sys.exit(1) + from graphify.serve import _find_node + from networkx.readwrite import json_graph + + label = sys.argv[2] + graph_path = _default_graph_path() + args = sys.argv[3:] + for i, a in enumerate(args): + if a == "--graph" and i + 1 < len(args): + graph_path = args[i + 1] + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) + _raw = json.loads(gp.read_text(encoding="utf-8")) + if "links" not in _raw and "edges" in _raw: + _raw = dict(_raw, links=_raw["edges"]) + # Force directed so the renderer can recover stored caller→callee direction. + _raw = {**_raw, "directed": True} + try: + G = json_graph.node_link_graph(_raw, edges="links") + except TypeError: + G = json_graph.node_link_graph(_raw) + matches = _find_node(G, label) + if not matches: + print(f"No node matching '{label}' found.") + sys.exit(0) + nid = matches[0] + d = G.nodes[nid] + print(f"Node: {d.get('label', nid)}") + print(f" ID: {nid}") + print( + f" Source: {d.get('source_file', '')} {d.get('source_location', '')}".rstrip() + ) + print(f" Type: {d.get('file_type', '')}") + print(f" Community: {d.get('community_name') or d.get('community', '')}") + # Work-memory overlay: a derived experiential hint from `graphify reflect`, + # merged in display-only from the .graphify_learning.json sidecar next to + # graph.json. No line when the node has no overlay entry. + try: + from graphify.reflect import load_learning_overlay as _llo + from graphify.security import sanitize_label as _sl + _overlay = _llo(gp) + _entry = _overlay.get(str(nid)) + if _entry: + _status = _sl(str(_entry.get("status", ""))) + if _status == "contested": + _line = (f" Lesson: contested (useful {_entry.get('uses', 0)} / " + f"dead-end {_entry.get('neg', 0)})") + elif _status == "preferred": + _line = (f" Lesson: preferred source (start here) — " + f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") + else: + _line = (f" Lesson: {_status or 'tentative'} — " + f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") + if _entry.get("stale"): + _line += " [code changed since — re-verify]" + print(_line) + except Exception: + pass + print(f" Degree: {G.degree(nid)}") + from graphify.build import edge_data + connections: list[tuple[str, str, dict]] = [] # (direction, neighbor_id, edge_data) + for nb in G.successors(nid): + connections.append(("out", nb, edge_data(G, nid, nb))) + for nb in G.predecessors(nid): + connections.append(("in", nb, edge_data(G, nb, nid))) + if connections: + print(f"\nConnections ({len(connections)}):") + connections.sort(key=lambda c: G.degree(c[1]), reverse=True) + for direction, nb, edata in connections[:20]: + rel = edata.get("relation", "") + conf = edata.get("confidence", "") + arrow = "-->" if direction == "out" else "<--" + print(f" {arrow} {G.nodes[nb].get('label', nb)} [{rel}] [{conf}]") + if len(connections) > 20: + print(f" ... and {len(connections) - 20} more") + from graphify import querylog + querylog.log_query( + kind="explain", + question=sys.argv[2], + corpus=str(gp), + nodes_returned=len(connections), + ) + + elif cmd == "diagnose": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd != "multigraph": + print( + "Usage: graphify diagnose multigraph " + "[--graph path] [--json] [--max-examples N] " + "[--directed] [--undirected] [--extract-path path]", + file=sys.stderr, + ) + sys.exit(1) + + graph_path = Path(_default_graph_path()) + max_examples = 5 + directed: bool | None = None + direction_flag: str | None = None + json_output = False + extract_path: Path | None = None + + i = 3 + while i < len(sys.argv): + arg = sys.argv[i] + if arg == "--graph": + i += 1 + if i >= len(sys.argv): + print("error: --graph requires a path", file=sys.stderr) + sys.exit(1) + graph_path = Path(sys.argv[i]) + elif arg == "--json": + json_output = True + elif arg == "--max-examples": + i += 1 + if i >= len(sys.argv): + print("error: --max-examples requires an integer", file=sys.stderr) + sys.exit(1) + try: + max_examples = int(sys.argv[i]) + except ValueError: + print("error: --max-examples requires an integer", file=sys.stderr) + sys.exit(1) + if max_examples < 0: + print("error: --max-examples must be >= 0", file=sys.stderr) + sys.exit(1) + elif arg == "--directed": + if direction_flag == "undirected": + print( + "error: --directed and --undirected are mutually exclusive", + file=sys.stderr, + ) + sys.exit(1) + direction_flag = "directed" + directed = True + elif arg == "--undirected": + if direction_flag == "directed": + print( + "error: --directed and --undirected are mutually exclusive", + file=sys.stderr, + ) + sys.exit(1) + direction_flag = "undirected" + directed = False + elif arg == "--extract-path": + i += 1 + if i >= len(sys.argv): + print("error: --extract-path requires a path", file=sys.stderr) + sys.exit(1) + extract_path = Path(sys.argv[i]) + else: + print(f"error: unknown diagnose option {arg}", file=sys.stderr) + sys.exit(1) + i += 1 + + from graphify.diagnostics import ( + diagnose_file, + format_diagnostic_json, + format_diagnostic_report, + ) + + try: + summary = diagnose_file( + graph_path, + directed=directed, + root=Path(".").resolve(), + max_examples=max_examples, + extract_path=extract_path, + ) + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + + if json_output: + print(json.dumps(format_diagnostic_json(summary), indent=2)) + else: + print(format_diagnostic_report(summary)) + + elif cmd == "add": + if len(sys.argv) < 3: + print( + "Usage: graphify add [--author Name] [--contributor Name] [--dir ./raw]", + file=sys.stderr, + ) + sys.exit(1) + from graphify.ingest import ingest as _ingest + + url = sys.argv[2] + author: str | None = None + contributor: str | None = None + target_dir = Path("raw") + args = sys.argv[3:] + i = 0 + while i < len(args): + if args[i] == "--author" and i + 1 < len(args): + author = args[i + 1] + i += 2 + elif args[i] == "--contributor" and i + 1 < len(args): + contributor = args[i + 1] + i += 2 + elif args[i] == "--dir" and i + 1 < len(args): + target_dir = Path(args[i + 1]) + i += 2 + else: + i += 1 + try: + saved = _ingest(url, target_dir, author=author, contributor=contributor) + print(f"Saved to {saved}") + print("Run /graphify --update in your AI assistant to update the graph.") + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + + elif cmd == "watch": + watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") + if not watch_path.exists(): + print(f"error: path not found: {watch_path}", file=sys.stderr) + sys.exit(1) + from graphify.watch import watch as _watch + + try: + _watch(watch_path) + except ImportError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + + elif cmd in ("cluster-only", "label"): + # `label` is `cluster-only` that always (re)generates community names with + # the configured backend, even when a .graphify_labels.json already exists. + force_relabel = cmd == "label" + # Mirror the tree/export arg-parsing pattern: walk argv so flags and + # the optional positional path can appear in any order (#724). + no_viz = "--no-viz" in sys.argv + no_label = "--no-label" in sys.argv + missing_only = "--missing-only" in sys.argv + co_timing = "--timing" in sys.argv + _backend_arg = next((a for a in sys.argv if a.startswith("--backend=")), None) + label_backend = _backend_arg.split("=", 1)[1] if _backend_arg else None + _model_arg = next((a for a in sys.argv if a.startswith("--model=")), None) + label_model = _model_arg.split("=", 1)[1] if _model_arg else None + _min_cs_arg = next((a for a in sys.argv if a.startswith("--min-community-size=")), None) + min_community_size = int(_min_cs_arg.split("=")[1]) if _min_cs_arg else 3 + args = sys.argv[2:] + watch_path: Path | None = None + graph_override: Path | None = None + co_resolution: float = 1.0 + co_exclude_hubs: float | None = None + label_max_concurrency: int = 4 + label_batch_size: int = 100 + i_arg = 0 + while i_arg < len(args): + a = args[i_arg] + if a == "--graph" and i_arg + 1 < len(args): + graph_override = Path(args[i_arg + 1]); i_arg += 2 + elif a == "--backend" and i_arg + 1 < len(args): + label_backend = args[i_arg + 1]; i_arg += 2 + elif a.startswith("--backend="): + label_backend = a.split("=", 1)[1]; i_arg += 1 + elif a == "--model" and i_arg + 1 < len(args): + label_model = args[i_arg + 1]; i_arg += 2 + elif a.startswith("--model="): + label_model = a.split("=", 1)[1]; i_arg += 1 + elif a == "--resolution" and i_arg + 1 < len(args): + co_resolution = float(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--resolution="): + co_resolution = float(a.split("=", 1)[1]); i_arg += 1 + elif a == "--exclude-hubs" and i_arg + 1 < len(args): + co_exclude_hubs = float(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--exclude-hubs="): + co_exclude_hubs = float(a.split("=", 1)[1]); i_arg += 1 + elif a == "--max-concurrency" and i_arg + 1 < len(args): + label_max_concurrency = int(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--max-concurrency="): + label_max_concurrency = int(a.split("=", 1)[1]); i_arg += 1 + elif a == "--batch-size" and i_arg + 1 < len(args): + label_batch_size = int(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--batch-size="): + label_batch_size = int(a.split("=", 1)[1]); i_arg += 1 + elif a in ("--no-viz", "--missing-only") or a.startswith("--min-community-size="): + i_arg += 1 + elif a.startswith("--"): + i_arg += 1 + elif watch_path is None: + watch_path = Path(a); i_arg += 1 + else: + i_arg += 1 + if watch_path is None: + watch_path = Path(".") + graph_json = graph_override if graph_override is not None else watch_path / _GRAPHIFY_OUT / "graph.json" + if not graph_json.exists(): + print( + f"error: no graph found at {graph_json} — run /graphify first", + file=sys.stderr, + ) + sys.exit(1) + from networkx.readwrite import json_graph as _jg + from graphify.build import build_from_json + from graphify.cluster import cluster, score_all, remap_communities_to_previous + from graphify.analyze import ( + god_nodes, + surprising_connections, + suggest_questions, + ) + from graphify.report import generate + from graphify.export import to_json, to_html + + stages = _StageTimer(co_timing) + print("Loading existing graph...") + # Solution 3 (#1019): don't hard-exit on an oversized graph.json here. + # Core outputs (graph.json + GRAPH_REPORT.md) still get written; the + # graph.html render below falls back to the community-aggregation view + # (node_limit=5000) when over the cap. + from graphify.security import check_graph_file_size_cap as _check_cap + _over_cap = False + try: + _check_cap(graph_json) + except ValueError: + _over_cap = True + try: + _over_cap_bytes = graph_json.stat().st_size + except OSError: + _over_cap_bytes = -1 + print( + f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " + f"falling back to community-aggregation view (node_limit=5000)", + file=sys.stderr, + ) + _raw = json.loads(graph_json.read_text(encoding="utf-8")) + _directed = bool(_raw.get("directed", False)) + G = build_from_json(_raw, directed=_directed) + print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") + stages.mark("load") + print("Re-clustering...") + communities = cluster(G, resolution=co_resolution, exclude_hubs_percentile=co_exclude_hubs) + # Mirror the watch/update path (#822): map new cids to prior ones by + # node-overlap so the existing .graphify_labels.json keeps attaching + # to the same conceptual community after re-clustering. Without this, + # labels follow raw cid index and become misaligned whenever the + # graph has changed between labeling and cluster-only (#1027). + previous_node_community = { + n["id"]: n["community"] + for n in _raw.get("nodes", []) + if n.get("community") is not None and n.get("id") is not None + } + if previous_node_community: + communities = remap_communities_to_previous(communities, previous_node_community) + stages.mark("cluster") + cohesion = score_all(G, communities) + gods = god_nodes(G) + surprises = surprising_connections(G, communities) + stages.mark("analyze") + out = watch_path / _GRAPHIFY_OUT + out.mkdir(parents=True, exist_ok=True) + labels_path = out / ".graphify_labels.json" + existing_labels: dict[int, str] = {} + if labels_path.exists(): + try: + existing_labels = { + int(k): v + for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items() + if isinstance(v, str) + } + except Exception: + existing_labels = {} + # Accumulate token usage from the labeling LLM calls so cluster-only mode + # reports real cost instead of a hardcoded zero (#1694). Stays {0, 0} on + # the reuse / no-label paths, which make no LLM calls. + label_token_usage = {"input": 0, "output": 0} + if labels_path.exists() and not force_relabel: + # Reuse saved labels, but don't blindly trust them: the graph may have + # been re-scoped/re-clustered since labeling, in which case a cid now + # covers a DIFFERENT community and its old (LLM) name is wrong (#label-stale). + # Validate each community against the membership signature saved beside the + # labels; any community that changed (or has no saved label) is renamed by + # its current hub — deterministic and correct-by-construction — and the user + # is told to `graphify label` for fresh LLM names. Unchanged communities keep + # their saved label. When no signature sidecar exists (labels predate this), + # fall back to hub-filling only the communities missing a label. + from graphify.cluster import community_member_sigs, label_communities_by_hub + sig_path = labels_path.parent / (labels_path.name + ".sig") + saved_sigs: dict[int, str] = {} + if sig_path.exists(): + try: + saved_sigs = { + int(k): v for k, v in + json.loads(sig_path.read_text(encoding="utf-8")).items() + if isinstance(v, str) + } + except Exception: + saved_sigs = {} + cur_sigs = community_member_sigs(communities) + count_mismatch = len(existing_labels) != len(communities) + labels = {} + hub_labels: dict[int, str] | None = None + changed = 0 + for cid in communities: + have_label = cid in existing_labels + if saved_sigs: + # Precise: the membership signature tells us if this exact + # community changed since it was labeled. + fresh = have_label and saved_sigs.get(cid) == cur_sigs.get(cid) + else: + # No signature sidecar (labels predate it). A differing community + # COUNT means the labels describe a different clustering, so a cid's + # old label can't be trusted; equal count is the best "same" signal. + fresh = have_label and not count_mismatch + if fresh: + labels[cid] = existing_labels[cid] + else: + if hub_labels is None: + hub_labels = label_communities_by_hub(G, communities) + labels[cid] = hub_labels[cid] + if have_label: + changed += 1 + if changed: + print( + f"[graphify] community set changed since labeling " + f"({len(existing_labels)} saved labels, {len(communities)} communities now; " + f"renamed {changed} community(ies) by their hub). " + f"Run `graphify label` to refresh names with the LLM.", + file=sys.stderr, + ) + elif no_label and not force_relabel: + labels = {cid: f"Community {cid}" for cid in communities} + else: + # No labels file yet (or `graphify label` forced a refresh). When run + # standalone there is no orchestrating agent to do skill.md Step 5, so + # auto-name communities rather than leave "Community N" (#1097). + from graphify.cluster import label_communities_by_hub + from graphify.llm import generate_community_labels + print("Labeling communities...") + # Deterministic, LLM-free base labels: name each community after its + # highest-degree hub, so the report is readable even with no backend + # (previously bare "Community N"). A configured LLM backend overrides these + # with richer names below; its no-backend placeholder fallback does NOT. + hub_labels = label_communities_by_hub(G, communities) + label_communities_input = communities + labels = dict(hub_labels) + if missing_only: + labels = { + cid: existing_labels.get(cid, hub_labels[cid]) + for cid in communities + } + label_communities_input = { + cid: members + for cid, members in communities.items() + if cid not in existing_labels or existing_labels.get(cid) == f"Community {cid}" + } + generated_labels, _ = generate_community_labels( + G, label_communities_input, backend=label_backend, model=label_model, gods=gods, + max_concurrency=label_max_concurrency, batch_size=label_batch_size, + usage_out=label_token_usage, + ) + # Only let the LLM OVERRIDE where it produced a real name — its no-backend + # fallback returns "Community {cid}" placeholders, which must not clobber + # the deterministic hub labels. + labels.update({ + cid: v for cid, v in generated_labels.items() + if v and v != f"Community {cid}" + }) + stages.mark("label") + questions = suggest_questions(G, communities, labels) + tokens = label_token_usage + from graphify.export import _git_head as _gh + _commit = _gh() + from graphify.report import load_learning_for_report as _llfr + report = generate(G, communities, cohesion, labels, gods, surprises, + {"warning": "cluster-only mode — file stats not available"}, + tokens, str(watch_path), suggested_questions=questions, + min_community_size=min_community_size, built_at_commit=_commit, + learning=_llfr(out / "graph.json")) + (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") + stages.mark("report") + from graphify.export import backup_if_protected as _backup + _backup(out) + analysis = { + "communities": {str(k): v for k, v in communities.items()}, + "cohesion": {str(k): v for k, v in cohesion.items()}, + "gods": gods, + "surprises": surprises, + "questions": questions, + } + (out / ".graphify_analysis.json").write_text( + json.dumps(analysis, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + to_json(G, communities, str(out / "graph.json"), community_labels=labels) + labels_path.write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8") + # Membership signatures beside the labels so a later cluster-only can detect + # which communities changed and avoid reusing a stale label (see reuse above). + from graphify.cluster import community_member_sigs as _cms + (labels_path.parent / (labels_path.name + ".sig")).write_text( + json.dumps({str(k): v for k, v in _cms(communities).items()}), encoding="utf-8") + + # Mirror watch.py pattern: gate to_html so core outputs (graph.json + + # GRAPH_REPORT.md) always land. Honor --no-viz explicitly; otherwise + # fall back to ValueError handling so an oversized graph doesn't crash + # the CLI mid-write and leave a stale graph.html on disk. + html_target = out / "graph.html" + if no_viz: + if html_target.exists(): + html_target.unlink() + stages.mark("export"); stages.total() + print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated (--no-viz; graph.html removed).") + else: + try: + # Over-cap fallback (#1019): force the community-aggregation + # path so an oversized graph still renders a usable graph.html. + _node_limit = 5000 if _over_cap else None + to_html(G, communities, str(html_target), community_labels=labels or None, + node_limit=_node_limit) + stages.mark("export"); stages.total() + print(f"Done - {len(communities)} communities. GRAPH_REPORT.md, graph.json and graph.html updated.") + except ValueError as viz_err: + if html_target.exists(): + html_target.unlink() + print(f"Skipped graph.html: {viz_err}") + stages.mark("export"); stages.total() + print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated.") + + elif cmd == "update": + force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") + no_cluster = False + args = sys.argv[2:] + watch_arg: str | None = None + for a in args: + if a == "--force": + force = True + continue + if a == "--no-cluster": + no_cluster = True + continue + if a.startswith("-"): + print(f"error: unknown update option: {a}", file=sys.stderr) + sys.exit(2) + if watch_arg is not None: + print("error: update accepts at most one path argument", file=sys.stderr) + sys.exit(2) + watch_arg = a + + if watch_arg is not None: + watch_path = Path(watch_arg) + else: + # Try to recover the scan root saved by the last full build + saved = Path(_GRAPHIFY_OUT) / ".graphify_root" + if saved.exists(): + watch_path = Path(saved.read_text(encoding="utf-8").strip()) + else: + watch_path = Path(".") + if not watch_path.exists(): + print(f"error: path not found: {watch_path}", file=sys.stderr) + sys.exit(1) + from graphify.watch import _rebuild_code + + print(f"Re-extracting code files in {watch_path} (no LLM needed)...") + # Interactive CLI: block on the per-repo lock rather than skip, so the + # user sees their explicit `graphify update` complete instead of + # exiting silently when a hook-driven rebuild happens to be running. + ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) + if ok: + print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") + if not ( + os.environ.get("GEMINI_API_KEY") + or os.environ.get("GOOGLE_API_KEY") + or os.environ.get("MOONSHOT_API_KEY") + or os.environ.get("DEEPSEEK_API_KEY") + or os.environ.get("GRAPHIFY_NO_TIPS") + ): + print("Tip: set GEMINI_API_KEY or GOOGLE_API_KEY to use Gemini for semantic extraction.") + else: + print( + "Nothing to update or rebuild failed — check output above.", + file=sys.stderr, + ) + sys.exit(1) + + elif cmd == "hook-check": + # Codex Desktop rejects hookSpecificOutput.additionalContext on PreToolUse. + # Keep this as a cross-platform no-op so installed hooks never break Bash + # tool calls. Graph guidance reaches the agent via AGENTS.md / skill instead. + sys.exit(0) + elif cmd == "hook-guard": + # Shell-agnostic Claude/Codebuddy PreToolUse guard (#522). Replaces the old + # inline-bash hooks that failed on Windows. Prints an additionalContext nudge + # toward graphify when a graph exists; always exits 0 (never blocks a tool). + _run_hook_guard(sys.argv[2] if len(sys.argv) > 2 else "") + sys.exit(0) + elif cmd == "check-update": + if len(sys.argv) < 3: + print("Usage: graphify check-update ", file=sys.stderr) + sys.exit(1) + from graphify.watch import check_update + + check_update(Path(sys.argv[2]).resolve()) + sys.exit(0) + elif cmd == "tree": + # Emit a D3 v7 collapsible-tree HTML view of graph.json: + # expand-all / collapse-all / reset-view buttons, multi-line + # wrapText labels with separately-coloured name + count, + # depth-based palette, click-to-toggle subtree, hover inspector + # showing top-K outbound edges per symbol. + from typing import Optional as _Opt + from graphify.tree_html import write_tree_html, DEFAULT_MAX_CHILDREN + graph_path = Path(_GRAPHIFY_OUT) / "graph.json" + output_path: "_Opt[Path]" = None + root: "_Opt[str]" = None + max_children = DEFAULT_MAX_CHILDREN + top_k_edges = 0 + project_label: "_Opt[str]" = None + args = sys.argv[2:] + i_arg = 0 + while i_arg < len(args): + a = args[i_arg] + if a == "--graph" and i_arg + 1 < len(args): + graph_path = Path(args[i_arg + 1]); i_arg += 2 + elif a == "--output" and i_arg + 1 < len(args): + output_path = Path(args[i_arg + 1]); i_arg += 2 + elif a == "--root" and i_arg + 1 < len(args): + root = args[i_arg + 1]; i_arg += 2 + elif a == "--max-children" and i_arg + 1 < len(args): + max_children = int(args[i_arg + 1]); i_arg += 2 + elif a == "--top-k-edges" and i_arg + 1 < len(args): + top_k_edges = int(args[i_arg + 1]); i_arg += 2 + elif a == "--label" and i_arg + 1 < len(args): + project_label = args[i_arg + 1]; i_arg += 2 + elif a in ("-h", "--help"): + print("Usage: graphify tree [--graph PATH] [--output HTML]") + print(" --graph PATH path to graph.json (default graphify-out/graph.json)") + print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") + print(" --root PATH filesystem root (default: longest common dir of all source_files)") + print(" --max-children N cap visible children per node (default 200)") + print(" --top-k-edges N pre-compute top-K outbound edges per symbol (default 12)") + print(" --label NAME project label shown in the page header") + return + else: + i_arg += 1 + if not graph_path.is_file(): + print(f"error: graph.json not found at {graph_path}", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(graph_path) + if output_path is None: + output_path = graph_path.parent / "GRAPH_TREE.html" + out = write_tree_html( + graph_path=graph_path, output_path=output_path, + root=root, max_children=max_children, + top_k_edges=top_k_edges, project_label=project_label, + ) + size_kb = out.stat().st_size / 1024 + print(f"wrote {out} ({size_kb:.1f} KB)") + print(f"open with: xdg-open {out} (or file://{out.resolve()})") + sys.exit(0) + + elif cmd == "merge-driver": + # git merge driver for graph.json — takes (base, current, other) and writes + # the union of current+other nodes/edges back to current. Exits 1 on + # corrupt input so git surfaces the conflict instead of silently + # accepting a poisoned merge (see F-005). + # Usage: graphify merge-driver %O %A %B (set in .git/config merge driver) + if len(sys.argv) < 5: + print("Usage: graphify merge-driver ", file=sys.stderr) + sys.exit(1) + _base_path, _current_path, _other_path = sys.argv[2], sys.argv[3], sys.argv[4] + # Hard caps so a malicious or corrupted graph.json cannot exhaust memory + # at parse time. 50 MB / 100k nodes are well above any realistic graph + # (typical graphs are <5 MB / <50k nodes); anything larger should fail + # the merge so a human can investigate. + _MERGE_MAX_BYTES = 50 * 1024 * 1024 + _MERGE_MAX_NODES = 100_000 + import networkx as _nx + from networkx.readwrite import json_graph as _jg + def _load_graph(p: str): + path_obj = Path(p) + try: + size = path_obj.stat().st_size + except OSError as exc: + raise RuntimeError(f"cannot stat {p}: {exc}") from exc + if size > _MERGE_MAX_BYTES: + raise RuntimeError( + f"graph.json {p} is {size} bytes, exceeds {_MERGE_MAX_BYTES}-byte cap" + ) + data = json.loads(path_obj.read_text(encoding="utf-8")) + try: + return _jg.node_link_graph(data, edges="links"), data + except TypeError: + return _jg.node_link_graph(data), data + try: + G_cur, _ = _load_graph(_current_path) + G_oth, _ = _load_graph(_other_path) + except Exception as exc: + print(f"[graphify merge-driver] error loading graphs: {exc}", file=sys.stderr) + sys.exit(1) # surface the conflict so git doesn't accept a corrupt merge + merged = _nx.compose(G_cur, G_oth) + if merged.number_of_nodes() > _MERGE_MAX_NODES: + print( + f"[graphify merge-driver] merged graph has {merged.number_of_nodes()} nodes, " + f"exceeds {_MERGE_MAX_NODES}-node cap; aborting merge.", + file=sys.stderr, + ) + sys.exit(1) + try: + out_data = _jg.node_link_data(merged, edges="links") + except TypeError: + out_data = _jg.node_link_data(merged) + Path(_current_path).write_text(json.dumps(out_data, indent=2), encoding="utf-8") + sys.exit(0) + + elif cmd == "merge-graphs": + # graphify merge-graphs graph1.json graph2.json ... --out merged.json + args = sys.argv[2:] + graph_paths: list[Path] = [] + out_path = Path(_GRAPHIFY_OUT) / "merged-graph.json" + i = 0 + while i < len(args): + if args[i] == "--out" and i + 1 < len(args): + out_path = Path(args[i + 1]) + i += 2 + else: + graph_paths.append(Path(args[i])) + i += 1 + if len(graph_paths) < 2: + print( + "Usage: graphify merge-graphs [...] [--out merged.json]", + file=sys.stderr, + ) + sys.exit(1) + import networkx as _nx + from networkx.readwrite import json_graph as _jg + from graphify.build import prefix_graph_for_global as _prefix, distinct_repo_tags as _repo_tags + graphs = [] + for gp in graph_paths: + if not gp.exists(): + print(f"error: not found: {gp}", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) + data = json.loads(gp.read_text(encoding="utf-8")) + # Normalize edges/links key before loading — graphify writes "links" + # via node_link_data but older runs may have used "edges" (#738). + if "links" not in data and "edges" in data: + data = dict(data, links=data["edges"]) + try: + G = _jg.node_link_graph(data, edges="links") + except TypeError: + G = _jg.node_link_graph(data) + graphs.append(G) + # nx.compose requires all graphs to be the same type. When input graphs + # come from different sources (e.g. an AST-only run vs a full LLM run) one + # may be a MultiGraph and another a Graph. Normalise everything to Graph + # (the graphify default) by converting MultiGraphs with nx.Graph(). + def _to_simple(g: "_nx.Graph") -> "_nx.Graph": + # nx.compose requires every graph to be the same type. Inputs may + # disagree on BOTH axes — directed vs undirected, and multi vs simple + # — because per-repo graph.json files are written by different extract + # paths at different times. Normalise everything to a plain undirected + # Graph (the merged cross-repo view is undirected anyway), which covers + # DiGraph / MultiGraph / MultiDiGraph. Without this a directed input + # crashed compose with "All graphs must be directed or undirected" (#1606). + if type(g) is not _nx.Graph: + return _nx.Graph(g) + return g + # Unique repo tag per graph. The bare `graphify-out/..` dir name is not + # unique across inputs (src/graphify-out and frontend/src/graphify-out both + # → "src"), which collides same-stem node ids and silently merges unrelated + # entities (#1729). distinct_repo_tags guarantees a distinct prefix per graph. + repo_tags = _repo_tags(graph_paths) + naive_tags = [gp.parent.parent.name for gp in graph_paths] + if len(set(naive_tags)) != len(naive_tags): + print(f" note: repo dir names collide; using distinct tags: {', '.join(repo_tags)}") + merged = _nx.Graph() + for G, repo_tag in zip(graphs, repo_tags): + prefixed = _to_simple(_prefix(G, repo_tag)) + merged = _nx.compose(merged, prefixed) + try: + out_data = _jg.node_link_data(merged, edges="links") + except TypeError: + out_data = _jg.node_link_data(merged) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(out_data, indent=2), encoding="utf-8") + print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges") + print(f"Written to: {out_path}") + + elif cmd == "clone": + if len(sys.argv) < 3: + print( + "Usage: graphify clone [--branch ] [--out ]", + file=sys.stderr, + ) + sys.exit(1) + url = sys.argv[2] + branch: str | None = None + out_dir: Path | None = None + args = sys.argv[3:] + i = 0 + while i < len(args): + if args[i] == "--branch" and i + 1 < len(args): + branch = args[i + 1] + i += 2 + elif args[i] == "--out" and i + 1 < len(args): + out_dir = Path(args[i + 1]) + i += 2 + else: + i += 1 + local_path = _clone_repo(url, branch=branch, out_dir=out_dir) + print(local_path) + + elif cmd == "export": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd not in ("html", "callflow-html", "obsidian", "wiki", "svg", "graphml", "neo4j", "falkordb"): + print("Usage: graphify export ", file=sys.stderr) + print(" html [--graph PATH] [--labels PATH] [--node-limit N] [--no-viz]", file=sys.stderr) + print(" callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH] [--report PATH] [--sections PATH] [--output HTML]", file=sys.stderr) + print(" [--lang auto|zh-CN|en] [--max-sections N] [--diagram-scale N]", file=sys.stderr) + print(" obsidian [--graph PATH] [--labels PATH] [--dir PATH]", file=sys.stderr) + print(" wiki [--graph PATH] [--labels PATH]", file=sys.stderr) + print(" svg [--graph PATH] [--labels PATH]", file=sys.stderr) + print(" graphml [--graph PATH]", file=sys.stderr) + print(" neo4j [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) + print(" (or set NEO4J_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) + print(" falkordb [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) + print(" (or set FALKORDB_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) + sys.exit(1) + + # Parse shared args + args = sys.argv[3:] + graph_path = Path(_GRAPHIFY_OUT) / "graph.json" + graph_path_explicit = False + labels_path = Path(_GRAPHIFY_OUT) / ".graphify_labels.json" + labels_path_explicit = False + report_path = Path(_GRAPHIFY_OUT) / "GRAPH_REPORT.md" + report_path_explicit = False + sections_path: Path | None = None + callflow_output: Path | None = None + callflow_lang = "auto" + callflow_max_sections = 15 + callflow_diagram_scale = 1.0 + callflow_max_diagram_nodes = 18 + callflow_max_diagram_edges = 24 + analysis_path = Path(_GRAPHIFY_OUT) / ".graphify_analysis.json" + node_limit = 5000 + no_viz = False + obsidian_dir = Path(_GRAPHIFY_OUT) / "obsidian" + # Shared push-connection settings for the graph-database sinks (neo4j, + # falkordb), parsed from the generic --push/--user/--password flags below. + push_uri: str | None = None + push_user = "neo4j" # Neo4j default user; FalkorDB auth is optional and ignores it + # F-031: prefer an env var so the password never appears on argv (visible + # in `ps` output / shell history). The explicit --password flag still + # overrides it. Each sink reads its own var: FALKORDB_PASSWORD for falkordb, + # NEO4J_PASSWORD otherwise. + push_password: str | None = ( + os.environ.get("FALKORDB_PASSWORD") if subcmd == "falkordb" + else os.environ.get("NEO4J_PASSWORD") + ) or None + i = 0 + while i < len(args): + a = args[i] + if a == "--graph" and i + 1 < len(args): + graph_path = Path(args[i + 1]) + graph_path_explicit = True + i += 2 + elif a == "--labels" and i + 1 < len(args): + labels_path = Path(args[i + 1]) + labels_path_explicit = True + i += 2 + elif a == "--report" and i + 1 < len(args): + report_path = Path(args[i + 1]) + report_path_explicit = True + i += 2 + elif a == "--sections" and i + 1 < len(args): + sections_path = Path(args[i + 1]); i += 2 + elif a == "--output" and i + 1 < len(args): + callflow_output = Path(args[i + 1]).expanduser() + if not callflow_output.is_absolute(): + callflow_output = Path.cwd() / callflow_output + i += 2 + elif a == "--lang" and i + 1 < len(args): + callflow_lang = args[i + 1]; i += 2 + elif a == "--max-sections" and i + 1 < len(args): + callflow_max_sections = int(args[i + 1]); i += 2 + elif a == "--diagram-scale" and i + 1 < len(args): + callflow_diagram_scale = float(args[i + 1]); i += 2 + elif a == "--max-diagram-nodes" and i + 1 < len(args): + callflow_max_diagram_nodes = int(args[i + 1]); i += 2 + elif a == "--max-diagram-edges" and i + 1 < len(args): + callflow_max_diagram_edges = int(args[i + 1]); i += 2 + elif a in ("-h", "--help") and subcmd == "callflow-html": + print("Usage: graphify export callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH]") + print(" --report PATH path to GRAPH_REPORT.md") + print(" --sections PATH JSON section definitions") + print(" --output HTML output path (default graphify-out/-callflow.html)") + print(" --lang LANG auto, zh-CN, en, etc. (default auto)") + print(" --max-sections N maximum auto-derived sections (default 15)") + print(" --diagram-scale N Mermaid diagram scale (default 1.0)") + print(" --max-diagram-nodes N representative nodes per section (default 18)") + print(" --max-diagram-edges N representative edges per section (default 24)") + sys.exit(0) + elif a == "--node-limit" and i + 1 < len(args): + node_limit = int(args[i + 1]); i += 2 + elif a == "--no-viz": + no_viz = True; i += 1 + elif a == "--dir" and i + 1 < len(args): + obsidian_dir = Path(args[i + 1]); i += 2 + elif a == "--push" and i + 1 < len(args): + push_uri = args[i + 1]; i += 2 + elif a == "--user" and i + 1 < len(args): + push_user = args[i + 1]; i += 2 + elif a == "--password" and i + 1 < len(args): + push_password = args[i + 1]; i += 2 + elif subcmd == "callflow-html" and not a.startswith("-") and not graph_path_explicit: + candidate = Path(a) + if candidate.name == "graph.json" or candidate.suffix.lower() == ".json": + graph_path = candidate + elif (candidate / "graph.json").exists(): + graph_path = candidate / "graph.json" + else: + graph_path = candidate / _GRAPHIFY_OUT / "graph.json" + graph_path_explicit = True + i += 1 + else: + i += 1 + + graph_path = graph_path.expanduser() + if graph_path_explicit: + graph_out_dir = graph_path.parent + if not labels_path_explicit: + labels_path = graph_out_dir / ".graphify_labels.json" + if not report_path_explicit: + report_path = graph_out_dir / "GRAPH_REPORT.md" + labels_path = labels_path.expanduser() + report_path = report_path.expanduser() + + if not graph_path.exists(): + print(f"error: graph not found: {graph_path}. Run /graphify first.", file=sys.stderr) + sys.exit(1) + + if subcmd == "callflow-html": + from graphify.callflow_html import write_callflow_html as _write_callflow_html + out = _write_callflow_html( + graph=graph_path, + report=report_path, + labels=labels_path, + sections=sections_path, + output=callflow_output, + lang=callflow_lang, + max_sections=callflow_max_sections, + diagram_scale=callflow_diagram_scale, + max_diagram_nodes=callflow_max_diagram_nodes, + max_diagram_edges=callflow_max_diagram_edges, + verbose=True, + ) + print(f"callflow HTML written - open in any browser: {out}") + sys.exit(0) + + from networkx.readwrite import json_graph as _jg + from graphify.build import build_from_json as _bfj + from graphify.security import check_graph_file_size_cap as _check_cap + + # Solution 3 (#1019): for the HTML view, an oversized graph.json should + # not be a hard error. Detect the over-cap condition here and fall back + # to the community-aggregation view (node_limit=5000) below instead of + # exiting 1. All other subcommands keep the hard cap. + _over_cap = False + try: + _check_cap(graph_path) + except ValueError as _cap_err: + if subcmd == "html": + _over_cap = True + try: + _over_cap_bytes = graph_path.stat().st_size + except OSError: + _over_cap_bytes = -1 + print( + f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " + f"falling back to community-aggregation view (node_limit=5000)", + file=sys.stderr, + ) + else: + print(f"error: {_cap_err}", file=sys.stderr) + sys.exit(1) + _raw = json.loads(graph_path.read_text(encoding="utf-8")) + if "links" not in _raw and "edges" in _raw: + _raw = dict(_raw, links=_raw["edges"]) + try: + G = _jg.node_link_graph(_raw, edges="links") + except TypeError: + G = _jg.node_link_graph(_raw) + + # Load optional analysis/labels + communities: dict[int, list[str]] = {} + if analysis_path.exists(): + _an = json.loads(analysis_path.read_text(encoding="utf-8")) + communities = {int(k): v for k, v in _an.get("communities", {}).items()} + cohesion: dict[int, float] = {int(k): v for k, v in _an.get("cohesion", {}).items()} + gods_data = _an.get("gods", []) + else: + cohesion = {} + gods_data = [] + + # Fallback: graph.json carries the per-node community as a node attribute + # (`to_json` writes it on every node). The analysis sidecar is the + # canonical source — but the post-commit / watch rebuild path doesn't + # regenerate it, and `extract` may have its temp files cleaned up. When + # that happens, `graphify export html` previously bailed with + # "Single community - aggregated view not useful." even though the + # per-node attribute had the right data all along. Reconstruct from + # the graph itself so downstream subcommands (html, obsidian, wiki, + # svg, graphml, neo4j) don't silently produce a degraded artifact. + if not communities: + reconstructed: dict[int, list[str]] = {} + for node_id, data in G.nodes(data=True): + cid_raw = data.get("community") + if cid_raw is None: + continue + try: + cid = int(cid_raw) + except (TypeError, ValueError): + continue + reconstructed.setdefault(cid, []).append(str(node_id)) + if reconstructed: + communities = reconstructed + + labels: dict[int, str] = {} + if labels_path.exists(): + labels = {int(k): v for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items()} + + out_dir = graph_path.parent + + if subcmd == "html": + from graphify.export import to_html as _to_html + if no_viz: + html_target = out_dir / "graph.html" + if html_target.exists(): + html_target.unlink() + print("--no-viz: skipped graph.html") + else: + # Over-cap fallback (#1019): force the community-aggregation + # path so the oversized graph still renders a usable artifact. + _effective_node_limit = 5000 if _over_cap else node_limit + _to_html(G, communities, str(out_dir / "graph.html"), + community_labels=labels or None, node_limit=_effective_node_limit) + if G.number_of_nodes() <= _effective_node_limit: + print(f"graph.html written - open in any browser, no server needed") + if _over_cap: + sys.exit(0) + + elif subcmd == "obsidian": + from graphify.export import to_obsidian as _to_obsidian, to_canvas as _to_canvas + n = _to_obsidian(G, communities, str(obsidian_dir), + community_labels=labels or None, cohesion=cohesion or None) + print(f"Obsidian vault: {n} notes in {obsidian_dir}/") + _to_canvas(G, communities, str(obsidian_dir / "graph.canvas"), + community_labels=labels or None) + print(f"Canvas: {obsidian_dir}/graph.canvas") + print(f"Open {obsidian_dir}/ as a vault in Obsidian.") + + elif subcmd == "wiki": + from graphify.wiki import to_wiki as _to_wiki + from graphify.analyze import god_nodes as _god_nodes + if not communities: + print( + "error: .graphify_analysis.json is missing or empty — refusing to export wiki to prevent data loss.\n" + "Run `graphify extract .` (or `graphify cluster-only .`) to regenerate community data first.", + file=sys.stderr, + ) + sys.exit(1) + if not gods_data: + gods_data = _god_nodes(G) + n = _to_wiki(G, communities, str(out_dir / "wiki"), + community_labels=labels or None, cohesion=cohesion or None, + god_nodes_data=gods_data) + print(f"Wiki: {n} articles written to {out_dir}/wiki/") + print(f" {out_dir}/wiki/index.md -> agent entry point") + + elif subcmd == "svg": + from graphify.export import to_svg as _to_svg + _to_svg(G, communities, str(out_dir / "graph.svg"), + community_labels=labels or None) + print(f"graph.svg written - embeds in Obsidian, Notion, GitHub READMEs") + + elif subcmd == "graphml": + from graphify.export import to_graphml as _to_graphml + _to_graphml(G, communities, str(out_dir / "graph.graphml")) + print(f"graph.graphml written - open in Gephi, yEd, or any GraphML tool") + + elif subcmd == "neo4j": + if push_uri: + from graphify.export import push_to_neo4j as _push + if push_password is None: + print("error: --password required for --push", file=sys.stderr) + sys.exit(1) + result = _push(G, uri=push_uri, user=push_user, + password=push_password, communities=communities) + print(f"Pushed to Neo4j: {result['nodes']} nodes, {result['edges']} edges") + else: + from graphify.export import to_cypher as _to_cypher + _to_cypher(G, str(out_dir / "cypher.txt")) + print(f"cypher.txt written - import with: cypher-shell < {out_dir}/cypher.txt") + + elif subcmd == "falkordb": + if push_uri: + from graphify.export import push_to_falkordb as _push + result = _push(G, uri=push_uri, user=push_user, + password=push_password, communities=communities) + print(f"Pushed to FalkorDB: {result['nodes']} nodes, {result['edges']} edges") + else: + from graphify.export import to_cypher as _to_cypher + _to_cypher(G, str(out_dir / "cypher.txt")) + print(f"cypher.txt written ({out_dir}/cypher.txt) - statements are OpenCypher. " + f"FalkorDB's GRAPH.QUERY runs one statement at a time (no bulk script " + f"import), so load a graph with: graphify export falkordb --push " + f"falkordb://localhost:6379") + + elif cmd == "benchmark": + from graphify.benchmark import run_benchmark, print_benchmark + + graph_path = sys.argv[2] if len(sys.argv) > 2 else _default_graph_path() + _enforce_graph_size_cap_or_exit(Path(graph_path)) + # Try to load corpus_words from detect output + corpus_words = None + detect_path = Path(".graphify_detect.json") + if detect_path.exists(): + try: + detect_data = json.loads(detect_path.read_text(encoding="utf-8")) + corpus_words = detect_data.get("total_words") + except Exception: + pass + result = run_benchmark(graph_path, corpus_words=corpus_words) + print_benchmark(result) + + elif cmd == "global": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + from graphify.global_graph import ( + global_add as _global_add, + global_remove as _global_remove, + global_list as _global_list, + global_path as _global_path, + ) + if subcmd == "add": + # graphify global add [--as ] + args = sys.argv[3:] + source = None + tag = None + i = 0 + while i < len(args): + if args[i] == "--as" and i + 1 < len(args): + tag = args[i + 1]; i += 2 + elif not source: + source = Path(args[i]); i += 1 + else: + i += 1 + if not source: + print("Usage: graphify global add [--as ]", file=sys.stderr) + sys.exit(1) + tag = tag or source.parent.parent.name + try: + result = _global_add(source, tag) + if result["skipped"]: + print(f"'{tag}' unchanged since last add - global graph not modified.") + else: + print(f"Added '{tag}' to global graph: +{result['nodes_added']} nodes, " + f"-{result['nodes_removed']} pruned. Global: {_global_path()}") + except Exception as exc: + print(f"error: {exc}", file=sys.stderr); sys.exit(1) + elif subcmd == "remove": + tag = sys.argv[3] if len(sys.argv) > 3 else "" + if not tag: + print("Usage: graphify global remove ", file=sys.stderr); sys.exit(1) + try: + removed = _global_remove(tag) + print(f"Removed '{tag}' from global graph ({removed} nodes pruned).") + except KeyError as exc: + print(f"error: {exc}", file=sys.stderr); sys.exit(1) + elif subcmd == "list": + repos = _global_list() + if not repos: + print("Global graph is empty. Use 'graphify global add' to add a project.") + else: + print(f"Global graph: {_global_path()}") + for tag, info in repos.items(): + print(f" {tag}: {info.get('node_count', '?')} nodes, added {info.get('added_at', '?')[:10]}") + elif subcmd == "path": + print(_global_path()) + else: + print("Usage: graphify global [add|remove|list|path]", file=sys.stderr); sys.exit(1) + + elif cmd == "extract": + # Headless full-pipeline extraction for CI / scripts (#698). + # Runs detect -> AST extraction on code -> semantic LLM extraction on + # docs/papers/images -> merge -> build -> cluster -> write outputs. + # Unlike the skill.md path (which runs through Claude Code subagents), + # this calls extract_corpus_parallel directly using whichever backend + # has an API key set. + if len(sys.argv) < 3: + print( + "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " + "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] " + "[--max-workers N] [--token-budget N] [--max-concurrency N] " + "[--api-timeout S] [--postgres DSN] [--cargo] [--timing]", + file=sys.stderr, + ) + sys.exit(1) + + has_path = True + if sys.argv[2].startswith("-"): + has_path = False + target = Path(".").resolve() + else: + target = Path(sys.argv[2]).resolve() + if not target.exists(): + print(f"error: path not found: {target}", file=sys.stderr) + sys.exit(1) + + backend: str | None = None + model: str | None = None + extract_mode: str | None = None + out_dir: Path | None = None + cli_postgres_dsn: str | None = None + cli_cargo: bool = False + no_cluster = False + dedup_llm = False + google_workspace = False + global_merge = False + code_only = False + global_repo_tag: str | None = None + # Performance/tuning knobs (issue #792). None means "use library default". + cli_max_workers: int | None = None + cli_token_budget: int | None = None + cli_max_concurrency: int | None = None + cli_api_timeout: float | None = None + # Clustering tuning knobs + cli_resolution: float = 1.0 + cli_exclude_hubs: float | None = None + cli_excludes: list[str] = [] + cli_timing: bool = False + + def _parse_int(name: str, raw: str) -> int: + try: + v = int(raw) + except ValueError: + print(f"error: {name} must be a positive integer (got {raw!r})", file=sys.stderr) + sys.exit(2) + if v <= 0: + print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) + sys.exit(2) + return v + + def _parse_float(name: str, raw: str) -> float: + try: + v = float(raw) + except ValueError: + print(f"error: {name} must be a positive number (got {raw!r})", file=sys.stderr) + sys.exit(2) + if v <= 0: + print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) + sys.exit(2) + return v + + args = sys.argv[3:] if has_path else sys.argv[2:] + i = 0 + while i < len(args): + a = args[i] + if a == "--backend" and i + 1 < len(args): + backend = args[i + 1]; i += 2 + elif a.startswith("--backend="): + backend = a.split("=", 1)[1]; i += 1 + elif a == "--model" and i + 1 < len(args): + model = args[i + 1]; i += 2 + elif a.startswith("--model="): + model = a.split("=", 1)[1]; i += 1 + elif a == "--mode" and i + 1 < len(args): + extract_mode = args[i + 1]; i += 2 + elif a.startswith("--mode="): + extract_mode = a.split("=", 1)[1]; i += 1 + elif a == "--out" and i + 1 < len(args): + out_dir = Path(args[i + 1]); i += 2 + elif a.startswith("--out="): + out_dir = Path(a.split("=", 1)[1]); i += 1 + elif a == "--no-cluster": + no_cluster = True; i += 1 + elif a == "--dedup-llm": + dedup_llm = True; i += 1 + elif a == "--code-only": + code_only = True; i += 1 + elif a == "--google-workspace": + google_workspace = True; i += 1 + elif a == "--global": + global_merge = True; i += 1 + elif a == "--as" and i + 1 < len(args): + global_repo_tag = args[i + 1]; i += 2 + elif a == "--max-workers" and i + 1 < len(args): + cli_max_workers = _parse_int("--max-workers", args[i + 1]); i += 2 + elif a.startswith("--max-workers="): + cli_max_workers = _parse_int("--max-workers", a.split("=", 1)[1]); i += 1 + elif a == "--token-budget" and i + 1 < len(args): + cli_token_budget = _parse_int("--token-budget", args[i + 1]); i += 2 + elif a.startswith("--token-budget="): + cli_token_budget = _parse_int("--token-budget", a.split("=", 1)[1]); i += 1 + elif a == "--max-concurrency" and i + 1 < len(args): + cli_max_concurrency = _parse_int("--max-concurrency", args[i + 1]); i += 2 + elif a.startswith("--max-concurrency="): + cli_max_concurrency = _parse_int("--max-concurrency", a.split("=", 1)[1]); i += 1 + elif a == "--api-timeout" and i + 1 < len(args): + cli_api_timeout = _parse_float("--api-timeout", args[i + 1]); i += 2 + elif a.startswith("--api-timeout="): + cli_api_timeout = _parse_float("--api-timeout", a.split("=", 1)[1]); i += 1 + elif a == "--resolution" and i + 1 < len(args): + cli_resolution = _parse_float("--resolution", args[i + 1]); i += 2 + elif a.startswith("--resolution="): + cli_resolution = _parse_float("--resolution", a.split("=", 1)[1]); i += 1 + elif a == "--exclude-hubs" and i + 1 < len(args): + cli_exclude_hubs = float(args[i + 1]); i += 2 + elif a.startswith("--exclude-hubs="): + cli_exclude_hubs = float(a.split("=", 1)[1]); i += 1 + elif a == "--exclude" and i + 1 < len(args): + cli_excludes.append(args[i + 1]); i += 2 + elif a.startswith("--exclude="): + cli_excludes.append(a.split("=", 1)[1]); i += 1 + elif a == "--postgres" and i + 1 < len(args): + cli_postgres_dsn = args[i + 1]; i += 2 + elif a.startswith("--postgres="): + cli_postgres_dsn = a.split("=", 1)[1]; i += 1 + elif a == "--cargo": + cli_cargo = True + i += 1 + elif a == "--timing": + cli_timing = True; i += 1 + else: + i += 1 + + if not has_path and cli_postgres_dsn is None: + print("error: must specify a path to scan or a --postgres DSN", file=sys.stderr) + sys.exit(1) + + _VALID_MODES = {"deep"} + if extract_mode is not None and extract_mode not in _VALID_MODES: + print( + f"error: unknown --mode '{extract_mode}'. " + f"Available: {', '.join(sorted(_VALID_MODES))}", + file=sys.stderr, + ) + sys.exit(2) + deep_mode = extract_mode == "deep" + if deep_mode: + print("[graphify extract] deep mode enabled: richer semantic extraction") + + # CLI flag wins over env var. Setting GRAPHIFY_API_TIMEOUT here so + # _call_openai_compat picks it up without needing a new kwarg path. + if cli_api_timeout is not None: + os.environ["GRAPHIFY_API_TIMEOUT"] = str(cli_api_timeout) + if cli_max_workers is not None: + os.environ["GRAPHIFY_MAX_WORKERS"] = str(cli_max_workers) + + # Resolve output dir. The user-facing contract is "/graphify-out/" + # so a fresh checkout writes graphify-out/ at the project root, matching + # the skill.md pipeline. + out_root = (out_dir.resolve() if out_dir else target) + graphify_out = out_root / _GRAPHIFY_OUT + graphify_out.mkdir(parents=True, exist_ok=True) + + stages = _StageTimer(cli_timing) + + from graphify.detect import ( + detect as _detect, + detect_incremental as _detect_incremental, + save_manifest as _save_manifest, + ) + manifest_path = graphify_out / "manifest.json" + existing_graph_path = graphify_out / "graph.json" + incremental_mode = manifest_path.exists() and existing_graph_path.exists() if has_path else False + + if not has_path: + code_files = [] + doc_files = [] + paper_files = [] + image_files = [] + deleted_files = [] + unchanged_total = 0 + files_by_type = {} + elif incremental_mode: + print(f"[graphify extract] incremental scan of {target}") + detection = _detect_incremental( + target, + manifest_path=str(manifest_path), + google_workspace=google_workspace or None, + extra_excludes=cli_excludes or None, + ) + files_by_type = detection.get("files", {}) + new_by_type = detection.get("new_files", {}) + code_files = [Path(p) for p in new_by_type.get("code", [])] + doc_files = [Path(p) for p in new_by_type.get("document", [])] + paper_files = [Path(p) for p in new_by_type.get("paper", [])] + image_files = [Path(p) for p in new_by_type.get("image", [])] + deleted_files = list(detection.get("deleted_files", [])) + unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) + else: + print(f"[graphify extract] scanning {target}") + detection = _detect(target, google_workspace=google_workspace or None, extra_excludes=cli_excludes or None) + files_by_type = detection.get("files", {}) + code_files = [Path(p) for p in files_by_type.get("code", [])] + doc_files = [Path(p) for p in files_by_type.get("document", [])] + paper_files = [Path(p) for p in files_by_type.get("paper", [])] + image_files = [Path(p) for p in files_by_type.get("image", [])] + deleted_files = [] + unchanged_total = 0 + + semantic_files = doc_files + paper_files + image_files + # --code-only: index code (pure local AST, no key) and skip the semantic + # (doc/paper/image) pass entirely, so a mixed repo doesn't hard-fail when no + # LLM backend is configured (#1734). Report what was skipped rather than + # silently dropping it. + if code_only and semantic_files: + print( + f"[graphify extract] --code-only: skipping {len(semantic_files)} " + f"non-code file(s) ({len(doc_files)} docs, {len(paper_files)} papers, " + f"{len(image_files)} images) — no LLM extraction" + ) + semantic_files = [] + doc_files = [] + paper_files = [] + image_files = [] + if incremental_mode: + print( + f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " + f"{len(paper_files)} papers, {len(image_files)} images changed; " + f"{unchanged_total} unchanged; {len(deleted_files)} deleted" + ) + else: + print( + f"[graphify extract] found {len(code_files)} code, " + f"{len(doc_files)} docs, {len(paper_files)} papers, " + f"{len(image_files)} images" + ) + # Surface files that were seen but not classified (extensionless non-shebang + # project files like Dockerfile/Makefile, or unsupported extensions), so they + # are no longer invisible in graphify's own output (#1692). + _unclassified = detection.get("unclassified", []) if isinstance(detection, dict) else [] + if _unclassified: + _names = ", ".join(sorted({Path(p).name for p in _unclassified})[:6]) + _more = f" (+{len(_unclassified) - 6} more)" if len(_unclassified) > 6 else "" + print( + f"[graphify extract] {len(_unclassified)} file(s) not classified " + f"(no supported extension or shebang), skipped: {_names}{_more}" + ) + stages.mark("detect") + + # Resolve the LLM backend only now that we know whether the corpus + # needs one. A code-only corpus is pure local AST and must not require + # an API key; the key is enforced below only when there's LLM work. + from graphify.llm import ( + BACKENDS as _BACKENDS, + detect_backend as _detect_backend, + estimate_cost as _estimate_cost, + extract_corpus_parallel as _extract_corpus_parallel, + _format_backend_env_keys, + _get_backend_api_key, + ) + needs_llm = bool(semantic_files) or dedup_llm + if backend is None and needs_llm: + backend = _detect_backend() + if backend is not None and backend not in _BACKENDS: + print( + f"error: unknown backend '{backend}'. " + f"Available: {', '.join(sorted(_BACKENDS))}", + file=sys.stderr, + ) + sys.exit(1) + if needs_llm: + if backend is None: + reasons = [] + if semantic_files: + reasons.append( + f"{len(semantic_files)} doc/paper/image file(s) need semantic extraction" + ) + if dedup_llm: + reasons.append("--dedup-llm was passed") + hint = "" + if semantic_files: + hint = (" Or pass --code-only to index just the code " + "(local AST, no key) and skip the non-code files.") + print( + "error: no LLM API key found (" + "; ".join(reasons) + "). " + "Set GEMINI_API_KEY or GOOGLE_API_KEY (gemini), MOONSHOT_API_KEY " + "(kimi), ANTHROPIC_API_KEY (claude), OPENAI_API_KEY (openai), " + "DEEPSEEK_API_KEY (deepseek), or pass --backend. A code-only " + "corpus needs no key." + hint, + file=sys.stderr, + ) + sys.exit(1) + if backend == "ollama": + from graphify.llm import _validate_ollama_base_url + _oll_url = os.environ.get("OLLAMA_BASE_URL", _BACKENDS["ollama"].get("base_url", "")) + try: + _validate_ollama_base_url(_oll_url, warn=False) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(2) + if not _get_backend_api_key(backend): + allow_no_key = False + if backend == "ollama": + from urllib.parse import urlparse + ollama_url = os.environ.get( + "OLLAMA_BASE_URL", + _BACKENDS["ollama"].get("base_url", ""), + ) + try: + host = (urlparse(ollama_url).hostname or "").lower() + except Exception: + host = "" + allow_no_key = ( + host in ("localhost", "127.0.0.1", "::1") + or host.startswith("127.") + ) + elif backend == "bedrock": + allow_no_key = bool( + os.environ.get("AWS_PROFILE") + or os.environ.get("AWS_REGION") + or os.environ.get("AWS_DEFAULT_REGION") + or os.environ.get("AWS_ACCESS_KEY_ID") + ) + elif backend == "claude-cli": + import shutil as _shutil + allow_no_key = _shutil.which("claude") is not None + if not allow_no_key: + print( + "error: backend 'claude-cli' requires the `claude` CLI on $PATH " + "(install Claude Code and run `claude` once to authenticate).", + file=sys.stderr, + ) + sys.exit(1) + if not allow_no_key: + print( + f"error: backend '{backend}' requires {_format_backend_env_keys(backend)} to be set.", + file=sys.stderr, + ) + sys.exit(1) + + # AST extraction on code files. Empty code list (docs-only corpus) is + # the issue #698 case — skip cleanly instead of crashing inside extract(). + ast_result: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} + if code_files: + from graphify.extract import extract as _ast_extract + # Anchor the cache at the output root, not the scanned project: + # with --out, a /graphify-out/cache/ would leak a + # graphify-out/ dir into a project that asked for external output. + ast_kwargs: dict = {"cache_root": out_root} + if cli_max_workers is not None: + ast_kwargs["max_workers"] = cli_max_workers + print(f"[graphify extract] AST extraction on {len(code_files)} code files...") + try: + ast_result = _ast_extract(code_files, **ast_kwargs) + except Exception as exc: + print(f"[graphify extract] AST extraction failed: {exc}", file=sys.stderr) + ast_result = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} + stages.mark("AST extract") + + # Semantic extraction on docs/papers/images. Check cache first. + from graphify.cache import ( + check_semantic_cache as _check_semantic_cache, + prune_semantic_cache as _prune_semantic_cache, + save_semantic_cache as _save_semantic_cache, + ) + sem_result: dict = { + "nodes": [], "edges": [], "hyperedges": [], + "input_tokens": 0, "output_tokens": 0, + } + sem_cache_hits = 0 + sem_cache_misses = 0 + if semantic_files: + sem_paths_str = [str(p) for p in semantic_files] + cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( + _check_semantic_cache(sem_paths_str, root=out_root) + ) + sem_cache_hits = len(semantic_files) - len(uncached_paths) + sem_cache_misses = len(uncached_paths) + sem_result["nodes"].extend(cached_nodes) + sem_result["edges"].extend(cached_edges) + sem_result["hyperedges"].extend(cached_hyperedges) + if sem_cache_hits: + print(f"[graphify extract] semantic cache: {sem_cache_hits} hit / {sem_cache_misses} miss") + + if uncached_paths: + print(f"[graphify extract] semantic extraction on {len(uncached_paths)} files via {backend}...") + corpus_kwargs: dict = { + "backend": backend, + "model": model, + "root": target, + } + if deep_mode: + corpus_kwargs["deep_mode"] = True + if cli_token_budget is not None: + corpus_kwargs["token_budget"] = cli_token_budget + if cli_max_concurrency is not None: + corpus_kwargs["max_concurrency"] = cli_max_concurrency + + # Minimal progress callback so the CLI is no longer silent + # during long local-inference runs (issue #792 addendum). + # Also track per-chunk success so we can fail loudly when + # every chunk errors (e.g. missing backend SDK package). + _chunk_stats = {"total": 0, "succeeded": 0} + def _progress(idx: int, total: int, _result: dict) -> None: + _chunk_stats["total"] = total + _chunk_stats["succeeded"] += 1 + print( + f"[graphify extract] chunk {idx + 1}/{total} done", + flush=True, + ) + corpus_kwargs["on_chunk_done"] = _progress + + try: + fresh = _extract_corpus_parallel( + [Path(p) for p in uncached_paths], + **corpus_kwargs, + ) + except ImportError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + except Exception as exc: + print( + f"[graphify extract] semantic extraction failed: {exc}", + file=sys.stderr, + ) + fresh = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + + # on_chunk_done only fires after a chunk succeeds. If fresh + # semantic extraction was requested and no chunks completed, + # fail instead of writing an AST-only graph with exit 0. + if uncached_paths and _chunk_stats["succeeded"] == 0: + print( + f"[graphify extract] error: all semantic chunks failed " + f"for backend '{backend}' ({len(uncached_paths)} uncached files) - " + f"see per-chunk errors above. If you see 'requires the X package', " + f"run `pip install X` and retry.", + file=sys.stderr, + ) + sys.exit(1) + try: + _save_semantic_cache( + fresh.get("nodes", []), + fresh.get("edges", []), + fresh.get("hyperedges", []), + root=out_root, + ) + except Exception as exc: + print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) + sem_result["nodes"].extend(fresh.get("nodes", [])) + sem_result["edges"].extend(fresh.get("edges", [])) + sem_result["hyperedges"].extend(fresh.get("hyperedges", [])) + sem_result["input_tokens"] += fresh.get("input_tokens", 0) + sem_result["output_tokens"] += fresh.get("output_tokens", 0) + + # Prune orphaned semantic cache entries. The semantic cache is + # content-hash-keyed and unversioned, so it is never swept by the AST + # version-cleanup: every content change or file deletion leaves a + # permanent orphan that accumulates unbounded (#1527). Sweep it against + # the FULL live document set (``files_by_type`` — present in both the + # incremental and full branches), NOT the incremental ``semantic_files`` + # changed-subset, which would delete every unchanged doc's valid entry. + # Best-effort: a prune failure must never break extraction. + try: + from graphify.cache import file_hash as _file_hash + _live_hashes: set[str] = set() + for _kind in ("document", "paper", "image"): + for _fp in files_by_type.get(_kind, []): + _abs = Path(_fp) + if not _abs.is_absolute(): + _abs = Path(out_root) / _abs + if not _abs.is_file(): + continue # deleted/missing — leave out so its entry is pruned + try: + _live_hashes.add(_file_hash(_abs, out_root)) + except OSError: + pass + _prune_semantic_cache(out_root, _live_hashes) + except Exception as exc: + print(f"[graphify extract] warning: could not prune semantic cache: {exc}", file=sys.stderr) + stages.mark("semantic extract") + + pg_result: dict = {"nodes": [], "edges": []} + if cli_postgres_dsn is not None: + from graphify.pg_introspect import introspect_postgres + print(f"[graphify extract] introspecting PostgreSQL schema...") + try: + pg_result = introspect_postgres(cli_postgres_dsn) + except (ConnectionError, ImportError) as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, " + f"{len(pg_result['edges'])} edges") + + cargo_result: dict = {"nodes": [], "edges": []} + if cli_cargo: + from graphify.cargo_introspect import introspect_cargo + print("[graphify extract] introspecting Cargo workspace...") + try: + cargo_result = introspect_cargo(target) + except (ConnectionError, ImportError, OSError) as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + print(f"[graphify extract] Cargo: {len(cargo_result['nodes'])} nodes, " + f"{len(cargo_result['edges'])} edges") + + # Merge AST + semantic + pg_result + cargo_result. Order matters for deduplication: passing AST + # first means semantic node attributes win on collision (richer labels + # for symbols also referenced in docs). Hyperedges only come from the + # semantic side. + merged: dict = { + "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), + "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), + "hyperedges": list(sem_result.get("hyperedges", [])), + "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), + "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), + } + + graph_json_path = graphify_out / "graph.json" + analysis_path = graphify_out / ".graphify_analysis.json" + + # Build a manifest-safe files dict: only stamp semantic_hash for files + # that actually produced output (cache hit or fresh extraction). Files + # whose chunk failed have no source_file entry in sem_result — leaving + # their semantic_hash empty so detect_incremental re-queues them (#933). + _sem_extracted: set[str] = { + n.get("source_file", "") for n in sem_result.get("nodes", []) + } | { + e.get("source_file", "") for e in sem_result.get("edges", []) + } + _sem_extracted.discard("") + _sem_types = {"document", "paper", "image"} + _manifest_files = { + ftype: [f for f in flist if ftype not in _sem_types or f in _sem_extracted] + for ftype, flist in files_by_type.items() + } + + if no_cluster: + # --no-cluster: dump the raw merged extraction as graph.json. + # No NetworkX, no community detection, no analysis sidecar. + # Dedupe nodes (by id) and parallel edges so the raw output matches the + # clustered path (whose DiGraph collapses both) and stays deterministic + # across modes (#1317; node dedup also collapses shared Swift module + # anchors emitted per importing file, #1327). + from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes + from graphify.export import backup_if_protected as _backup + if ( + incremental_mode + and not code_files + and not semantic_files + and not deleted_files + and not pg_result.get("nodes") + and not pg_result.get("edges") + and not cargo_result.get("nodes") + and not cargo_result.get("edges") + ): + print( + "[graphify extract] no incremental changes detected " + "(--no-cluster); outputs left untouched." + ) + try: + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) + except Exception as exc: + print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) + stages.total() + sys.exit(0) + + merged["nodes"] = _dedupe_nodes(merged["nodes"]) + merged["edges"] = _dedupe_edges(merged["edges"]) + # Backfill source_file from endpoint nodes — this raw path bypasses + # build_from_json's backfill, and semantic edges sometimes omit it (#1279). + _node_sf = {n.get("id"): n.get("source_file") for n in merged["nodes"]} + for _e in merged["edges"]: + if not _e.get("source_file"): + _e["source_file"] = ( + _node_sf.get(_e.get("source")) or _node_sf.get(_e.get("target")) or "" + ) + _backup(graphify_out) + graph_json_path.write_text( + json.dumps(merged, indent=2), encoding="utf-8" + ) + stages.mark("write") + cost = _estimate_cost( + backend, merged["input_tokens"], merged["output_tokens"] + ) + print( + f"[graphify extract] wrote {graph_json_path} — " + f"{len(merged['nodes'])} nodes, {len(merged['edges'])} edges " + f"(no clustering)" + ) + if merged["input_tokens"] or merged["output_tokens"]: + print( + f"[graphify extract] tokens: " + f"{merged['input_tokens']:,} in / " + f"{merged['output_tokens']:,} out, " + f"est. cost: ${cost:.4f}" + ) + try: + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) + except Exception as exc: + print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) + if global_merge: + from graphify.global_graph import global_add as _global_add + _tag = global_repo_tag or target.name + try: + result = _global_add(graphify_out / "graph.json", _tag) + if result["skipped"]: + print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") + else: + print(f"[graphify global] '{_tag}' merged into global graph " + f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") + except Exception as exc: + print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) + stages.total() + sys.exit(0) + + # Build graph + cluster + score + write. + from graphify.build import ( + build as _build, + build_from_json as _build_from_json, + build_merge as _build_merge, + ) + from graphify.cluster import cluster as _cluster, score_all as _score_all + from graphify.export import to_json as _to_json + from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising + dedup_backend = backend if dedup_llm else None + if incremental_mode: + G = _build_merge( + [merged], + graph_path=existing_graph_path, + prune_sources=deleted_files or None, + dedup=True, + dedup_llm_backend=dedup_backend, + root=target, + ) + else: + G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target) + stages.mark("build") + if G.number_of_nodes() == 0: + print( + "[graphify extract] graph is empty — extraction produced no nodes. " + "Possible causes: all files skipped, binary-only corpus, or LLM " + "returned no edges.", + file=sys.stderr, + ) + sys.exit(1) + + communities = _cluster(G, resolution=cli_resolution, exclude_hubs_percentile=cli_exclude_hubs) + stages.mark("cluster") + cohesion = _score_all(G, communities) + try: + gods = _god_nodes(G) + except Exception: + gods = [] + try: + surprises = _surprising(G, communities) + except Exception: + surprises = [] + stages.mark("analyze") + + from graphify.export import backup_if_protected as _backup + _backup(graphify_out) + _to_json(G, communities, str(graph_json_path), force=True) + stages.mark("export") + if merged.get("output_tokens", 0) > 0: + (graphify_out / ".graphify_semantic_marker").write_text( + json.dumps({"output_tokens": merged["output_tokens"]}), encoding="utf-8" + ) + if global_merge: + from graphify.global_graph import global_add as _global_add + _tag = global_repo_tag or target.name + try: + result = _global_add(graphify_out / "graph.json", _tag) + if result["skipped"]: + print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") + else: + print(f"[graphify global] '{_tag}' merged into global graph " + f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") + except Exception as exc: + print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) + analysis = { + "communities": {str(k): v for k, v in communities.items()}, + "cohesion": {str(k): v for k, v in cohesion.items()}, + "gods": gods, + "surprises": surprises, + "tokens": { + "input": merged["input_tokens"], + "output": merged["output_tokens"], + }, + } + analysis_path.write_text(json.dumps(analysis, indent=2), encoding="utf-8") + try: + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) + except Exception as exc: + print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) + + cost = _estimate_cost(backend, merged["input_tokens"], merged["output_tokens"]) + print( + f"[graphify extract] wrote {graph_json_path}: " + f"{G.number_of_nodes()} nodes, {G.number_of_edges()} edges, " + f"{len(communities)} communities" + ) + print(f"[graphify extract] wrote {analysis_path}") + if incremental_mode: + print( + f"[graphify extract] incremental summary: " + f"{sem_cache_hits + unchanged_total} files cached/unchanged, " + f"{len(code_files) + sem_cache_misses} re-extracted, " + f"{len(deleted_files)} deleted" + ) + elif sem_cache_hits: + print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") + if merged["input_tokens"] or merged["output_tokens"]: + print( + f"[graphify extract] tokens: " + f"{merged['input_tokens']:,} in / " + f"{merged['output_tokens']:,} out, " + f"est. cost (~{backend}): ${cost:.4f}" + ) + # extract intentionally stops at graph.json + analysis; the report and + # community labels are produced by `cluster-only` (or an agent's Step 5). + # Point standalone users at it so communities get named (#1097). + print( + "[graphify extract] next: run " + f"`graphify cluster-only {graphify_out.parent}` " + "to generate GRAPH_REPORT.md and name communities" + ) + stages.total() + + elif cmd == "cache-check": + # graphify cache-check [--root ] + # Reads file paths (one per line) from , checks semantic cache. + # Writes: + # graphify-out/.graphify_cached.json — already-cached nodes/edges/hyperedges + # graphify-out/.graphify_uncached.txt — paths that need extraction + # Stdout: "Cache: N hit, M miss" + from graphify.cache import check_semantic_cache + if len(sys.argv) < 3: + print("Usage: graphify cache-check [--root ]", file=sys.stderr) + sys.exit(1) + files_from = Path(sys.argv[2]) + root = Path(".") + i = 3 + while i < len(sys.argv): + if sys.argv[i] == "--root" and i + 1 < len(sys.argv): + root = Path(sys.argv[i + 1]) + i += 2 + else: + i += 1 + files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()] + cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(files, root) + out = root / _GRAPHIFY_OUT + out.mkdir(parents=True, exist_ok=True) + if cached_nodes or cached_edges or cached_hyperedges: + (out / ".graphify_cached.json").write_text( + json.dumps({"nodes": cached_nodes, "edges": cached_edges, "hyperedges": cached_hyperedges}, + ensure_ascii=False), + encoding="utf-8", + ) + (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8") + print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss") + + elif cmd == "merge-chunks": + # graphify merge-chunks --out + # Concatenates .graphify_chunk_*.json files written by semantic subagents. + # Deduplicates nodes by id (first writer wins). Sums token counts. + import glob as _glob + if len(sys.argv) < 3: + print("Usage: graphify merge-chunks --out ", file=sys.stderr) + sys.exit(1) + out_path: Path | None = None + chunk_args: list[str] = [] + i = 2 + while i < len(sys.argv): + if sys.argv[i] == "--out" and i + 1 < len(sys.argv): + out_path = Path(sys.argv[i + 1]) + i += 2 + else: + chunk_args.append(sys.argv[i]) + i += 1 + if not out_path: + print("error: --out required", file=sys.stderr) + sys.exit(1) + chunk_files: list[str] = [] + for arg in chunk_args: + expanded = _glob.glob(arg) + chunk_files.extend(sorted(expanded) if expanded else [arg]) + merged: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + seen_ids: set[str] = set() + for cf in chunk_files: + try: + chunk = json.loads(Path(cf).read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + print(f"[graphify merge-chunks] warning: skipping {cf}: {exc}", file=sys.stderr) + continue + for n in chunk.get("nodes", []): + if n.get("id") not in seen_ids: + seen_ids.add(n["id"]) + merged["nodes"].append(n) + merged["edges"].extend(chunk.get("edges", [])) + merged["hyperedges"].extend(chunk.get("hyperedges", [])) + merged["input_tokens"] += chunk.get("input_tokens", 0) + merged["output_tokens"] += chunk.get("output_tokens", 0) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(merged, ensure_ascii=False), encoding="utf-8") + print( + f"Merged {len(chunk_files)} chunks: {len(merged['nodes'])} nodes, {len(merged['edges'])} edges, " + f"{merged['input_tokens']:,} in / {merged['output_tokens']:,} out tokens" + ) + + elif cmd == "merge-semantic": + # graphify merge-semantic --cached --new --out + # Merges cached semantic results with freshly-extracted chunk results. + # Deduplicates nodes by id (cached entries take priority over new ones). + if len(sys.argv) < 3: + print("Usage: graphify merge-semantic --cached --new --out ", file=sys.stderr) + sys.exit(1) + cached_path: Path | None = None + new_path: Path | None = None + out_path2: Path | None = None + i = 2 + while i < len(sys.argv): + if sys.argv[i] == "--cached" and i + 1 < len(sys.argv): + cached_path = Path(sys.argv[i + 1]); i += 2 + elif sys.argv[i] == "--new" and i + 1 < len(sys.argv): + new_path = Path(sys.argv[i + 1]); i += 2 + elif sys.argv[i] == "--out" and i + 1 < len(sys.argv): + out_path2 = Path(sys.argv[i + 1]); i += 2 + else: + i += 1 + if not out_path2: + print("error: --out required", file=sys.stderr) + sys.exit(1) + empty: dict = {"nodes": [], "edges": [], "hyperedges": []} + cached_data = json.loads(cached_path.read_text(encoding="utf-8")) if cached_path and cached_path.exists() else empty + new_data = json.loads(new_path.read_text(encoding="utf-8")) if new_path and new_path.exists() else empty + seen_ids2: set[str] = set() + all_nodes: list[dict] = [] + for n in cached_data.get("nodes", []) + new_data.get("nodes", []): + if n.get("id") not in seen_ids2: + seen_ids2.add(n["id"]) + all_nodes.append(n) + merged2 = { + "nodes": all_nodes, + "edges": cached_data.get("edges", []) + new_data.get("edges", []), + "hyperedges": cached_data.get("hyperedges", []) + new_data.get("hyperedges", []), + } + out_path2.parent.mkdir(parents=True, exist_ok=True) + out_path2.write_text(json.dumps(merged2, ensure_ascii=False), encoding="utf-8") + print(f"Merged: {len(merged2['nodes'])} nodes, {len(merged2['edges'])} edges") + + elif Path(cmd).exists() or cmd in (".", "..") or cmd.startswith(("./", "../", "/", "~")): + # User ran `graphify ` directly — treat as `graphify extract `. + # Common when following the PowerShell note in README (`graphify .`) or + # copy-pasting skill invocations without the leading slash. + sys.argv.insert(2, sys.argv[1]) + sys.argv[1] = "extract" + _reenter_main() + else: + print(f"error: unknown command '{cmd}'", file=sys.stderr) + print("Run 'graphify --help' for usage.", file=sys.stderr) + sys.exit(1) diff --git a/graphify/export.py b/graphify/export.py index 36e4f9949..bf27eedc4 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -17,6 +17,8 @@ from graphify.analyze import _node_community_map from graphify.build import edge_data +from graphify.exporters.graphdb import push_to_falkordb, push_to_neo4j # noqa: E402,F401 + # Artifacts worth preserving across rebuilds (non-regenerable without LLM or curation). _BACKUP_ARTIFACTS = [ @@ -149,313 +151,9 @@ def _yaml_str(s: str) -> str: return "".join(out) -COMMUNITY_COLORS = [ - "#4E79A7", "#F28E2B", "#E15759", "#76B7B2", "#59A14F", - "#EDC948", "#B07AA1", "#FF9DA7", "#9C755F", "#BAB0AC", -] - -MAX_NODES_FOR_VIZ = 5_000 +from graphify.exporters.base import COMMUNITY_COLORS # noqa: E402,F401 - -def _viz_node_limit() -> int: - """Return the effective viz node limit, honoring GRAPHIFY_VIZ_NODE_LIMIT env var. - - Falls back to MAX_NODES_FOR_VIZ when the env var is unset, empty, or non-integer. - Set to 0 to disable HTML viz unconditionally (useful for CI runners). - """ - import os - raw = os.environ.get("GRAPHIFY_VIZ_NODE_LIMIT") - if raw is None or not raw.strip(): - return MAX_NODES_FOR_VIZ - try: - return int(raw) - except ValueError: - return MAX_NODES_FOR_VIZ - - -def _html_styles() -> str: - return """""" - - -def _hyperedge_script(hyperedges_json: str) -> str: - return f"""""" - - -def _html_script(nodes_json: str, edges_json: str, legend_json: str) -> str: - return f"""""" +from graphify.exporters.html import to_html # noqa: E402,F401 _CONFIDENCE_SCORE_DEFAULTS = {"EXTRACTED": 1.0, "INFERRED": 0.5, "AMBIGUOUS": 0.2} @@ -629,244 +327,6 @@ def to_cypher(G: nx.Graph, output_path: str) -> None: f.write("\n".join(lines)) -def to_html( - G: nx.Graph, - communities: dict[int, list[str]], - output_path: str, - community_labels: dict[int, str] | None = None, - member_counts: dict[int, int] | None = None, - node_limit: int | None = None, - learning_overlay: dict | None = None, -) -> None: - """Generate an interactive vis.js HTML visualization of the graph. - - Features: node size by degree, click-to-inspect panel, search box, - community filter, physics clustering by community, confidence-styled edges. - Raises ValueError if graph exceeds MAX_NODES_FOR_VIZ. - - If member_counts is provided (aggregated community view), node sizes are - based on community member counts rather than graph degree. - - If node_limit is set and the graph exceeds it, automatically builds an - aggregated community-level meta-graph instead of raising ValueError. - """ - limit = node_limit if node_limit is not None else _viz_node_limit() - if G.number_of_nodes() > limit: - if node_limit is not None: - # Build aggregated community meta-graph - from collections import Counter as _Counter - import networkx as _nx - print(f"Graph has {G.number_of_nodes()} nodes (above {limit} limit). Building aggregated community view...") - node_to_community = {nid: cid for cid, members in communities.items() for nid in members} - meta = _nx.Graph() - for cid, members in communities.items(): - meta.add_node(str(cid), label=(community_labels or {}).get(cid, f"Community {cid}")) - edge_counts = _Counter() - for u, v in G.edges(): - cu, cv = node_to_community.get(u), node_to_community.get(v) - if cu is not None and cv is not None and cu != cv: - edge_counts[(min(cu, cv), max(cu, cv))] += 1 - for (cu, cv), w in edge_counts.items(): - meta.add_edge(str(cu), str(cv), weight=w, - relation=f"{w} cross-community edges", confidence="AGGREGATED") - if meta.number_of_nodes() <= 1: - print("Single community - aggregated view not useful. Skipping graph.html.") - return - meta_communities = {cid: [str(cid)] for cid in communities} - mc = {cid: len(members) for cid, members in communities.items()} - # Remap hyperedges from semantic node IDs to community IDs - raw_hyperedges = G.graph.get("hyperedges", []) - if raw_hyperedges: - remapped = [] - for he in raw_hyperedges: - he_members = he.get("nodes", []) - comm_ids, seen = [], set() - for nid in he_members: - c = node_to_community.get(nid) - if c is None: - continue - s = str(c) - if s in seen: - continue - seen.add(s) - comm_ids.append(s) - if len(comm_ids) < 2: - continue - remapped.append({ - "id": he.get("id", ""), - "label": he.get("label") or he.get("relation", "").replace("_", " "), - "nodes": comm_ids, - }) - meta.graph["hyperedges"] = remapped - to_html(meta, meta_communities, output_path, - community_labels=community_labels, member_counts=mc) - print(f"graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)") - print("Tip: run with --obsidian for full node-level detail.") - return - raise ValueError( - f"Graph has {G.number_of_nodes()} nodes - too large for HTML viz " - f"(limit: {limit}). Use --no-viz, raise GRAPHIFY_VIZ_NODE_LIMIT, " - f"or reduce input size." - ) - - node_community = _node_community_map(communities) - degree = dict(G.degree()) - max_deg = max(degree.values(), default=1) or 1 - max_mc = (max(member_counts.values(), default=1) or 1) if member_counts else 1 - - # Work-memory overlay (derived sidecar). When not passed explicitly, load it - # best-effort from the sibling .graphify_learning.json next to the output - # graph.html (which lives beside graph.json). Empty/missing => no learning - # fields, so the un-annotated render is byte-identical to pre-feature. - if learning_overlay is None: - learning_overlay = {} - try: - from graphify.reflect import load_learning_overlay as _llo - learning_overlay = _llo(Path(output_path)) - except Exception: - learning_overlay = {} - # Status -> ring color. preferred=green, contested=amber. Tentative gets no - # ring (it's not yet trustworthy enough to highlight in the map). - _RING = {"preferred": "#22c55e", "contested": "#f59e0b"} - - # Build nodes list for vis.js - vis_nodes = [] - for node_id, data in G.nodes(data=True): - cid = node_community.get(node_id, 0) - color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)] - label = sanitize_label(data.get("label", node_id)) - deg = degree.get(node_id, 1) - if member_counts: - mc = member_counts.get(cid, 1) - size = 10 + 30 * (mc / max_mc) - font_size = 12 - else: - size = 10 + 30 * (deg / max_deg) - # Only show label for high-degree nodes by default; others show on hover - font_size = 12 if deg >= max_deg * 0.15 else 0 - node = { - "id": node_id, - "label": label, - "color": {"background": color, "border": color, "highlight": {"background": "#ffffff", "border": color}}, - "size": round(size, 1), - "font": {"size": font_size, "color": "#ffffff"}, - "title": _html.escape(label), - "community": cid, - "community_name": sanitize_label((community_labels or {}).get(cid, f"Community {cid}")), - "source_file": sanitize_label(str(data.get("source_file") or "")), - "file_type": data.get("file_type", ""), - "degree": deg, - } - # Conditional learning fields — only present for annotated nodes, so - # un-annotated output keeps the exact pre-feature node dict shape. - entry = learning_overlay.get(str(node_id)) if learning_overlay else None - if entry: - status = sanitize_label(str(entry.get("status", ""))) - stale = bool(entry.get("stale")) - node["learning_status"] = status - node["learning_stale"] = stale - ring = _RING.get(status) - if ring: - # Status-colored ring via the border; stale => desaturated + - # dashed (vis.js supports per-node `shapeProperties.borderDashes`). - if stale: - ring = "#9ca3af" - node["shapeProperties"] = {"borderDashes": [4, 4]} - node["borderWidth"] = 3 - node["color"] = { - "background": color, "border": ring, - "highlight": {"background": "#ffffff", "border": ring}, - } - # Lesson line appended to the hover title. - if status == "contested": - lesson = f"Lesson: contested (useful {entry.get('uses', 0)} / dead-end {entry.get('neg', 0)})" - elif status == "preferred": - lesson = f"Lesson: preferred source ({entry.get('uses', 0)} useful, score={entry.get('score', 0)})" - else: - lesson = f"Lesson: {status} ({entry.get('uses', 0)} useful)" - if stale: - lesson += " [code changed — re-verify]" - node["title"] = _html.escape(label) + "\n" + _html.escape(sanitize_label(lesson)) - vis_nodes.append(node) - - # Build edges list. Restore original edge direction from _src/_tgt - # (stashed by build.py for exactly this reason): undirected NetworkX - # canonicalizes endpoint order, which would otherwise flip the arrow - # for `calls` and `rationale_for` in the rendered graph (#563). - vis_edges = [] - for u, v, data in G.edges(data=True): - confidence = data.get("confidence", "EXTRACTED") - relation = data.get("relation", "") - true_src = data.get("_src", u) - true_tgt = data.get("_tgt", v) - vis_edges.append({ - "from": true_src, - "to": true_tgt, - "label": relation, - "title": _html.escape(f"{relation} [{confidence}]"), - "dashes": confidence != "EXTRACTED", - "width": 2 if confidence == "EXTRACTED" else 1, - "color": {"opacity": 0.7 if confidence == "EXTRACTED" else 0.35}, - "confidence": confidence, - }) - - # Build community legend data - legend_data = [] - for cid in sorted((community_labels or {}).keys()): - color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)] - lbl = _html.escape(sanitize_label((community_labels or {}).get(cid, f"Community {cid}"))) - n = member_counts.get(cid, len(communities.get(cid, []))) if member_counts else len(communities.get(cid, [])) - legend_data.append({"cid": cid, "color": color, "label": lbl, "count": n}) - - # Escape sequences so embedded JSON cannot break out of the script tag - def _js_safe(obj) -> str: - return json.dumps(obj).replace(" - - - -graphify - {title} - -{_html_styles()} - - -
- -{_html_script(nodes_json, edges_json, legend_json)} -{_hyperedge_script(hyperedges_json)} - -""" - - Path(output_path).write_text(html, encoding="utf-8") # nosec - - # Keep backward-compatible alias - skill.md calls generate_html generate_html = to_html @@ -1397,174 +857,6 @@ def safe_name(label: str) -> str: Path(output_path).write_text(json.dumps(canvas_data, indent=2), encoding="utf-8") # nosec -def push_to_neo4j( - G: nx.Graph, - uri: str, - user: str, - password: str, - communities: dict[int, list[str]] | None = None, -) -> dict[str, int]: - """Push graph directly to a running Neo4j instance via the Python driver. - - Requires: pip install neo4j - - Uses MERGE so re-running is safe - nodes and edges are upserted, not duplicated. - Returns a dict with counts of nodes and edges pushed. - """ - try: - from neo4j import GraphDatabase - except ImportError as e: - raise ImportError( - "neo4j driver not installed. Run: pip install neo4j" - ) from e - - node_community = _node_community_map(communities) if communities else {} - - def _safe_rel(relation: str) -> str: - return re.sub(r"[^A-Z0-9_]", "_", relation.upper().replace(" ", "_").replace("-", "_")) or "RELATED_TO" - - def _safe_label(label: str) -> str: - """Sanitize a Neo4j node label to prevent Cypher injection.""" - sanitized = re.sub(r"[^A-Za-z0-9_]", "", label) - return sanitized if sanitized else "Entity" - - driver = GraphDatabase.driver(uri, auth=(user, password)) - nodes_pushed = 0 - edges_pushed = 0 - - with driver.session() as session: - for node_id, data in G.nodes(data=True): - props = { - k: v for k, v in data.items() - if isinstance(v, (str, int, float, bool)) and not k.startswith("_") - } - props["id"] = node_id - cid = node_community.get(node_id) - if cid is not None: - props["community"] = cid - ftype = _safe_label(data.get("file_type", "Entity").capitalize()) - session.run( - f"MERGE (n:{ftype} {{id: $id}}) SET n += $props", - id=node_id, - props=props, - ) - nodes_pushed += 1 - - for u, v, data in G.edges(data=True): - rel = _safe_rel(data.get("relation", "RELATED_TO")) - props = { - k: v for k, v in data.items() - if isinstance(v, (str, int, float, bool)) and not k.startswith("_") - } - session.run( - f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) " - f"MERGE (a)-[r:{rel}]->(b) SET r += $props", - src=u, - tgt=v, - props=props, - ) - edges_pushed += 1 - - driver.close() - return {"nodes": nodes_pushed, "edges": edges_pushed} - - -def push_to_falkordb( - G: nx.Graph, - uri: str, - user: str | None = None, - password: str | None = None, - communities: dict[int, list[str]] | None = None, - graph_name: str = "graphify", -) -> dict[str, int]: - """Push graph directly to a running FalkorDB instance via the Python SDK. - - Requires: pip install falkordb - - FalkorDB is OpenCypher-compatible, so the MERGE/SET upsert queries are - identical to push_to_neo4j. Differences from the Neo4j path: - - connects with FalkorDB(host, port, username, password) instead of a bolt - driver; only the host/port are read from the URI, so the scheme is - informational - "falkordb://localhost:6379", "redis://localhost:6379" - and a bare "localhost:6379" are all equivalent (default port 6379). - - a named graph is selected via db.select_graph(graph_name) (default - "graphify"); FalkorDB keys each graph by name in the same instance. - - queries run via graph.query(cypher, params) - there is no session object. - - auth is optional (FalkorDB runs without credentials by default), so user - and password may be None. - - no APOC: the Neo4j path does not use APOC either, so nothing to port. - - Uses MERGE so re-running is safe - nodes and edges are upserted, not - duplicated. Returns a dict with counts of nodes and edges pushed. - """ - try: - from falkordb import FalkorDB - except ImportError as e: - raise ImportError( - "falkordb SDK not installed. Run: pip install falkordb" - ) from e - - from urllib.parse import urlparse - - node_community = _node_community_map(communities) if communities else {} - - def _safe_rel(relation: str) -> str: - return re.sub(r"[^A-Z0-9_]", "_", relation.upper().replace(" ", "_").replace("-", "_")) or "RELATED_TO" - - def _safe_label(label: str) -> str: - """Sanitize a FalkorDB node label to prevent Cypher injection.""" - sanitized = re.sub(r"[^A-Za-z0-9_]", "", label) - return sanitized if sanitized else "Entity" - - parsed = urlparse(uri if "://" in uri else f"redis://{uri}") - # FalkorDB auth is optional. Only send credentials when a password is - # provided; otherwise connect anonymously and ignore any bolt-style default - # username (e.g. Neo4j's "neo4j"), which FalkorDB rejects as an unknown ACL - # user. Credentials embedded in the URI take precedence over the args. - connect_user = parsed.username or (user if password else None) - connect_password = parsed.password or (password or None) - db = FalkorDB( - host=parsed.hostname or "localhost", - port=parsed.port or 6379, - username=connect_user, - password=connect_password, - ) - graph = db.select_graph(graph_name) - nodes_pushed = 0 - edges_pushed = 0 - - for node_id, data in G.nodes(data=True): - props = { - k: v for k, v in data.items() - if isinstance(v, (str, int, float, bool)) and not k.startswith("_") - } - props["id"] = node_id - cid = node_community.get(node_id) - if cid is not None: - props["community"] = cid - ftype = _safe_label(data.get("file_type", "Entity").capitalize()) - graph.query( - f"MERGE (n:{ftype} {{id: $id}}) SET n += $props", - {"id": node_id, "props": props}, - ) - nodes_pushed += 1 - - for u, v, data in G.edges(data=True): - rel = _safe_rel(data.get("relation", "RELATED_TO")) - props = { - k: v for k, v in data.items() - if isinstance(v, (str, int, float, bool)) and not k.startswith("_") - } - graph.query( - f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) " - f"MERGE (a)-[r:{rel}]->(b) SET r += $props", - {"src": u, "tgt": v, "props": props}, - ) - edges_pushed += 1 - - return {"nodes": nodes_pushed, "edges": edges_pushed} - - def to_graphml( G: nx.Graph, communities: dict[int, list[str]], diff --git a/graphify/exporters/__init__.py b/graphify/exporters/__init__.py new file mode 100644 index 000000000..41e1aeff9 --- /dev/null +++ b/graphify/exporters/__init__.py @@ -0,0 +1 @@ +"""exporters package.""" diff --git a/graphify/exporters/base.py b/graphify/exporters/base.py new file mode 100644 index 000000000..9c9419646 --- /dev/null +++ b/graphify/exporters/base.py @@ -0,0 +1,14 @@ +"""Shared constants/helpers for the graphify exporters package. + +Symbols used by more than one exporter live here so each exporter module can be +split out of graphify/export.py without a circular import (export.py and the +per-format modules both import from here, never from each other). +""" +from __future__ import annotations + +# Categorical palette for community coloring, shared by the HTML, SVG, and +# Obsidian exporters. Moved verbatim from graphify/export.py. +COMMUNITY_COLORS = [ + "#4E79A7", "#F28E2B", "#E15759", "#76B7B2", "#59A14F", + "#EDC948", "#B07AA1", "#FF9DA7", "#9C755F", "#BAB0AC", +] diff --git a/graphify/exporters/graphdb.py b/graphify/exporters/graphdb.py new file mode 100644 index 000000000..14c47f0d5 --- /dev/null +++ b/graphify/exporters/graphdb.py @@ -0,0 +1,173 @@ +"""graphdb — moved verbatim from graphify/export.py.""" +from __future__ import annotations + +from graphify.analyze import _node_community_map +import networkx as nx +import re + + +def push_to_neo4j( + G: nx.Graph, + uri: str, + user: str, + password: str, + communities: dict[int, list[str]] | None = None, +) -> dict[str, int]: + """Push graph directly to a running Neo4j instance via the Python driver. + + Requires: pip install neo4j + + Uses MERGE so re-running is safe - nodes and edges are upserted, not duplicated. + Returns a dict with counts of nodes and edges pushed. + """ + try: + from neo4j import GraphDatabase + except ImportError as e: + raise ImportError( + "neo4j driver not installed. Run: pip install neo4j" + ) from e + + node_community = _node_community_map(communities) if communities else {} + + def _safe_rel(relation: str) -> str: + return re.sub(r"[^A-Z0-9_]", "_", relation.upper().replace(" ", "_").replace("-", "_")) or "RELATED_TO" + + def _safe_label(label: str) -> str: + """Sanitize a Neo4j node label to prevent Cypher injection.""" + sanitized = re.sub(r"[^A-Za-z0-9_]", "", label) + return sanitized if sanitized else "Entity" + + driver = GraphDatabase.driver(uri, auth=(user, password)) + nodes_pushed = 0 + edges_pushed = 0 + + with driver.session() as session: + for node_id, data in G.nodes(data=True): + props = { + k: v for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } + props["id"] = node_id + cid = node_community.get(node_id) + if cid is not None: + props["community"] = cid + ftype = _safe_label(data.get("file_type", "Entity").capitalize()) + session.run( + f"MERGE (n:{ftype} {{id: $id}}) SET n += $props", + id=node_id, + props=props, + ) + nodes_pushed += 1 + + for u, v, data in G.edges(data=True): + rel = _safe_rel(data.get("relation", "RELATED_TO")) + props = { + k: v for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } + session.run( + f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) " + f"MERGE (a)-[r:{rel}]->(b) SET r += $props", + src=u, + tgt=v, + props=props, + ) + edges_pushed += 1 + + driver.close() + return {"nodes": nodes_pushed, "edges": edges_pushed} + +def push_to_falkordb( + G: nx.Graph, + uri: str, + user: str | None = None, + password: str | None = None, + communities: dict[int, list[str]] | None = None, + graph_name: str = "graphify", +) -> dict[str, int]: + """Push graph directly to a running FalkorDB instance via the Python SDK. + + Requires: pip install falkordb + + FalkorDB is OpenCypher-compatible, so the MERGE/SET upsert queries are + identical to push_to_neo4j. Differences from the Neo4j path: + - connects with FalkorDB(host, port, username, password) instead of a bolt + driver; only the host/port are read from the URI, so the scheme is + informational - "falkordb://localhost:6379", "redis://localhost:6379" + and a bare "localhost:6379" are all equivalent (default port 6379). + - a named graph is selected via db.select_graph(graph_name) (default + "graphify"); FalkorDB keys each graph by name in the same instance. + - queries run via graph.query(cypher, params) - there is no session object. + - auth is optional (FalkorDB runs without credentials by default), so user + and password may be None. + - no APOC: the Neo4j path does not use APOC either, so nothing to port. + + Uses MERGE so re-running is safe - nodes and edges are upserted, not + duplicated. Returns a dict with counts of nodes and edges pushed. + """ + try: + from falkordb import FalkorDB + except ImportError as e: + raise ImportError( + "falkordb SDK not installed. Run: pip install falkordb" + ) from e + + from urllib.parse import urlparse + + node_community = _node_community_map(communities) if communities else {} + + def _safe_rel(relation: str) -> str: + return re.sub(r"[^A-Z0-9_]", "_", relation.upper().replace(" ", "_").replace("-", "_")) or "RELATED_TO" + + def _safe_label(label: str) -> str: + """Sanitize a FalkorDB node label to prevent Cypher injection.""" + sanitized = re.sub(r"[^A-Za-z0-9_]", "", label) + return sanitized if sanitized else "Entity" + + parsed = urlparse(uri if "://" in uri else f"redis://{uri}") + # FalkorDB auth is optional. Only send credentials when a password is + # provided; otherwise connect anonymously and ignore any bolt-style default + # username (e.g. Neo4j's "neo4j"), which FalkorDB rejects as an unknown ACL + # user. Credentials embedded in the URI take precedence over the args. + connect_user = parsed.username or (user if password else None) + connect_password = parsed.password or (password or None) + db = FalkorDB( + host=parsed.hostname or "localhost", + port=parsed.port or 6379, + username=connect_user, + password=connect_password, + ) + graph = db.select_graph(graph_name) + nodes_pushed = 0 + edges_pushed = 0 + + for node_id, data in G.nodes(data=True): + props = { + k: v for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } + props["id"] = node_id + cid = node_community.get(node_id) + if cid is not None: + props["community"] = cid + ftype = _safe_label(data.get("file_type", "Entity").capitalize()) + graph.query( + f"MERGE (n:{ftype} {{id: $id}}) SET n += $props", + {"id": node_id, "props": props}, + ) + nodes_pushed += 1 + + for u, v, data in G.edges(data=True): + rel = _safe_rel(data.get("relation", "RELATED_TO")) + props = { + k: v for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } + graph.query( + f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) " + f"MERGE (a)-[r:{rel}]->(b) SET r += $props", + {"src": u, "tgt": v, "props": props}, + ) + edges_pushed += 1 + + return {"nodes": nodes_pushed, "edges": edges_pushed} diff --git a/graphify/exporters/html.py b/graphify/exporters/html.py new file mode 100644 index 000000000..5ba61ce24 --- /dev/null +++ b/graphify/exporters/html.py @@ -0,0 +1,547 @@ +"""html — moved verbatim from graphify/export.py.""" +from __future__ import annotations + +from graphify.exporters.base import COMMUNITY_COLORS # noqa: E402,F401 +from pathlib import Path +import html as _html +from graphify.analyze import _node_community_map +import json +import networkx as nx +from graphify.security import sanitize_label + + +MAX_NODES_FOR_VIZ = 5_000 + +def _viz_node_limit() -> int: + """Return the effective viz node limit, honoring GRAPHIFY_VIZ_NODE_LIMIT env var. + + Falls back to MAX_NODES_FOR_VIZ when the env var is unset, empty, or non-integer. + Set to 0 to disable HTML viz unconditionally (useful for CI runners). + """ + import os + raw = os.environ.get("GRAPHIFY_VIZ_NODE_LIMIT") + if raw is None or not raw.strip(): + return MAX_NODES_FOR_VIZ + try: + return int(raw) + except ValueError: + return MAX_NODES_FOR_VIZ + +def _html_styles() -> str: + return """""" + +def _hyperedge_script(hyperedges_json: str) -> str: + return f"""""" + +def _html_script(nodes_json: str, edges_json: str, legend_json: str) -> str: + return f"""""" + +def to_html( + G: nx.Graph, + communities: dict[int, list[str]], + output_path: str, + community_labels: dict[int, str] | None = None, + member_counts: dict[int, int] | None = None, + node_limit: int | None = None, + learning_overlay: dict | None = None, +) -> None: + """Generate an interactive vis.js HTML visualization of the graph. + + Features: node size by degree, click-to-inspect panel, search box, + community filter, physics clustering by community, confidence-styled edges. + Raises ValueError if graph exceeds MAX_NODES_FOR_VIZ. + + If member_counts is provided (aggregated community view), node sizes are + based on community member counts rather than graph degree. + + If node_limit is set and the graph exceeds it, automatically builds an + aggregated community-level meta-graph instead of raising ValueError. + """ + limit = node_limit if node_limit is not None else _viz_node_limit() + if G.number_of_nodes() > limit: + if node_limit is not None: + # Build aggregated community meta-graph + from collections import Counter as _Counter + import networkx as _nx + print(f"Graph has {G.number_of_nodes()} nodes (above {limit} limit). Building aggregated community view...") + node_to_community = {nid: cid for cid, members in communities.items() for nid in members} + meta = _nx.Graph() + for cid, members in communities.items(): + meta.add_node(str(cid), label=(community_labels or {}).get(cid, f"Community {cid}")) + edge_counts = _Counter() + for u, v in G.edges(): + cu, cv = node_to_community.get(u), node_to_community.get(v) + if cu is not None and cv is not None and cu != cv: + edge_counts[(min(cu, cv), max(cu, cv))] += 1 + for (cu, cv), w in edge_counts.items(): + meta.add_edge(str(cu), str(cv), weight=w, + relation=f"{w} cross-community edges", confidence="AGGREGATED") + if meta.number_of_nodes() <= 1: + print("Single community - aggregated view not useful. Skipping graph.html.") + return + meta_communities = {cid: [str(cid)] for cid in communities} + mc = {cid: len(members) for cid, members in communities.items()} + # Remap hyperedges from semantic node IDs to community IDs + raw_hyperedges = G.graph.get("hyperedges", []) + if raw_hyperedges: + remapped = [] + for he in raw_hyperedges: + he_members = he.get("nodes", []) + comm_ids, seen = [], set() + for nid in he_members: + c = node_to_community.get(nid) + if c is None: + continue + s = str(c) + if s in seen: + continue + seen.add(s) + comm_ids.append(s) + if len(comm_ids) < 2: + continue + remapped.append({ + "id": he.get("id", ""), + "label": he.get("label") or he.get("relation", "").replace("_", " "), + "nodes": comm_ids, + }) + meta.graph["hyperedges"] = remapped + to_html(meta, meta_communities, output_path, + community_labels=community_labels, member_counts=mc) + print(f"graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)") + print("Tip: run with --obsidian for full node-level detail.") + return + raise ValueError( + f"Graph has {G.number_of_nodes()} nodes - too large for HTML viz " + f"(limit: {limit}). Use --no-viz, raise GRAPHIFY_VIZ_NODE_LIMIT, " + f"or reduce input size." + ) + + node_community = _node_community_map(communities) + degree = dict(G.degree()) + max_deg = max(degree.values(), default=1) or 1 + max_mc = (max(member_counts.values(), default=1) or 1) if member_counts else 1 + + # Work-memory overlay (derived sidecar). When not passed explicitly, load it + # best-effort from the sibling .graphify_learning.json next to the output + # graph.html (which lives beside graph.json). Empty/missing => no learning + # fields, so the un-annotated render is byte-identical to pre-feature. + if learning_overlay is None: + learning_overlay = {} + try: + from graphify.reflect import load_learning_overlay as _llo + learning_overlay = _llo(Path(output_path)) + except Exception: + learning_overlay = {} + # Status -> ring color. preferred=green, contested=amber. Tentative gets no + # ring (it's not yet trustworthy enough to highlight in the map). + _RING = {"preferred": "#22c55e", "contested": "#f59e0b"} + + # Build nodes list for vis.js + vis_nodes = [] + for node_id, data in G.nodes(data=True): + cid = node_community.get(node_id, 0) + color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)] + label = sanitize_label(data.get("label", node_id)) + deg = degree.get(node_id, 1) + if member_counts: + mc = member_counts.get(cid, 1) + size = 10 + 30 * (mc / max_mc) + font_size = 12 + else: + size = 10 + 30 * (deg / max_deg) + # Only show label for high-degree nodes by default; others show on hover + font_size = 12 if deg >= max_deg * 0.15 else 0 + node = { + "id": node_id, + "label": label, + "color": {"background": color, "border": color, "highlight": {"background": "#ffffff", "border": color}}, + "size": round(size, 1), + "font": {"size": font_size, "color": "#ffffff"}, + "title": _html.escape(label), + "community": cid, + "community_name": sanitize_label((community_labels or {}).get(cid, f"Community {cid}")), + "source_file": sanitize_label(str(data.get("source_file") or "")), + "file_type": data.get("file_type", ""), + "degree": deg, + } + # Conditional learning fields — only present for annotated nodes, so + # un-annotated output keeps the exact pre-feature node dict shape. + entry = learning_overlay.get(str(node_id)) if learning_overlay else None + if entry: + status = sanitize_label(str(entry.get("status", ""))) + stale = bool(entry.get("stale")) + node["learning_status"] = status + node["learning_stale"] = stale + ring = _RING.get(status) + if ring: + # Status-colored ring via the border; stale => desaturated + + # dashed (vis.js supports per-node `shapeProperties.borderDashes`). + if stale: + ring = "#9ca3af" + node["shapeProperties"] = {"borderDashes": [4, 4]} + node["borderWidth"] = 3 + node["color"] = { + "background": color, "border": ring, + "highlight": {"background": "#ffffff", "border": ring}, + } + # Lesson line appended to the hover title. + if status == "contested": + lesson = f"Lesson: contested (useful {entry.get('uses', 0)} / dead-end {entry.get('neg', 0)})" + elif status == "preferred": + lesson = f"Lesson: preferred source ({entry.get('uses', 0)} useful, score={entry.get('score', 0)})" + else: + lesson = f"Lesson: {status} ({entry.get('uses', 0)} useful)" + if stale: + lesson += " [code changed — re-verify]" + node["title"] = _html.escape(label) + "\n" + _html.escape(sanitize_label(lesson)) + vis_nodes.append(node) + + # Build edges list. Restore original edge direction from _src/_tgt + # (stashed by build.py for exactly this reason): undirected NetworkX + # canonicalizes endpoint order, which would otherwise flip the arrow + # for `calls` and `rationale_for` in the rendered graph (#563). + vis_edges = [] + for u, v, data in G.edges(data=True): + confidence = data.get("confidence", "EXTRACTED") + relation = data.get("relation", "") + true_src = data.get("_src", u) + true_tgt = data.get("_tgt", v) + vis_edges.append({ + "from": true_src, + "to": true_tgt, + "label": relation, + "title": _html.escape(f"{relation} [{confidence}]"), + "dashes": confidence != "EXTRACTED", + "width": 2 if confidence == "EXTRACTED" else 1, + "color": {"opacity": 0.7 if confidence == "EXTRACTED" else 0.35}, + "confidence": confidence, + }) + + # Build community legend data + legend_data = [] + for cid in sorted((community_labels or {}).keys()): + color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)] + lbl = _html.escape(sanitize_label((community_labels or {}).get(cid, f"Community {cid}"))) + n = member_counts.get(cid, len(communities.get(cid, []))) if member_counts else len(communities.get(cid, [])) + legend_data.append({"cid": cid, "color": color, "label": lbl, "count": n}) + + # Escape sequences so embedded JSON cannot break out of the script tag + def _js_safe(obj) -> str: + return json.dumps(obj).replace(" + + + +graphify - {title} + +{_html_styles()} + + +
+ +{_html_script(nodes_json, edges_json, legend_json)} +{_hyperedge_script(hyperedges_json)} + +""" + + Path(output_path).write_text(html, encoding="utf-8") # nosec diff --git a/graphify/extract.py b/graphify/extract.py index ed0550432..a4bc25bc7 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -28,17 +28,116 @@ _make_id, _read_text, ) +from graphify.extractors.apex import extract_apex # noqa: F401 +from graphify.extractors.bash import extract_bash # noqa: F401 from graphify.extractors.blade import extract_blade # noqa: F401 from graphify.extractors.csharp import ( _resolve_cross_file_csharp_imports, _resolve_csharp_type_references, ) +from graphify.extractors.dart import extract_dart # noqa: F401 +from graphify.extractors.dm import extract_dm, extract_dmf, extract_dmi, extract_dmm # noqa: F401 from graphify.extractors.elixir import extract_elixir # noqa: F401 +from graphify.extractors.fortran import _cpp_preprocess, extract_fortran # noqa: F401 +from graphify.extractors.go import extract_go # noqa: F401 +from graphify.extractors.json_config import extract_json # noqa: F401 +from graphify.extractors.markdown import extract_markdown # noqa: F401 +from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 +from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 from graphify.extractors.razor import extract_razor # noqa: F401 +from graphify.extractors.rust import extract_rust # noqa: F401 +from graphify.extractors.sln import extract_sln # noqa: F401 +from graphify.extractors.sql import extract_sql # noqa: F401 +from graphify.extractors.terraform import extract_terraform # noqa: F401 +from graphify.extractors.verilog import extract_verilog # noqa: F401 from graphify.extractors.zig import extract_zig # noqa: F401 from graphify.security import sanitize_metadata from graphify.paths import disambiguate_ambiguous_candidates +from graphify.extractors.models import LanguageConfig, _JS_CACHE_BYPASS_SUFFIXES, _NamespaceExportFact, _StarExportFact, _SymbolAliasFact, _SymbolDeclarationFact, _SymbolExportFact, _SymbolImportFact, _SymbolResolutionFacts, _SymbolUseFact, _WORKSPACE_PACKAGE_CACHE # noqa: E402,F401 + +from graphify.extractors.resolution import ( # noqa: E402,F401 + _DECLDEF_HEADER_SUFFIXES, + _DECLDEF_IMPL_SUFFIXES, + _EXPORT_CONDITION_PRIORITY, + _JS_INDEX_FILES, + _JS_PRIMITIVE_TYPES, + _JS_RESOLVE_EXTS, + _TSCONFIG_ALIAS_CACHE, + _VUE_SCRIPT_LANG_RE, + _VUE_SCRIPT_RE, + _WORKSPACE_MANIFEST_NAMES, + _apply_symbol_resolution_facts, + _augment_symbol_resolution_edges, + _collect_js_symbol_resolution_facts, + _collect_python_symbol_resolution_facts, + _contained_in_package, + _decldef_class_stem, + _disambiguate_colliding_node_ids, + _find_workspace_root, + _is_type_like_definition, + _js_call_identifier, + _js_default_export_name, + _js_default_import_name, + _js_export_clause, + _js_export_statement_is_star, + _js_exported_declaration_names, + _js_lexical_aliases, + _js_module_specifier, + _js_named_specifiers, + _js_namespace_export_name, + _js_source_path, + _js_top_level_function_bodies, + _load_tsconfig_aliases, + _load_workspace_packages, + _match_tsconfig_alias, + _merge_decl_def_classes, + _node_disambiguation_source_key, + _package_entry_candidates, + _parse_js_tree, + _parse_python_tree, + _pascal_class_stem_cache, + _pascal_project_root, + _pascal_resolve_class, + _pascal_resolve_unit, + _pascal_unit_cache, + _pnpm_workspace_globs, + _python_call_identifier, + _python_import_from_module, + _python_imported_names, + _python_top_level_function_bodies, + _read_tsconfig_aliases, + _resolve_c_include_path, + _resolve_cross_file_imports, + _resolve_cross_file_java_imports, + _resolve_export_target, + _resolve_java_type_references, + _resolve_js_import_path, + _resolve_js_import_target, + _resolve_js_module_path, + _resolve_lua_import_target, + _resolve_python_module_path, + _resolve_tsconfig_alias, + _resolve_workspace_import, + _source_key, + _strip_jsonc, + _ts_collect_type_refs, + _ts_heritage_clause_entries, + _ts_walk_class_members, + _vue_mask_non_script, + _walk_js_tree, + _walk_python_tree, + _workspace_globs, +) + +from graphify.extractors.engine import REFERENCE_CONTEXTS, _CSHARP_TYPE_PARAMETER_SCOPE_DECLARATIONS, _C_PRIMITIVE_TYPE_NODES, _JAVA_BUILTIN_TYPES, _JAVA_TYPE_PARAMETER_SCOPE_DECLARATIONS, _JS_FUNCTION_VALUE_TYPES, _JS_SCOPE_BOUNDARY, _PYTHON_ANNOTATION_NOISE, _PYTHON_TYPE_CONTAINERS, _RUBY_CLASS_FACTORIES, _c_collect_type_refs, _cpp_collect_type_refs, _cpp_declarator_name, _cpp_local_var_types, _csharp_attribute_names, _csharp_classify_base, _csharp_collect_type_refs, _csharp_extra_walk, _csharp_member_type_table, _csharp_namespace_id, _csharp_namespace_name, _csharp_pre_scan_interfaces, _csharp_type_parameters_in_scope, _dynamic_import_js, _extract_generic, _find_body, _find_require_call, _get_cpp_func_name, _java_annotation_names, _java_collect_type_refs, _java_extra_walk, _java_type_parameters_in_scope, _js_collect_pattern_idents, _js_dispatch_value_idents, _js_extra_walk, _js_local_bound_names, _js_member_assignment_target, _js_module_bound_names, _kotlin_collect_type_refs, _kotlin_function_return_type_node, _kotlin_property_type_node, _kotlin_user_type_name, _php_collect_type_refs, _php_method_return_type_node, _php_name_text, _python_collect_assignment_targets, _python_collect_param_refs, _python_collect_type_refs, _python_local_bound_names, _python_module_bound_names, _python_param_names, _read_csharp_type_name, _require_imports_js, _ruby_const_last_name, _ruby_extra_walk, _ruby_local_class_bindings, _ruby_new_class_name, _scala_collect_type_refs, _semantic_reference_edge, _source_location, _swift_classify_base, _swift_collect_type_refs, _swift_constructor_type, _swift_declaration_keyword, _swift_extra_walk, _swift_local_var_types, _swift_pre_scan, _swift_property_name, _swift_property_type_node, _swift_receiver_name, _swift_user_type_name, _ts_decorator_name, _ts_descendant_decorators, _ts_emit_decorator_edges, _ts_extra_walk, _ts_method_name, _ts_receiver_type_table # noqa: E402,F401 + +from graphify.extractors.pascal import _PAS_BEGIN_END_TOKEN_RE, _PAS_CALL_RE, _PAS_END_SEMI_RE, _PAS_IMPL_HEADER_RE, _PAS_KEYWORDS, _PAS_METHOD_DECL_RE, _PAS_MODULE_RE, _PAS_TOKEN_RE, _PAS_TYPE_HEADER_RE, _PAS_USES_RE, _extract_pascal_regex, _pascal_find_body, _pascal_split_bases, _pascal_split_sections, _pascal_split_uses, _pascal_strip_comments, extract_pascal # noqa: E402,F401 + +from graphify.extractors.objc import _objc_local_var_types, extract_objc # noqa: E402,F401 + +from graphify.extractors.julia import extract_julia # noqa: E402,F401 + _RECURSION_LIMIT = 10_000 # Language built-in globals that AST may classify as call targets when used as @@ -67,10 +166,6 @@ def _safe_extract(extractor: Callable, path: Path) -> dict: return {"nodes": [], "edges": [], "error": f"{type(e).__name__}: {e}"} - - - - def _file_node_id(rel_path: Path) -> str: """File-level node ID matching the skill.md spec: ``{parent_dir}_{stem}`` — one parent directory level, no extension. ``rel_path`` MUST be relative to @@ -81,791 +176,26 @@ def _file_node_id(rel_path: Path) -> str: return _make_id(_file_stem(rel_path)) -def _csharp_namespace_id(dotted_name: str) -> str: - digest = hashlib.sha1(dotted_name.encode("utf-8")).hexdigest()[:16] - return f"csharp_namespace:{digest}" - - -_TSCONFIG_ALIAS_CACHE: dict[str, dict[str, list[str]]] = {} -_WORKSPACE_PACKAGE_CACHE: dict[str, dict[str, Path]] = {} -_WORKSPACE_MANIFEST_NAMES = ("pnpm-workspace.yaml", "package.json") -_JS_CACHE_BYPASS_SUFFIXES = {".js", ".jsx", ".mjs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte"} -_JS_RESOLVE_EXTS = (".ts", ".tsx", ".mts", ".cts", ".svelte", ".js", ".jsx", ".mjs") -_JS_INDEX_FILES = ("index.ts", "index.tsx", "index.svelte", "index.js", "index.jsx", "index.mjs") - - SEMANTIC_RELATIONS = frozenset({ "inherits", "implements", "mixes_in", "embeds", "references", "calls", "imports", "imports_from", "re_exports", "contains", "method", }) -REFERENCE_CONTEXTS = frozenset({ - "field", "parameter_type", "return_type", "generic_arg", "attribute", "value", "type", -}) - - -def _source_location(line: int | str | None) -> str | None: - if line is None: - return None - if isinstance(line, str): - return line if line.startswith("L") else f"L{line}" - return f"L{line}" - - -def _semantic_reference_edge( - source: str, - target: str, - context: str, - source_file: str, - line: int | str | None, -) -> dict: - if context not in REFERENCE_CONTEXTS: - raise ValueError(f"unknown reference context: {context}") - return { - "source": source, - "target": target, - "relation": "references", - "context": context, - "confidence": "EXTRACTED", - "source_file": source_file, - "source_location": _source_location(line), - "weight": 1.0, - } - - -def _resolve_js_import_path(candidate: Path) -> Path: - """Resolve a JS/TS/Svelte import target to a local file when it exists.""" - candidate = Path(os.path.normpath(candidate)) - if candidate.is_file(): - return candidate - - # TS ESM convention: imports often spell .js/.jsx while source is .ts/.tsx. - if candidate.suffix == ".js": - ts_candidate = candidate.with_suffix(".ts") - if ts_candidate.is_file(): - return ts_candidate - elif candidate.suffix == ".jsx": - tsx_candidate = candidate.with_suffix(".tsx") - if tsx_candidate.is_file(): - return tsx_candidate - - # Append extensions to the full filename, which covers extensionless imports, - # multi-dot helpers, and Svelte 5 rune files like Foo.svelte.ts. - for ext in _JS_RESOLVE_EXTS: - with_ext = candidate.parent / f"{candidate.name}{ext}" - if with_ext.is_file(): - return with_ext - - # Only fall back to directory indexes after file candidates lose. - if candidate.is_dir(): - for index_name in _JS_INDEX_FILES: - index_candidate = candidate / index_name - if index_candidate.is_file(): - return index_candidate - - return candidate - - -def _strip_jsonc(text: str) -> str: - """Strip // line comments, /* */ block comments, and trailing commas from JSONC. - - Preserves string contents (including // and /* inside strings) by skipping over - quoted spans first. Required for tsconfig.json files generated by SvelteKit, - NestJS, Vite, T3, Astro, etc., which use JSONC by default (#700). - """ - # Remove block and line comments while leaving string literals untouched. - pattern = re.compile( - r'"(?:\\.|[^"\\])*"' # double-quoted string (with escapes) - r"|/\*.*?\*/" # /* block comment */ - r"|//[^\n]*", # // line comment - re.DOTALL, - ) - - def _replace(match: re.Match) -> str: - token = match.group(0) - if token.startswith('"'): - return token - return "" - - stripped = pattern.sub(_replace, text) - # Remove trailing commas before } or ] (allowing whitespace between). - stripped = re.sub(r",(\s*[}\]])", r"\1", stripped) - return stripped - - -def _read_tsconfig_aliases(tsconfig: Path, base_dir: Path, seen: set) -> dict[str, list[str]]: - """Recursively read path aliases from a tsconfig, following extends chains. - - Child config paths override parent. Circular extends are detected via seen set. - npm package configs (e.g. @tsconfig/svelte) are skipped since they're not on disk. - Handles JSONC (comments + trailing commas) which is the default tsconfig format - for SvelteKit, NestJS, Vite, T3, Astro, etc. (#700). - """ - if str(tsconfig) in seen: - return {} - seen.add(str(tsconfig)) - try: - raw = tsconfig.read_text(encoding="utf-8") - except Exception as e: - print(f" warning: could not read {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True) - return {} - try: - data = json.loads(raw) - except json.JSONDecodeError: - try: - data = json.loads(_strip_jsonc(raw)) - except json.JSONDecodeError as e: - print(f" warning: failed to parse {tsconfig} as JSON/JSONC ({e.msg} at line {e.lineno} col {e.colno})", file=sys.stderr, flush=True) - return {} - except Exception as e: - print(f" warning: failed to parse {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True) - return {} - - aliases: dict[str, list[str]] = {} - # `extends` may be a string or, since TypeScript 5.0, an array of paths. - # For an array, parents are processed in order with later entries - # overriding earlier ones; the extending config (paths below) overrides - # all parents. Without the list branch, an array `extends` raised - # `AttributeError: 'list' object has no attribute 'startswith'`, which - # _safe_extract turned into a skip of the whole file. - extends = data.get("extends") - if isinstance(extends, str): - extends_list = [extends] - elif isinstance(extends, list): - extends_list = [e for e in extends if isinstance(e, str)] - else: - extends_list = [] - for ext in extends_list: - # Skip scoped npm package configs (e.g. @tsconfig/svelte) — not on disk. - if not ext or ext.startswith("@"): - continue - extended_path = (base_dir / ext).resolve() - if not extended_path.suffix: - extended_path = extended_path.with_suffix(".json") - if extended_path.exists(): - aliases.update(_read_tsconfig_aliases(extended_path, extended_path.parent, seen)) - - # tsconfig `paths` are resolved relative to `baseUrl` (itself relative to - # the tsconfig's directory), not the tsconfig directory directly. Honoring - # baseUrl is required for the common monorepo / NestJS layout where - # baseUrl points at a subdirectory, e.g. baseUrl "./src" with - # "@services/*": ["services/*"] must resolve to /src/services rather - # than /services. Defaults to "." so configs without baseUrl (paths - # relative to the tsconfig dir, the TS 4.1+ behavior) keep working. - compiler_options = data.get("compilerOptions", {}) - base_url = compiler_options.get("baseUrl") or "." - paths_base = base_dir / base_url - paths = compiler_options.get("paths", {}) - for alias, targets in paths.items(): - if not targets: - continue - # Keep ALL targets in declared order — tsc tries each until one resolves - # on disk. Discarding the fallbacks (#1531) misresolved/dropped imports - # whose file lived at a non-first target. Preserve wildcard tokens in - # both sides until the resolver substitutes the captured segment, then - # normalizes the concrete path (#927). Empty/non-string entries are skipped. - target_patterns = [ - str(paths_base / t) - for t in targets - if isinstance(t, str) and t - ] - if target_patterns: - aliases[alias] = target_patterns - - return aliases - - -def _load_tsconfig_aliases(start_dir: Path) -> dict[str, list[str]]: - """Walk up from start_dir to find tsconfig.json and return compilerOptions.paths aliases. - - Follows extends chains so SvelteKit/Nuxt/NestJS inherited aliases are included. - Returns a dict mapping alias patterns to ordered resolved target patterns; - wildcard tokens remain intact for substitution during resolution (#927). - Result is cached by tsconfig path string. - """ - current = start_dir.resolve() - for candidate in [current, *current.parents]: - tsconfig = candidate / "tsconfig.json" - if tsconfig.exists(): - key = str(tsconfig) - if key not in _TSCONFIG_ALIAS_CACHE: - _TSCONFIG_ALIAS_CACHE[key] = _read_tsconfig_aliases(tsconfig, candidate, seen=set()) - return _TSCONFIG_ALIAS_CACHE[key] - return {} - - -def _match_tsconfig_alias(raw: str, pattern: str) -> "tuple[tuple[int, int], str, bool] | None": - """Return (specificity, captured text, is_wildcard) when pattern matches raw. - - Exact aliases win first. Wildcard aliases follow TypeScript's longest-prefix - rule. The final branch preserves Graphify's existing support for treating a - non-wildcard alias as a directory prefix, but only after real wildcard matches. - """ - if "*" in pattern: - if pattern.count("*") != 1: - return None - prefix, suffix = pattern.split("*", 1) - if not raw.startswith(prefix) or not raw.endswith(suffix): - return None - end = len(raw) - len(suffix) if suffix else len(raw) - if end < len(prefix): - return None - return (1, -len(prefix)), raw[len(prefix):end], True - - if raw == pattern: - return (0, -len(pattern)), "", False - - prefix = pattern.rstrip("/") - if prefix and raw.startswith(prefix + "/"): - return (2, -len(prefix)), raw[len(prefix):].lstrip("/"), False - return None - - -def _resolve_tsconfig_alias(raw: str, aliases: dict[str, list[str]]) -> "Path | None": - """Resolve `raw` against the most specific matching tsconfig alias pattern. - - Within that pattern, try targets in declared order and return the first whose - candidate resolves to a real file. If none exist, return the first candidate - so existing phantom/external-edge behavior stays unchanged. - """ - best: "tuple[tuple[int, int], str, bool, list[str]] | None" = None - for pattern, targets in aliases.items(): - match = _match_tsconfig_alias(raw, pattern) - if match is None: - continue - specificity, captured, is_wildcard = match - if best is None or specificity < best[0]: - best = specificity, captured, is_wildcard, targets - - if best is None: - return None - - _, captured, is_wildcard, targets = best - first = None - for target in targets: - if is_wildcard: - # TypeScript substitutes only when the matched star is non-empty. - substituted = target.replace("*", captured, 1) if captured else target - cand = Path(os.path.normpath(substituted)) - else: - cand = Path(target) - if captured: - cand = Path(os.path.normpath(cand / captured)) - resolved = _resolve_js_import_path(cand) - if resolved.is_file(): - return resolved - if first is None: - first = cand - return first - - -def _find_workspace_root(start_dir: Path) -> Path | None: - current = start_dir.resolve() - for candidate in [current, *current.parents]: - if (candidate / "pnpm-workspace.yaml").exists(): - return candidate - package_json = candidate / "package.json" - if package_json.is_file(): - try: - data = json.loads(package_json.read_text(encoding="utf-8")) - except Exception: - continue - if "workspaces" in data: - return candidate - return None - - -def _pnpm_workspace_globs(workspace_file: Path) -> list[str]: - globs: list[str] = [] - in_packages = False - for raw_line in workspace_file.read_text(encoding="utf-8", errors="replace").splitlines(): - line = raw_line.strip() - if not line or line.startswith("#"): - continue - if line.startswith("packages:"): - in_packages = True - continue - if in_packages and line.startswith("-"): - value = line[1:].strip().strip("'\"") - if value and not value.startswith("!"): - globs.append(value) - continue - if in_packages and not raw_line.startswith((" ", "\t")): - break - return globs - - -def _workspace_globs(root: Path) -> list[str]: - pnpm_workspace = root / "pnpm-workspace.yaml" - if pnpm_workspace.exists(): - return _pnpm_workspace_globs(pnpm_workspace) - - package_json = root / "package.json" - try: - data = json.loads(package_json.read_text(encoding="utf-8")) - except Exception: - return [] - - workspaces = data.get("workspaces") - if isinstance(workspaces, list): - return [item for item in workspaces if isinstance(item, str) and not item.startswith("!")] - if isinstance(workspaces, dict): - packages = workspaces.get("packages") - if isinstance(packages, list): - return [item for item in packages if isinstance(item, str) and not item.startswith("!")] - return [] - - -def _load_workspace_packages(start_dir: Path) -> dict[str, Path]: - root = _find_workspace_root(start_dir) - if root is None: - return {} - manifest_mtimes = tuple( - (name, (root / name).stat().st_mtime_ns) - for name in _WORKSPACE_MANIFEST_NAMES - if (root / name).is_file() - ) - key = str((root, manifest_mtimes)) - if key in _WORKSPACE_PACKAGE_CACHE: - return _WORKSPACE_PACKAGE_CACHE[key] - - packages: dict[str, Path] = {} - for pattern in _workspace_globs(root): - package_dirs: list[Path] = [root] if pattern in (".", "./") else list(root.glob(pattern)) - for package_dir in package_dirs: - manifest = package_dir / "package.json" - if not manifest.is_file(): - continue - try: - data = json.loads(manifest.read_text(encoding="utf-8")) - except Exception: - continue - name = data.get("name") - if isinstance(name, str) and name: - packages[name] = package_dir - _WORKSPACE_PACKAGE_CACHE[key] = packages - return packages - # Condition keys consulted when resolving an `exports` target, in priority # order. `default` is Node's catch-all and must be consulted LAST so a more # specific condition (source/import/module/etc.) wins when several match. -_EXPORT_CONDITION_PRIORITY = ( - "source", "import", "module", "svelte", "types", "require", "default", -) - - -def _resolve_export_target(value: Any) -> str | None: - """Resolve an `exports` map value (string or condition object) to a - relative target string, honouring _EXPORT_CONDITION_PRIORITY for objects - and recursing into nested condition objects.""" - if isinstance(value, str): - return value - if isinstance(value, dict): - for cond in _EXPORT_CONDITION_PRIORITY: - v = value.get(cond) - if isinstance(v, str): - return v - if isinstance(v, dict): - nested = _resolve_export_target(v) - if nested: - return nested - return None - - -def _contained_in_package(resolved: Path, package_dir: Path) -> bool: - """Guard against `exports` targets that escape the package directory - (e.g. "./evil": "../../../etc/passwd"). Only accept paths that stay - within package_dir after resolution.""" - try: - return resolved.resolve().is_relative_to(package_dir.resolve()) - except ValueError: - return False - - -def _package_entry_candidates(package_dir: Path, subpath: str) -> list[Path]: - manifest = package_dir / "package.json" - manifest_data: dict[str, Any] = {} - try: - manifest_data = json.loads(manifest.read_text(encoding="utf-8")) - except Exception: - pass - - if subpath: - # Consult the package's `exports` subpath map before the bare-path - # fallback (#1308): "./browser" -> conditions -> file, plus single - # wildcard "./*" patterns. Targets that escape the package dir are - # rejected; resolution then falls through to the bare path. - exports = manifest_data.get("exports") - if isinstance(exports, dict): - subpath_key = "./" + subpath - target = _resolve_export_target(exports.get(subpath_key)) - if target: - candidate = package_dir / target - if _contained_in_package(candidate, package_dir): - return [candidate] - else: - for pattern, pattern_value in exports.items(): - if "*" in pattern and pattern.count("*") == 1: - prefix, suffix = pattern.split("*", 1) - if (subpath_key.startswith(prefix) - and (not suffix or subpath_key.endswith(suffix))): - matched = subpath_key[len(prefix):len(subpath_key) - len(suffix) if suffix else None] - resolved = _resolve_export_target(pattern_value) - if resolved and "*" in resolved: - candidate = package_dir / resolved.replace("*", matched) - if _contained_in_package(candidate, package_dir): - return [candidate] - return [package_dir / subpath] - - exports = manifest_data.get("exports") - if isinstance(exports, str): - return [package_dir / exports] - if isinstance(exports, dict): - dot_target = _resolve_export_target(exports.get(".")) - if dot_target: - return [package_dir / dot_target] - - candidates: list[Path] = [] - for key in ("svelte", "module", "main", "types"): - value = manifest_data.get(key) - if isinstance(value, str): - candidates.append(package_dir / value) - candidates.append(package_dir / "src/index") - candidates.append(package_dir / "index") - return candidates - - -def _resolve_workspace_import(raw: str, start_dir: Path) -> Path | None: - packages = _load_workspace_packages(start_dir) - for package_name, package_dir in packages.items(): - if raw == package_name: - subpath = "" - elif raw.startswith(package_name + "/"): - subpath = raw[len(package_name) + 1:] - else: - continue - for candidate in _package_entry_candidates(package_dir, subpath): - resolved = _resolve_js_import_path(candidate) - if resolved.is_file(): - return resolved - return None - - -def _resolve_js_module_path(raw: str | Path, start_dir: Path | None = None) -> Path | None: - """Resolve a JS/TS module path or specifier to a local source file. - - With a Path argument this preserves the path-based helper API used by - import-extension tests. With a string plus start_dir it resolves JS/TS - module specifiers including relative paths, tsconfig aliases, and workspace - packages. - """ - if isinstance(raw, Path): - return _resolve_js_import_path(raw) - if start_dir is None: - return _resolve_js_import_path(Path(raw)) - if raw.startswith("."): - return _resolve_js_import_path(start_dir / raw) - - aliases = _load_tsconfig_aliases(start_dir) - hit = _resolve_tsconfig_alias(raw, aliases) - if hit is not None: - return _resolve_js_import_path(hit) - - return _resolve_workspace_import(raw, start_dir) # ── LanguageConfig dataclass ───────────────────────────────────────────────── -@dataclass -class LanguageConfig: - ts_module: str # e.g. "tree_sitter_python" - ts_language_fn: str = "language" # attr to call: e.g. tslang.language() - - class_types: frozenset = frozenset() - function_types: frozenset = frozenset() - import_types: frozenset = frozenset() - call_types: frozenset = frozenset() - static_prop_types: frozenset = frozenset() - helper_fn_names: frozenset = frozenset() - container_bind_methods: frozenset = frozenset() - event_listener_properties: frozenset = frozenset() - - # Name extraction - name_field: str = "name" - name_fallback_child_types: tuple = () - - # Body detection - body_field: str = "body" - body_fallback_child_types: tuple = () # e.g. ("declaration_list", "compound_statement") - - # Call name extraction - call_function_field: str = "function" # field on call node for callee - call_accessor_node_types: frozenset = frozenset() # member/attribute nodes - call_accessor_field: str = "attribute" # field on accessor for method name - call_accessor_object_field: str = "" # field on accessor for the receiver/object - - # Stop recursion at these types in walk_calls - function_boundary_types: frozenset = frozenset() - - # Import handler: called for import nodes instead of generic handling - import_handler: Callable | None = None - - # Optional custom name resolver for functions (C, C++ declarator unwrapping) - resolve_function_name_fn: Callable | None = None - - # Extra label formatting for functions: if True, functions get "name()" label - function_label_parens: bool = True - - # Extra walk hook called after generic dispatch (for JS arrow functions, C# namespaces, etc.) - extra_walk_fn: Callable | None = None - # ── Generic helpers ─────────────────────────────────────────────────────────── - -_PYTHON_TYPE_CONTAINERS = frozenset({ - "list", "dict", "set", "tuple", "frozenset", "type", - "List", "Dict", "Set", "Tuple", "FrozenSet", "Type", - "Optional", "Union", "Sequence", "Iterable", "Mapping", "MutableMapping", - "Iterator", "Callable", "Awaitable", "AsyncIterable", "AsyncIterator", "Coroutine", - "Generator", "AsyncGenerator", "ContextManager", "AsyncContextManager", - "Annotated", "ClassVar", "Final", "Literal", "Concatenate", "ParamSpec", "TypeVar", - "None", "Ellipsis", -}) - # Scalar builtins and test-mock names that appear as type annotations but carry # no useful semantic meaning as graph nodes (#1147). Suppressed at the annotation # walker level so they are never created as nodes or emitted as edges. -_PYTHON_ANNOTATION_NOISE = frozenset({ - # scalar builtins - "str", "int", "float", "bool", "bytes", "bytearray", "complex", "object", - "True", "False", - # unittest.mock - "MagicMock", "Mock", "AsyncMock", "NonCallableMock", - "NonCallableMagicMock", "PropertyMock", "patch", "sentinel", -}) - - -def _python_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a Python type annotation; append (name, role) where role is 'type' or 'generic_arg'. - - Builtin/typing containers (list, dict, Optional, Union, …) are not emitted as refs themselves, - but their nested type arguments still count as generic_arg. - """ - if node is None: - return - t = node.type - if t == "type": - for c in node.children: - if c.is_named: - _python_collect_type_refs(c, source, generic, out) - return - if t == "identifier": - name = _read_text(node, source) - if name and name not in _PYTHON_TYPE_CONTAINERS and name not in _PYTHON_ANNOTATION_NOISE: - out.append((name, "generic_arg" if generic else "type")) - return - if t == "attribute": - tail = _read_text(node, source).rsplit(".", 1)[-1] - if tail and tail not in _PYTHON_TYPE_CONTAINERS and tail not in _PYTHON_ANNOTATION_NOISE: - out.append((tail, "generic_arg" if generic else "type")) - return - if t == "generic_type": - for c in node.children: - if c.type == "identifier": - container = _read_text(c, source) - if container and container not in _PYTHON_TYPE_CONTAINERS and container not in _PYTHON_ANNOTATION_NOISE: - out.append((container, "generic_arg" if generic else "type")) - elif c.type == "type_parameter": - for sub in c.children: - if sub.is_named: - _python_collect_type_refs(sub, source, True, out) - return - if t == "subscript": - value = node.child_by_field_name("value") - if value is not None: - _python_collect_type_refs(value, source, generic, out) - for c in node.children: - if c is value or not c.is_named: - continue - _python_collect_type_refs(c, source, True, out) - return - if node.is_named: - for c in node.children: - if c.is_named: - _python_collect_type_refs(c, source, generic, out) - - -def _csharp_pre_scan_interfaces(root_node, source: bytes) -> set[str]: - """Return names declared as `interface` in this C# compilation unit.""" - out: set[str] = set() - stack = [root_node] - while stack: - n = stack.pop() - if n.type == "interface_declaration": - name_node = n.child_by_field_name("name") - if name_node is not None: - text = _read_text(name_node, source) - if text: - out.add(text) - stack.extend(n.children) - return out - - -def _csharp_classify_base(name: str, interface_names: set[str]) -> str: - """`implements` if the base name is an interface (declared or by I-prefix convention), else `inherits`.""" - if name in interface_names: - return "implements" - if len(name) >= 2 and name[0] == "I" and name[1].isupper(): - return "implements" - return "inherits" - - -_CSHARP_TYPE_PARAMETER_SCOPE_DECLARATIONS = frozenset({ - "class_declaration", - "interface_declaration", - "record_declaration", - "struct_declaration", - "method_declaration", -}) - - -def _csharp_type_parameters_in_scope(node, source: bytes) -> frozenset[str]: - """Return C# type-parameter names visible from ``node``.""" - names: set[str] = set() - scope = node - while scope is not None: - if scope.type in _CSHARP_TYPE_PARAMETER_SCOPE_DECLARATIONS: - for child in scope.children: - if child.type != "type_parameter_list": - continue - for param in child.children: - if param.type == "type_parameter": - name_node = next( - (sub for sub in param.children if sub.type == "identifier"), - None, - ) - if name_node is not None: - name = _read_text(name_node, source) - if name: - names.add(name) - elif param.type == "identifier": - name = _read_text(param, source) - if name: - names.add(name) - scope = scope.parent - return frozenset(names) - - -def _csharp_collect_type_refs( - node, - source: bytes, - generic: bool, - out: list[tuple[str, str, bool, str]], - skip: frozenset[str] | None = None, -) -> None: - """Walk a C# type expression; append (name, role, qualified, qualifier) tuples.""" - if node is None: - return - if skip is None: - skip = _csharp_type_parameters_in_scope(node, source) - t = node.type - if t == "predefined_type": - return - if t == "identifier": - name = _read_text(node, source) - if name and name not in skip: - out.append((name, "generic_arg" if generic else "type", False, "")) - return - if t == "qualified_name": - prefix, _, text = _read_text(node, source).rpartition(".") - text = text.split("<", 1)[0] - if text and text not in skip: - out.append((text, "generic_arg" if generic else "type", True, prefix)) - return - if t == "generic_name": - name_child = node.child_by_field_name("name") - if name_child is None: - for sub in node.children: - if sub.type == "identifier": - name_child = sub - break - if name_child is not None: - qualified = name_child.type == "qualified_name" - prefix, _, name = _read_text(name_child, source).rpartition(".") - if name and name not in skip: - out.append((name, "generic_arg" if generic else "type", qualified, prefix if qualified else "")) - for sub in node.children: - if sub.type == "type_argument_list": - for arg in sub.children: - if arg.is_named: - _csharp_collect_type_refs(arg, source, True, out, skip) - return - if t in ("nullable_type", "array_type", "pointer_type", "ref_type"): - for c in node.children: - if c.is_named: - _csharp_collect_type_refs(c, source, generic, out, skip) - return - if node.is_named: - for c in node.children: - if c.is_named: - _csharp_collect_type_refs(c, source, generic, out, skip) - - -def _csharp_attribute_names(method_node, source: bytes) -> list[tuple[str, bool, str]]: - """Collect attribute names from a C# method/declaration's attribute_list children.""" - names: list[tuple[str, bool, str]] = [] - skip = _csharp_type_parameters_in_scope(method_node, source) - for child in method_node.children: - if child.type != "attribute_list": - continue - for attr in child.children: - if attr.type != "attribute": - continue - name_node = attr.child_by_field_name("name") - if name_node is None: - for sub in attr.children: - if sub.type in ("identifier", "qualified_name"): - name_node = sub - break - if name_node is not None: - qualified = name_node.type == "qualified_name" - prefix, _, text = _read_text(name_node, source).rpartition(".") - if text and text not in skip: - names.append((text, qualified, prefix if qualified else "")) - return names - - -_JAVA_TYPE_PARAMETER_SCOPE_DECLARATIONS = frozenset({ - "class_declaration", - "interface_declaration", - "record_declaration", - "method_declaration", - "constructor_declaration", -}) - - -def _java_type_parameters_in_scope(node, source: bytes) -> frozenset[str]: - """Return Java type-parameter names visible from ``node``.""" - names: set[str] = set() - scope = node - while scope is not None: - if scope.type in _JAVA_TYPE_PARAMETER_SCOPE_DECLARATIONS: - params = scope.child_by_field_name("type_parameters") - if params is not None: - for param in params.children: - if param.type != "type_parameter": - continue - name_node = next( - (child for child in param.children if child.type == "type_identifier"), - None, - ) - if name_node is not None: - names.add(_read_text(name_node, source)) - scope = scope.parent - return frozenset(names) # java.lang (auto-imported) plus the ubiquitous java.util / java.io / java.time / @@ -876,909 +206,31 @@ def _java_type_parameters_in_scope(node, source: bytes) -> frozenset[str]: # type-ref walker so they are never created as nodes or emitted as edges. The # boxed-scalar/`void` primitives are already dropped by grammar node type above; # these are the class/interface names the grammar reports as identifiers. -_JAVA_BUILTIN_TYPES = frozenset({ - # java.lang — core - "Object", "String", "CharSequence", "StringBuilder", "StringBuffer", - "Number", "Byte", "Short", "Integer", "Long", "Float", "Double", - "Boolean", "Character", "Void", "Class", "Enum", "Record", "Math", - "System", "Thread", "Runnable", "Comparable", "Iterable", "Cloneable", - "AutoCloseable", "Appendable", "Readable", "Process", "ProcessBuilder", - "Runtime", "Package", "ThreadLocal", "InheritableThreadLocal", - # java.lang — throwables - "Throwable", "Exception", "RuntimeException", "Error", - "IllegalArgumentException", "IllegalStateException", "NullPointerException", - "IndexOutOfBoundsException", "ArrayIndexOutOfBoundsException", - "ClassCastException", "NumberFormatException", "ArithmeticException", - "UnsupportedOperationException", "InterruptedException", - "CloneNotSupportedException", "SecurityException", "StackOverflowError", - "OutOfMemoryError", "AssertionError", - # java.util — collections & core - "Collection", "List", "ArrayList", "LinkedList", "Vector", "Stack", - "Set", "HashSet", "LinkedHashSet", "TreeSet", "SortedSet", "NavigableSet", - "EnumSet", "Map", "HashMap", "LinkedHashMap", "TreeMap", "SortedMap", - "NavigableMap", "Hashtable", "EnumMap", "Properties", "Queue", "Deque", - "ArrayDeque", "PriorityQueue", "Iterator", "ListIterator", "Comparator", - "Optional", "OptionalInt", "OptionalLong", "OptionalDouble", "Collections", - "Arrays", "Objects", "Date", "Calendar", "Random", "UUID", "Scanner", - "StringJoiner", "StringTokenizer", "BitSet", "Spliterator", "Locale", - "NoSuchElementException", "ConcurrentModificationException", - # java.util.stream - "Stream", "IntStream", "LongStream", "DoubleStream", "Collector", - "Collectors", - # java.util.function - "Function", "BiFunction", "Consumer", "BiConsumer", "Supplier", - "Predicate", "BiPredicate", "UnaryOperator", "BinaryOperator", - "IntFunction", "ToIntFunction", "ToLongFunction", "ToDoubleFunction", - # java.util.concurrent - "Callable", "Future", "CompletableFuture", "CompletionStage", "Executor", - "ExecutorService", "Executors", "ScheduledExecutorService", "TimeUnit", - "ConcurrentHashMap", "ConcurrentMap", "CopyOnWriteArrayList", - "BlockingQueue", "CountDownLatch", "Semaphore", "CyclicBarrier", - "AtomicInteger", "AtomicLong", "AtomicBoolean", "AtomicReference", - # java.time - "Instant", "Duration", "Period", "LocalDate", "LocalTime", "LocalDateTime", - "ZonedDateTime", "OffsetDateTime", "ZoneId", "ZoneOffset", "DayOfWeek", - "Month", "Year", "Clock", "DateTimeFormatter", - # java.io / java.nio.file - "IOException", "UncheckedIOException", "FileNotFoundException", "File", - "InputStream", "OutputStream", "Reader", "Writer", "BufferedReader", - "BufferedWriter", "InputStreamReader", "OutputStreamWriter", "FileReader", - "FileWriter", "PrintStream", "PrintWriter", "ByteArrayInputStream", - "ByteArrayOutputStream", "Serializable", "Closeable", "Path", "Paths", - "Files", - # java.math - "BigDecimal", "BigInteger", -}) - - -def _java_collect_type_refs( - node, - source: bytes, - generic: bool, - out: list[tuple[str, str]], - skip: frozenset[str] | None = None, -) -> None: - """Walk a Java type expression; append (name, role) tuples.""" - if node is None: - return - if skip is None: - skip = _java_type_parameters_in_scope(node, source) - t = node.type - if t in ("integral_type", "floating_point_type", "boolean_type", "void_type"): - return - if t == "type_identifier": - name = _read_text(node, source) - if name and name not in skip and name not in _JAVA_BUILTIN_TYPES: - out.append((name, "generic_arg" if generic else "type")) - return - if t == "scoped_type_identifier": - text = _read_text(node, source).rsplit(".", 1)[-1] - if text and text not in _JAVA_BUILTIN_TYPES: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "generic_type": - for c in node.children: - if c.type in ("type_identifier", "scoped_type_identifier"): - text = _read_text(c, source).rsplit(".", 1)[-1] - if ( - text - and text not in _JAVA_BUILTIN_TYPES - and (c.type == "scoped_type_identifier" or text not in skip) - ): - out.append((text, "generic_arg" if generic else "type")) - break - for c in node.children: - if c.type == "type_arguments": - for arg in c.children: - if arg.is_named: - _java_collect_type_refs(arg, source, True, out, skip) - return - if t == "array_type": - for c in node.children: - if c.is_named: - _java_collect_type_refs(c, source, generic, out, skip) - return - if node.is_named: - for c in node.children: - if c.is_named: - _java_collect_type_refs(c, source, generic, out, skip) - - -def _java_annotation_names(declaration_node, source: bytes) -> list[str]: - """Collect annotation names from a Java declaration's `modifiers` child.""" - names: list[str] = [] - modifiers = None - for child in declaration_node.children: - if child.type == "modifiers": - modifiers = child - break - if modifiers is None: - return names - for anno in modifiers.children: - if anno.type not in ("marker_annotation", "annotation"): - continue - name_node = anno.child_by_field_name("name") - if name_node is None: - for sub in anno.children: - if sub.type in ("identifier", "scoped_identifier", "type_identifier"): - name_node = sub - break - if name_node is not None: - text = _read_text(name_node, source).rsplit(".", 1)[-1] - if text: - names.append(text) - return names - - -_GO_PREDECLARED_TYPES = frozenset({ - "bool", "byte", "complex64", "complex128", "error", "float32", "float64", - "int", "int8", "int16", "int32", "int64", "rune", "string", - "uint", "uint8", "uint16", "uint32", "uint64", "uintptr", "any", "comparable", -}) - - -def _go_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a Go type expression; append (name, role) tuples.""" - if node is None: - return - t = node.type - if t == "type_identifier": - text = _read_text(node, source) - if text and text not in _GO_PREDECLARED_TYPES: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "qualified_type": - text = _read_text(node, source).rsplit(".", 1)[-1] - if text and text not in _GO_PREDECLARED_TYPES: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "generic_type": - type_field = node.child_by_field_name("type") - if type_field is not None: - sub: list[tuple[str, str]] = [] - _go_collect_type_refs(type_field, source, generic, sub) - out.extend(sub) - for c in node.children: - if c.type == "type_arguments": - for arg in c.children: - if arg.is_named: - _go_collect_type_refs(arg, source, True, out) - return - if t in ("pointer_type", "slice_type", "array_type", "map_type", - "channel_type", "parenthesized_type"): - for c in node.children: - if c.is_named: - _go_collect_type_refs(c, source, generic, out) - return - if node.is_named: - for c in node.children: - if c.is_named: - _go_collect_type_refs(c, source, generic, out) -def _rust_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a Rust type expression; append (name, role) tuples.""" - if node is None: - return - t = node.type - if t == "primitive_type": - return - if t == "type_identifier": - text = _read_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "scoped_type_identifier": - text = _read_text(node, source).rsplit("::", 1)[-1] - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "generic_type": - name_node = node.child_by_field_name("type") - if name_node is None: - for c in node.children: - if c.type in ("type_identifier", "scoped_type_identifier"): - name_node = c - break - if name_node is not None: - text = _read_text(name_node, source).rsplit("::", 1)[-1] - if text: - out.append((text, "generic_arg" if generic else "type")) - for c in node.children: - if c.type == "type_arguments": - for arg in c.children: - if arg.is_named: - _rust_collect_type_refs(arg, source, True, out) - return - if t in ("reference_type", "pointer_type", "array_type", "tuple_type", "slice_type"): - for c in node.children: - if c.is_named: - _rust_collect_type_refs(c, source, generic, out) - return - if node.is_named: - for c in node.children: - if c.is_named: - _rust_collect_type_refs(c, source, generic, out) +# ── C / C++ type-ref helpers ───────────────────────────────────────────────── -def _php_name_text(node, source: bytes) -> str | None: - """Return the unqualified name text from a PHP `name`/`qualified_name` node.""" - if node is None: - return None - return _read_text(node, source).rsplit("\\", 1)[-1] or None +# ── Scala type-ref helpers ─────────────────────────────────────────────────── -def _php_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a PHP type expression; append (name, role) tuples.""" - if node is None: - return - t = node.type - if t == "primitive_type": - return - if t == "named_type": - for c in node.children: - if c.type in ("name", "qualified_name"): - text = _php_name_text(c, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - return - if t in ("name", "qualified_name"): - text = _php_name_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t in ("nullable_type", "union_type", "intersection_type", "optional_type"): - for c in node.children: - if c.is_named: - _php_collect_type_refs(c, source, generic, out) - return - if node.is_named: - for c in node.children: - if c.is_named: - _php_collect_type_refs(c, source, generic, out) - - -def _php_method_return_type_node(method_node): - """Return the named_type/primitive_type node sitting after formal_parameters.""" - saw_params = False - for c in method_node.children: - if c.type == "formal_parameters": - saw_params = True - continue - if saw_params and c.is_named and c.type not in ("compound_statement",): - if c.type in ("named_type", "primitive_type", "nullable_type", - "union_type", "intersection_type", "optional_type"): - return c +def _resolve_name(node, source: bytes, config: LanguageConfig) -> str | None: + """Get the name from a node using config.name_field, falling back to child types.""" + if config.resolve_function_name_fn is not None: + # For C/C++ where the name is inside a declarator + return None # caller handles this separately + n = node.child_by_field_name(config.name_field) + if n: + return _read_text(n, source) + for child in node.children: + if child.type in config.name_fallback_child_types: + return _read_text(child, source) return None -def _kotlin_user_type_name(user_type_node, source: bytes) -> str | None: - """Return the head identifier text from a Kotlin user_type node (without generics).""" - if user_type_node is None: - return None - for c in user_type_node.children: - if c.type == "type_identifier": - text = _read_text(c, source) - return text or None - if c.type == "identifier": - text = _read_text(c, source) - return text or None - if c.type == "simple_user_type": - for sub in c.children: - if sub.type in ("identifier", "type_identifier"): - text = _read_text(sub, source) - return text or None - return None - +# ── Import handlers ─────────────────────────────────────────────────────────── -def _kotlin_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a Kotlin type expression; append (name, role) tuples.""" - if node is None: - return - t = node.type - if t in ("integral_literal", "boolean_literal"): - return - if t == "user_type": - for c in node.children: - if c.type in ("identifier", "type_identifier"): - text = _read_text(c, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - break - if c.type == "simple_user_type": - for sub in c.children: - if sub.type in ("identifier", "type_identifier"): - text = _read_text(sub, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - break - break - for c in node.children: - if c.type == "type_arguments": - for arg in c.children: - if arg.type == "type_projection": - for sub in arg.children: - if sub.is_named: - _kotlin_collect_type_refs(sub, source, True, out) - elif arg.is_named: - _kotlin_collect_type_refs(arg, source, True, out) - return - if t in ("identifier", "type_identifier"): - text = _read_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t in ("nullable_type", "parenthesized_type", "type_reference"): - for c in node.children: - if c.is_named: - _kotlin_collect_type_refs(c, source, generic, out) - return - if node.is_named: - for c in node.children: - if c.is_named: - _kotlin_collect_type_refs(c, source, generic, out) - - -def _kotlin_property_type_node(property_node): - """Find the user_type node within a Kotlin property_declaration.""" - for c in property_node.children: - if c.type == "variable_declaration": - for sub in c.children: - if sub.type in ("user_type", "nullable_type", "type_reference"): - return sub - if c.type in ("user_type", "nullable_type", "type_reference"): - return c - return None - - -def _kotlin_function_return_type_node(func_node): - """Find the return-type node of a Kotlin function_declaration (the type after `: ` post-params).""" - saw_params = False - saw_colon = False - for c in func_node.children: - if c.type == "function_value_parameters": - saw_params = True - continue - if saw_params and c.type == ":": - saw_colon = True - continue - if saw_colon: - if c.is_named: - return c - return None - - -def _swift_declaration_keyword(node) -> str | None: - """Return the leading kind token for a Swift class_declaration: class/struct/enum/extension/actor.""" - for c in node.children: - if not c.is_named and c.type in ("class", "struct", "enum", "extension", "actor"): - return c.type - return None - - -def _swift_pre_scan(root_node, source: bytes) -> tuple[set[str], set[str]]: - """Pre-scan a Swift compilation unit and return (protocol_names, class_like_names).""" - protocols: set[str] = set() - classes: set[str] = set() - stack = [root_node] - while stack: - n = stack.pop() - if n.type == "protocol_declaration": - name_node = n.child_by_field_name("name") - if name_node is None: - for c in n.children: - if c.type == "type_identifier": - name_node = c - break - if name_node is not None: - text = _read_text(name_node, source) - if text: - protocols.add(text) - elif n.type == "class_declaration": - kw = _swift_declaration_keyword(n) - if kw in ("class", "struct", "enum", "actor"): - name_node = n.child_by_field_name("name") - if name_node is not None: - text = _read_text(name_node, source) - if text: - classes.add(text) - stack.extend(n.children) - return protocols, classes - - -def _swift_classify_base(name: str, kind: str | None, is_first: bool, - protocols: set[str], classes: set[str]) -> str: - """Classify a Swift inheritance_specifier entry as `inherits` or `implements`.""" - if name in protocols: - return "implements" - if name in classes: - return "inherits" - # struct/enum/extension/actor cannot inherit a class — all conformances are protocols. - if kind in ("struct", "enum", "extension", "actor"): - return "implements" - # `class`: first entry is conventionally the base class; subsequent are protocols. - return "inherits" if is_first else "implements" - - -def _swift_user_type_name(user_type_node, source: bytes) -> str | None: - """Return the head type_identifier text from a Swift user_type node (without generics).""" - if user_type_node is None: - return None - for c in user_type_node.children: - if c.type == "type_identifier": - text = _read_text(c, source) - return text or None - return None - - -def _swift_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a Swift type expression; append (name, role) tuples (role 'type' or 'generic_arg').""" - if node is None: - return - t = node.type - if t == "type_annotation": - for c in node.children: - if c.is_named: - _swift_collect_type_refs(c, source, generic, out) - return - if t == "user_type": - for c in node.children: - if c.type == "type_identifier": - text = _read_text(c, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - break - for c in node.children: - if c.type == "type_arguments": - for arg in c.children: - if arg.is_named: - _swift_collect_type_refs(arg, source, True, out) - return - if t == "type_identifier": - text = _read_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t in ("optional_type", "implicitly_unwrapped_optional_type", "array_type", - "dictionary_type", "tuple_type"): - for c in node.children: - if c.is_named: - _swift_collect_type_refs(c, source, generic, out) - return - if node.is_named: - for c in node.children: - if c.is_named: - _swift_collect_type_refs(c, source, generic, out) - - -def _swift_property_type_node(property_node): - """Return the type_annotation child of a Swift property_declaration, if any.""" - for c in property_node.children: - if c.type == "type_annotation": - return c - return None - - -def _swift_property_name(property_node, source: bytes) -> str | None: - """Return the bound name of a Swift property (``let x``/``var x = ...``).""" - for c in property_node.children: - if c.type == "pattern": - for sc in c.children: - if sc.type == "simple_identifier": - return _read_text(sc, source) - if c.type == "simple_identifier": - return _read_text(c, source) - return None - - -def _swift_constructor_type(call_node, source: bytes) -> str | None: - """If a Swift call expression is a constructor (``Foo()``), return the type name. - - Only upper-cased callees are treated as types so a free-function call like - ``configure()`` in an initializer is not mistaken for a constructor. - """ - first = call_node.children[0] if call_node.children else None - if first is not None and first.type == "simple_identifier": - text = _read_text(first, source) - if text and text[:1].isupper(): - return text - return None - - -def _swift_receiver_name(recv_node, source: bytes) -> str | None: - """Return the depth-1 receiver name of a Swift member call (``recv.method()``). - - ``vm.update()`` -> ``vm``; ``Type.staticMethod()`` -> ``Type``; - ``Singleton.shared.method()`` -> ``Singleton`` (head of the chain); - ``self.svc.fetch()`` -> ``svc`` (the property the call is reached through). - Returns None for anything deeper, so resolution stays depth-1. - """ - if recv_node is None: - return None - if recv_node.type == "simple_identifier": - return _read_text(recv_node, source) - if recv_node.type == "navigation_expression": - head = recv_node.children[0] if recv_node.children else None - if head is not None and head.type == "simple_identifier": - return _read_text(head, source) - if head is not None and head.type == "self_expression": - for child in recv_node.children: - if child.type == "navigation_suffix": - for sc in child.children: - if sc.type == "simple_identifier": - return _read_text(sc, source) - return None - - -# ── C / C++ type-ref helpers ───────────────────────────────────────────────── - -_C_PRIMITIVE_TYPE_NODES = frozenset({ - "primitive_type", "sized_type_specifier", "auto", "placeholder_type_specifier", -}) - - -def _c_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a C type expression; append (name, role) tuples for user-defined types. - Skips primitive types and qualifiers; recognises type_identifier.""" - if node is None or node.type in _C_PRIMITIVE_TYPE_NODES: - return - t = node.type - if t == "type_identifier": - text = _read_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t in ("pointer_declarator", "reference_declarator", "array_declarator", - "type_qualifier", "type_descriptor", "abstract_pointer_declarator", - "abstract_reference_declarator", "abstract_array_declarator"): - for c in node.children: - if c.is_named: - _c_collect_type_refs(c, source, generic, out) - - -def _cpp_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a C++ type expression; append (name, role) tuples. - Resolves qualified_identifier tails (std::string → string) and template_type - base + arguments (std::vector → vector + HttpClient as generic_arg).""" - if node is None or node.type in _C_PRIMITIVE_TYPE_NODES: - return - t = node.type - if t == "type_identifier": - text = _read_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "qualified_identifier": - name_node = node.child_by_field_name("name") - if name_node is not None: - _cpp_collect_type_refs(name_node, source, generic, out) - return - if t == "template_type": - name_node = node.child_by_field_name("name") - if name_node is not None: - text = _read_text(name_node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - args_node = node.child_by_field_name("arguments") - if args_node is not None: - for c in args_node.children: - if c.is_named: - _cpp_collect_type_refs(c, source, True, out) - return - if t in ("type_descriptor", "pointer_declarator", "reference_declarator", - "array_declarator", "type_qualifier", "abstract_pointer_declarator", - "abstract_reference_declarator", "abstract_array_declarator"): - for c in node.children: - if c.is_named: - _cpp_collect_type_refs(c, source, generic, out) - - -# ── Scala type-ref helpers ─────────────────────────────────────────────────── - -def _scala_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a Scala type expression; append (name, role) tuples. - Handles type_identifier, generic_type (List[T]), and common type wrappers.""" - if node is None: - return - t = node.type - if t == "type_identifier": - text = _read_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "generic_type": - base = node.child_by_field_name("type") - if base is None: - for c in node.children: - if c.type == "type_identifier": - base = c - break - if base is not None and base.type == "type_identifier": - text = _read_text(base, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - for c in node.children: - if c.type == "type_arguments": - for arg in c.children: - if arg.is_named: - _scala_collect_type_refs(arg, source, True, out) - return - if t in ("compound_type", "infix_type", "function_type", "tuple_type", - "annotated_type", "projected_type"): - for c in node.children: - if c.is_named: - _scala_collect_type_refs(c, source, generic, out) - - -def _python_collect_param_refs(params_node, source: bytes) -> list[tuple[str, str]]: - """Collect type refs from each typed parameter under a `parameters` node.""" - out: list[tuple[str, str]] = [] - if params_node is None: - return out - for child in params_node.children: - if child.type in ("typed_parameter", "typed_default_parameter"): - type_node = child.child_by_field_name("type") - _python_collect_type_refs(type_node, source, False, out) - return out - - -def _python_param_names(params_node, source: bytes) -> set[str]: - """Plain parameter identifiers declared on a Python `parameters` node. - - Covers positional/keyword params plus `*args` / `**kwargs` and typed or - default forms — anything that binds a local name the function body can shadow - a module-level definition with. - """ - out: set[str] = set() - if params_node is None: - return out - for child in params_node.children: - if child.type == "identifier": - out.add(_read_text(child, source)) - elif child.type in ( - "typed_parameter", - "default_parameter", - "typed_default_parameter", - "list_splat_pattern", - "dictionary_splat_pattern", - ): - # The bound name is the first identifier child (the rest is type/default). - name_n = child.child_by_field_name("name") - if name_n is None: - name_n = next( - (c for c in child.children if c.type == "identifier"), None - ) - if name_n is not None: - out.add(_read_text(name_n, source)) - return out - - -def _python_collect_assignment_targets(node, source: bytes, out: set[str]) -> None: - """Identifiers bound as `pattern` targets under a Python AST subtree. - - Recurses through `pattern_list` / `tuple_pattern` / `list_pattern` so tuple - unpacking (`a, b = ...`, `for a, b in ...`) contributes every bound name. - """ - if node is None: - return - if node.type == "identifier": - out.add(_read_text(node, source)) - return - if node.type in ("pattern_list", "tuple_pattern", "list_pattern"): - for c in node.children: - _python_collect_assignment_targets(c, source, out) - - -def _python_local_bound_names(func_def_node, source: bytes) -> set[str]: - """Names bound LOCALLY inside a Python function: parameters plus assignment, - `for`, `with ... as`, and comprehension targets. - - Used by the indirect-dispatch guard to reject a call-argument identifier that - is a parameter or a local binding — it names a local value, not the module- - level function/class that happens to share the name. Nested `function_definition` - and `class_definition` subtrees are NOT descended into: their bindings belong - to a different scope. - """ - bound: set[str] = set() - bound |= _python_param_names(func_def_node.child_by_field_name("parameters"), source) - - def walk(n) -> None: - for child in n.children: - t = child.type - if t in ("function_definition", "class_definition", "lambda"): - continue # inner scope — its bindings are not this function's locals - if t == "assignment": - _python_collect_assignment_targets( - child.child_by_field_name("left"), source, bound - ) - elif t in ("for_statement", "for_in_clause"): - _python_collect_assignment_targets( - child.child_by_field_name("left"), source, bound - ) - elif t == "with_statement": - for item in child.children: - if item.type == "with_clause": - for wi in item.children: - if wi.type == "with_item": - alias = wi.child_by_field_name("alias") - _python_collect_assignment_targets(alias, source, bound) - elif t == "named_expression": # walrus := - _python_collect_assignment_targets( - child.child_by_field_name("name"), source, bound - ) - walk(child) - - body = func_def_node.child_by_field_name("body") - if body is not None: - walk(body) - return bound - - -def _python_module_bound_names(root, source: bytes) -> set[str]: - """Names rebound by assignment at MODULE scope (top-level `x = ...`, `for`, walrus). - - The module-scope analogue of the per-function shadow set: a dispatch-table value - whose name is reassigned to data at module level (`handler = build()`) names that - value, not a same-named function, so it must not manufacture an indirect edge. - Function and class bodies are not descended into — their bindings are local. - """ - bound: set[str] = set() - - def walk(n) -> None: - for child in n.children: - t = child.type - if t in ("function_definition", "class_definition", "lambda"): - continue # inner scope — not a module-level binding - if t == "assignment": - _python_collect_assignment_targets( - child.child_by_field_name("left"), source, bound - ) - elif t in ("for_statement", "for_in_clause"): - _python_collect_assignment_targets( - child.child_by_field_name("left"), source, bound - ) - elif t == "named_expression": # walrus := - _python_collect_assignment_targets( - child.child_by_field_name("name"), source, bound - ) - walk(child) - - walk(root) - return bound - - -_JS_SCOPE_BOUNDARY = frozenset({ - "function_declaration", "function_expression", "function", "arrow_function", - "method_definition", "class_declaration", "class", "generator_function", - "generator_function_declaration", -}) - - -def _js_collect_pattern_idents(node, source: bytes, bound: set) -> None: - """Collect binding identifier names from a JS/TS pattern (a parameter, or a - declarator LHS). Recurses through destructuring (object/array patterns, rest) - but never into the default-value side of `x = default` or a type annotation, - so only names actually bound by the pattern are collected.""" - t = node.type - if t in ("identifier", "shorthand_property_identifier_pattern"): - bound.add(_read_text(node, source)) - return - if t == "type_annotation": - return # `(h: Handler)` — Handler is a type, not a bound name - if t == "assignment_pattern": # `x = default` — only x is bound - left = node.child_by_field_name("left") - if left is not None: - _js_collect_pattern_idents(left, source, bound) - return - if t == "pair_pattern": # `{ a: localName }` — localName is bound - val = node.child_by_field_name("value") - if val is not None: - _js_collect_pattern_idents(val, source, bound) - return - for c in node.children: - if c.is_named: - _js_collect_pattern_idents(c, source, bound) - - -def _js_local_bound_names(func_node, source: bytes) -> set[str]: - """Names bound locally inside a JS/TS function: parameters plus `const`/`let`/ - `var` declarator targets. Mirrors `_python_local_bound_names`: an argument that - is a parameter or local binding names a local value, not a same-named module - function, so it must not manufacture an indirect_call edge. Nested function and - class scopes are not descended into.""" - bound: set[str] = set() - params = func_node.child_by_field_name("parameters") - if params is not None: - _js_collect_pattern_idents(params, source, bound) - - def walk(n) -> None: - for c in n.children: - if c.type in _JS_SCOPE_BOUNDARY: - continue # inner scope — its bindings are not this function's locals - if c.type == "variable_declarator": - name = c.child_by_field_name("name") - if name is not None: - _js_collect_pattern_idents(name, source, bound) - walk(c) - - body = func_node.child_by_field_name("body") - if body is not None: - walk(body) - return bound - - -def _js_module_bound_names(root, source: bytes) -> set[str]: - """Module-scope names rebound to NON-function data (`const X = {...}`, `let y = 5`). - - The JS/TS module-scope shadow set. Unlike the per-function set, a declarator - whose value is itself a function (`const cb = () => {}`) is EXCLUDED: that name - IS a callable we want dispatch tables to resolve to, not a data shadow. - """ - bound: set[str] = set() - - def walk(n) -> None: - for c in n.children: - if c.type in _JS_SCOPE_BOUNDARY: - continue - if c.type == "variable_declarator": - value = c.child_by_field_name("value") - if value is None or value.type not in _JS_FUNCTION_VALUE_TYPES: - name = c.child_by_field_name("name") - if name is not None: - _js_collect_pattern_idents(name, source, bound) - walk(c) - - walk(root) - return bound - - -def _js_dispatch_value_idents(coll_node): - """Yield identifier value-nodes of a JS/TS object/array literal that are - function-reference candidates: object property VALUES and shorthand properties - (`{ handler }`), and array elements. Keys and inline methods are not references.""" - if coll_node.type == "object": - for c in coll_node.children: - if c.type == "pair": - val = c.child_by_field_name("value") - if val is not None and val.type == "identifier": - yield val - elif c.type == "shorthand_property_identifier": - yield c - else: # array - for el in coll_node.children: - if el.type == "identifier": - yield el - - -def _resolve_name(node, source: bytes, config: LanguageConfig) -> str | None: - """Get the name from a node using config.name_field, falling back to child types.""" - if config.resolve_function_name_fn is not None: - # For C/C++ where the name is inside a declarator - return None # caller handles this separately - n = node.child_by_field_name(config.name_field) - if n: - return _read_text(n, source) - for child in node.children: - if child.type in config.name_fallback_child_types: - return _read_text(child, source) - return None - - -def _find_body(node, config: LanguageConfig): - """Find the body node using config.body_field, falling back to child types.""" - b = node.child_by_field_name(config.body_field) - if b: - return b - for child in node.children: - if child.type in config.body_fallback_child_types: - return child - return None - - -# ── Import handlers ─────────────────────────────────────────────────────────── - -def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: +def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: t = node.type if t == "import_statement": for child in node.children: @@ -1823,34 +275,6 @@ def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, s }) -def _resolve_js_import_target(raw: str, str_path: str) -> "tuple[str, Path | None] | None": - """Resolve a JS/TS import path string to (target_nid, resolved_path). - - Handles relative paths, tsconfig path aliases, workspace packages, and - bare/scoped imports. - Returns None if `raw` is empty. - """ - if not raw: - return None - resolved_path = _resolve_js_module_path(raw, Path(str_path).parent) - if resolved_path is not None: - return _make_id(str(resolved_path)), resolved_path - module_name = raw.split("/")[-1] - if not module_name: - return None - # Unresolved: relative/absolute, tsconfig-alias and workspace resolution have - # all run and failed, so this is an external package (or a dangling local - # path). Namespace the id with the "ref" prefix — the J-4 convention already - # used for tsconfig `extends`/`$ref` externals — so it can NEVER collapse to - # the same _make_id as a local file/symbol node. Without it, the bare - # last-segment id (e.g. "tailwindcss/colors" -> "colors") collides with any - # unrelated local file of that stem via build.py's pre-migration alias index, - # producing a confident (EXTRACTED) cross-language phantom imports_from edge - # (#1638). The ref-namespaced target has no node, so build drops it as an - # external reference — the correct outcome for a third-party import. - return _make_id("ref", raw), None - - def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: is_reexport = node.type == "export_statement" # Only handle export_statement if it has a `from` clause (re-export). @@ -1948,73 +372,6 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p }) -def _dynamic_import_js(node, source: bytes, caller_nid: str, str_path: str, edges: list, - seen_dyn_pairs: set) -> bool: - """Detect dynamic import() calls in JS/TS and emit imports_from edges. - - Handles patterns like: - await import('./foo.js') - import('./foo.js').then(...) - const m = await import(`./foo`) - - Returns True if the node was a dynamic import (caller should skip normal call handling). - """ - # Dynamic import is a call_expression whose function child is the keyword "import". - # tree-sitter-typescript parses `import('...')` as call_expression with first child - # being an "import" token (type="import"). - func_node = node.child_by_field_name("function") - if func_node is None: - # Fallback: check first child directly (some TS versions) - if node.children and _read_text(node.children[0], source) == "import": - func_node = node.children[0] - else: - return False - if _read_text(func_node, source) != "import": - return False - - # Extract the module path from the arguments - args = node.child_by_field_name("arguments") - if args is None: - return True # It's an import() but no args — skip - for arg in args.children: - if arg.type == "template_string": - # Skip dynamic template literals — path can't be statically resolved - if any(c.type == "template_substitution" for c in arg.children): - break - raw = _read_text(arg, source).strip("`") - elif arg.type == "string": - raw = _read_text(arg, source).strip("'\" ") - else: - continue - if not raw: - break - # Resolve path using the same logic as static imports. - resolved = _resolve_js_import_target(raw, str_path) - if resolved is None: - break - tgt_nid, _ = resolved - pair = (caller_nid, tgt_nid) - if pair not in seen_dyn_pairs: - seen_dyn_pairs.add(pair) - edges.append({ - "source": caller_nid, - "target": tgt_nid, - # A deferred `import(...)` is a real dependency, so keep it as an - # `imports_from` edge (visible in the graph) but mark it `deferred` - # so find_import_cycles does not treat it as a static import and - # report a phantom file cycle (#1241). - "relation": "imports_from", - "context": "import", - "deferred": True, - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - }) - break - return True - - def _import_java(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: def _walk_scoped(n) -> str: parts: list[str] = [] @@ -2054,20 +411,6 @@ def _walk_scoped(n) -> str: break -def _resolve_c_include_path(raw: str, str_path: str) -> "Path | None": - """Resolve a quoted #include path to a real file on disk. - - Searches relative to the including file's directory. Returns None for - system headers (<...>) or paths that don't exist on disk. - """ - if not raw: - return None - candidate = (Path(str_path).parent / raw).resolve() - if candidate.is_file(): - return candidate - return None - - def _import_c(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: for child in node.children: if child.type in ("string_literal", "system_lib_string", "string"): @@ -2227,730 +570,69 @@ def _get_c_func_name(node, source: bytes) -> str | None: return None -def _get_cpp_func_name(node, source: bytes) -> str | None: - """Recursively unwrap declarator to find the innermost identifier (C++).""" - if node.type == "identifier": - return _read_text(node, source) - if node.type in ("field_identifier", "destructor_name", "operator_name"): - return _read_text(node, source) - if node.type == "qualified_identifier": - # An out-of-class DEFINITION (`void Foo::bar() {}`) carries a - # qualified_identifier declarator. Retaining the `Foo::` qualifier makes - # _make_id(stem, "Foo::bar") normalize to the same id as the in-class - # member _make_id(class_nid, "bar"), so the decl in Foo.h and the def in - # Foo.cpp resolve to ONE method node instead of two (#1547). The full - # qualified text also handles nested scopes (`A::B::bar`). Free functions - # never have a qualified_identifier here, so their bare-name ids are - # unchanged; only qualified definitions shift onto their owning class. - return _read_text(node, source) - decl = node.child_by_field_name("declarator") - if decl: - return _get_cpp_func_name(decl, source) - for child in node.children: - if child.type == "identifier": - return _read_text(child, source) - return None - +# ── JS/TS extra walk for arrow functions ────────────────────────────────────── -def _cpp_declarator_name(node, source: bytes) -> str | None: - """Return the bare variable name from a C++ declaration declarator, unwrapping - pointer/reference/init wrappers (``*f``, ``&r``, ``f = Foo()``). Returns None - for anything that isn't a plain named local (arrays, function pointers, - structured bindings) so the type table never records a guessed receiver.""" - t = node.type - if t == "identifier": - return _read_text(node, source) - if t in ("pointer_declarator", "reference_declarator", "init_declarator"): - inner = node.child_by_field_name("declarator") - if inner is None: - for c in node.children: - if c.type in ("identifier", "pointer_declarator", - "reference_declarator"): - inner = c - break - if inner is not None: - return _cpp_declarator_name(inner, source) - return None +# Node types whose value is a callable, for the JS/TS assignment / class-field +# / function-expression forms below. Older tree-sitter-javascript grammars +# label a function expression `function`; current ones use `function_expression`. -def _cpp_local_var_types(body_node, source: bytes, table: dict[str, str]) -> None: - """Collect ``var -> ClassName`` from local variable declarations in a C++ - function body, for receiver-type inference in the cross-file member-call pass - (#1547). Handles ``Foo f;``, ``Foo* f;``, ``Foo *f = ...;``, ``Foo f = Foo();``. - Only a class-like (``type_identifier``/``qualified_identifier``) type with a - single named declarator is recorded — PRECISION over recall: a built-in type - (``int x``), an ambiguous multi-declarator line, or an un-nameable declarator - contributes nothing rather than a guess. A qualified type ``ns::Foo`` records - its simple tail ``Foo`` so it keys to the type's definition node label. - """ - stack = [body_node] - while stack: - n = stack.pop() - if n.type in ("function_definition", "lambda_expression"): - # Don't descend into a nested function/lambda: its locals are scoped - # away and would pollute this body's table. - if n is not body_node: - continue - if n.type == "declaration": - type_node = n.child_by_field_name("type") - if type_node is not None and type_node.type in ( - "type_identifier", "qualified_identifier" - ): - type_name = _read_text(type_node, source).split("::")[-1].strip() - declarators = [ - c for c in n.children - if c.type in ("identifier", "pointer_declarator", - "reference_declarator", "init_declarator") - ] - # A single declarator only: `Foo a, b;` is ambiguous to attribute - # to one receiver name cleanly, so skip multi-declarator lines. - if type_name and type_name[:1].isupper() and len(declarators) == 1: - var = _cpp_declarator_name(declarators[0], source) - if var and var not in table: - table[var] = type_name - for c in n.children: - stack.append(c) - - -def _swift_local_var_types(body_node, source: bytes, table: dict[str, str]) -> None: - """Collect ``var -> Type`` from local ``let``/``var`` bindings in a Swift - function body, so a member call on the local (``x.method()``) resolves to Type - in the cross-file member-call pass (#1604). - - Two initializer shapes are recorded, PRECISION over recall: - - a constructor call ``let x = Type()`` (``_swift_constructor_type``); - - a static-member access ``let x = Type.shared`` (a navigation_expression - with an upper-cased head) — the singleton-cached-into-a-local idiom, one - of the most common Swift call patterns and previously resolved to nothing. - Nested function declarations are not descended into (their locals are scoped - away); the first binding for a name wins, so a class property of the same name - already in the table is not overwritten. - """ - stack = [body_node] - while stack: - n = stack.pop() - if n.type == "function_declaration" and n is not body_node: - continue - if n.type == "property_declaration": - prop_type: str | None = None - for child in n.children: - if child.type == "call_expression": - prop_type = _swift_constructor_type(child, source) - break - if child.type == "navigation_expression": - head = child.children[0] if child.children else None - if head is not None and head.type == "simple_identifier": - htext = _read_text(head, source) - if htext and htext[:1].isupper(): - prop_type = htext - break - name = _swift_property_name(n, source) - if name and prop_type and name not in table: - table[name] = prop_type - for c in n.children: - stack.append(c) - - -def _csharp_member_type_table(root, source: bytes) -> dict[str, str]: - """Collect ``name -> TypeName`` for C# receiver typing (#1609): class fields, - properties, method parameters, and local variable declarations. - - File-scoped, first-binding-wins (like the C++ table): a field declared once at - class scope is visible to every method's `field.Method()`, and a param/local - shadowing the same name is a conservative approximation graphify already accepts - for receiver typing. Only a resolvable, non-`var` type name is recorded; `var` - without a `new T()` initializer, and predefined/lower-cased primitives, are - skipped (precision over recall — an untypable receiver is left for the resolver - to drop rather than guess). `var v = new T()` is typed from the object-creation. - """ - table: dict[str, str] = {} +# ── TS extra walk for namespace / module declarations ───────────────────────── - def _typed(type_node) -> str | None: - info = _read_csharp_type_name(type_node, source) - if not info: - return None - name = info[0] - # A genuine C# class name is Pascal-cased; skip predefined primitives - # (int/bool/string) which never own a resolvable method definition here. - return name if name and name[:1].isupper() else None - - def _decl_names(var_decl): - for c in var_decl.children: - if c.type == "variable_declarator": - nm = c.child_by_field_name("name") or next( - (g for g in c.children if g.type == "identifier"), None) - if nm is not None: - yield _read_text(nm, source), c - - def _new_type(declarator) -> str | None: - # `var v = new Server()` — recover the type from the object_creation_expression. - for g in declarator.children: - if g.type == "object_creation_expression": - return _typed(g.child_by_field_name("type")) - return None - stack = [root] - while stack: - n = stack.pop() - t = n.type - if t in ("field_declaration", "local_declaration_statement"): - vd = next((c for c in n.children if c.type == "variable_declaration"), None) - if vd is not None: - type_node = vd.child_by_field_name("type") - declared = _typed(type_node) - for name, decl in _decl_names(vd): - resolved = declared or _new_type(decl) - if name and resolved and name not in table: - table[name] = resolved - elif t == "property_declaration": - nm = n.child_by_field_name("name") - resolved = _typed(n.child_by_field_name("type")) - if nm is not None and resolved: - pname = _read_text(nm, source) - if pname not in table: - table[pname] = resolved - elif t == "parameter": - nm = n.child_by_field_name("name") - resolved = _typed(n.child_by_field_name("type")) - if nm is not None and resolved: - pname = _read_text(nm, source) - if pname not in table: - table[pname] = resolved - for c in n.children: - stack.append(c) - return table - - -def _ts_receiver_type_table(root, source: bytes, table: dict[str, str]) -> None: - """Add TS/JS receiver bindings to ``table`` (name -> TypeName), for member-call - resolution beyond the constructor-injected `this.field` case (#1630): - - * local ``const/let/var x = new Foo()`` -> ``x: Foo`` (Pattern A); - * a type-annotated parameter ``(svc: Svc)`` -> ``svc: Svc`` (Pattern B), so a - call on the param — including inside a returned closure — resolves. - - File-scoped, first-binding-wins (merged into the constructor-injection table, - which is populated first and therefore wins on a name clash). Only a bare - ``type_identifier`` (a single class/interface name) is recorded — an array, - union, generic, qualified, or predefined type is skipped (precision over - recall, matching the receiver-typed resolvers for Swift/C#/C++).""" - def _bare_type_ident(type_annotation): - # type_annotation -> ": T"; accept only a single type_identifier child. - idents = [c for c in type_annotation.children if c.type == "type_identifier"] - others = [c for c in type_annotation.children - if c.is_named and c.type not in ("type_identifier",)] - if len(idents) == 1 and not others: - return _read_text(idents[0], source) - return None +# ── C# extra walk for namespace declarations ────────────────────────────────── - stack = [root] - while stack: - n = stack.pop() - t = n.type - if t == "variable_declarator": - name_n = n.child_by_field_name("name") - value = n.child_by_field_name("value") - if (name_n is not None and name_n.type == "identifier" - and value is not None and value.type == "new_expression"): - ctor = value.child_by_field_name("constructor") - if ctor is not None and ctor.type in ("identifier", "type_identifier"): - name = _read_text(name_n, source) - tname = _read_text(ctor, source) - if name and tname and name not in table: - table[name] = tname - elif t == "required_parameter" or t == "optional_parameter": - pat = n.child_by_field_name("pattern") - ann = n.child_by_field_name("type") - if pat is not None and pat.type == "identifier" and ann is not None: - tname = _bare_type_ident(ann) - name = _read_text(pat, source) - if name and tname and name not in table: - table[name] = tname - for c in n.children: - stack.append(c) - - -def _objc_local_var_types(body_node, source: bytes, table: dict[str, str]) -> None: - """Collect ``var -> ClassName`` from ObjC local declarations (``Foo *f = ...;``) - in a method body, for receiver typing in the cross-file message-send pass - (#1556). Only a capitalized ``type_identifier`` with a single named declarator - is recorded; a built-in/lower-cased type or an un-nameable declarator is skipped - (precision over recall). Reuses the C++ declarator unwrapper (identical grammar). - """ - stack = [body_node] - while stack: - n = stack.pop() - if n.type == "method_definition" and n is not body_node: - continue - if n.type == "declaration": - type_node = n.child_by_field_name("type") - if type_node is None: - for c in n.children: - if c.type == "type_identifier": - type_node = c - break - if type_node is not None and type_node.type == "type_identifier": - type_name = _read_text(type_node, source).strip() - declarators = [ - c for c in n.children - if c.type in ("identifier", "pointer_declarator", "init_declarator") - ] - if type_name and type_name[:1].isupper() and len(declarators) == 1: - var = _cpp_declarator_name(declarators[0], source) - if var and var not in table: - table[var] = type_name - for c in n.children: - stack.append(c) +# ── Swift extra walk for enum cases ────────────────────────────────────────── -# ── JS/TS extra walk for arrow functions ────────────────────────────────────── -def _find_require_call(value_node): - """Return the call_expression node if `value_node` is a `require(...)` call - or `require(...).x` member access. Otherwise None.""" - if value_node is None: - return None - if value_node.type == "call_expression": - fn = value_node.child_by_field_name("function") - if fn is not None and fn.type == "identifier": - return value_node - if value_node.type == "member_expression": - obj = value_node.child_by_field_name("object") - return _find_require_call(obj) - return None +# ── Java extra walk for enum constants ─────────────────────────────────────── +def _java_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, + nodes: list, edges: list, seen_ids: set, function_bodies: list, + parent_class_nid: str | None, add_node_fn, add_edge_fn, + walk_fn) -> bool: + """Handle enum_constant for Java. Returns True if handled.""" + if node.type == "enum_constant" and parent_class_nid: + name_node = node.child_by_field_name("name") + if name_node is None: + return True + const_name = _read_text(name_node, source) + line = node.start_point[0] + 1 + const_nid = _make_id(parent_class_nid, const_name) + add_node_fn(const_nid, const_name, line) + add_edge_fn(parent_class_nid, const_nid, "case_of", line) + # Anonymous-body constants (`MONDAY { void greet(){} }`): descend so the + # body's methods aren't dropped; const_nid attaches them to the constant. + for child in node.children: + if child.type == "class_body": + for member in child.children: + walk_fn(member, parent_class_nid=const_nid) + return True + return False -def _require_imports_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> bool: - """Detect CommonJS require imports inside lexical_declaration / variable_declaration. - Handles three patterns: - const { foo, bar } = require('./mod') → file → mod (imports_from), file → foo, file → bar - const mod = require('./mod') → file → mod (imports_from) - const x = require('./mod').y → file → mod (imports_from), file → y +# ── Kotlin extra walk for enum entries ─────────────────────────────────────── - Returns True if any require import was found. - """ - if node.type not in ("lexical_declaration", "variable_declaration"): - return False - found = False - for child in node.children: - if child.type != "variable_declarator": - continue - value = child.child_by_field_name("value") - call = _find_require_call(value) - if call is None: - continue - fn = call.child_by_field_name("function") - if fn is None or _read_text(fn, source) != "require": - continue - args = call.child_by_field_name("arguments") - if args is None: - continue - raw = None - for arg in args.children: - if arg.type == "string": - raw = _read_text(arg, source).strip("'\"` ") +def _kotlin_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, + nodes: list, edges: list, seen_ids: set, function_bodies: list, + parent_class_nid: str | None, add_node_fn, add_edge_fn, + walk_fn) -> bool: + """Handle enum_entry for Kotlin. Returns True if handled.""" + if node.type == "enum_entry" and parent_class_nid: + name_node = None + for child in node.children: + if child.type in ("simple_identifier", "identifier"): + name_node = child break - if not raw: - continue - resolved = _resolve_js_import_target(raw, str_path) - if resolved is None: - continue - tgt_nid, resolved_path = resolved - line = node.start_point[0] + 1 - edges.append({ - "source": file_nid, - "target": tgt_nid, - "relation": "imports_from", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - }) - found = True - - # Symbol-level edges for destructured / accessor binders. - target_stem = _file_stem(resolved_path) if resolved_path is not None else None - name_node = child.child_by_field_name("name") - sym_names: list[str] = [] - if name_node is not None and name_node.type == "object_pattern": - # `const { a, b: alias } = require('./m')` — emit edges for each property key - for prop in name_node.children: - if prop.type == "shorthand_property_identifier_pattern": - sym_names.append(_read_text(prop, source)) - elif prop.type == "pair_pattern": - key = prop.child_by_field_name("key") - if key is not None: - sym_names.append(_read_text(key, source)) - elif value is not None and value.type == "member_expression": - # `const x = require('./m').y` — symbol is the property accessed - prop = value.child_by_field_name("property") - if prop is not None: - sym_names.append(_read_text(prop, source)) - if target_stem is not None: - for sym in sym_names: - edges.append({ - "source": file_nid, - "target": _make_id(target_stem, sym), - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - }) - return found - - -# Node types whose value is a callable, for the JS/TS assignment / class-field -# / function-expression forms below. Older tree-sitter-javascript grammars -# label a function expression `function`; current ones use `function_expression`. -_JS_FUNCTION_VALUE_TYPES = frozenset({"arrow_function", "function_expression", "function", "generator_function"}) - - -def _js_member_assignment_target(left, source: bytes): - """Classify the symbol an `assignment_expression` LHS defines when its RHS - is a function. Returns (kind, owner_name, member_name) or None. - - this.foo = fn → ("this", None, "foo") - exports.foo = fn → ("exports", None, "foo") - module.exports.foo = fn → ("exports", None, "foo") - Foo.prototype.bar = fn → ("prototype", "Foo", "bar") - - Any other shape (an arbitrary `obj.x = fn`) returns None and is skipped — - capturing those would reintroduce the bare-named / phantom-god-node class - of bug the module-level scope guard (#1077) exists to prevent. - """ - if left is None or left.type != "member_expression": - return None - prop = left.child_by_field_name("property") - if prop is None: - return None - member_name = _read_text(prop, source) - if not member_name: - return None - obj = left.child_by_field_name("object") - if obj is None: - return None - if obj.type == "this": - return ("this", None, member_name) - if obj.type == "identifier": - if _read_text(obj, source) == "exports": - return ("exports", None, member_name) - return None - if obj.type == "member_expression": - # module.exports.X or Foo.prototype.X - inner_obj = obj.child_by_field_name("object") - inner_prop = obj.child_by_field_name("property") - if inner_obj is None or inner_prop is None: - return None - inner_prop_name = _read_text(inner_prop, source) - if inner_obj.type == "identifier": - inner_obj_name = _read_text(inner_obj, source) - if inner_obj_name == "module" and inner_prop_name == "exports": - return ("exports", None, member_name) - if inner_prop_name == "prototype": - return ("prototype", inner_obj_name, member_name) - return None - - -def _js_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, - nodes: list, edges: list, seen_ids: set, function_bodies: list, - parent_class_nid: str | None, add_node_fn, add_edge_fn, - callable_def_nids: set | None = None, - local_bound_names: dict | None = None) -> bool: - """Handle lexical_declaration (arrow functions, CJS requires, module-level const literals) for JS/TS. Returns True if handled.""" - # CommonJS / prototype member assignments whose value is a function: - # exports.X = () => {} → file-contained function X() - # module.exports.X = fn → file-contained function X() - # Foo.prototype.bar = fn → method bar() owned by Foo - # (`this.X = fn` lives inside a function body, which is not recursed here; - # it is captured at the enclosing function — see the function branch.) - if node.type == "expression_statement": - assign = next((c for c in node.children - if c.type == "assignment_expression"), None) - if assign is not None: - value = assign.child_by_field_name("right") - if value is not None and value.type in _JS_FUNCTION_VALUE_TYPES: - target = _js_member_assignment_target( - assign.child_by_field_name("left"), source) - if target is not None: - kind, owner_name, member_name = target - line = node.start_point[0] + 1 - handled = False - if kind == "exports": - nid = _make_id(stem, member_name) - add_node_fn(nid, f"{member_name}()", line) - add_edge_fn(file_nid, nid, "contains", line) - handled = True - elif kind == "prototype": - owner_nid = _make_id(stem, owner_name) - nid = _make_id(owner_nid, member_name) - add_node_fn(nid, f".{member_name}()", line) - add_edge_fn(owner_nid, nid, "method", line) - handled = True - if handled: - if callable_def_nids is not None: - callable_def_nids.add(nid) # CJS/prototype fn is callable - if local_bound_names is not None: - local_bound_names[nid] = _js_local_bound_names(value, source) - body = value.child_by_field_name("body") - if body: - function_bodies.append((nid, body)) - return True - - # Class fields whose value is a function: - # class C { handler = () => {} } → method handler() owned by C - # Reaches here with parent_class_nid set because class bodies are recursed - # with the class nid as parent. - if parent_class_nid and node.type in ("field_definition", "public_field_definition"): - prop = node.child_by_field_name("property") or node.child_by_field_name("name") - value = node.child_by_field_name("value") - if (prop is not None and value is not None - and value.type in _JS_FUNCTION_VALUE_TYPES): - field_name = _read_text(prop, source) - if field_name: - line = node.start_point[0] + 1 - nid = _make_id(parent_class_nid, field_name) - add_node_fn(nid, f".{field_name}()", line) - add_edge_fn(parent_class_nid, nid, "method", line) - if callable_def_nids is not None: - callable_def_nids.add(nid) # arrow class-field is callable - if local_bound_names is not None: - local_bound_names[nid] = _js_local_bound_names(value, source) - body = value.child_by_field_name("body") - if body: - function_bodies.append((nid, body)) - return True - - if node.type in ("lexical_declaration", "variable_declaration"): - # CJS require imports — emit edges, do not block other lexical_declaration handling - require_found = _require_imports_js(node, source, file_nid, stem, edges, str_path) - - # Scope guard (#1077): only emit nodes for module-level declarations. - # Without this, `const x = ...` inside an arrow callback (e.g. inside - # `describe(() => { const set = new Set(...) })`) emits a bare-named - # node, and the same name collides across unrelated files producing - # phantom god-nodes. Bodies of arrow functions are walked separately - # via function_bodies, so we never need to emit nodes for locals here. - parent = node.parent - is_module_level = parent is not None and ( - parent.type == "program" - or (parent.type == "export_statement" - and parent.parent is not None - and parent.parent.type == "program") - ) - - # Arrow function declarations and module-level const literals (lexical_declaration only) - arrow_found = False - const_found = False - if node.type == "lexical_declaration" and is_module_level: - for child in node.children: - if child.type == "variable_declarator": - value = child.child_by_field_name("value") - if value and value.type in _JS_FUNCTION_VALUE_TYPES: - # `const f = () => {}` and `const f = function(){}` - name_node = child.child_by_field_name("name") - if name_node: - func_name = _read_text(name_node, source) - line = child.start_point[0] + 1 - func_nid = _make_id(stem, func_name) - add_node_fn(func_nid, f"{func_name}()", line) - add_edge_fn(file_nid, func_nid, "contains", line) - if callable_def_nids is not None: - callable_def_nids.add(func_nid) # `const f = () =>` is callable - if local_bound_names is not None: - local_bound_names[func_nid] = _js_local_bound_names(value, source) - body = value.child_by_field_name("body") - if body: - function_bodies.append((func_nid, body)) - arrow_found = True - elif value and value.type in ( - "object", "array", "as_expression", "call_expression", "new_expression", - ): - # Module-level const with literal/object/array/factory value - name_node = child.child_by_field_name("name") - if name_node: - const_name = _read_text(name_node, source) - line = child.start_point[0] + 1 - const_nid = _make_id(stem, const_name) - add_node_fn(const_nid, const_name, line) - add_edge_fn(file_nid, const_nid, "contains", line) - const_found = True - if arrow_found: - return True - if const_found: - return True - if require_found: - return True - return False - - -# ── TS extra walk for namespace / module declarations ───────────────────────── - -def _ts_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, - nodes: list, edges: list, seen_ids: set, function_bodies: list, - parent_class_nid: str | None, add_node_fn, add_edge_fn, - walk_fn) -> bool: - """Emit a container node for a TS `namespace`/`module` declaration. - - `namespace Foo {}` parses as `internal_module` (with `name`/`body` fields); - `module Bar {}` and ambient `declare module "pkg" {}` parse as a named - `module` node that exposes no fields, so its name and body are found - positionally. Without this the container was never a node — its members were - still reached by the default recurse but lost their namespace context. The - members stay file-contained (parity with C#'s `_csharp_extra_walk`); the - namespace becomes a sibling marker node so it is queryable. Returns True if - handled. - - The guard requires `is_named` because the anonymous `module` keyword token - shares the `module` type string and would otherwise match here. - """ - if node.is_named and node.type in ("internal_module", "module"): - name_node = node.child_by_field_name("name") - if name_node is None: - for child in node.children: - if child.is_named and child.type in ( - "identifier", "nested_identifier", "string"): - name_node = child - break - body = node.child_by_field_name("body") - if body is None: - for child in node.children: - if child.type == "statement_block": - body = child - break - if name_node is not None: - ns_name = _read_text(name_node, source) - if name_node.type == "string": - ns_name = ns_name.strip("'\"`") - if ns_name: - ns_nid = _make_id(stem, ns_name) - line = node.start_point[0] + 1 - add_node_fn(ns_nid, ns_name, line) - add_edge_fn(file_nid, ns_nid, "contains", line) - if body is not None: - for child in body.children: - walk_fn(child, parent_class_nid) - return True - return False - - -# ── C# extra walk for namespace declarations ────────────────────────────────── - -def _csharp_namespace_name(node, source: bytes) -> str: - name_node = node.child_by_field_name("name") - if name_node is not None: - return _read_text(name_node, source).strip() - for child in node.children: - if child.type in ("identifier", "qualified_name"): - return _read_text(child, source).strip() - return "" - - -def _csharp_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, - nodes: list, edges: list, seen_ids: set, function_bodies: list, - parent_class_nid: str | None, add_node_fn, add_edge_fn, - walk_fn, namespace_stack: list[str], scope_stack: list[str]) -> bool: - """Handle namespace declarations for C#. Returns True if handled.""" - if node.type == "namespace_declaration": - ns_name = _csharp_namespace_name(node, source) - pushed = False - if ns_name: - namespace_stack.append(ns_name) - scope_stack.append(f"s{node.start_byte}") - pushed = True - ns_label = ".".join(namespace_stack) - ns_nid = _csharp_namespace_id(ns_label) - line = node.start_point[0] + 1 - add_node_fn(ns_nid, ns_label, line, node_type="namespace", metadata={"kind": "csharp_namespace"}) - add_edge_fn(file_nid, ns_nid, "contains", line) - body = node.child_by_field_name("body") - if body: - try: - for child in body.children: - walk_fn(child, parent_class_nid) - finally: - if pushed: - namespace_stack.pop() - scope_stack.pop() - elif pushed: - namespace_stack.pop() - scope_stack.pop() - return True - if node.type == "file_scoped_namespace_declaration": - ns_name = _csharp_namespace_name(node, source) - if ns_name: - namespace_stack.append(ns_name) - scope_stack.append(f"s{node.start_byte}") - ns_label = ".".join(namespace_stack) - ns_nid = _csharp_namespace_id(ns_label) - line = node.start_point[0] + 1 - add_node_fn(ns_nid, ns_label, line, node_type="namespace", metadata={"kind": "csharp_namespace"}) - add_edge_fn(file_nid, ns_nid, "contains", line) - return True - return False - - -# ── Swift extra walk for enum cases ────────────────────────────────────────── - -def _swift_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, - nodes: list, edges: list, seen_ids: set, function_bodies: list, - parent_class_nid: str | None, add_node_fn, add_edge_fn, - ensure_named_node_fn) -> bool: - """Handle enum_entry for Swift. Returns True if handled.""" - if node.type == "enum_entry" and parent_class_nid: - line = node.start_point[0] + 1 - for child in node.children: - if child.type == "simple_identifier": - case_name = _read_text(child, source) - case_nid = _make_id(parent_class_nid, case_name) - add_node_fn(case_nid, case_name, line) - add_edge_fn(parent_class_nid, case_nid, "case_of", line) - # Associated-value types nest as `enum_type_parameters -> user_type -> - # type_identifier` (a sibling of the case-name simple_identifier). The - # case-name loop above never descends into them, so `case started(Session)` - # used to drop the Event -> Session reference entirely. Mirror the Swift - # property/parameter emit style: collect the type refs and emit a - # `references` edge from the ENUM node to each collected type. - for child in node.children: - if child.type != "enum_type_parameters": - continue - for grand in child.children: - if not grand.is_named: - continue - refs: list[tuple[str, str]] = [] - _swift_collect_type_refs(grand, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "type" - target_nid = ensure_named_node_fn(ref_name, line) - if target_nid != parent_class_nid: - add_edge_fn(parent_class_nid, target_nid, "references", - line, context=ctx) - return True - return False - - -# ── Java extra walk for enum constants ─────────────────────────────────────── - -def _java_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, - nodes: list, edges: list, seen_ids: set, function_bodies: list, - parent_class_nid: str | None, add_node_fn, add_edge_fn, - walk_fn) -> bool: - """Handle enum_constant for Java. Returns True if handled.""" - if node.type == "enum_constant" and parent_class_nid: - name_node = node.child_by_field_name("name") - if name_node is None: - return True - const_name = _read_text(name_node, source) + if name_node is None: + return True + const_name = _read_text(name_node, source) line = node.start_point[0] + 1 const_nid = _make_id(parent_class_nid, const_name) add_node_fn(const_nid, const_name, line) add_edge_fn(parent_class_nid, const_nid, "case_of", line) - # Anonymous-body constants (`MONDAY { void greet(){} }`): descend so the - # body's methods aren't dropped; const_nid attaches them to the constant. for child in node.children: if child.type == "class_body": for member in child.children: @@ -3141,7 +823,7 @@ def _java_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: st # older forks use `simple_identifier`. Accept both so the extractor # works across grammar generations. name_fallback_child_types=("simple_identifier", "identifier"), - body_fallback_child_types=("function_body", "class_body"), + body_fallback_child_types=("function_body", "class_body", "enum_class_body"), function_boundary_types=frozenset({"function_declaration"}), import_handler=_import_kotlin, ) @@ -3182,44 +864,6 @@ def _java_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: st ) -def _resolve_lua_import_target(raw_module: str, str_path: str) -> str: - """Resolve a Lua require() module name to a node id. - - Lua module names use dots as path separators: `require("pkg.b")` looks for - `pkg/b.lua` (or `pkg/b/init.lua`) relative to a package root. We probe the - importing file's directory and walk upward looking for a matching file on - disk; if found, the returned id matches the file node id `_extract_generic` - assigns to that file (`_make_id(str(path))`), so the edge lands on a real - node. When nothing matches, fall back to `_make_id` of the full dotted - module name so cross-file resolution can still complete via the symbol - resolution pass instead of dropping the edge entirely (#1075). - """ - if not raw_module: - return "" - rel = raw_module.replace(".", "/") - try: - start_dir = Path(str_path).parent - except Exception: - start_dir = None - if start_dir is not None: - probe = start_dir - # Walk up a few levels so requires from nested files still resolve when - # the package root is above the importing file. - for _ in range(6): - for suffix in (".lua", ".luau"): - cand = probe / f"{rel}{suffix}" - if cand.is_file(): - return _make_id(str(cand)) - for suffix in (".lua", ".luau"): - cand = probe / rel / f"init{suffix}" - if cand.is_file(): - return _make_id(str(cand)) - if probe.parent == probe: - break - probe = probe.parent - return _make_id(raw_module) - - def _import_lua(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: """Extract require('module') from Lua variable_declaration nodes.""" text = _read_text(node, source) @@ -3289,31 +933,6 @@ def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, st return modules -def _read_csharp_type_name(node, source: bytes) -> tuple[str, bool, str] | None: - """Resolve a C# type name, whether it was qualified, and its qualifier prefix.""" - if node is None: - return None - if node.type in ("identifier", "predefined_type"): - return (_read_text(node, source), False, "") - if node.type == "qualified_name": - prefix, _, tail = _read_text(node, source).rpartition(".") - tail = tail.split("<", 1)[0] - return (tail, True, prefix) - if node.type == "generic_name": - name_node = node.child_by_field_name("name") - if name_node is not None: - qualified = name_node.type == "qualified_name" - prefix, _, tail = _read_text(name_node, source).rpartition(".") - return (tail, qualified, prefix if qualified else "") - for child in node.children: - if not child.is_named: - continue - result = _read_csharp_type_name(child, source) - if result: - return result - return None - - _SWIFT_CONFIG = LanguageConfig( ts_module="tree_sitter_swift", class_types=frozenset({"class_declaration", "protocol_declaration"}), @@ -3332,150 +951,9 @@ def _read_csharp_type_name(node, source: bytes) -> tuple[str, bool, str] | None: # ── Ruby local type inference (for member-call resolution) ───────────────────── -def _ruby_new_class_name(node, source: bytes) -> str | None: - """Return ``ClassName`` if ``node`` is a ``ClassName.new(...)`` call, else None. - - Only a bare capitalized constant receiver counts (``Processor.new``); - namespaced (``A::B.new``) and dynamic receivers are intentionally ignored so - the binding stays unambiguous. - """ - if node is None or node.type != "call": - return None - recv = node.child_by_field_name("receiver") - meth = node.child_by_field_name("method") - if recv is None or meth is None: - return None - if recv.type != "constant" or _read_text(meth, source) != "new": - return None - return _read_text(recv, source) - - -def _ruby_local_class_bindings(body_node, source: bytes) -> dict[str, str | None]: - """Map ``local_var -> ClassName`` for ``var = ClassName.new`` within one Ruby - method body, not descending into nested method definitions. - - 100%-confidence contract: a variable assigned more than once, or to anything - other than a single ``Constant.new``, maps to ``None`` (ambiguous) so callers - never resolve it. Only the certain single-binding case carries a type. - """ - bindings: dict[str, str | None] = {} - boundary = {"method", "singleton_method"} - - def visit(n) -> None: - for child in n.children: - if child.type in boundary: - continue # nested method has its own scope - if child.type == "assignment": - left = child.child_by_field_name("left") - right = child.child_by_field_name("right") - if left is not None and left.type == "identifier": - var = _read_text(left, source) - cls = _ruby_new_class_name(right, source) if right is not None else None - if cls is None: - # assigned to something we can't type: poison if it was typed - if var in bindings: - bindings[var] = None - elif var in bindings: - if bindings[var] != cls: - bindings[var] = None # reassigned to a different class - else: - bindings[var] = cls - visit(child) - - visit(body_node) - return bindings - - -def _ruby_const_last_name(node, source: bytes) -> str: - """Last constant of a ``constant`` or ``scope_resolution`` (``A::B::C`` -> ``C``).""" - if node is None: - return "" - if node.type == "constant": - return _read_text(node, source) - if node.type == "scope_resolution": - consts = [c for c in node.children if c.type == "constant"] - if consts: - return _read_text(consts[-1], source) - return "" - - # `Const = (...)` shapes that define a lightweight class named after the # constant. tree-sitter parses each as an `assignment`, not a `class`, so the # generic class branch never saw them (#1640). -_RUBY_CLASS_FACTORIES = frozenset({("Struct", "new"), ("Class", "new"), ("Data", "define")}) - - -def _ruby_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, - nodes: list, edges: list, seen_ids: set, function_bodies: list, - parent_class_nid: str | None, add_node, add_edge, walk, - callable_def_nids: set) -> bool: - """Ruby: a constant assignment whose RHS is ``Struct.new(...)``, - ``Class.new(Super)`` or ``Data.define(...)`` defines a class named after the - constant (#1640). Synthesize the class node, attach block-defined methods via - ``method`` (by recursing the block with the new node as parent), and emit an - ``inherits`` edge for ``Class.new(Super)``. Returns True if handled. - """ - if node.type != "assignment": - return False - left = node.child_by_field_name("left") - right = node.child_by_field_name("right") - if left is None or right is None or left.type != "constant" or right.type != "call": - return False - recv = right.child_by_field_name("receiver") - meth = right.child_by_field_name("method") - if recv is None or meth is None or recv.type != "constant": - return False - if (_read_text(recv, source), _read_text(meth, source)) not in _RUBY_CLASS_FACTORIES: - return False - - const_name = _read_text(left, source) - if not const_name: - return False - line = node.start_point[0] + 1 - class_nid = _make_id(stem, const_name) - add_node(class_nid, const_name, line) - callable_def_nids.add(class_nid) # a class is callable (its constructor) - # Mirror the generic class branch: containment always hangs off the file node. - add_edge(file_nid, class_nid, "contains", line) - - # `Class.new(Super)` — the first positional constant argument is the superclass. - if _read_text(recv, source) == "Class": - args = next((c for c in right.children if c.type == "argument_list"), None) - if args is not None: - for arg in args.children: - if arg.type in ("constant", "scope_resolution"): - base = _ruby_const_last_name(arg, source) - if base: - base_nid = _make_id(stem, base) - if base_nid not in seen_ids: - base_nid = _make_id(base) - if base_nid not in seen_ids: - # origin_file lets _disambiguate_colliding_node_ids - # tell this file's unresolved reference apart from - # another file's same-named one, instead of every - # file's stub collapsing onto one shared bare id - # (see ensure_named_node(), which sets the same - # field for this exact reason). - nodes.append({ - "id": base_nid, "label": base, - "file_type": "code", "source_file": "", - "source_location": "", "origin_file": str_path, - }) - seen_ids.add(base_nid) - add_edge(class_nid, base_nid, "inherits", line) - break - - # Recurse the do/brace block so block-defined methods attach to the class. - # The block wraps its statements in a `body_statement` (like a class body); - # descend into it so the method handler sees parent_class_nid — otherwise the - # default recurse resets the parent to None and the method hangs off the file - # with a dot-less label. - block = next((c for c in right.children if c.type in ("do_block", "block")), None) - if block is not None: - body = next((c for c in block.children if c.type == "body_statement"), block) - for child in body.children: - walk(child, parent_class_nid=class_nid) - return True # ── Generic extractor ───────────────────────────────────────────────────────── @@ -4849,6 +2327,12 @@ def _emit_java_parent_type(type_node, rel: str, at_line: int) -> None: parent_class_nid, add_node, add_edge, walk): return + if config.ts_module == "tree_sitter_kotlin": + if _kotlin_extra_walk(node, source, file_nid, stem, str_path, + nodes, edges, seen_ids, function_bodies, + parent_class_nid, add_node, add_edge, walk): + return + if config.ts_module == "tree_sitter_ruby": if _ruby_extra_walk(node, source, file_nid, stem, str_path, nodes, edges, seen_ids, function_bodies, @@ -6224,40 +3708,6 @@ def extract_astro(path: Path) -> dict: # The open-tag matcher skips over quoted attribute values so a `>` inside one # (e.g. Vue 3.3+ generic components: ` close tag - pos = m.end() - if lang is None: - lang_m = _VUE_SCRIPT_LANG_RE.search(m.group(1)) - if lang_m: - lang = lang_m.group(1).lower() - out.append(_blank(src[pos:])) - return "".join(out), lang def extract_vue(path: Path) -> dict: @@ -6474,7336 +3924,1026 @@ def extract_csharp(path: Path) -> dict: return _extract_generic(path, _CSHARP_CONFIG) -def extract_apex(path: Path) -> dict: - """Extract classes, interfaces, enums, methods, and Salesforce constructs from - Apex .cls and .trigger files using regex (no tree-sitter grammar on PyPI).""" - import re as _re - try: - source = path.read_text(encoding="utf-8", errors="replace") - except OSError: - return {"nodes": [], "edges": []} - - str_path = str(path) - stem = _file_stem(path) - file_nid = _make_id(str_path) - - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() +def extract_kotlin(path: Path) -> dict: + """Extract classes, objects, functions, and imports from a .kt/.kts file.""" + return _extract_generic(path, _KOTLIN_CONFIG) - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": label, - "file_type": "code", - "source_file": str_path, - "source_location": f"L{line}", - }) - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED") -> None: - edges.append({ - "source": src, - "target": tgt, - "relation": relation, - "confidence": confidence, - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - }) +def extract_scala(path: Path) -> dict: + """Extract classes, objects, functions, and imports from a .scala file.""" + return _extract_generic(path, _SCALA_CONFIG) - add_node(file_nid, path.name, 1) - lines = source.splitlines() +def extract_php(path: Path) -> dict: + """Extract classes, functions, methods, namespace uses, and calls from a .php file.""" + return _extract_generic(path, _PHP_CONFIG) - _ACCESS = r"(?:public|private|protected|global|webService)?" - _SHARING = r"(?:\s+(?:with|without|inherited)\s+sharing)?" - _MOD = r"(?:\s+(?:abstract|virtual|override|static|final|transient|testMethod))?" - _ANNOTATION = r"(?:\s*@\w+(?:\s*\([^)]*\))?\s*)*" - cls_re = _re.compile( - rf"^{_ANNOTATION}\s*{_ACCESS}{_SHARING}{_MOD}\s*class\s+(\w+)" - rf"(?:\s+extends\s+(\w+))?(?:\s+implements\s+([\w,\s]+))?\s*\{{?", - _re.IGNORECASE, - ) - iface_re = _re.compile( - rf"^{_ANNOTATION}\s*{_ACCESS}{_SHARING}{_MOD}\s*interface\s+(\w+)" - rf"(?:\s+extends\s+([\w,\s]+))?\s*\{{?", - _re.IGNORECASE, - ) - enum_re = _re.compile( - rf"^{_ANNOTATION}\s*{_ACCESS}{_SHARING}{_MOD}\s*enum\s+(\w+)\s*\{{?", - _re.IGNORECASE, - ) - trigger_re = _re.compile( - r"^\s*trigger\s+(\w+)\s+on\s+(\w+)\s*\(", - _re.IGNORECASE, - ) - method_re = _re.compile( - rf"^{_ANNOTATION}\s*{_ACCESS}{_MOD}\s*(?:static\s+)?[\w<>\[\]]+\s+(\w+)\s*\([^)]*\)\s*(?:throws\s+\w+\s*)?\{{?", - _re.IGNORECASE, - ) - annotation_re = _re.compile(r"@(\w+)", _re.IGNORECASE) - soql_re = _re.compile(r"\[\s*SELECT\b[^\]]+FROM\s+(\w+)", _re.IGNORECASE) - dml_re = _re.compile(r"\b(insert|update|delete|upsert|merge|undelete)\s+\w", _re.IGNORECASE) - - _CONTROL_FLOW = frozenset({ - "if", "else", "for", "while", "do", "switch", "try", "catch", - "finally", "return", "throw", "new", "void", "null", - "true", "false", "this", "super", "class", "interface", "enum", - "trigger", "on", - }) +# One level of balanced parens (e.g. `Foo #(Bar #(int))`) — bounded so malformed +# input cannot trigger pathological backtracking. - current_class_nid: str | None = None - pending_annotations: list[str] = [] - for lineno, line_text in enumerate(lines, start=1): - stripped = line_text.strip() +def extract_lua(path: Path) -> dict: + """Extract functions, methods, require() imports, and calls from a .lua file.""" + return _extract_generic(path, _LUA_CONFIG) - if stripped.startswith("@"): - for m in annotation_re.finditer(stripped): - pending_annotations.append(m.group(1).lower()) - continue - tm = trigger_re.match(stripped) - if tm: - trig_name, sobject = tm.group(1), tm.group(2) - trig_nid = _make_id(stem, trig_name) - add_node(trig_nid, trig_name, lineno) - add_edge(file_nid, trig_nid, "contains", lineno) - sob_nid = _make_id(sobject) - if sob_nid not in seen_ids: - add_node(sob_nid, sobject, lineno) - add_edge(trig_nid, sob_nid, "uses", lineno, confidence="INFERRED") - current_class_nid = trig_nid - pending_annotations = [] - continue +def extract_swift(path: Path) -> dict: + """Extract classes, structs, protocols, functions, imports, and calls from a .swift file.""" + return _extract_generic(path, _SWIFT_CONFIG) - cm = cls_re.match(stripped) - if cm: - class_name = cm.group(1) - if class_name.lower() in _CONTROL_FLOW: - pending_annotations = [] - continue - class_nid = _make_id(stem, class_name) - add_node(class_nid, class_name, lineno) - add_edge(file_nid, class_nid, "contains", lineno) - if cm.group(2): - base = cm.group(2).strip() - base_nid = _make_id(stem, base) - if base_nid not in seen_ids: - base_nid = _make_id(base) - if base_nid not in seen_ids: - add_node(base_nid, base, lineno) - add_edge(class_nid, base_nid, "extends", lineno, confidence="INFERRED") - if cm.group(3): - for iface in cm.group(3).split(","): - iface = iface.strip() - if iface: - iface_nid = _make_id(stem, iface) - if iface_nid not in seen_ids: - iface_nid = _make_id(iface) - if iface_nid not in seen_ids: - add_node(iface_nid, iface, lineno) - add_edge(class_nid, iface_nid, "implements", lineno, confidence="INFERRED") - current_class_nid = class_nid - pending_annotations = [] - continue - im = iface_re.match(stripped) - if im: - iface_name = im.group(1) - if iface_name.lower() in _CONTROL_FLOW: - pending_annotations = [] - continue - iface_nid = _make_id(stem, iface_name) - add_node(iface_nid, iface_name, lineno) - add_edge(file_nid if current_class_nid is None else current_class_nid, - iface_nid, "contains", lineno) - if im.group(2): - for parent in im.group(2).split(","): - parent = parent.strip() - if parent: - parent_nid = _make_id(stem, parent) - if parent_nid not in seen_ids: - parent_nid = _make_id(parent) - if parent_nid not in seen_ids: - add_node(parent_nid, parent, lineno) - add_edge(iface_nid, parent_nid, "extends", lineno, confidence="INFERRED") - pending_annotations = [] - continue +# ── Julia extractor (custom walk) ──────────────────────────────────────────── - em = enum_re.match(stripped) - if em: - enum_name = em.group(1) - if enum_name.lower() in _CONTROL_FLOW: - pending_annotations = [] - continue - enum_nid = _make_id(stem, enum_name) - add_node(enum_nid, enum_name, lineno) - add_edge(file_nid if current_class_nid is None else current_class_nid, - enum_nid, "contains", lineno) - pending_annotations = [] - continue - if current_class_nid is not None: - mm = method_re.match(stripped) - if mm: - method_name = mm.group(1) - if method_name.lower() not in _CONTROL_FLOW: - method_nid = _make_id(current_class_nid, method_name) - method_label = f".{method_name}()" - add_node(method_nid, method_label, lineno) - add_edge(current_class_nid, method_nid, "method", lineno) - if "auraenabled" in pending_annotations or "invocablemethod" in pending_annotations: - add_edge(file_nid, method_nid, "contains", lineno, confidence="INFERRED") - pending_annotations = [] - continue +# ── Go extractor (custom walk) ──────────────────────────────────────────────── - pending_annotations = [] - for sm in soql_re.finditer(line_text): - sobject = sm.group(1) - sob_nid = _make_id(sobject) - if sob_nid not in seen_ids: - add_node(sob_nid, sobject, lineno) - src = current_class_nid or file_nid - add_edge(src, sob_nid, "uses", lineno, confidence="INFERRED") +# ── Rust extractor (custom walk) ────────────────────────────────────────────── - for dm in dml_re.finditer(line_text): - dml_op = dm.group(1).lower() - dml_nid = _make_id(f"dml_{dml_op}") - if dml_nid not in seen_ids: - add_node(dml_nid, dml_op, lineno) - src = current_class_nid or file_nid - add_edge(src, dml_nid, "uses", lineno, confidence="INFERRED") +# Common Rust trait/stdlib method names that appear in virtually every codebase. +# Resolving these cross-file produces spurious INFERRED edges across crate +# boundaries (issue #908) — skip them from the unresolved-call queue entirely. - return {"nodes": nodes, "edges": edges} +# ── Zig ─────────────────────────────────────────────────────────────────────── -def extract_kotlin(path: Path) -> dict: - """Extract classes, objects, functions, and imports from a .kt/.kts file.""" - return _extract_generic(path, _KOTLIN_CONFIG) +# ── PowerShell ──────────────────────────────────────────────────────────────── -def extract_scala(path: Path) -> dict: - """Extract classes, objects, functions, and imports from a .scala file.""" - return _extract_generic(path, _SCALA_CONFIG) +# ── PowerShell manifest (.psd1) ────────────────────────────────────────────── -def extract_php(path: Path) -> dict: - """Extract classes, functions, methods, namespace uses, and calls from a .php file.""" - return _extract_generic(path, _PHP_CONFIG) +# Keys in a .psd1 whose values are module names/paths we treat as imports. +# ── Cross-file import resolution ────────────────────────────────────────────── -def extract_dart(path: Path) -> dict: - """Extract classes, mixins, functions, imports, generic calls, and annotations from a .dart file using regex.""" - try: - src = path.read_text(encoding="utf-8", errors="replace") - except OSError: - return {"error": f"cannot read {path}"} - - # Remove inline and multi-line comments while leaving string literals untouched to prevent stripping URLs/paths inside strings - comment_string_pattern = re.compile( - r'"""(?:\\.|[\s\S])*?"""' - r"|'''(?:\\.|[\s\S])*?'''" - r'|"(?:\\.|[^"\\])*"' - r"|'(?:\\.|[^'\\])*'" - r"|/\*[\s\S]*?\*/" - r"|//[^\n]*" - ) - def _comment_replace(match: re.Match) -> str: - token = match.group(0) - if token.startswith("/"): - return "" - return token - src_clean = comment_string_pattern.sub(_comment_replace, src) - - stem = _file_stem(path) - file_nid = _make_id(str(path)) - - # Check if this is a part-of file and redirect to parent - part_of_match = re.search(r"^\s*part\s+of\s+['\"]([^'\"]+)['\"]", src_clean, re.MULTILINE) - is_part = False - if part_of_match: - parent_ref = part_of_match.group(1) - if parent_ref.endswith(".dart"): - try: - parent_path = (path.parent / parent_ref).resolve() - if parent_path.exists(): - stem = _file_stem(parent_path) - file_nid = _make_id(str(parent_path)) - is_part = True - except Exception: - pass - - nodes = [] - if not is_part: - nodes.append({"id": file_nid, "label": path.name, "file_type": "code", - "source_file": str(path), "source_location": None}) - edges = [] - defined: set[str] = set() - - def add_node(nid: str, label: str, ftype: str = "code", source_file: str | None = str(path)) -> None: - if nid not in defined: - nodes.append({"id": nid, "label": label, "file_type": ftype, - "source_file": source_file, "source_location": None}) - defined.add(nid) - - def add_edge(src_id: str, tgt_id: str, relation: str, weight: float = 1.0, context: str | None = None) -> None: - edge = {"source": src_id, "target": tgt_id, "relation": relation, - "confidence": "EXTRACTED", "confidence_score": 1.0, - "source_file": str(path), "source_location": None, "weight": weight} - if context: - edge["context"] = context - edges.append(edge) +def _canonicalize_csharp_namespace_nodes(all_nodes: list[dict], all_edges: list[dict]) -> None: + """Collapse duplicate C# namespace node entries to one canonical node per label.""" + by_label: dict[str, list[dict]] = {} + for node in all_nodes: + if node.get("type") != "namespace": + continue + label = node.get("label") + if isinstance(label, str): + by_label.setdefault(label, []).append(node) - def _split_types(text: str) -> list[str]: - parts = [] - current = [] - depth = 0 - for char in text: - if char == "<": - depth += 1 - current.append(char) - elif char == ">": - depth -= 1 - current.append(char) - elif char == "," and depth == 0: - parts.append("".join(current).strip()) - current = [] - else: - current.append(char) - if current: - parts.append("".join(current).strip()) - return [p for p in parts if p] - - def _find_matching_brace(text: str, start_pos: int) -> int: - brace_count = 0 - in_double_quote = False - in_single_quote = False - escape = False - - first_brace = text.find("{", start_pos) - if first_brace == -1: - return len(text) - - brace_count = 1 - i = first_brace + 1 - n = len(text) - while i < n: - char = text[i] - if escape: - escape = False - i += 1 - continue - if char == "\\": - escape = True - i += 1 - continue - if text[i:i+3] == '"""' and not in_single_quote: - i += 3 - end = text.find('"""', i) - i = end + 3 if end != -1 else n - continue - if text[i:i+3] == "'''" and not in_double_quote: - i += 3 - end = text.find("'''", i) - i = end + 3 if end != -1 else n - continue - if char == '"' and not in_single_quote: - in_double_quote = not in_double_quote - elif char == "'" and not in_double_quote: - in_single_quote = not in_single_quote - elif not in_double_quote and not in_single_quote: - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - return i + 1 - i += 1 - return len(text) - - # 1. Classes, mixins, and enums declarations (with inheritance, mixins, interfaces, and generics) - # Supports multiple combined modifiers (e.g., abstract base class, mixin class) without capturing "class" as a name - class_pattern = r"^\s*(?:(?:abstract|sealed|base|interface|final|mixin)\s+)*(?:class|mixin|enum|extension\s+type)\s+(\w+)" - for m in re.finditer(class_pattern, src_clean, re.MULTILINE): - class_name = m.group(1) - class_nid = _make_id(stem, class_name) - add_node(class_nid, class_name) - add_edge(file_nid, class_nid, "defines") - - # Manually parse extends/on, with, and implements in header to handle nested generics brackets balanced - start_idx = m.end() - rest = src_clean[start_idx : start_idx + 500] - - # Skip class generic parameters - if rest.lstrip().startswith("<"): - offset = rest.find("<") - depth = 1 - i = offset + 1 - while i < len(rest) and depth > 0: - if rest[i] == "<": depth += 1 - elif rest[i] == ">": depth -= 1 - i += 1 - rest = rest[i:] - - # Skip primary constructor (e.g. extension type MyExt(int id)) - if rest.lstrip().startswith("("): - offset = rest.find("(") - depth = 1 - i = offset + 1 - while i < len(rest) and depth > 0: - if rest[i] == "(": depth += 1 - elif rest[i] == ")": depth -= 1 - i += 1 - rest = rest[i:] - - header_end = rest.find("{") - if header_end == -1: - header_end = rest.find(";") - if header_end == -1: - header_end = len(rest) - header = rest[:header_end] - - base_class = None - generics = None - mixins_list = [] - interfaces_list = [] - - # Parse extends or on - extends_m = re.search(r"^\s*(?:extends|on)\s+([a-zA-Z0-9_.]+)", header) - if extends_m: - base_class = extends_m.group(1) - rest_header = header[extends_m.end():] - if rest_header.strip().startswith("<"): - start_idx = rest_header.find("<") - depth = 1 - i = start_idx + 1 - while i < len(rest_header) and depth > 0: - if rest_header[i] == "<": - depth += 1 - elif rest_header[i] == ">": - depth -= 1 - if depth == 0: - generics = rest_header[start_idx + 1 : i] - break - i += 1 - if generics is not None: - header = rest_header[i + 1:] - else: - header = rest_header - else: - header = rest_header - - # Parse with - with_m = re.search(r"^\s*with\s+", header) - if with_m: - rest_header = header[with_m.end():] - impl_idx = rest_header.find("implements") - if impl_idx != -1: - mixins_str = rest_header[:impl_idx] - header = rest_header[impl_idx:] - else: - mixins_str = rest_header - header = "" - mixins_list = _split_types(mixins_str) - - # Parse implements - impl_m = re.search(r"^\s*implements\s+", header) - if impl_m: - interfaces_list = _split_types(header[impl_m.end():]) - - # Map extends inheritance relation - if base_class: - base_nid = _make_id(base_class) - add_node(base_nid, base_class, source_file=None) - add_edge(class_nid, base_nid, "inherits") - - # Map generic type arguments (e.g. MyBloc extends Bloc) - if generics: - for gen in _split_types(generics): - gen_clean = gen.split("<")[0].strip() - if gen_clean not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}: - gen_nid = _make_id(gen_clean) - add_node(gen_nid, gen_clean, source_file=None) - add_edge(class_nid, gen_nid, "references") - - # Map mixins - for mixin in mixins_list: - mixin_clean = mixin.split("<")[0].strip() - mixin_nid = _make_id(mixin_clean) - add_node(mixin_nid, mixin_clean, source_file=None) - add_edge(class_nid, mixin_nid, "mixes_in") - - # Map interfaces - for interface in interfaces_list: - interface_clean = interface.split("<")[0].strip() - interface_nid = _make_id(interface_clean) - add_node(interface_nid, interface_clean, source_file=None) - add_edge(class_nid, interface_nid, "implements") - - # Extract class body for precise framework dependencies and event handling - start_idx = m.start() - brace_pos = src_clean.find("{", start_idx) - semi_pos = src_clean.find(";", start_idx) - - has_body = brace_pos != -1 - if has_body and semi_pos != -1 and semi_pos < brace_pos: - has_body = False - - if has_body: - end_pos = _find_matching_brace(src_clean, start_idx) - class_body = src_clean[brace_pos:end_pos] - - # Bloc event registration: on() - for em in re.finditer(r"\bon<(\w+)>\s*\(", class_body): - event_name = em.group(1) - event_nid = _make_id(event_name) - add_node(event_nid, event_name, source_file=None) - add_edge(class_nid, event_nid, "calls", context="bloc_event") - - # Bloc state emissions: emit(MyState) or yield MyState - for sm in re.finditer(r"\b(?:emit|yield)\s*\(?\s*(?:const\s+)?([A-Z]\w*)\b", class_body): - state_name = sm.group(1) - if state_name not in {"String", "List", "Map", "Set", "Future", "Stream", "Object"}: - state_nid = _make_id(state_name) - add_node(state_nid, state_name, source_file=None) - add_edge(class_nid, state_nid, "calls", context="emit_state") - - # Bloc event additions: widget.add(MyEvent()) or bloc.add(MyEvent()) - for am in re.finditer(r"\b(?:\w*[Bb]loc\w*|context\.read<\w+>\(\))\.add\(\s*(?:const\s+)?([A-Z]\w*)\b", class_body): - event_name = am.group(1) - if event_name not in {"String", "List", "Map", "Set", "Future", "Stream", "Object"}: - event_nid = _make_id(event_name) - add_node(event_nid, event_name, source_file=None) - add_edge(class_nid, event_nid, "calls", context="bloc_add_event") - - # Riverpod provider references: ref.watch(provider) - for rm in re.finditer(r"\bref\.(?:watch|read|listen)\s*\(\s*(\w+)\b", class_body): - provider_name = rm.group(1) - provider_nid = _make_id(provider_name) - add_node(provider_nid, provider_name, source_file=None) - add_edge(class_nid, provider_nid, "references", context="riverpod_reference") - - # Widget to Bloc references: BlocBuilder - for bm in re.finditer(r"\bBloc(?:Builder|Listener|Consumer|Provider|Selector)\s*<\s*([a-zA-Z0-9_]+)\b", class_body): - bloc_name = bm.group(1) - if bloc_name not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}: - bloc_nid = _make_id(bloc_name) - add_node(bloc_nid, bloc_name, source_file=None) - add_edge(class_nid, bloc_nid, "references", context="bloc_widget_binding") - - # context.read() or BlocProvider.of(context) - for lm in re.finditer(r"\b(?:read|watch|select|of)\s*<([a-zA-Z0-9_]+)>", class_body): - bloc_name = lm.group(1) - if bloc_name not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}: - bloc_nid = _make_id(bloc_name) - add_node(bloc_nid, bloc_name, source_file=None) - add_edge(class_nid, bloc_nid, "references", context="bloc_lookup") - - # 2. Annotations mapping (class, mixin, enum, or function level annotations) - # Support: @riverpod, @Riverpod(...), @injectable, @singleton, @RoutePage(), @HiveType(typeId: 0), @RestApi() - # Matches `@annotation` and links it to the next class/mixin/enum/function declaration in the file - annotation_pattern = r"@(\w+)(?:\([^)]*\))?" - for am in re.finditer(annotation_pattern, src_clean): - annotation_name = am.group(1) - if annotation_name in {"override", "deprecated", "required", "protected", "mustCallSuper"}: + remap: dict[str, str] = {} + drop_node_ids: set[int] = set() + for group in by_label.values(): + if len(group) < 2: continue - annotation_pos = am.end() - intervening_text = src_clean[annotation_pos : annotation_pos + 300] + canonical = sorted( + group, + key=lambda node: ( + str(node.get("source_file") or ""), + str(node.get("source_location") or ""), + str(node.get("id") or ""), + ), + )[0] + canonical_id = canonical.get("id") + for node in group: + if node is canonical: + continue + drop_node_ids.add(id(node)) + dup_id = node.get("id") + if isinstance(dup_id, str) and isinstance(canonical_id, str): + remap[dup_id] = canonical_id - class_m = re.search(r"^\s*(?:(?:abstract|sealed|base|interface|final|mixin)\s+)*(?:class|mixin|enum|extension\s+type)\s+(\w+)", intervening_text, re.MULTILINE) - func_m = re.search(r"^\s*(?:factory\s+|static\s+|async\s+|external\s+|abstract\s+)?(?:\([^)]+\)|[a-zA-Z0-9_<>,.?]+)(?:\s+[a-zA-Z0-9_<>,.?]+){0,3}\s+(\w+)\s*\(", intervening_text, re.MULTILINE) + if remap: + for edge in all_edges: + if edge.get("source") in remap: + edge["source"] = remap[str(edge["source"])] + if edge.get("target") in remap: + edge["target"] = remap[str(edge["target"])] - target_nid = None - target_name = None - target_type = None + if drop_node_ids: + all_nodes[:] = [node for node in all_nodes if id(node) not in drop_node_ids] - if class_m and func_m: - if class_m.start() < func_m.start(): - target_name = class_m.group(1) - target_type = "class" - target_nid = _make_id(stem, target_name) - else: - target_name = func_m.group(1) - target_type = "function" - target_nid = _make_id(stem, target_name) - elif class_m: - target_name = class_m.group(1) - target_type = "class" - target_nid = _make_id(stem, target_name) - elif func_m: - target_name = func_m.group(1) - target_type = "function" - target_nid = _make_id(stem, target_name) - - if target_nid and target_name: - actual_intervening = intervening_text[:min(class_m.start() if class_m else 300, func_m.start() if func_m else 300)] - if ";" not in actual_intervening and "}" not in actual_intervening and "{" not in actual_intervening: - annotation_nid = _make_id("annotation", annotation_name.lower()) - add_node(annotation_nid, f"@{annotation_name}", ftype="concept", source_file=None) - add_edge(target_nid, annotation_nid, "configures") - - # Riverpod specific provider generation mapping (supports camelCase class and functional providers) - if annotation_name.lower() == "riverpod": - if target_type == "class": - provider_name = target_name[0].lower() + target_name[1:] + "Provider" if len(target_name) > 1 else target_name.lower() + "Provider" - else: - provider_name = target_name + "Provider" - provider_nid = _make_id(provider_name) - add_node(provider_nid, provider_name, ftype="concept", source_file=str(path)) - add_edge(target_nid, provider_nid, "defines", context="riverpod_provider") - - # 2.5 Typedefs (Type Aliases) - typedef_pattern = r"^\s*typedef\s+(\w+)\s*(?:<[^>]+>)?\s*=\s*([a-zA-Z0-9_<>,.?\s]+);" - for m in re.finditer(typedef_pattern, src_clean, re.MULTILINE): - typedef_name = m.group(1) - target_type = m.group(2).split("<")[0].split(".")[-1].strip() - if target_type not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "List", "Map", "Set", "void", "Function"}: - typedef_nid = _make_id(stem, typedef_name) - add_node(typedef_nid, typedef_name) - add_edge(file_nid, typedef_nid, "defines") - target_nid = _make_id(target_type) - add_node(target_nid, target_type, source_file=None) - add_edge(typedef_nid, target_nid, "references", context="typedef") - - # 3. Extensions (extension MyExt on MyClass) - ext_pattern = r"^\s{0,4}extension\s+(\w+)?(?:<[^>]+>)?\s+on\s+(\w+)" - for m in re.finditer(ext_pattern, src_clean, re.MULTILINE): - ext_name = m.group(1) or f"{stem}_anonymous_extension" - target_class = m.group(2) - - ext_nid = _make_id(stem, ext_name) - label = m.group(1) or f"Extension on {target_class}" - add_node(ext_nid, label) - add_edge(file_nid, ext_nid, "defines") - - target_nid = _make_id(target_class) - add_node(target_nid, target_class, source_file=None) - add_edge(ext_nid, target_nid, "extends") - - # 4. Top-level and class-level variable declarations (generic variables, records, late, and destructuring) - # Restrict indentation to 0-2 spaces to avoid matching local variables inside functions or switch expressions - var_pattern = r"^\s{0,2}(?:late\s+)?(?:(?:final|const|var)\s+)?(?:\([^)]+\)\s+|([a-zA-Z0-9_<>,.?]+(?:\s+[a-zA-Z0-9_<>,.?]+){0,3})\s+)?(?:(\w+)|(?:\w+\s*)?\(([^)]+)\))\s*(?:=|$|;)" - for m in re.finditer(var_pattern, src_clean, re.MULTILINE): - var_type = m.group(1) - single_name = m.group(2) - destructured_names = m.group(3) - - if not re.match(r"^\s*(?:late|final|const|var)\b", m.group(0)) and not var_type: - continue - if single_name: - if single_name not in {"if", "for", "while", "switch", "catch", "return"}: - var_nid = _make_id(stem, single_name) - add_node(var_nid, single_name) - add_edge(file_nid, var_nid, "defines") - - if var_type and var_type not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "List", "Map", "Set", "void"}: - clean_type = var_type.split("<")[0].split(".")[-1].strip() - type_nid = _make_id(clean_type) - add_node(type_nid, clean_type, source_file=None) - add_edge(file_nid, type_nid, "references", context="variable_type") - elif destructured_names: - for name in [n.strip() for n in destructured_names.split(",") if n.strip()]: - if ":" in name: - name = name.split(":")[-1].strip() - if re.match(r"^[a-zA-Z_]\w*$", name) and not re.match(r"^[A-Z]", name): - if name not in {"if", "for", "while", "switch", "catch", "return"}: - var_nid = _make_id(stem, name) - add_node(var_nid, name) - add_edge(file_nid, var_nid, "defines") - - # 5. Top-level and member functions/methods (supports typed/generic/record return types and Riverpod/Bloc references) - # Restrict indentation to 0-2 spaces to avoid matching nested local functions or methods inside multiline switch statements - method_pattern = r"^\s{0,2}(?:factory\s+|static\s+|async\s+|external\s+|abstract\s+)?(?:\([^)]+\)|[a-zA-Z0-9_<>,.?]+)(?:\s+[a-zA-Z0-9_<>,.?]+){0,3}\s+(\w+(?:\.\w+)?)\s*\(" - for m in re.finditer(method_pattern, src_clean, re.MULTILINE): - raw_name = m.group(1) - name = raw_name.split(".")[-1] - if name in {"if", "for", "while", "switch", "catch", "return", "void", "dynamic", "final", "const", "get", "set"}: - continue - if re.match(r"^[A-Z]", name): - continue - nid = _make_id(stem, name) - add_node(nid, name) - add_edge(file_nid, nid, "defines") - - # Get function body using matching brace to extract Riverpod reference patterns - start_idx = m.start() - brace_pos = src_clean.find("{", start_idx) - semi_pos = src_clean.find(";", start_idx) - arrow_pos = src_clean.find("=>", start_idx) - - has_body = brace_pos != -1 - if has_body and semi_pos != -1 and semi_pos < brace_pos: - has_body = False - if has_body and arrow_pos != -1 and arrow_pos < brace_pos: - has_body = False - - if has_body: - end_pos = _find_matching_brace(src_clean, start_idx) - func_body = src_clean[brace_pos:end_pos] - - # Extract Riverpod provider references: ref.watch(provider) - for rm in re.finditer(r"\bref\.(?:watch|read|listen)\s*\(\s*(\w+)\b", func_body): - provider_name = rm.group(1) - provider_nid = _make_id(provider_name) - add_node(provider_nid, provider_name, source_file=None) - add_edge(nid, provider_nid, "references", context="riverpod_reference") - - # Extract Bloc event additions: widget.add(MyEvent()) or bloc.add(MyEvent()) - for am in re.finditer(r"\b(?:\w*[Bb]loc\w*|context\.read<\w+>\(\))\.add\(\s*(?:const\s+)?([A-Z]\w*)\b", func_body): - event_name = am.group(1) - if event_name not in {"String", "List", "Map", "Set", "Future", "Stream", "Object"}: - event_nid = _make_id(event_name) - add_node(event_nid, event_name, source_file=None) - add_edge(nid, event_nid, "calls", context="bloc_add_event") - - # context.read() or BlocProvider.of(context) - for lm in re.finditer(r"\b(?:read|watch|select|of)\s*<([a-zA-Z0-9_]+)>", func_body): - bloc_name = lm.group(1) - if bloc_name not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}: - bloc_nid = _make_id(bloc_name) - add_node(bloc_nid, bloc_name, source_file=None) - add_edge(nid, bloc_nid, "references", context="bloc_lookup") - - # Universal Navigation Patters (GoRouter, AutoRoute, Navigator) - for nm in re.finditer(r"\b(?:go|push|goNamed|pushNamed|replace|replaceNamed)\s*\(\s*(?:context\s*,\s*)?['\"]([a-zA-Z0-9_/?=&%-]+)['\"]", func_body): - route_path = nm.group(1) - route_nid = _make_id("route", route_path.replace("/", "_").replace("?", "_").replace("=", "_").replace("&", "_")) - add_node(route_nid, f"Route {route_path}", ftype="concept", source_file=None) - add_edge(nid, route_nid, "navigates", context="route_path") - - for cm in re.finditer(r"\b(?:go|push|goNamed|pushNamed|replace|replaceNamed)\s*\(\s*(?:context\s*,\s*)?([A-Z][a-zA-Z0-9_]*\.[a-zA-Z0-9_]+)", func_body): - route_const = cm.group(1) - route_nid = _make_id("route", route_const.replace(".", "_")) - add_node(route_nid, route_const, ftype="concept", source_file=None) - add_edge(nid, route_nid, "navigates", context="route_const") - - for om in re.finditer(r"\b(?:push|replace)\s*\(\s*(?:context\s*,\s*)?.*?\b([A-Z]\w*(?:Route|Screen|Page))\b", func_body): - route_class = om.group(1) - route_nid = _make_id(route_class) - add_node(route_nid, route_class, source_file=None) - add_edge(nid, route_nid, "navigates", context="route_object") - - # 6. Imports and Exports - for m in re.finditer(r"""^\s*import\s+['"]([^'"]+)['"]""", src_clean, re.MULTILINE): - pkg = m.group(1) - tgt_nid = _make_id(pkg) - add_node(tgt_nid, pkg, source_file=None) - add_edge(file_nid, tgt_nid, "imports") - - for m in re.finditer(r"""^\s*export\s+['"]([^'"]+)['"]""", src_clean, re.MULTILINE): - pkg = m.group(1) - tgt_nid = _make_id(pkg) - add_node(tgt_nid, pkg, source_file=None) - add_edge(file_nid, tgt_nid, "exports") - - # 7. Generic Invocations / Type Lookups (Universal Dependency Lookup) - # Matches any method call with type parameters: methodName() or object.methodName() - # Automatically extracts GetIt, Injectable, Riverpod, Provider, BlocProvider, and InheritedWidget type lookups! - generic_call_pattern = r"\b\w+<([a-zA-Z0-9_.]+(?:<[a-zA-Z0-9_.,\s<>]+>)?)\s*>\s*\(" - type_blacklist = {"String", "int", "double", "bool", "num", "dynamic", "Object", "List", "Map", "Set", "Future", "Stream", "void"} - for m in re.finditer(generic_call_pattern, src_clean): - type_name = m.group(1).split(".")[-1].strip() - clean_name = type_name.split("<")[0].strip() - if clean_name not in type_blacklist: - target_nid = _make_id(clean_name) - add_node(target_nid, clean_name, source_file=None) - add_edge(file_nid, target_nid, "references", context="type_lookup") +# Languages whose identifiers are case-insensitive, so cross-file name resolution +# may fold case. Everywhere else, case is semantic (`Path` the class vs `PATH` the +# env var are distinct) and folding manufactures false edges / super-hubs (#1581). +_CASE_INSENSITIVE_EXTS = frozenset({ + ".php", ".phtml", ".php3", ".php4", ".php5", ".php7", ".phps", # PHP fns/classes + ".sql", # SQL identifiers + ".nim", ".nims", ".nimble", # Nim (style-insensitive) +}) - return {"nodes": nodes, "edges": edges} +def _lang_is_case_insensitive(source_file: object) -> bool: + """True when the file's language resolves identifiers case-insensitively (#1581).""" + if not source_file: + return False + return Path(str(source_file)).suffix.lower() in _CASE_INSENSITIVE_EXTS -def _sv_first_identifier(node, source: bytes) -> str | None: - """First `simple_identifier` under node in pre-order, or None. - tree-sitter-verilog 1.0.3 nests declaration names a few levels deep instead - of exposing a `name` field. Scope the search to the right child node (e.g. - `function_identifier`) or this returns the return-type instead of the name. - """ - if node is None: - return None - for child in node.children: - if child.type == "simple_identifier": - return _read_text(child, source) - found = _sv_first_identifier(child, source) - if found: - return found - return None +# Language interop families for cross-file call resolution. A call in one language +# can never bind by name to a definition in another family — a TSX component does +# not invoke a Kotlin method, and a Python function does not invoke a Java one. +# Families are grouped by REAL interop so legitimate cross-language resolution +# keeps working: Kotlin/Java/Scala/Groovy share the JVM, C/C++/Objective-C/CUDA +# share headers and symbols (Swift bridges to Objective-C), and JS/TS variants +# (plus Vue/Svelte/Astro SFC script blocks) compile into one module graph. +# Extensions absent from this map (docs, configs, unknown languages) resolve to +# no family and are never filtered — same permissive default as before. +_LANG_FAMILY_BY_EXT: dict[str, str] = { + # JS/TS module graph (SFCs embed JS/TS) + ".js": "jsts", ".jsx": "jsts", ".mjs": "jsts", ".cjs": "jsts", + ".ts": "jsts", ".tsx": "jsts", ".mts": "jsts", ".cts": "jsts", + ".vue": "jsts", ".svelte": "jsts", ".astro": "jsts", + # JVM interop + ".java": "jvm", ".kt": "jvm", ".kts": "jvm", + ".scala": "jvm", ".groovy": "jvm", ".gradle": "jvm", + # C-family: shared headers, Objective-C/C++ mix, Swift↔ObjC bridging + ".c": "native", ".h": "native", ".cpp": "native", ".cc": "native", + ".cxx": "native", ".hpp": "native", ".cu": "native", ".cuh": "native", + ".metal": "native", ".m": "native", ".mm": "native", ".swift": "native", + # Single-language families + ".py": "python", + ".go": "go", + ".rs": "rust", + ".rb": "ruby", + ".php": "php", ".phtml": "php", ".php3": "php", ".php4": "php", + ".php5": "php", ".php7": "php", ".phps": "php", + ".cs": "dotnet", ".razor": "dotnet", ".cshtml": "dotnet", ".xaml": "dotnet", + ".lua": "lua", ".luau": "lua", + ".zig": "zig", + ".ex": "elixir", ".exs": "elixir", + ".jl": "julia", + ".dart": "dart", + ".sh": "shell", ".bash": "shell", + ".ps1": "powershell", ".psm1": "powershell", ".psd1": "powershell", +} -def _sv_child(node, type_name: str) -> object | None: - if node is None: +def _lang_family(source_file: object) -> str | None: + """Interop family of the file's language, or None when unknown/not code.""" + if not source_file: return None - for child in node.children: - if child.type == type_name: - return child - return None - + return _LANG_FAMILY_BY_EXT.get(Path(str(source_file)).suffix.lower()) -_SV_BUILTIN_TYPES = frozenset({ - "bit", "logic", "reg", "wire", "int", "integer", "shortint", "longint", - "byte", "time", "real", "shortreal", "void", "string", "type", "event", - "mailbox", "semaphore", "process", "chandle", -}) -_SV_NON_TYPE_WORDS = frozenset({ - "return", "if", "else", "for", "foreach", "while", "case", "begin", "end", - "function", "task", "class", "endclass", "endfunction", "endtask", -}) +def _node_label_key(node: dict, fold: bool = False) -> str: + label = str(node.get("label", "")).strip() + key = re.sub(r"[^a-zA-Z0-9]+", "", label) + return key.lower() if fold else key -# One level of balanced parens (e.g. `Foo #(Bar #(int))`) — bounded so malformed -# input cannot trigger pathological backtracking. -_SV_PARENS_INNER = r"(?:[^()]|\([^()]*\))*" -_SV_PARENS = r"\(" + _SV_PARENS_INNER + r"\)" -_SV_FUNC_RE = re.compile( - r"\bfunction\s+([A-Za-z_]\w*(?:\s*#\s*" + _SV_PARENS + r")?)\s+(\w+)\s*" - r"\((" + _SV_PARENS_INNER + r")\)\s*;", - re.MULTILINE, -) +def _rewire_unique_stub_nodes(nodes: list[dict], edges: list[dict]) -> None: + """Map unresolved no-source stubs to a unique real definition with the same label.""" + real_by_label: dict[str, list[dict]] = {} # exact-case (all languages) + real_by_label_ci: dict[str, list[dict]] = {} # case-INSENSITIVE-language reals only + stubs: list[dict] = [] -_SV_PARAM_RE = re.compile( - r"\s*(?:input|output|inout|ref|const\s+ref)?\s*" - r"([A-Za-z_]\w*(?:\s*#\s*" + _SV_PARENS + r")?)\s+\w+" -) + for node in nodes: + key = _node_label_key(node) + if not key: + continue + if node.get("source_file"): + if _is_type_like_definition(node): + # Match stubs case-SENSITIVELY: a `Path` reference must not rewire to a + # `PATH` env var (#1581). Fold only for genuinely case-insensitive + # languages, where `foo` legitimately resolves to `Foo`. + real_by_label.setdefault(key, []).append(node) + if _lang_is_case_insensitive(node.get("source_file")): + real_by_label_ci.setdefault( + _node_label_key(node, fold=True), []).append(node) + continue + stubs.append(node) + remap: dict[str, str] = {} + for stub in stubs: + stub_id = str(stub.get("id", "")) + if not stub_id: + continue + candidates = real_by_label.get(_node_label_key(stub), []) + if len(candidates) != 1: + # No unique exact match — fall back to a case-insensitive match, but + # only against case-insensitive-language definitions (so a case-sensitive + # `PATH` can never absorb a `Path` reference). + candidates = real_by_label_ci.get(_node_label_key(stub, fold=True), []) + if len(candidates) != 1: + continue + target_id = candidates[0].get("id") + if isinstance(target_id, str) and target_id and target_id != stub_id: + remap[stub_id] = target_id -def _sv_strip_comments(text: str) -> str: - text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL) - return re.sub(r"//.*", "", text) + if not remap: + return + by_id = {node.get("id"): node for node in nodes if node.get("id")} + csharp_scoped_relations = {"inherits", "implements", "references", "imports"} + for edge in edges: + is_csharp_scoped_edge = ( + str(edge.get("source_file", "")).endswith(".cs") + and edge.get("relation") in csharp_scoped_relations + ) + source = edge.get("source") + if source in remap: + remapped_source = remap[str(source)] + if not ( + is_csharp_scoped_edge + and str(by_id.get(remapped_source, {}).get("source_file", "")).endswith(".cs") + ): + edge["source"] = remapped_source + target = edge.get("target") + if target in remap: + remapped_target = remap[str(target)] + if not ( + is_csharp_scoped_edge + and str(by_id.get(remapped_target, {}).get("source_file", "")).endswith(".cs") + ): + edge["target"] = remapped_target -def _sv_split_type_list(text: str) -> list[str]: - parts: list[str] = [] - depth = 0 - start = 0 - for idx, ch in enumerate(text): - if ch == "(": - depth += 1 - elif ch == ")": - depth = max(0, depth - 1) - elif ch == "," and depth == 0: - item = text[start:idx].strip() - if item: - parts.append(item) - start = idx + 1 - item = text[start:].strip() - if item: - parts.append(item) - return parts + referenced = {x for e in edges for x in (e.get("source"), e.get("target"))} + drop_ids = {stub_id for stub_id in remap if stub_id not in referenced} + nodes[:] = [node for node in nodes if node.get("id") not in drop_ids] -def _sv_collect_type_refs(type_text: str, generic: bool = False, - skip: frozenset[str] = frozenset()) -> list[tuple[str, str]]: - refs: list[tuple[str, str]] = [] - text = type_text.strip() - if not text: - return refs - head = re.match(r"([A-Za-z_]\w*)", text) - if head: - name = head.group(1) - # `skip` carries the enclosing class's `#(type T = ...)` parameters so - # they are not mistaken for referenced types. - if name not in _SV_BUILTIN_TYPES and name not in _SV_NON_TYPE_WORDS and name not in skip: - refs.append((name, "generic_arg" if generic else "type")) - params = re.search(r"#\s*\((" + _SV_PARENS_INNER + r")\)", text) - if params: - for arg in _sv_split_type_list(params.group(1)): - refs.extend(_sv_collect_type_refs(arg, generic=True, skip=skip)) - return refs - - -def _augment_systemverilog_semantics( - raw: str, - stem: str, - str_path: str, - file_nid: str, +def _augment_js_reexport_edges( + paths: list[Path], nodes: list[dict], edges: list[dict], - seen_ids: set[str], + root: Path, ) -> None: - label_to_nid = {node["label"]: node["id"] for node in nodes} + """Compatibility wrapper for the JS/TS symbol-resolution post-pass.""" + facts = _SymbolResolutionFacts() + _collect_js_symbol_resolution_facts(paths, facts) + _apply_symbol_resolution_facts(paths, nodes, edges, root, facts) - def line_for(offset: int) -> int: - return raw.count("\n", 0, offset) + 1 - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({"id": nid, "label": label, "file_type": "code", - "source_file": str_path, "source_location": f"L{line}", - "confidence_score": 1.0}) - label_to_nid[label] = nid - - def ensure_type(label: str, line: int) -> str: - if label in label_to_nid: - return label_to_nid[label] - nid = _make_id(stem, label) - add_node(nid, label, line) - return nid +# Header / implementation file-extension pairing for the decl/def class merge. - def add_edge(src: str, target_label: str, relation: str, line: int, context: str | None = None) -> None: - tgt = ensure_type(target_label, line) - edge = {"source": src, "target": tgt, "relation": relation, - "confidence": "EXTRACTED", "confidence_score": 1.0, - "source_file": str_path, "source_location": f"L{line}", "weight": 1.0} - if context: - edge["context"] = context - edges.append(edge) - text = _sv_strip_comments(raw) - # Consuming `endclass` (rather than a lookahead) makes each match own its - # terminator, so back-to-back or malformed classes cannot bleed bodies. - class_re = re.compile( - r"\b(?:(interface)\s+)?class\s+(\w+)([^;{]*)\s*;(.*?)\bendclass\b", - re.DOTALL, - ) - for match in class_re.finditer(text): - class_name = match.group(2) - header = match.group(3) or "" - body = match.group(4) or "" - line = line_for(match.start()) - # `#(type T = Payload)` declares `T` as a class type parameter, not a - # referenced type — collect these to skip below. - type_params = frozenset(re.findall(r"\btype\s+(\w+)", header)) - class_nid = _make_id(stem, class_name) - add_node(class_nid, class_name, line) - edges.append({"source": file_nid, "target": class_nid, "relation": "defines", - "confidence": "EXTRACTED", "confidence_score": 1.0, - "source_file": str_path, "source_location": f"L{line}", "weight": 1.0}) - - ext = re.search(r"\bextends\s+(\w+)", header) - if ext: - add_edge(class_nid, ext.group(1), "inherits", line) - impl = re.search(r"\bimplements\s+([^;{]+)", header) - if impl: - for iface_name in _sv_split_type_list(impl.group(1)): - add_edge(class_nid, iface_name.split("#", 1)[0].strip(), "implements", line) - - body_without_functions = re.sub( - r"\bfunction\b.*?\bendfunction\b", - lambda m: "\n" * m.group(0).count("\n"), - body, - flags=re.DOTALL, - ) - # Optional leading class-property qualifiers (rand/local/protected/etc.) - # must be consumed: otherwise a qualified field like `rand Config x;` - # (three tokens) fails the ` ;` shape and its type reference - # is silently dropped. - for field in re.finditer(r"^\s*(?:(?:rand|randc|local|protected|static|const|automatic|var)\s+)*([A-Za-z_]\w*(?:\s*#\s*\([^;]+?\))?)\s+\w+\s*;", body_without_functions, re.MULTILINE): - # Count to the start of the type token (group 1), not the match - # start: `^\s*` consumes the leading newline(s), so field.start() - # would resolve to the class's line instead of the field's. - field_line = line + body_without_functions.count("\n", 0, field.start(1)) - for ref_name, role in _sv_collect_type_refs(field.group(1), skip=type_params): - add_edge(class_nid, ref_name, "references", field_line, "generic_arg" if role == "generic_arg" else "field") - - for fm in _SV_FUNC_RE.finditer(body): - return_type, func_name, params = fm.group(1), fm.group(2), fm.group(3) - func_line = line + body.count("\n", 0, fm.start()) - func_nid = _make_id(class_nid, func_name) - add_node(func_nid, func_name, func_line) - edges.append({"source": class_nid, "target": func_nid, "relation": "method", - "confidence": "EXTRACTED", "confidence_score": 1.0, - "source_file": str_path, "source_location": f"L{func_line}", "weight": 1.0}) - for ref_name, role in _sv_collect_type_refs(return_type, skip=type_params): - add_edge(func_nid, ref_name, "references", func_line, "generic_arg" if role == "generic_arg" else "return_type") - for param in _sv_split_type_list(params): - pm = _SV_PARAM_RE.match(param) - if not pm: - continue - for ref_name, role in _sv_collect_type_refs(pm.group(1), skip=type_params): - add_edge(func_nid, ref_name, "references", func_line, "generic_arg" if role == "generic_arg" else "parameter_type") +def _merge_swift_extensions( + per_file: list[dict], + all_nodes: list[dict], + all_edges: list[dict], +) -> None: + """Collapse cross-file Swift `extension Foo` nodes into the canonical `Foo`. + tree-sitter-swift reuses `class_declaration` for both `class Foo` and + `extension Foo`, and node ids carry the file stem, so each file that + extends `Foo` produces its own `Foo` node. The match is done by label: + when exactly one non-extension declaration shares the label, extension + nodes redirect onto it. Extensions of types outside the corpus (no match) + and ambiguous labels (more than one match) are left untouched — picking + arbitrarily would invent edges. + """ + extension_nids: set[str] = set() + extension_labels: dict[str, str] = {} + for result in per_file: + for ext in result.get("swift_extensions", []) or []: + extension_nids.add(ext["nid"]) + extension_labels[ext["nid"]] = ext["label"] -def extract_verilog(path: Path) -> dict: - """Extract modules, functions, tasks, package imports, instantiations, and - SystemVerilog class semantics (inherits/implements edges, field/parameter/ - return-type references) from .v/.sv files.""" - try: - import tree_sitter_verilog as tsverilog - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree_sitter_verilog not installed"} + if not extension_nids: + return - try: - language = Language(tsverilog.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} - - stem = _file_stem(path) - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({"id": nid, "label": label, "file_type": "code", - "source_file": str_path, "source_location": f"L{line}", - "confidence_score": 1.0}) - - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", score: float = 1.0) -> None: - edges.append({"source": src, "target": tgt, "relation": relation, - "confidence": confidence, "confidence_score": score, - "source_file": str_path, "source_location": f"L{line}", "weight": 1.0}) - - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - def walk(node, module_nid: str | None = None) -> None: - t = node.type - - # SystemVerilog class bodies are handled by _augment_systemverilog_semantics - # (regex over source text). Skip their subtrees so in-class methods are not - # double-emitted here — and with the wrong, return-type-derived name. - if t in ("class_declaration", "interface_class_declaration"): - return - - if t == "module_declaration": - mod_name = _sv_first_identifier(_sv_child(node, "module_header"), source) - if mod_name: - line = node.start_point[0] + 1 - nid = _make_id(stem, mod_name) - add_node(nid, mod_name, line) - add_edge(file_nid, nid, "defines", line) - for child in node.children: - walk(child, nid) - return - - # `function_prototype` only appears inside class/interface-class bodies - # (skipped above) and nests its name differently; it is intentionally not - # handled here. - elif t == "function_declaration": - fn_body = _sv_child(node, "function_body_declaration") - func_name = _sv_first_identifier(_sv_child(fn_body, "function_identifier"), source) - if func_name: - line = node.start_point[0] + 1 - parent = module_nid or file_nid - nid = _make_id(parent, func_name) - add_node(nid, f"{func_name}()", line) - add_edge(parent, nid, "contains", line) - - elif t == "task_declaration": - tk_body = _sv_child(node, "task_body_declaration") - task_name = _sv_first_identifier(_sv_child(tk_body, "task_identifier"), source) - if task_name: - line = node.start_point[0] + 1 - parent = module_nid or file_nid - nid = _make_id(parent, task_name) - add_node(nid, task_name, line) - add_edge(parent, nid, "contains", line) + label_to_canonical: dict[str, list[str]] = {} + for n in all_nodes: + if n.get("id") in extension_nids: + continue + label = n.get("label") + if not label: + continue + label_to_canonical.setdefault(label, []).append(n["id"]) - elif t == "package_import_declaration": - for child in node.children: - if child.type == "package_import_item": - pkg_text = _read_text(child, source) - pkg_name = pkg_text.split("::")[0].strip() - if pkg_name: - line = node.start_point[0] + 1 - tgt_nid = _make_id(pkg_name) - add_node(tgt_nid, pkg_name, line) - src_nid = module_nid or file_nid - add_edge(src_nid, tgt_nid, "imports_from", line) - - elif t in ("module_instantiation", "checker_instantiation"): - # `leaf u_leaf();` parses as checker_instantiation in 1.0.3; - # module_instantiation (when it occurs) exposes a `module_type` field. - # Both reduce to the first identifier under the node — the instantiated - # type, not the instance name (which appears later). - if module_nid: - type_node = node.child_by_field_name("module_type") - inst_type = (_read_text(type_node, source).strip() if type_node - else _sv_first_identifier(node, source)) - if inst_type: - line = node.start_point[0] + 1 - tgt_nid = _make_id(inst_type) - add_node(tgt_nid, inst_type, line) - add_edge(module_nid, tgt_nid, "instantiates", line) + remap: dict[str, str] = {} + for ext_nid in extension_nids: + candidates = label_to_canonical.get(extension_labels[ext_nid], []) + if len(candidates) != 1: + continue + canonical_nid = candidates[0] + if canonical_nid != ext_nid: + remap[ext_nid] = canonical_nid - for child in node.children: - walk(child, module_nid) + if not remap: + return - walk(root) - _augment_systemverilog_semantics( - source.decode("utf-8", errors="replace"), - stem, - str_path, - file_nid, - nodes, - edges, - seen_ids, - ) - return {"nodes": nodes, "edges": edges} + all_nodes[:] = [n for n in all_nodes if n.get("id") not in remap] + # Each extension file's `contains` edge ends up pointing at the canonical + # type — multiple files containing the same node is the intended shape: + # the type owns the methods, the files own their slice. Self-loops are + # dropped (e.g. an in-file extension method whose call already pointed at + # the canonical type). + rewritten: list[dict] = [] + seen_keys: set[tuple] = set() + for e in all_edges: + src = remap.get(e.get("source"), e.get("source")) + tgt = remap.get(e.get("target"), e.get("target")) + if src == tgt: + continue + e["source"] = src + e["target"] = tgt + key = (src, tgt, e.get("relation"), e.get("source_file"), e.get("source_location")) + if key in seen_keys: + continue + seen_keys.add(key) + rewritten.append(e) + all_edges[:] = rewritten -def extract_sql(path: Path, content: str | bytes | None = None) -> dict: - """Extract tables, views, functions, and relationships from .sql files via tree-sitter.""" - try: - import tree_sitter_sql as tssql - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree_sitter_sql not installed. Run: pip install tree-sitter-sql"} - try: - language = Language(tssql.language()) - parser = Parser(language) - source = ( - content.encode("utf-8") if isinstance(content, str) - else content if content is not None - else path.read_bytes() - ) - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} +def _resolve_swift_member_calls( + per_file: list[dict], + all_nodes: list[dict], + all_edges: list[dict], +) -> None: + """Resolve cross-file Swift member calls (``recv.method()``) to the real + definition of the receiver's type (#1356). + The shared cross-file call pass drops every ``is_member_call`` because a bare + method name (``update``) collides across the corpus and inflates god-nodes + (#543/#1219). Swift extractors record the receiver of each member call and a + per-file ``name -> type`` table (``swift_type_table``); this pass uses them to + type the receiver, then emits an edge ONLY when that type name resolves to + exactly one definition. A type-qualified call (``Type.staticMethod()``) is + EXTRACTED (the type is named explicitly in source); an instance call typed via + local inference (``obj.method()``) is INFERRED. The shared-pass member-call drop + stays intact: this is purely additive and fires only on receiver-typed Swift calls. - stem = _file_stem(path) - str_path = str(path) - file_nid = _make_id(str_path) - nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code", - "source_file": str_path, "source_location": None}] - edges: list[dict] = [] - seen_ids: set[str] = {file_nid} - table_nids: dict[str, str] = {} # name → nid for reference resolution + Must run after id-disambiguation so node ids and caller_nids are final. + """ + type_table_by_file: dict[str, dict[str, str]] = {} + for result in per_file: + tt = result.get("swift_type_table") + if tt and tt.get("path"): + type_table_by_file[tt["path"]] = tt.get("table", {}) + if not type_table_by_file: + return - def _read(n) -> str: - return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace") + def _key(label: str) -> str: + return re.sub(r"[^a-zA-Z0-9]+", "", str(label)).lower() - def _obj_name(n) -> str | None: - for c in n.children: - if c.type == "object_reference": - return _read(c) - return None + # A genuine Swift type is the target of a `contains` edge from its file node. + # Bare type references create a same-label shadow node (via ensure_named_node) + # that carries a source_file but is NOT contained; excluding non-contained + # nodes keeps that shadow from making a real type name look ambiguous. + contained = {e.get("target") for e in all_edges if e.get("relation") == "contains"} - def _add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({"id": nid, "label": label, "file_type": "code", - "source_file": str_path, "source_location": f"L{line}"}) - edges.append({"source": file_nid, "target": nid, "relation": "contains", - "confidence": "EXTRACTED", "source_file": str_path, - "source_location": f"L{line}", "weight": 1.0}) - - def _add_edge(src: str, tgt: str, relation: str, line: int) -> None: - edges.append({"source": src, "target": tgt, "relation": relation, - "confidence": "EXTRACTED", "source_file": str_path, - "source_location": f"L{line}", "weight": 1.0}) - - def walk(node) -> None: - t = node.type - line = node.start_point[0] + 1 + # Type name -> definition node ids (real, source-backed, type-like defs only). + # len != 1 is the god-node guard: an ambiguous type name bails. + type_def_nids: dict[str, list[str]] = {} + node_by_id: dict[str, dict] = {} + for n in all_nodes: + node_by_id[n.get("id")] = n + if n.get("source_file") and n.get("id") in contained and _is_type_like_definition(n): + type_def_nids.setdefault(_key(n.get("label", "")), []).append(n["id"]) - if t == "create_table": - name = _obj_name(node) - if name: - nid = _make_id(stem, name) - _add_node(nid, name, line) - table_nids[name.lower()] = nid - # Foreign key REFERENCES - for col in node.children: - if col.type == "column_definitions": - has_error = any(cd.type == "ERROR" for cd in col.children) - seen_refs: set[str] = set() - for cd in col.children: - if cd.type == "column_definition": - # Inline column-level REFERENCES - ref_name: str | None = None - found_ref = False - for cc in cd.children: - if cc.type == "keyword_references": - found_ref = True - elif found_ref and cc.type == "object_reference": - ref_name = _read(cc) - break - if ref_name: - ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name) - _add_edge(nid, ref_nid, "references", line) - seen_refs.add(ref_name.lower()) - elif cd.type == "constraints": - # Table-level FOREIGN KEY ... REFERENCES ... constraints - for constraint in cd.children: - if constraint.type != "constraint": - continue - ref_name = None - found_ref = False - for cc in constraint.children: - if cc.type == "keyword_references": - found_ref = True - elif found_ref and cc.type == "object_reference": - ref_name = _read(cc) - break - if ref_name: - ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name) - _add_edge(nid, ref_nid, "references", line) - seen_refs.add(ref_name.lower()) - if has_error: - # Dialect-specific syntax (e.g. Firebird COMPUTED BY) causes ERROR - # nodes that make the parser drop the trailing constraints block. - # Regex-scan the raw column_definitions text as fallback. - col_text = _read(col) - for rm in re.finditer(r"\bREFERENCES\s+([\w$]+)", col_text, re.IGNORECASE): - ref_name = rm.group(1) - if ref_name.lower() not in seen_refs: - ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name) - _add_edge(nid, ref_nid, "references", line) - seen_refs.add(ref_name.lower()) - - elif t == "create_view": - name = _obj_name(node) - if name: - nid = _make_id(stem, name) - _add_node(nid, name, line) - table_nids[name.lower()] = nid - # FROM/JOIN table references inside view body - _walk_from_refs(node, nid, line) - - elif t == "create_function": - name = _obj_name(node) - if name: - nid = _make_id(stem, name) - _add_node(nid, f"{name}()", line) - _walk_from_refs(node, nid, line) - - elif t == "create_procedure": - name = _obj_name(node) - if name: - nid = _make_id(stem, name) - _add_node(nid, f"{name}()", line) - _walk_from_refs(node, nid, line) - - elif t == "alter_table": - name = _obj_name(node) - if name: - src_nid = table_nids.get(name.lower()) - if not src_nid: - src_nid = _make_id(stem, name) - _add_node(src_nid, name, line) - table_nids[name.lower()] = src_nid - for child in node.children: - if child.type == "add_constraint": - for cc in child.children: - if cc.type != "constraint": - continue - found_ref = False - ref_name: str | None = None - for ccc in cc.children: - if ccc.type == "keyword_references": - found_ref = True - elif found_ref and ccc.type == "object_reference": - ref_name = _read(ccc) - break - if ref_name: - ref_nid = table_nids.get(ref_name.lower()) - if not ref_nid: - ref_nid = _make_id(stem, ref_name) - _add_edge(src_nid, ref_nid, "references", line) - - elif t == "create_trigger": - trig_name: str | None = None - tbl_name: str | None = None - after_trigger = False - after_for = False - for c in node.children: - if c.type == "keyword_trigger": - after_trigger = True - elif after_trigger and not trig_name and c.type == "object_reference": - trig_name = _read(c) - elif c.type == "keyword_for": - after_for = True - elif after_for and not tbl_name and c.type == "object_reference": - tbl_name = _read(c) - if trig_name: - trig_nid = _make_id(stem, trig_name) - _add_node(trig_nid, trig_name, line) - if tbl_name: - tbl_nid = table_nids.get(tbl_name.lower()) or _make_id(stem, tbl_name) - _add_edge(trig_nid, tbl_nid, "triggers", line) - - elif t == "fb_proc_or_trigger": - text = _read(node) - m = re.match( - r"CREATE\s+(?:OR\s+(?:REPLACE|ALTER)\s+)?" - r"(PROCEDURE|TRIGGER|FUNCTION)\s+([\w$]+)", - text, re.IGNORECASE, - ) - if m: - obj_type = m.group(1).upper() - obj_name = m.group(2) - obj_nid = _make_id(stem, obj_name) - label = obj_name if obj_type == "TRIGGER" else f"{obj_name}()" - _add_node(obj_nid, label, line) - if obj_type == "TRIGGER": - fm = re.search(r"\bFOR\s+([\w$]+)", text, re.IGNORECASE) - if fm: - tbl = fm.group(1) - tbl_nid = table_nids.get(tbl.lower()) or _make_id(stem, tbl) - _add_edge(obj_nid, tbl_nid, "triggers", line) - _NON_TABLES = { - "select", "where", "set", "dual", "null", "true", "false", - "first", "skip", "rows", "next", "only", "lateral", - } - seen_tbls: set[str] = set() - for rm in re.finditer(r"\b(?:FROM|JOIN|INTO)\s+([\w$]+)", text, re.IGNORECASE): - tbl = rm.group(1) - if tbl.lower() not in _NON_TABLES and tbl.lower() not in seen_tbls: - seen_tbls.add(tbl.lower()) - tbl_nid = table_nids.get(tbl.lower()) or _make_id(stem, tbl) - _add_edge(obj_nid, tbl_nid, "reads_from", line) - for rm in re.finditer(r"\bUPDATE\s+([\w$]+)", text, re.IGNORECASE): - tbl = rm.group(1) - if tbl.lower() not in _NON_TABLES and tbl.lower() not in seen_tbls: - seen_tbls.add(tbl.lower()) - tbl_nid = table_nids.get(tbl.lower()) or _make_id(stem, tbl) - _add_edge(obj_nid, tbl_nid, "reads_from", line) + # (type_node_id, method_key) -> method_node_id, from `method` edges. + method_index: dict[tuple[str, str], str] = {} + for e in all_edges: + if e.get("relation") != "method": + continue + src, tgt = e.get("source"), e.get("target") + tnode = node_by_id.get(tgt) + if tnode is not None: + method_index[(src, _key(tnode.get("label", "")))] = tgt - for child in node.children: - walk(child) + all_raw_calls: list[dict] = [] + for result in per_file: + all_raw_calls.extend(result.get("raw_calls", [])) - def _walk_from_refs(node, caller_nid: str, line: int) -> None: - """Recursively find FROM/JOIN table references inside a node.""" - if node.type in ("from", "join"): - for c in node.children: - if c.type == "relation": - for cc in c.children: - if cc.type == "object_reference": - tbl = _read(cc) - tbl_nid = _make_id(stem, tbl) - _add_edge(caller_nid, tbl_nid, "reads_from", - c.start_point[0] + 1) - for child in node.children: - _walk_from_refs(child, caller_nid, line) - - for stmt in root.children: - if stmt.type == "statement": - for child in stmt.children: - walk(child) - elif stmt.type in ("fb_proc_or_trigger", "set_term", "declare_external_function"): - walk(stmt) - - # Global regex fallback: catch any REFERENCES missed due to ERROR nodes in the parse tree - # (e.g. Firebird COMPUTED BY columns push constraints out of the tree entirely). - # Snapshot after tree walk so we don't re-emit edges already captured above. - emitted = {(e["source"], e["target"]) for e in edges if e["relation"] == "references"} - src_text = source.decode("utf-8", errors="replace") - for m in re.finditer(r"CREATE\s+TABLE\s+([\w$]+)\s*\(", src_text, re.IGNORECASE): - tbl_name = m.group(1) - tbl_nid = table_nids.get(tbl_name.lower()) - if tbl_nid is None: + existing_pairs = {(e.get("source"), e.get("target")) for e in all_edges} + for rc in all_raw_calls: + if not rc.get("is_member_call"): continue - tbl_line = src_text[: m.start()].count("\n") + 1 - tail = src_text[m.start():] - end = re.search(r"(?:^|\n)(?:CREATE|SET\s+TERM|ALTER)\s", tail[1:], re.IGNORECASE) - block = tail[: end.start() + 1] if end else tail - for rm in re.finditer(r"\bREFERENCES\s+([\w$]+)", block, re.IGNORECASE): - ref_name = rm.group(1) - ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name) - if (tbl_nid, ref_nid) not in emitted: - _add_edge(tbl_nid, ref_nid, "references", tbl_line) - emitted.add((tbl_nid, ref_nid)) - - return {"nodes": nodes, "edges": edges} + receiver = rc.get("receiver") + callee = rc.get("callee") + if not receiver or not callee: + continue + # Determine the receiver's type. An upper-cased receiver is itself a type + # (Type.staticMethod(), Singleton.shared.x()); otherwise look it up in the + # declaring file's local type table. + if receiver[:1].isupper(): + type_name = receiver + type_qualified = True + else: + type_name = type_table_by_file.get(rc.get("source_file", ""), {}).get(receiver) + type_qualified = False + if not type_name: + continue + type_defs = type_def_nids.get(_key(type_name), []) + if len(type_defs) != 1: # ambiguous or absent -> bail (god-node guard) + continue + type_nid = type_defs[0] + caller = rc.get("caller_nid") + if not caller: + continue + method_nid = method_index.get((type_nid, _key(callee))) + target = method_nid or type_nid + relation = "calls" if method_nid else "references" + if target == caller or (caller, target) in existing_pairs: + continue + existing_pairs.add((caller, target)) + # A type-qualified call (`Type.staticMethod()`) names the receiver type + # explicitly in source, so it is an exact reference — EXTRACTED, matching + # the Python qualified-class-method pass (#1533). An instance call whose + # receiver type came from local inference (`obj.method()`) stays INFERRED. + all_edges.append({ + "source": caller, + "target": target, + "relation": relation, + "context": "call", + "confidence": "EXTRACTED" if type_qualified else "INFERRED", + "confidence_score": 1.0 if type_qualified else 0.8, + "source_file": rc.get("source_file", ""), + "source_location": rc.get("source_location"), + "weight": 1.0, + }) -def extract_lua(path: Path) -> dict: - """Extract functions, methods, require() imports, and calls from a .lua file.""" - return _extract_generic(path, _LUA_CONFIG) +def _resolve_python_member_calls( + per_file: list[dict], + all_nodes: list[dict], + all_edges: list[dict], +) -> None: + """Resolve cross-file Python qualified class-method calls (``ClassName.method()``) + to the class-qualified method node (#1446). + The shared cross-file call pass drops every ``is_member_call`` because a bare + method name (``log``) collides across the corpus and inflates god-nodes + (#543/#1219). That guard is right for *instance* calls (``obj.method()``) but + misses *class-qualified* calls (``ClassName.method()``), where the receiver is + an explicitly-named class — an exact, unambiguous reference. This pass uses the + receiver captured by the extractor, and when it is a capitalized name resolving + to exactly one class node that owns the called method, emits an EXTRACTED + ``calls`` edge. Purely additive (only member calls the shared pass skipped), + with a single-definition god-node guard. -def extract_swift(path: Path) -> dict: - """Extract classes, structs, protocols, functions, imports, and calls from a .swift file.""" - return _extract_generic(path, _SWIFT_CONFIG) + Must run after id-disambiguation so node ids and caller_nids are final. + """ + def _key(label: str) -> str: + return re.sub(r"[^a-zA-Z0-9]+", "", str(label)).lower() + node_by_id: dict[str, dict] = {n.get("id"): n for n in all_nodes} -# ── Julia extractor (custom walk) ──────────────────────────────────────────── + # A class owns methods: it is the source of one or more `method` edges. Index + # class label -> owning class node ids (len != 1 is the god-node guard), and + # (class_node_id, method_key) -> method_node_id. + class_def_nids: dict[str, list[str]] = {} + method_index: dict[tuple[str, str], str] = {} + for e in all_edges: + if e.get("relation") != "method": + continue + src, tgt = e.get("source"), e.get("target") + cnode = node_by_id.get(src) + if cnode is not None: + class_def_nids.setdefault(_key(cnode.get("label", "")), []).append(src) + tnode = node_by_id.get(tgt) + if tnode is not None: + method_index[(src, _key(tnode.get("label", "")))] = tgt + if not class_def_nids: + return + # A class with N methods produced N entries; collapse to a unique set. + for k in list(class_def_nids): + class_def_nids[k] = sorted(set(class_def_nids[k])) -def extract_julia(path: Path) -> dict: - """Extract modules, structs, functions, imports, and calls from a .jl file.""" - try: - import tree_sitter_julia as tsjulia - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree-sitter-julia not installed"} + all_raw_calls: list[dict] = [] + for result in per_file: + all_raw_calls.extend(result.get("raw_calls", [])) - try: - language = Language(tsjulia.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} - - stem = _file_stem(path) - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - function_bodies: list[tuple[str, object]] = [] - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": label, - "file_type": "code", - "source_file": str_path, - "source_location": f"L{line}", - }) - - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0, - context: str | None = None) -> None: - edge = { - "source": src, - "target": tgt, - "relation": relation, - "confidence": confidence, - "source_file": str_path, - "source_location": f"L{line}", - "weight": weight, - } - if context: - edge["context"] = context - edges.append(edge) - - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - def ensure_named_node(name: str, line: int) -> str: - nid = _make_id(stem, name) - if nid in seen_ids: - return nid - nid = _make_id(name) - if nid not in seen_ids: - # The name isn't defined in this file, so this is a cross-file reference - # (e.g. a `Thing` type annotation imported from another module). Emit a - # SOURCELESS stub — like the inheritance-base path below — so the - # corpus-level rewire can collapse it onto the real definition. A sourced - # stub here makes _disambiguate_colliding_node_ids bake the referencing - # file's path (with extension) into the id and blocks the rewire, which is - # the phantom-duplicate-node bug (#1402). - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": name, - "file_type": "code", - "source_file": "", - "source_location": "", - "origin_file": str_path, - }) - return nid - - def _func_name_from_signature(sig_node) -> str | None: - """Extract function name from a Julia signature node (call_expression > identifier).""" - for child in sig_node.children: - if child.type == "call_expression": - callee = child.children[0] if child.children else None - if callee and callee.type == "identifier": - return _read_text(callee, source) - return None - - def walk_calls(body_node, func_nid: str) -> None: - if body_node is None: - return - t = body_node.type - if t in ("function_definition", "short_function_definition"): - return - if t == "call_expression" and body_node.children: - callee = body_node.children[0] - # Direct call: foo(...) - if callee.type == "identifier": - callee_name = _read_text(callee, source) - target_nid = _make_id(stem, callee_name) - add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1, - confidence="EXTRACTED", context="call") - # Method call: obj.method(...) - elif callee.type == "field_expression" and len(callee.children) >= 3: - method_node = callee.children[-1] - method_name = _read_text(method_node, source) - target_nid = _make_id(stem, method_name) - add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1, - confidence="EXTRACTED", context="call") - for child in body_node.children: - walk_calls(child, func_nid) - - def walk(node, scope_nid: str) -> None: - t = node.type - - # Module - if t == "module_definition": - name_node = next((c for c in node.children if c.type == "identifier"), None) - if name_node: - mod_name = _read_text(name_node, source) - mod_nid = _make_id(stem, mod_name) - line = node.start_point[0] + 1 - add_node(mod_nid, mod_name, line) - add_edge(file_nid, mod_nid, "defines", line) - for child in node.children: - walk(child, mod_nid) - return - - # Struct (struct / mutable struct — both map to struct_definition in tree-sitter-julia) - if t == "struct_definition": - # type_head may contain: identifier (simple) or binary_expression (Foo <: Bar) - type_head = next((c for c in node.children if c.type == "type_head"), None) - if not type_head: - return - struct_name: str | None = None - super_name: str | None = None - bin_expr = next((c for c in type_head.children if c.type == "binary_expression"), None) - if bin_expr: - identifiers = [c for c in bin_expr.children if c.type == "identifier"] - if identifiers: - struct_name = _read_text(identifiers[0], source) - if len(identifiers) >= 2: - super_name = _read_text(identifiers[-1], source) - else: - name_node = next((c for c in type_head.children if c.type == "identifier"), None) - if name_node: - struct_name = _read_text(name_node, source) - if not struct_name: - return - struct_nid = _make_id(stem, struct_name) - line = node.start_point[0] + 1 - add_node(struct_nid, struct_name, line) - add_edge(scope_nid, struct_nid, "defines", line) - if super_name: - add_edge(struct_nid, ensure_named_node(super_name, line), - "inherits", line, confidence="EXTRACTED") - # Field types: each `name::Type` lowers to a typed_expression child of struct_definition - for child in node.children: - if child.type == "typed_expression": - type_ids = [c for c in child.children if c.type == "identifier"] - if len(type_ids) >= 2: - field_line = child.start_point[0] + 1 - type_name = _read_text(type_ids[-1], source) - type_nid = ensure_named_node(type_name, field_line) - edges.append(_semantic_reference_edge( - struct_nid, type_nid, "field", str_path, field_line)) - return - - # Abstract type - if t == "abstract_definition": - # type_head > identifier - type_head = next((c for c in node.children if c.type == "type_head"), None) - if type_head: - name_node = next((c for c in type_head.children if c.type == "identifier"), None) - if name_node: - abs_name = _read_text(name_node, source) - abs_nid = _make_id(stem, abs_name) - line = node.start_point[0] + 1 - add_node(abs_nid, abs_name, line) - add_edge(scope_nid, abs_nid, "defines", line) - return - - # Function: function foo(...) ... end - if t == "function_definition": - sig_node = next((c for c in node.children if c.type == "signature"), None) - if sig_node: - func_name = _func_name_from_signature(sig_node) - if func_name: - func_nid = _make_id(stem, func_name) - line = node.start_point[0] + 1 - add_node(func_nid, f"{func_name}()", line) - add_edge(scope_nid, func_nid, "defines", line) - function_bodies.append((func_nid, node)) - return - - # Short function: foo(x) = expr - if t == "assignment": - lhs = node.children[0] if node.children else None - if lhs and lhs.type == "call_expression" and lhs.children: - callee = lhs.children[0] - if callee.type == "identifier": - func_name = _read_text(callee, source) - func_nid = _make_id(stem, func_name) - line = node.start_point[0] + 1 - add_node(func_nid, f"{func_name}()", line) - add_edge(scope_nid, func_nid, "defines", line) - # Only walk the RHS (index 2 after lhs and operator) to avoid self-loops - rhs = node.children[-1] if len(node.children) >= 3 else None - if rhs: - function_bodies.append((func_nid, rhs)) - return - - # Using / Import - if t in ("using_statement", "import_statement"): - line = node.start_point[0] + 1 - - def _julia_mod_name(n): - # identifier (`Foo`), scoped_identifier (`Base.Threads`), or - # import_path (relative `..Sibling`) -> the module name. Only bare - # identifiers were handled, so qualified/relative imports — and the - # scoped package of a `selected_import` — were silently dropped. - if n.type == "import_path": - ids = [c for c in n.children if c.type == "identifier"] - return _read_text(ids[-1], source) if ids else None - if n.type in ("identifier", "scoped_identifier"): - return _read_text(n, source) - return None - - def _emit_import(name): - if not name: - return - imp_nid = _make_id(name) - add_node(imp_nid, name, line) - add_edge(scope_nid, imp_nid, "imports", line, context="import") - - for child in node.children: - if child.type in ("identifier", "scoped_identifier", "import_path"): - _emit_import(_julia_mod_name(child)) - elif child.type == "selected_import": - # `import Base.Threads: nthreads` — the package (first named - # child) may itself be a scoped_identifier/import_path. - pkg = next( - (c for c in child.children - if c.type in ("identifier", "scoped_identifier", "import_path")), - None, - ) - if pkg is not None: - _emit_import(_julia_mod_name(pkg)) - return - - for child in node.children: - walk(child, scope_nid) - - walk(root, file_nid) - - for func_nid, body_node in function_bodies: - # For function_definition nodes, walk children directly to avoid - # the boundary check returning early on the top-level node itself. - # Skip the "signature" child — it contains the function's own call_expression - # which would create a self-loop. - if body_node.type == "function_definition": - for child in body_node.children: - if child.type != "signature": - walk_calls(child, func_nid) - else: - walk_calls(body_node, func_nid) - - return {"nodes": nodes, "edges": edges} - - -_FORTRAN_CPP_EXTS = {".F", ".F90", ".F95", ".F03", ".F08"} - - -def _cpp_preprocess(path: Path) -> bytes: - """Run cpp -w -P on a capital-F Fortran file and return preprocessed bytes. - - Falls back to raw file bytes if cpp is not available. Capital-F extensions - conventionally require C preprocessor expansion (#ifdef MPI, #define REAL8, etc.) - before parsing. - - Security (F-007): we pass `-nostdinc` and `-I /dev/null` so a malicious - source file containing `#include "/home/victim/.ssh/id_rsa"` (or any other - include directive) cannot inline arbitrary host files into the output that - we then ship to an LLM. Without these flags `cpp` happily resolves any - relative or absolute include path it can read, which is a corpus-side - file-exfiltration vector. - """ - import shutil - import subprocess - if not shutil.which("cpp"): - return path.read_bytes() - try: - # Pass an absolute path so a corpus file named like "-I/etc/x.F90" cannot - # be parsed by cpp as an option (cpp does not accept a "--" end-of-options - # terminator). An absolute path always begins with "/". - result = subprocess.run( - ["cpp", "-w", "-P", "-nostdinc", "-I", "/dev/null", str(path.resolve())], - capture_output=True, - timeout=30, - ) - if result.returncode == 0 and result.stdout: - return result.stdout - except Exception: - pass - return path.read_bytes() - - -def extract_fortran(path: Path) -> dict: - """Extract programs, modules, subroutines, functions, use statements, and calls from Fortran files. - - Capital-F extensions (.F, .F90, etc.) are run through the C preprocessor before - parsing so #ifdef/#define macros are resolved. - """ - try: - import tree_sitter_fortran as tsfortran - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree-sitter-fortran not installed"} - - try: - language = Language(tsfortran.language()) - parser = Parser(language) - source = _cpp_preprocess(path) if path.suffix in _FORTRAN_CPP_EXTS else path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} - - stem = _file_stem(path) - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - scope_bodies: list[tuple[str, object]] = [] - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": label, - "file_type": "code", - "source_file": str_path, - "source_location": f"L{line}", - }) - - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0, - context: str | None = None) -> None: - edge = { - "source": src, - "target": tgt, - "relation": relation, - "confidence": confidence, - "source_file": str_path, - "source_location": f"L{line}", - "weight": weight, - } - if context: - edge["context"] = context - edges.append(edge) - - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - def _fortran_name(stmt_node) -> str | None: - """Extract name from a *_statement node. Fortran is case-insensitive; lowercase.""" - for child in stmt_node.children: - if child.type in ("name", "identifier"): - return _read_text(child, source).lower() - return None - - def ensure_named_node(name: str, line: int) -> str: - nid = _make_id(stem, name) - if nid in seen_ids: - return nid - nid = _make_id(name) - if nid not in seen_ids: - # The name isn't defined in this file, so this is a cross-file reference - # (e.g. a `Thing` type annotation imported from another module). Emit a - # SOURCELESS stub — like the inheritance-base path below — so the - # corpus-level rewire can collapse it onto the real definition. A sourced - # stub here makes _disambiguate_colliding_node_ids bake the referencing - # file's path (with extension) into the id and blocks the rewire, which is - # the phantom-duplicate-node bug (#1402). - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": name, - "file_type": "code", - "source_file": "", - "source_location": "", - "origin_file": str_path, - }) - return nid - - def emit_signature_refs(scope_node, fn_nid: str, is_function: bool) -> None: - """Emit references[parameter_type] / references[return_type] edges for - a subroutine/function based on its variable_declaration siblings.""" - stmt_type = "function_statement" if is_function else "subroutine_statement" - stmt = next((c for c in scope_node.children if c.type == stmt_type), None) - if stmt is None: - return - param_names: set[str] = set() - params_node = next((c for c in stmt.children if c.type == "parameters"), None) - if params_node is not None: - for c in params_node.children: - if c.type == "identifier": - param_names.add(_read_text(c, source).lower()) - result_name: str | None = None - if is_function: - result_node = next((c for c in stmt.children if c.type == "function_result"), None) - if result_node is not None: - res_id = next((c for c in result_node.children if c.type == "identifier"), None) - if res_id is not None: - result_name = _read_text(res_id, source).lower() - else: - # implicit result variable: same name as the function - result_name = _fortran_name(stmt) - for child in scope_node.children: - if child.type != "variable_declaration": - continue - derived = next((c for c in child.children if c.type == "derived_type"), None) - if derived is None: - continue - type_name_node = next((c for c in derived.children if c.type == "type_name"), None) - if type_name_node is None: - continue - type_name = _read_text(type_name_node, source).lower() - for var in child.children: - if var.type != "identifier": - continue - var_name = _read_text(var, source).lower() - var_line = var.start_point[0] + 1 - if var_name in param_names: - tgt = ensure_named_node(type_name, var_line) - if tgt != fn_nid: - add_edge(fn_nid, tgt, "references", var_line, context="parameter_type") - elif is_function and var_name == result_name: - tgt = ensure_named_node(type_name, var_line) - if tgt != fn_nid: - add_edge(fn_nid, tgt, "references", var_line, context="return_type") - - def walk_calls(node, scope_nid: str) -> None: - if node is None: - return - t = node.type - if t in ("subroutine", "function", "module", "program", "internal_procedures"): - return - # call FOO(args) — tree-sitter-fortran uses subroutine_call - if t == "subroutine_call": - name_node = next((c for c in node.children if c.type == "identifier"), None) - if name_node: - callee = _read_text(name_node, source).lower() - target_nid = _make_id(stem, callee) - add_edge(scope_nid, target_nid, "calls", node.start_point[0] + 1, - confidence="EXTRACTED", context="call") - # x = compute(args) — function invocations are `call_expression`, which - # shares Fortran's `name(...)` syntax with array indexing. Only emit a - # call edge when the callee resolves to a procedure defined in this file - # (an array variable produces no matching node), so array accesses can't - # fabricate spurious `calls` edges. - elif t == "call_expression": - name_node = next((c for c in node.children if c.type == "identifier"), None) - if name_node: - callee = _read_text(name_node, source).lower() - target_nid = _make_id(stem, callee) - if target_nid in seen_ids and target_nid != scope_nid: - add_edge(scope_nid, target_nid, "calls", node.start_point[0] + 1, - confidence="EXTRACTED", context="call") - for child in node.children: - walk_calls(child, scope_nid) - - def walk(node, scope_nid: str) -> None: - t = node.type - - if t == "program": - stmt = next((c for c in node.children if c.type == "program_statement"), None) - name = _fortran_name(stmt) if stmt else None - if name: - nid = _make_id(stem, name) - line = node.start_point[0] + 1 - add_node(nid, name, line) - add_edge(file_nid, nid, "defines", line) - scope_bodies.append((nid, node)) - for child in node.children: - walk(child, nid) - return - - if t == "module": - stmt = next((c for c in node.children if c.type == "module_statement"), None) - name = _fortran_name(stmt) if stmt else None - if name: - nid = _make_id(stem, name) - line = node.start_point[0] + 1 - add_node(nid, name, line) - add_edge(file_nid, nid, "defines", line) - for child in node.children: - walk(child, nid) - return - - # subroutines/functions inside a module live under internal_procedures - if t == "internal_procedures": - for child in node.children: - walk(child, scope_nid) - return - - if t == "derived_type_definition": - stmt = next((c for c in node.children if c.type == "derived_type_statement"), None) - if stmt is not None: - name_node = next((c for c in stmt.children if c.type == "type_name"), None) - if name_node is not None: - type_name = _read_text(name_node, source).lower() - type_nid = _make_id(stem, type_name) - line = node.start_point[0] + 1 - add_node(type_nid, type_name, line) - add_edge(scope_nid, type_nid, "defines", line) - return - - if t == "subroutine": - stmt = next((c for c in node.children if c.type == "subroutine_statement"), None) - name = _fortran_name(stmt) if stmt else None - if name: - nid = _make_id(stem, name) - line = node.start_point[0] + 1 - add_node(nid, f"{name}()", line) - add_edge(scope_nid, nid, "defines", line) - scope_bodies.append((nid, node)) - emit_signature_refs(node, nid, is_function=False) - for child in node.children: - walk(child, nid) - return - - if t == "function": - stmt = next((c for c in node.children if c.type == "function_statement"), None) - name = _fortran_name(stmt) if stmt else None - if name: - nid = _make_id(stem, name) - line = node.start_point[0] + 1 - add_node(nid, f"{name}()", line) - add_edge(scope_nid, nid, "defines", line) - scope_bodies.append((nid, node)) - emit_signature_refs(node, nid, is_function=True) - for child in node.children: - walk(child, nid) - return - - if t == "use_statement": - line = node.start_point[0] + 1 - # tree-sitter-fortran uses module_name node for the used module - name_node = next((c for c in node.children if c.type in ("module_name", "name", "identifier")), None) - if name_node: - mod_name = _read_text(name_node, source).lower() - imp_nid = _make_id(mod_name) - add_node(imp_nid, mod_name, line) - add_edge(scope_nid, imp_nid, "imports", line, context="use") - return - - for child in node.children: - walk(child, scope_nid) - - walk(root, file_nid) - - _stmt_headers = { - "subroutine_statement", "function_statement", - "program_statement", "module_statement", - } - for scope_nid, body_node in scope_bodies: - for child in body_node.children: - if child.type not in _stmt_headers: - walk_calls(child, scope_nid) - - return {"nodes": nodes, "edges": edges} - - -# ── Go extractor (custom walk) ──────────────────────────────────────────────── - -def extract_go(path: Path) -> dict: - """Extract functions, methods, type declarations, and imports from a .go file.""" - try: - import tree_sitter_go as tsgo - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree-sitter-go not installed"} - - try: - language = Language(tsgo.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} - - stem = _file_stem(path) - # Use directory name as package scope so methods on the same type across - # multiple files in a package share one canonical type node. - pkg_scope = path.parent.name or stem - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - function_bodies: list[tuple[str, object]] = [] - go_imported_pkgs: set[str] = set() # local names of imported packages - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": label, - "file_type": "code", - "source_file": str_path, - "source_location": f"L{line}", - }) - - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0, - context: str | None = None) -> None: - edge = { - "source": src, - "target": tgt, - "relation": relation, - "confidence": confidence, - "source_file": str_path, - "source_location": f"L{line}", - "weight": weight, - } - if context: - edge["context"] = context - edges.append(edge) - - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - def ensure_named_node(name: str, line: int) -> str: - nid = _make_id(pkg_scope, name) - if nid in seen_ids: - return nid - nid = _make_id(name) - if nid not in seen_ids: - # The name isn't declared in this file, so this is a cross-file reference - # (e.g. a type defined in another file of the package). Emit a SOURCELESS - # stub — like the inheritance-base path in the other extractors — so the - # corpus-level rewire can collapse it onto the real definition. A sourced - # stub here makes _disambiguate_colliding_node_ids bake the referencing - # file's path (with extension) into the id and blocks the rewire, which is - # the phantom-duplicate-node bug (#1402). - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": name, - "file_type": "code", - "source_file": "", - "source_location": "", - "origin_file": str_path, - }) - return nid - - def emit_go_method_refs(func_node, func_nid: str, line: int) -> None: - params = func_node.child_by_field_name("parameters") - if params is not None: - for p in params.children: - if p.type != "parameter_declaration": - continue - type_node = p.child_by_field_name("type") - refs: list[tuple[str, str]] = [] - _go_collect_type_refs(type_node, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "parameter_type" - tgt = ensure_named_node(ref_name, line) - if tgt != func_nid: - add_edge(func_nid, tgt, "references", line, context=ctx) - result = func_node.child_by_field_name("result") - if result is not None: - if result.type == "parameter_list": - for p in result.children: - if p.type != "parameter_declaration": - continue - type_node = p.child_by_field_name("type") - if type_node is None: - for c in p.children: - if c.is_named: - type_node = c - break - refs = [] - _go_collect_type_refs(type_node, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "return_type" - tgt = ensure_named_node(ref_name, line) - if tgt != func_nid: - add_edge(func_nid, tgt, "references", line, context=ctx) - else: - refs = [] - _go_collect_type_refs(result, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "return_type" - tgt = ensure_named_node(ref_name, line) - if tgt != func_nid: - add_edge(func_nid, tgt, "references", line, context=ctx) - - def walk(node) -> None: - t = node.type - - if t == "function_declaration": - name_node = node.child_by_field_name("name") - if name_node: - func_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - func_nid = _make_id(stem, func_name) - add_node(func_nid, f"{func_name}()", line) - add_edge(file_nid, func_nid, "contains", line) - emit_go_method_refs(node, func_nid, line) - body = node.child_by_field_name("body") - if body: - function_bodies.append((func_nid, body)) - return - - if t == "method_declaration": - receiver = node.child_by_field_name("receiver") - receiver_type: str | None = None - if receiver: - for param in receiver.children: - if param.type == "parameter_declaration": - type_node = param.child_by_field_name("type") - if type_node: - receiver_type = _read_text(type_node, source).lstrip("*").strip() - break - name_node = node.child_by_field_name("name") - if not name_node: - return - method_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - - if receiver_type: - parent_nid = _make_id(pkg_scope, receiver_type) - add_node(parent_nid, receiver_type, line) - method_nid = _make_id(parent_nid, method_name) - add_node(method_nid, f".{method_name}()", line) - add_edge(parent_nid, method_nid, "method", line) - else: - method_nid = _make_id(stem, method_name) - add_node(method_nid, f"{method_name}()", line) - add_edge(file_nid, method_nid, "contains", line) - - emit_go_method_refs(node, method_nid, line) - body = node.child_by_field_name("body") - if body: - function_bodies.append((method_nid, body)) - return - - if t == "type_declaration": - for child in node.children: - if child.type != "type_spec": - continue - name_node = child.child_by_field_name("name") - if not name_node: - continue - type_name = _read_text(name_node, source) - line = child.start_point[0] + 1 - type_nid = _make_id(pkg_scope, type_name) - add_node(type_nid, type_name, line) - add_edge(file_nid, type_nid, "contains", line) - # Type body: struct fields (with embeds) or interface embedding. - type_body = None - for tc in child.children: - if tc.type in ("struct_type", "interface_type"): - type_body = tc - break - if type_body is None: - continue - if type_body.type == "struct_type": - for fdl in type_body.children: - if fdl.type != "field_declaration_list": - continue - for field in fdl.children: - if field.type != "field_declaration": - continue - has_name = any( - fc.type == "field_identifier" for fc in field.children - ) - type_node = field.child_by_field_name("type") - if type_node is None: - for fc in field.children: - if fc.is_named and fc.type != "field_identifier": - type_node = fc - break - refs: list[tuple[str, str]] = [] - _go_collect_type_refs(type_node, source, False, refs) - for ref_name, role in refs: - tgt = ensure_named_node(ref_name, field.start_point[0] + 1) - if tgt == type_nid: - continue - if not has_name and role == "type": - add_edge(type_nid, tgt, "embeds", - field.start_point[0] + 1) - else: - ctx = "generic_arg" if role == "generic_arg" else "field" - add_edge(type_nid, tgt, "references", - field.start_point[0] + 1, context=ctx) - elif type_body.type == "interface_type": - for elem in type_body.children: - if elem.type != "type_elem": - continue - refs = [] - for sub in elem.children: - if sub.is_named: - _go_collect_type_refs(sub, source, False, refs) - for ref_name, role in refs: - tgt = ensure_named_node(ref_name, elem.start_point[0] + 1) - if tgt == type_nid: - continue - if role == "type": - add_edge(type_nid, tgt, "embeds", - elem.start_point[0] + 1) - else: - add_edge(type_nid, tgt, "references", - elem.start_point[0] + 1, context="generic_arg") - return - - if t == "import_declaration": - for child in node.children: - if child.type == "import_spec_list": - for spec in child.children: - if spec.type == "import_spec": - path_node = spec.child_by_field_name("path") - if path_node: - raw = _read_text(path_node, source).strip('"') - # Prefix with go_pkg_ so stdlib names (e.g. "context") - # don't collide with local files of the same basename. - tgt_nid = _make_id("go", "pkg", raw) - add_edge(file_nid, tgt_nid, "imports_from", spec.start_point[0] + 1, context="import") - # Track local name (alias or last path segment) - alias = spec.child_by_field_name("name") - local_name = _read_text(alias, source) if alias else raw.split("/")[-1] - if local_name and local_name != "_" and local_name != ".": - go_imported_pkgs.add(local_name) - elif child.type == "import_spec": - path_node = child.child_by_field_name("path") - if path_node: - raw = _read_text(path_node, source).strip('"') - tgt_nid = _make_id("go", "pkg", raw) - add_edge(file_nid, tgt_nid, "imports_from", child.start_point[0] + 1, context="import") - alias = child.child_by_field_name("name") - local_name = _read_text(alias, source) if alias else raw.split("/")[-1] - if local_name and local_name != "_" and local_name != ".": - go_imported_pkgs.add(local_name) - return - - for child in node.children: - walk(child) - - walk(root) - - label_to_nid: dict[str, str] = {} - for n in nodes: - raw = n["label"] - normalised = raw.strip("()").lstrip(".") - label_to_nid[normalised] = n["id"] - - seen_call_pairs: set[tuple[str, str]] = set() - raw_calls: list[dict] = [] - - def walk_calls(node, caller_nid: str) -> None: - if node.type in ("function_declaration", "method_declaration"): - return - if node.type == "call_expression": - func_node = node.child_by_field_name("function") - callee_name: str | None = None - is_member_call: bool = False - if func_node: - if func_node.type == "identifier": - callee_name = _read_text(func_node, source) - elif func_node.type == "selector_expression": - field = func_node.child_by_field_name("field") - operand = func_node.child_by_field_name("operand") - receiver_name = _read_text(operand, source) if operand else "" - # Package-qualified call (e.g. fmt.Println) → allow cross-file resolution. - # Receiver method call (e.g. s.logger.Log) → skip, no import evidence. - is_member_call = receiver_name not in go_imported_pkgs - if field: - callee_name = _read_text(field, source) - if callee_name and callee_name not in _LANGUAGE_BUILTIN_GLOBALS: - tgt_nid = label_to_nid.get(callee_name) - if tgt_nid and tgt_nid != caller_nid: - pair = (caller_nid, tgt_nid) - if pair not in seen_call_pairs: - seen_call_pairs.add(pair) - line = node.start_point[0] + 1 - edges.append({ - "source": caller_nid, - "target": tgt_nid, - "relation": "calls", - "context": "call", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - }) - elif callee_name: - raw_calls.append({ - "caller_nid": caller_nid, - "callee": callee_name, - "is_member_call": is_member_call, - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - }) - for child in node.children: - walk_calls(child, caller_nid) - - for caller_nid, body_node in function_bodies: - walk_calls(body_node, caller_nid) - - valid_ids = seen_ids - clean_edges = [] - for edge in edges: - src, tgt = edge["source"], edge["target"] - if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from")): - clean_edges.append(edge) - - return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls} - - -# ── Rust extractor (custom walk) ────────────────────────────────────────────── - -# Common Rust trait/stdlib method names that appear in virtually every codebase. -# Resolving these cross-file produces spurious INFERRED edges across crate -# boundaries (issue #908) — skip them from the unresolved-call queue entirely. -_RUST_TRAIT_METHOD_BLOCKLIST: frozenset[str] = frozenset({ - "new", "default", "parse", "from_str", "now", "clone", "into", "from", - "to_string", "to_owned", "len", "is_empty", "iter", "next", "build", - "start", "run", "init", "app", "get", "set", "push", "pop", "insert", - "remove", "contains", "collect", "map", "filter", "unwrap", "expect", - "ok", "err", "some", "none", "send", "recv", "lock", "read", "write", -}) - -def extract_rust(path: Path) -> dict: - """Extract functions, structs, enums, traits, impl methods, and use declarations from a .rs file.""" - try: - import tree_sitter_rust as tsrust - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree-sitter-rust not installed"} - - try: - language = Language(tsrust.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} - - stem = _file_stem(path) - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - function_bodies: list[tuple[str, object]] = [] - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": label, - "file_type": "code", - "source_file": str_path, - "source_location": f"L{line}", - }) - - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0, - context: str | None = None) -> None: - edge = { - "source": src, - "target": tgt, - "relation": relation, - "confidence": confidence, - "source_file": str_path, - "source_location": f"L{line}", - "weight": weight, - } - if context: - edge["context"] = context - edges.append(edge) - - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - def ensure_named_node(name: str, line: int) -> str: - nid = _make_id(stem, name) - if nid in seen_ids: - return nid - nid = _make_id(name) - if nid not in seen_ids: - # The name isn't defined in this file, so this is a cross-file reference - # (e.g. a `Thing` type annotation imported from another module). Emit a - # SOURCELESS stub — like the inheritance-base path below — so the - # corpus-level rewire can collapse it onto the real definition. A sourced - # stub here makes _disambiguate_colliding_node_ids bake the referencing - # file's path (with extension) into the id and blocks the rewire, which is - # the phantom-duplicate-node bug (#1402). - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": name, - "file_type": "code", - "source_file": "", - "source_location": "", - "origin_file": str_path, - }) - return nid - - def emit_param_return_refs(func_node, func_nid: str, line: int) -> None: - params = func_node.child_by_field_name("parameters") - if params is not None: - for p in params.children: - if p.type != "parameter": - continue - type_node = p.child_by_field_name("type") - refs: list[tuple[str, str]] = [] - _rust_collect_type_refs(type_node, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "parameter_type" - tgt = ensure_named_node(ref_name, line) - if tgt != func_nid: - add_edge(func_nid, tgt, "references", line, context=ctx) - return_type = func_node.child_by_field_name("return_type") - if return_type is not None: - refs = [] - _rust_collect_type_refs(return_type, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "return_type" - tgt = ensure_named_node(ref_name, line) - if tgt != func_nid: - add_edge(func_nid, tgt, "references", line, context=ctx) - - def walk(node, parent_impl_nid: str | None = None) -> None: - t = node.type - - if t == "function_item": - name_node = node.child_by_field_name("name") - if name_node: - func_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - if parent_impl_nid: - func_nid = _make_id(parent_impl_nid, func_name) - add_node(func_nid, f".{func_name}()", line) - add_edge(parent_impl_nid, func_nid, "method", line) - else: - func_nid = _make_id(stem, func_name) - add_node(func_nid, f"{func_name}()", line) - add_edge(file_nid, func_nid, "contains", line) - emit_param_return_refs(node, func_nid, line) - body = node.child_by_field_name("body") - if body: - function_bodies.append((func_nid, body)) - return - - if t in ("struct_item", "enum_item", "trait_item"): - name_node = node.child_by_field_name("name") - if name_node: - item_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - item_nid = _make_id(stem, item_name) - add_node(item_nid, item_name, line) - add_edge(file_nid, item_nid, "contains", line) - if t == "trait_item": - for c in node.children: - if c.type != "trait_bounds": - continue - for sub in c.children: - if not sub.is_named: - continue - refs: list[tuple[str, str]] = [] - _rust_collect_type_refs(sub, source, False, refs) - for idx, (ref_name, _role) in enumerate(refs): - tgt = ensure_named_node(ref_name, line) - if tgt == item_nid: - continue - rel = "inherits" if idx == 0 else "references" - if rel == "inherits": - add_edge(item_nid, tgt, "inherits", line) - else: - add_edge(item_nid, tgt, "references", line, - context="generic_arg") - if t == "struct_item": - for c in node.children: - if c.type != "field_declaration_list": - continue - for field in c.children: - if field.type != "field_declaration": - continue - type_node = field.child_by_field_name("type") - if type_node is None: - for fc in field.children: - if fc.type in ("type_identifier", "generic_type", - "scoped_type_identifier", - "reference_type", "primitive_type"): - type_node = fc - break - refs = [] - _rust_collect_type_refs(type_node, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "field" - tgt = ensure_named_node(ref_name, field.start_point[0] + 1) - if tgt != item_nid: - add_edge(item_nid, tgt, "references", - field.start_point[0] + 1, context=ctx) - # Tuple structs (`struct Wrapper(pub Logger, Config);`) nest their - # positional field types directly under ordered_field_declaration_list - # with no field_declaration wrapper -- the same shape handled for tuple - # enum variants below. Without this branch these field type references - # are silently dropped. - for c in node.children: - if c.type != "ordered_field_declaration_list": - continue - fline = c.start_point[0] + 1 - for tc in c.children: - if tc.type not in ("type_identifier", "generic_type", - "scoped_type_identifier", "reference_type", - "primitive_type", "tuple_type", "array_type"): - continue - refs = [] - _rust_collect_type_refs(tc, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "field" - tgt = ensure_named_node(ref_name, fline) - if tgt != item_nid: - add_edge(item_nid, tgt, "references", fline, context=ctx) - if t == "enum_item": - # Variant payload types nest under enum_variant_list -> - # enum_variant -> ordered_field_declaration_list (tuple variant, - # `Click(Logger)`) | field_declaration_list (struct variant, - # `Resize { size: Dim }`). Neither was traversed, so every - # enum-variant type reference was silently dropped. - _TYPE_NODES = ("type_identifier", "generic_type", - "scoped_type_identifier", "reference_type", - "primitive_type", "tuple_type", "array_type") - - def _emit_enum_type(type_node, at_line): - if type_node is None: - return - refs2: list[tuple[str, str]] = [] - _rust_collect_type_refs(type_node, source, False, refs2) - for ref_name, role in refs2: - ctx = "generic_arg" if role == "generic_arg" else "field" - tgt = ensure_named_node(ref_name, at_line) - if tgt != item_nid: - add_edge(item_nid, tgt, "references", at_line, context=ctx) - - for c in node.children: - if c.type != "enum_variant_list": - continue - for variant in c.children: - if variant.type != "enum_variant": - continue - vline = variant.start_point[0] + 1 - for vc in variant.children: - if vc.type == "ordered_field_declaration_list": - for tc in vc.children: - if tc.type in _TYPE_NODES: - _emit_enum_type(tc, vline) - elif vc.type == "field_declaration_list": - for field in vc.children: - if field.type != "field_declaration": - continue - type_node = field.child_by_field_name("type") - _emit_enum_type(type_node, field.start_point[0] + 1) - return - - if t == "impl_item": - type_node = node.child_by_field_name("type") - trait_node = node.child_by_field_name("trait") - impl_nid: str | None = None - if type_node: - type_name = _read_text(type_node, source).strip() - impl_nid = _make_id(stem, type_name) - add_node(impl_nid, type_name, node.start_point[0] + 1) - if trait_node is not None and impl_nid is not None: - refs: list[tuple[str, str]] = [] - _rust_collect_type_refs(trait_node, source, False, refs) - for idx, (ref_name, _role) in enumerate(refs): - tgt = ensure_named_node(ref_name, node.start_point[0] + 1) - if tgt == impl_nid: - continue - if idx == 0: - add_edge(impl_nid, tgt, "implements", node.start_point[0] + 1) - else: - add_edge(impl_nid, tgt, "references", node.start_point[0] + 1, - context="generic_arg") - body = node.child_by_field_name("body") - if body: - for child in body.children: - walk(child, parent_impl_nid=impl_nid) - return - - if t == "use_declaration": - arg = node.child_by_field_name("argument") - if arg: - raw = _read_text(arg, source) - clean = raw.split("{")[0].rstrip(":").rstrip("*").rstrip(":") - module_name = clean.split("::")[-1].strip() - if module_name: - tgt_nid = _make_id(module_name) - add_edge(file_nid, tgt_nid, "imports_from", node.start_point[0] + 1, context="import") - return - - for child in node.children: - walk(child, parent_impl_nid=None) - - walk(root) - - label_to_nid: dict[str, str] = {} - for n in nodes: - raw = n["label"] - normalised = raw.strip("()").lstrip(".") - label_to_nid[normalised] = n["id"] - - seen_call_pairs: set[tuple[str, str]] = set() - raw_calls: list[dict] = [] - - def walk_calls(node, caller_nid: str) -> None: - if node.type == "function_item": - return - if node.type == "call_expression": - func_node = node.child_by_field_name("function") - callee_name: str | None = None - is_member_call: bool = False - is_scoped_call: bool = False - if func_node: - if func_node.type == "identifier": - callee_name = _read_text(func_node, source) - elif func_node.type == "field_expression": - is_member_call = True - field = func_node.child_by_field_name("field") - if field: - callee_name = _read_text(field, source) - elif func_node.type == "scoped_identifier": - # Type::method() — still allow in-file EXTRACTED match, but - # skip cross-file resolution: bare last-segment lookup ignores - # crate boundaries and produces spurious INFERRED edges (#908). - is_scoped_call = True - name = func_node.child_by_field_name("name") - if name: - callee_name = _read_text(name, source) - if callee_name and callee_name not in _LANGUAGE_BUILTIN_GLOBALS: - tgt_nid = label_to_nid.get(callee_name) - if tgt_nid and tgt_nid != caller_nid: - pair = (caller_nid, tgt_nid) - if pair not in seen_call_pairs: - seen_call_pairs.add(pair) - line = node.start_point[0] + 1 - edges.append({ - "source": caller_nid, - "target": tgt_nid, - "relation": "calls", - "context": "call", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - }) - elif not is_scoped_call and callee_name.lower() not in _RUST_TRAIT_METHOD_BLOCKLIST: - raw_calls.append({ - "caller_nid": caller_nid, - "callee": callee_name, - "is_member_call": is_member_call, - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - }) - for child in node.children: - walk_calls(child, caller_nid) - - for caller_nid, body_node in function_bodies: - walk_calls(body_node, caller_nid) - - valid_ids = seen_ids - clean_edges = [] - for edge in edges: - src, tgt = edge["source"], edge["target"] - if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from")): - clean_edges.append(edge) - - return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls} - - -# ── Zig ─────────────────────────────────────────────────────────────────────── - - - -# ── PowerShell ──────────────────────────────────────────────────────────────── - -def extract_powershell(path: Path) -> dict: - """Extract functions, classes, methods, and using statements from a .ps1 file.""" - try: - import tree_sitter_powershell as tsps - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree_sitter_powershell not installed"} - - try: - language = Language(tsps.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} - - stem = _file_stem(path) - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - function_bodies: list[tuple[str, Any]] = [] - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({"id": nid, "label": label, "file_type": "code", - "source_file": str_path, "source_location": f"L{line}"}) - - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0, - context: str | None = None) -> None: - edge = {"source": src, "target": tgt, "relation": relation, - "confidence": confidence, "source_file": str_path, - "source_location": f"L{line}", "weight": weight} - if context: - edge["context"] = context - edges.append(edge) - - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - _PS_SKIP = frozenset({ - "using", "return", "if", "else", "elseif", "foreach", "for", - "while", "do", "switch", "try", "catch", "finally", "throw", - "break", "continue", "exit", "param", "begin", "process", "end", - # Import commands — handled as import edges, not function calls - "import-module", - }) - - def _find_script_block_body(node): - for child in node.children: - if child.type == "script_block": - for sc in child.children: - if sc.type == "script_block_body": - return sc - return child - return None - - def ensure_named_node(name: str, line: int) -> str: - nid = _make_id(stem, name) - if nid in seen_ids: - return nid - nid = _make_id(name) - if nid not in seen_ids: - # The name isn't defined in this file, so this is a cross-file reference - # (e.g. a `Thing` type annotation imported from another module). Emit a - # SOURCELESS stub — like the inheritance-base path below — so the - # corpus-level rewire can collapse it onto the real definition. A sourced - # stub here makes _disambiguate_colliding_node_ids bake the referencing - # file's path (with extension) into the id and blocks the rewire, which is - # the phantom-duplicate-node bug (#1402). - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": name, - "file_type": "code", - "source_file": "", - "source_location": "", - "origin_file": str_path, - }) - return nid - - def _ps_type_name(type_literal_node) -> str | None: - """Drill into a type_literal node and return the inner type_identifier text.""" - if type_literal_node is None: - return None - for spec in type_literal_node.children: - if spec.type != "type_spec": - continue - for tname in spec.children: - if tname.type != "type_name": - continue - for tid in tname.children: - if tid.type == "type_identifier": - return _read_text(tid, source) - return None - - def walk(node, parent_class_nid: str | None = None) -> None: - t = node.type - - if t == "function_statement": - name_node = next((c for c in node.children if c.type == "function_name"), None) - if name_node: - func_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - func_nid = _make_id(stem, func_name) - add_node(func_nid, f"{func_name}()", line) - add_edge(file_nid, func_nid, "contains", line) - body = _find_script_block_body(node) - if body: - function_bodies.append((func_nid, body)) - # Also walk the body during the main pass so that - # Import-Module / dot-source inside functions emit - # file-level imports_from edges (#1331). - walk(body, parent_class_nid) - return - - if t == "class_statement": - name_node = next((c for c in node.children if c.type == "simple_name"), None) - if name_node: - class_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - class_nid = _make_id(stem, class_name) - add_node(class_nid, class_name, line) - add_edge(file_nid, class_nid, "contains", line) - # Base type(s) after ':'. PowerShell has no syntactic base vs - # interface split, so (matching the C# convention) treat the - # first base as the superclass (inherits) and the rest as - # interfaces (implements). Bases are the simple_name children - # after the ':' token. - colon_seen = False - base_index = 0 - for child in node.children: - if child.type == ":": - colon_seen = True - elif colon_seen and child.type == "simple_name": - base_nid = ensure_named_node(_read_text(child, source), line) - if base_nid != class_nid: - rel = "inherits" if base_index == 0 else "implements" - add_edge(class_nid, base_nid, rel, line) - base_index += 1 - for child in node.children: - walk(child, parent_class_nid=class_nid) - return - - if t == "class_property_definition" and parent_class_nid: - type_literal = next((c for c in node.children if c.type == "type_literal"), None) - type_name = _ps_type_name(type_literal) - if type_name: - line = node.start_point[0] + 1 - target_nid = ensure_named_node(type_name, line) - if target_nid != parent_class_nid: - add_edge(parent_class_nid, target_nid, "references", - line, context="field") - return - - if t == "class_method_definition": - name_node = next((c for c in node.children if c.type == "simple_name"), None) - if name_node: - method_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - if parent_class_nid: - method_nid = _make_id(parent_class_nid, method_name) - add_node(method_nid, f".{method_name}()", line) - add_edge(parent_class_nid, method_nid, "method", line) - else: - method_nid = _make_id(stem, method_name) - add_node(method_nid, f"{method_name}()", line) - add_edge(file_nid, method_nid, "contains", line) - # Return type: type_literal sibling of simple_name - return_type_literal = next( - (c for c in node.children if c.type == "type_literal"), None) - return_type_name = _ps_type_name(return_type_literal) - if return_type_name: - target_nid = ensure_named_node(return_type_name, line) - if target_nid != method_nid: - add_edge(method_nid, target_nid, "references", - line, context="return_type") - # Parameter types: class_method_parameter_list - param_list = next( - (c for c in node.children if c.type == "class_method_parameter_list"), None) - if param_list is not None: - for p in param_list.children: - if p.type != "class_method_parameter": - continue - ptype_literal = next( - (c for c in p.children if c.type == "type_literal"), None) - ptype_name = _ps_type_name(ptype_literal) - if not ptype_name: - continue - p_line = p.start_point[0] + 1 - target_nid = ensure_named_node(ptype_name, p_line) - if target_nid != method_nid: - add_edge(method_nid, target_nid, "references", - p_line, context="parameter_type") - body = _find_script_block_body(node) - if body: - function_bodies.append((method_nid, body)) - return - - if t == "command": - # Dot-sourcing: `. ./Shared.psm1` - # Uses command_invokation_operator '.' + command_name_expr (not command_name) - invoke_op = next( - (c for c in node.children if c.type == "command_invokation_operator"), None - ) - if invoke_op is not None and _read_text(invoke_op, source).strip() == ".": - name_expr = next( - (c for c in node.children if c.type == "command_name_expr"), None - ) - if name_expr is not None: - name_node = next( - (c for c in name_expr.children if c.type == "command_name"), None - ) - if name_node: - raw_path = _read_text(name_node, source) - # Strip relative path prefix (./ or .\ or just the dot) - module_stem = re.sub(r'^[./\\]+', '', raw_path) - # Drop extension to get bare module name - module_stem = re.sub(r'\.[^.]+$', '', module_stem).replace('\\', '/') - module_name = module_stem.split('/')[-1] - if module_name: - add_edge(file_nid, _make_id(module_name), "imports_from", - node.start_point[0] + 1) - return - - cmd_name_node = next((c for c in node.children if c.type == "command_name"), None) - if cmd_name_node: - cmd_text = _read_text(cmd_name_node, source).lower() - if cmd_text == "using": - tokens = [] - for child in node.children: - if child.type == "command_elements": - for el in child.children: - if el.type == "generic_token": - tokens.append(_read_text(el, source)) - module_tokens = [t for t in tokens - if t.lower() not in ("namespace", "module", "assembly")] - if module_tokens: - module_name = module_tokens[-1].split(".")[-1] - add_edge(file_nid, _make_id(module_name), "imports_from", - node.start_point[0] + 1) - elif cmd_text == "import-module": - # Collect generic_token args; skip command_parameter flags like -Name - # The module name is the first generic_token (or the one after -Name) - module_name: str | None = None - expect_name = False - for child in node.children: - if child.type != "command_elements": - continue - for el in child.children: - if el.type == "command_parameter": - param_text = _read_text(el, source).lstrip("-").lower() - expect_name = param_text in ("name", "n") - elif el.type == "generic_token": - token = _read_text(el, source) - if module_name is None or expect_name: - module_name = token - expect_name = False - if module_name: - # Strip extension; keep only the stem for the node ID - bare = re.sub(r'\.[^.]+$', '', module_name).split('/')[-1].split('\\')[-1] - if bare: - add_edge(file_nid, _make_id(bare), "imports_from", - node.start_point[0] + 1) - return - - for child in node.children: - walk(child, parent_class_nid) - - walk(root) - - label_to_nid = {n["label"].strip("()").lstrip(".").lower(): n["id"] for n in nodes} - seen_call_pairs: set[tuple[str, str]] = set() - raw_calls: list[dict] = [] - - def walk_calls(node, caller_nid: str) -> None: - if node.type in ("function_statement", "class_statement"): - return - if node.type == "command": - cmd_name_node = next((c for c in node.children if c.type == "command_name"), None) - if cmd_name_node: - cmd_text = _read_text(cmd_name_node, source) - if cmd_text.lower() not in _PS_SKIP: - tgt_nid = label_to_nid.get(cmd_text.lower()) - if tgt_nid and tgt_nid != caller_nid: - pair = (caller_nid, tgt_nid) - if pair not in seen_call_pairs: - seen_call_pairs.add(pair) - add_edge(caller_nid, tgt_nid, "calls", - node.start_point[0] + 1, - confidence="EXTRACTED", weight=1.0) - elif cmd_text: - raw_calls.append({ - "caller_nid": caller_nid, - "callee": cmd_text, - "is_member_call": False, - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - }) - for child in node.children: - walk_calls(child, caller_nid) - - for caller_nid, body_node in function_bodies: - walk_calls(body_node, caller_nid) - - clean_edges = [e for e in edges if e["source"] in seen_ids and - (e["target"] in seen_ids or e["relation"] in ("imports_from", "imports"))] - return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls} - - -# ── PowerShell manifest (.psd1) ────────────────────────────────────────────── - -# Keys in a .psd1 whose values are module names/paths we treat as imports. -_PSD1_IMPORT_KEYS = frozenset({"RootModule", "NestedModules", "RequiredModules"}) - - -def _psd1_collect_string_literals(node, source: bytes) -> list[str]: - """Recursively collect all string_literal text values under *node*.""" - results: list[str] = [] - - def _walk(n) -> None: - if n.type == "string_literal": - raw = source[n.start_byte:n.end_byte].decode(errors="replace") - # Strip surrounding quote chars (' or ") - results.append(raw.strip("'\"")) - return - for child in n.children: - _walk(child) - - _walk(node) - return results - - -def _psd1_module_name(raw: str) -> str: - """Derive a bare module name from a raw string value. - - e.g. 'MyModule.psm1' → 'MyModule', './sub/Util.psm1' → 'Util', 'PSReadLine' → 'PSReadLine' - """ - # Strip path prefix and extension - name = raw.replace("\\", "/").split("/")[-1] - name = re.sub(r"\.[^.]+$", "", name) # remove last extension - return name.strip() - - -def extract_powershell_manifest(path: Path) -> dict: - """Extract module dependency edges from a PowerShell .psd1 manifest file. - - .psd1 files are PowerShell data hashtables, not scripts. tree-sitter-powershell - parses them correctly (they are syntactically valid PS). We walk the AST looking - for RootModule, NestedModules, and RequiredModules keys and emit imports_from - edges for every referenced module. - - RequiredModules supports two forms: - - Simple string: 'PSReadLine' - - Module specification: @{ ModuleName = 'Pester'; ModuleVersion = '5.0' } - For the hashtable form we only follow the ModuleName key. - """ - try: - import tree_sitter_powershell as tsps - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree_sitter_powershell not installed"} - - try: - language = Language(tsps.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} - - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({"id": nid, "label": label, "file_type": "code", - "source_file": str_path, "source_location": f"L{line}"}) - - def add_import_edge(src: str, module_raw: str, line: int) -> None: - name = _psd1_module_name(module_raw) - if not name: - return - tgt_nid = _make_id(name) - edges.append({ - "source": src, - "target": tgt_nid, - "relation": "imports_from", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - "context": "import", - }) - - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - def walk_manifest(node) -> None: - """Walk the AST and emit edges for import-relevant hash_entry nodes.""" - if node.type != "hash_entry": - for child in node.children: - walk_manifest(child) - return - - # Identify the key - key_node = next((c for c in node.children if c.type == "key_expression"), None) - if key_node is None: - return - key_text = source[key_node.start_byte:key_node.end_byte].decode(errors="replace").strip() - - if key_text not in _PSD1_IMPORT_KEYS: - # Still recurse in case there are nested hashes (e.g. ModuleVersion entries - # contain sub-hashes, but we only care about top-level keys for imports) - return - - line = node.start_point[0] + 1 - value_node = next((c for c in node.children if c.type == "pipeline"), None) - if value_node is None: - return - - if key_text == "RootModule": - # Value is a single string - strings = _psd1_collect_string_literals(value_node, source) - for s in strings: - add_import_edge(file_nid, s, line) - - elif key_text == "NestedModules": - # Value is a string or @('a', 'b', ...) array — collect all string literals - strings = _psd1_collect_string_literals(value_node, source) - for s in strings: - add_import_edge(file_nid, s, line) - - elif key_text == "RequiredModules": - # Two forms: - # 1) 'SimpleModule' — direct string literals in the array - # 2) @{ ModuleName = 'Foo'; ModuleVersion = '2.0' } — use ModuleName only - # - # Strategy: walk the value for hash_entry nodes whose key is 'ModuleName'; - # collect their string values. For the remaining string_literal nodes that - # are NOT inside a hash_entry subtree, treat them as simple module names. - module_name_strings: list[str] = [] - inside_hash_entries: set[int] = set() # byte offsets of handled strings - - def find_modulename_entries(n) -> None: - if n.type == "hash_entry": - sub_key = next((c for c in n.children if c.type == "key_expression"), None) - if sub_key is not None: - sk_text = source[sub_key.start_byte:sub_key.end_byte].decode(errors="replace").strip() - # Collect strings inside *all* sub-keys so we can exclude them - for c in n.children: - if c.type == "pipeline": - for s_node in _collect_string_nodes(c): - inside_hash_entries.add(s_node.start_byte) - if sk_text == "ModuleName": - for c in n.children: - if c.type == "pipeline": - for s in _psd1_collect_string_literals(c, source): - module_name_strings.append(s) - return # don't recurse further into this hash_entry - for child in n.children: - find_modulename_entries(child) - - def _collect_string_nodes(n): - """Return all string_literal nodes in subtree.""" - if n.type == "string_literal": - yield n - return - for child in n.children: - yield from _collect_string_nodes(child) - - find_modulename_entries(value_node) - - # Now gather direct string literals not inside hash entries - direct_strings: list[str] = [] - for s_node in _collect_string_nodes(value_node): - if s_node.start_byte not in inside_hash_entries: - raw = source[s_node.start_byte:s_node.end_byte].decode(errors="replace") - direct_strings.append(raw.strip("'\"")) - - for s in direct_strings + module_name_strings: - add_import_edge(file_nid, s, line) - - walk_manifest(root) - - return {"nodes": nodes, "edges": edges, "raw_calls": []} - - -# ── Cross-file import resolution ────────────────────────────────────────────── - -def _source_key(source_file: str, root: Path) -> str: - if not source_file: - return "" - source_path = Path(source_file) - try: - return str(source_path.resolve().relative_to(root)) - except Exception: - return str(source_path) - - -def _node_disambiguation_source_key(node: dict, root: Path) -> str: - source_file = str(node.get("source_file", "")) - if source_file: - return _source_key(source_file, root) - return _source_key(str(node.get("origin_file", "")), root) - - -def _disambiguate_colliding_node_ids( - nodes: list[dict], - edges: list[dict], - raw_calls: list[dict], - root: Path, -) -> None: - """Rewrite only colliding node IDs, using source path as the disambiguator. - - Module anchor nodes (#1327) are exempt: ``import CoreKit`` from three files - yields three ``type=module`` nodes with the same id but different - source_files. Those are the *same* module, not distinct same-named symbols, - so they must collapse to one shared node — disambiguating them by path would - scatter a single module across N file-qualified duplicates. - """ - by_id: dict[str, list[dict]] = {} - for node in nodes: - if node.get("type") in ("module", "namespace"): - continue - nid = node.get("id") - if isinstance(nid, str) and nid: - by_id.setdefault(nid, []).append(node) - - remap: dict[tuple[str, str], str] = {} - ambiguous_ids: set[str] = set() - for old_id, group in by_id.items(): - source_keys = {_node_disambiguation_source_key(node, root) for node in group} - if len(group) < 2 or len(source_keys) < 2: - continue - ambiguous_ids.add(old_id) - # Salt the colliding id with the *path* it came from. The naive salt is - # ``_make_id(source_key, old_id)`` — source_key is the raw repo-relative - # path. But _make_id collapses every separator, so two DISTINCT paths - # whose only difference is a separator-vs-inner-punctuation swap - # (``a/b/c.md`` vs ``a.b/c.md``, ``foo/bar_baz.md`` vs ``foo_bar/baz.md``) - # normalize to the SAME salted id and still collide (#1522 — the residual - # of #1504 the 0.9.0 full-path stem didn't reach). When that happens, - # append a short stable hash of the *raw* source_key, which IS injective - # over distinct paths, so the colliders separate. Computed in code from - # source_file (never trusted from the LLM), so AST↔semantic parity holds. - naive: dict[str, str] = {} # source_key -> _make_id(source_key, old_id) - for source_key in source_keys: - if source_key: - naive[source_key] = _make_id(source_key, old_id) - # source_keys that, after normalization, are not unique among themselves. - seen: dict[str, int] = {} - for nid in naive.values(): - seen[nid] = seen.get(nid, 0) + 1 - needs_hash = {sk for sk, nid in naive.items() if seen.get(nid, 0) > 1} - for node in group: - source_key = _node_disambiguation_source_key(node, root) - if not source_key: - continue - if source_key in needs_hash: - salt = hashlib.sha1(source_key.encode("utf-8")).hexdigest()[:6] - new_id = _make_id(source_key, old_id, salt) - else: - new_id = naive.get(source_key) or _make_id(source_key, old_id) - remap[(old_id, source_key)] = new_id - if new_id != old_id: - node["id"] = new_id - - if not remap: - return - - unambiguous_remaps: dict[str, str] = {} - for old_id, group in by_id.items(): - if old_id in ambiguous_ids: - continue - candidates = { - node["id"] for node in group - if isinstance(node.get("id"), str) and node["id"] != old_id - } - if len(candidates) == 1: - unambiguous_remaps[old_id] = next(iter(candidates)) - - # A C/ObjC/C++ `#include "foo.h"` / `#import "foo.h"` resolves to the header's - # file node, but `foo.h` and its sibling `foo.c`/`foo.m`/`foo.cpp` collapse to - # the same `foo` file id, so disambiguation salts them apart by path. A - # cross-file import edge from a THIRD file carries neither salt's source_key, so - # the (target, edge_source_key) lookup misses and the edge dangles on the now - # dead `foo` id. Repoint those import edges to the HEADER variant (the include - # always targeted the header), keyed by the original colliding id (#1475). - _HEADER_SUFFIXES = (".h", ".hpp", ".hh", ".hxx") - header_remaps: dict[str, str] = {} - for old_id in ambiguous_ids: - for node in by_id.get(old_id, []): - sk = _node_disambiguation_source_key(node, root) - if sk and Path(sk).suffix.lower() in _HEADER_SUFFIXES: - new_id = remap.get((old_id, sk)) - if new_id: - header_remaps[old_id] = new_id - break - - for edge in edges: - edge_source_key = _source_key(str(edge.get("source_file", "")), root) - source_key = (edge.get("source", ""), edge_source_key) - target_key = (edge.get("target", ""), edge_source_key) - if source_key in remap: - edge["source"] = remap[source_key] - elif edge.get("source") in unambiguous_remaps: - edge["source"] = unambiguous_remaps[str(edge["source"])] - # imports/imports_from always target a header file, so they must resolve to - # the header variant BEFORE the same-source-file salt is considered. Keying - # the import target by the importer's own source file mis-points a `.m` - # importing its own `.h` back at itself (self-loop), and is wrong for any - # cross-file import whose importer shares the colliding id (#1475). - if (edge.get("relation") in ("imports", "imports_from") - and edge.get("target") in header_remaps): - edge["target"] = header_remaps[str(edge["target"])] - elif target_key in remap: - edge["target"] = remap[target_key] - elif edge.get("target") in unambiguous_remaps: - edge["target"] = unambiguous_remaps[str(edge["target"])] - - for raw_call in raw_calls: - call_source_key = _source_key(str(raw_call.get("source_file", "")), root) - caller_key = (raw_call.get("caller_nid", ""), call_source_key) - if caller_key in remap: - raw_call["caller_nid"] = remap[caller_key] - elif raw_call.get("caller_nid") in unambiguous_remaps: - raw_call["caller_nid"] = unambiguous_remaps[str(raw_call["caller_nid"])] - - -def _canonicalize_csharp_namespace_nodes(all_nodes: list[dict], all_edges: list[dict]) -> None: - """Collapse duplicate C# namespace node entries to one canonical node per label.""" - by_label: dict[str, list[dict]] = {} - for node in all_nodes: - if node.get("type") != "namespace": - continue - label = node.get("label") - if isinstance(label, str): - by_label.setdefault(label, []).append(node) - - remap: dict[str, str] = {} - drop_node_ids: set[int] = set() - for group in by_label.values(): - if len(group) < 2: - continue - canonical = sorted( - group, - key=lambda node: ( - str(node.get("source_file") or ""), - str(node.get("source_location") or ""), - str(node.get("id") or ""), - ), - )[0] - canonical_id = canonical.get("id") - for node in group: - if node is canonical: - continue - drop_node_ids.add(id(node)) - dup_id = node.get("id") - if isinstance(dup_id, str) and isinstance(canonical_id, str): - remap[dup_id] = canonical_id - - if remap: - for edge in all_edges: - if edge.get("source") in remap: - edge["source"] = remap[str(edge["source"])] - if edge.get("target") in remap: - edge["target"] = remap[str(edge["target"])] - - if drop_node_ids: - all_nodes[:] = [node for node in all_nodes if id(node) not in drop_node_ids] - - -# Languages whose identifiers are case-insensitive, so cross-file name resolution -# may fold case. Everywhere else, case is semantic (`Path` the class vs `PATH` the -# env var are distinct) and folding manufactures false edges / super-hubs (#1581). -_CASE_INSENSITIVE_EXTS = frozenset({ - ".php", ".phtml", ".php3", ".php4", ".php5", ".php7", ".phps", # PHP fns/classes - ".sql", # SQL identifiers - ".nim", ".nims", ".nimble", # Nim (style-insensitive) -}) - - -def _lang_is_case_insensitive(source_file: object) -> bool: - """True when the file's language resolves identifiers case-insensitively (#1581).""" - if not source_file: - return False - return Path(str(source_file)).suffix.lower() in _CASE_INSENSITIVE_EXTS - - -# Language interop families for cross-file call resolution. A call in one language -# can never bind by name to a definition in another family — a TSX component does -# not invoke a Kotlin method, and a Python function does not invoke a Java one. -# Families are grouped by REAL interop so legitimate cross-language resolution -# keeps working: Kotlin/Java/Scala/Groovy share the JVM, C/C++/Objective-C/CUDA -# share headers and symbols (Swift bridges to Objective-C), and JS/TS variants -# (plus Vue/Svelte/Astro SFC script blocks) compile into one module graph. -# Extensions absent from this map (docs, configs, unknown languages) resolve to -# no family and are never filtered — same permissive default as before. -_LANG_FAMILY_BY_EXT: dict[str, str] = { - # JS/TS module graph (SFCs embed JS/TS) - ".js": "jsts", ".jsx": "jsts", ".mjs": "jsts", ".cjs": "jsts", - ".ts": "jsts", ".tsx": "jsts", ".mts": "jsts", ".cts": "jsts", - ".vue": "jsts", ".svelte": "jsts", ".astro": "jsts", - # JVM interop - ".java": "jvm", ".kt": "jvm", ".kts": "jvm", - ".scala": "jvm", ".groovy": "jvm", ".gradle": "jvm", - # C-family: shared headers, Objective-C/C++ mix, Swift↔ObjC bridging - ".c": "native", ".h": "native", ".cpp": "native", ".cc": "native", - ".cxx": "native", ".hpp": "native", ".cu": "native", ".cuh": "native", - ".metal": "native", ".m": "native", ".mm": "native", ".swift": "native", - # Single-language families - ".py": "python", - ".go": "go", - ".rs": "rust", - ".rb": "ruby", - ".php": "php", ".phtml": "php", ".php3": "php", ".php4": "php", - ".php5": "php", ".php7": "php", ".phps": "php", - ".cs": "dotnet", ".razor": "dotnet", ".cshtml": "dotnet", ".xaml": "dotnet", - ".lua": "lua", ".luau": "lua", - ".zig": "zig", - ".ex": "elixir", ".exs": "elixir", - ".jl": "julia", - ".dart": "dart", - ".sh": "shell", ".bash": "shell", - ".ps1": "powershell", ".psm1": "powershell", ".psd1": "powershell", -} - - -def _lang_family(source_file: object) -> str | None: - """Interop family of the file's language, or None when unknown/not code.""" - if not source_file: - return None - return _LANG_FAMILY_BY_EXT.get(Path(str(source_file)).suffix.lower()) - - -def _node_label_key(node: dict, fold: bool = False) -> str: - label = str(node.get("label", "")).strip() - key = re.sub(r"[^a-zA-Z0-9]+", "", label) - return key.lower() if fold else key - - -def _is_type_like_definition(node: dict) -> bool: - if node.get("type") == "namespace": - return False - label = str(node.get("label", "")).strip() - if not label: - return False - if label.endswith(")") or label.startswith("."): - return False - if "." in label: - return False - return node.get("file_type") == "code" - - -def _rewire_unique_stub_nodes(nodes: list[dict], edges: list[dict]) -> None: - """Map unresolved no-source stubs to a unique real definition with the same label.""" - real_by_label: dict[str, list[dict]] = {} # exact-case (all languages) - real_by_label_ci: dict[str, list[dict]] = {} # case-INSENSITIVE-language reals only - stubs: list[dict] = [] - - for node in nodes: - key = _node_label_key(node) - if not key: - continue - if node.get("source_file"): - if _is_type_like_definition(node): - # Match stubs case-SENSITIVELY: a `Path` reference must not rewire to a - # `PATH` env var (#1581). Fold only for genuinely case-insensitive - # languages, where `foo` legitimately resolves to `Foo`. - real_by_label.setdefault(key, []).append(node) - if _lang_is_case_insensitive(node.get("source_file")): - real_by_label_ci.setdefault( - _node_label_key(node, fold=True), []).append(node) - continue - stubs.append(node) - - remap: dict[str, str] = {} - for stub in stubs: - stub_id = str(stub.get("id", "")) - if not stub_id: - continue - candidates = real_by_label.get(_node_label_key(stub), []) - if len(candidates) != 1: - # No unique exact match — fall back to a case-insensitive match, but - # only against case-insensitive-language definitions (so a case-sensitive - # `PATH` can never absorb a `Path` reference). - candidates = real_by_label_ci.get(_node_label_key(stub, fold=True), []) - if len(candidates) != 1: - continue - target_id = candidates[0].get("id") - if isinstance(target_id, str) and target_id and target_id != stub_id: - remap[stub_id] = target_id - - if not remap: - return - - by_id = {node.get("id"): node for node in nodes if node.get("id")} - csharp_scoped_relations = {"inherits", "implements", "references", "imports"} - for edge in edges: - is_csharp_scoped_edge = ( - str(edge.get("source_file", "")).endswith(".cs") - and edge.get("relation") in csharp_scoped_relations - ) - source = edge.get("source") - if source in remap: - remapped_source = remap[str(source)] - if not ( - is_csharp_scoped_edge - and str(by_id.get(remapped_source, {}).get("source_file", "")).endswith(".cs") - ): - edge["source"] = remapped_source - target = edge.get("target") - if target in remap: - remapped_target = remap[str(target)] - if not ( - is_csharp_scoped_edge - and str(by_id.get(remapped_target, {}).get("source_file", "")).endswith(".cs") - ): - edge["target"] = remapped_target - - referenced = {x for e in edges for x in (e.get("source"), e.get("target"))} - drop_ids = {stub_id for stub_id in remap if stub_id not in referenced} - nodes[:] = [node for node in nodes if node.get("id") not in drop_ids] - - -def _js_source_path(source_file: str, root: Path) -> Path | None: - if not source_file: - return None - path = Path(source_file) - if not path.is_absolute(): - path = root / path - try: - return path.resolve() - except Exception: - return path - - -@dataclass(frozen=True) -class _SymbolDeclarationFact: - file_path: Path - name: str - line: int - - -@dataclass(frozen=True) -class _SymbolImportFact: - file_path: Path - local_name: str - target_path: Path - imported_name: str - line: int - - -@dataclass(frozen=True) -class _SymbolAliasFact: - file_path: Path - alias: str - target_name: str - line: int - - -@dataclass(frozen=True) -class _SymbolExportFact: - file_path: Path - exported_name: str - line: int - local_name: str | None = None - target_path: Path | None = None - target_name: str | None = None - - -@dataclass(frozen=True) -class _StarExportFact: - file_path: Path - target_path: Path - line: int - - -@dataclass(frozen=True) -class _NamespaceExportFact: - file_path: Path - exported_name: str - target_path: Path - line: int - - -@dataclass(frozen=True) -class _SymbolUseFact: - file_path: Path - source_id: str - local_name: str - relation: str - context: str - line: int - - -@dataclass -class _SymbolResolutionFacts: - declarations: list[_SymbolDeclarationFact] = field(default_factory=list) - imports: list[_SymbolImportFact] = field(default_factory=list) - aliases: list[_SymbolAliasFact] = field(default_factory=list) - exports: list[_SymbolExportFact] = field(default_factory=list) - star_exports: list[_StarExportFact] = field(default_factory=list) - namespace_exports: list[_NamespaceExportFact] = field(default_factory=list) - uses: list[_SymbolUseFact] = field(default_factory=list) - # File-to-file submodule imports from `from pkg import submod` (#1146). - # Each entry is (importing_file, submodule_file, line). - module_imports: list[tuple[Path, Path, int]] = field(default_factory=list) - - -def _apply_symbol_resolution_facts( - paths: list[Path], - nodes: list[dict], - edges: list[dict], - root: Path, - facts: _SymbolResolutionFacts, -) -> None: - """Apply language-provided import/export/use facts to graph edges.""" - if not ( - facts.declarations - or facts.imports - or facts.aliases - or facts.exports - or facts.star_exports - or facts.namespace_exports - or facts.uses - or facts.module_imports - ): - return - - path_by_resolved = {path.resolve(): path for path in paths} - source_file_id = {path.resolve(): _make_id(str(path)) for path in paths} - symbol_nodes: dict[tuple[Path, str], str] = {} - for node in nodes: - source_path = _js_source_path(str(node.get("source_file", "")), root) - if source_path is None: - continue - label = str(node.get("label", "")).strip().strip("()").lstrip(".") - if label and node.get("id"): - symbol_nodes[(source_path, label)] = str(node["id"]) - - def ensure_symbol_node(path: Path, name: str, line: int) -> str: - resolved_path = path.resolve() - existing = symbol_nodes.get((resolved_path, name)) - if existing is not None: - return existing - node_id = _make_id(_file_stem(path), name) - symbol_nodes[(resolved_path, name)] = node_id - nodes.append({ - "id": node_id, - "label": name, - "file_type": "code", - "source_file": str(path), - "source_location": f"L{line}", - }) - return node_id - - existing_edges = { - ( - str(edge.get("source")), - str(edge.get("target")), - str(edge.get("relation")), - str(edge.get("context") or ""), - ) - for edge in edges - } - - def add_edge(source: str, target: str, relation: str, context: str, line: int, source_path: Path) -> None: - key = (source, target, relation, context or "") - if key in existing_edges: - return - existing_edges.add(key) - edges.append({ - "source": source, - "target": target, - "relation": relation, - "context": context, - "confidence": "EXTRACTED", - "source_file": str(source_path), - "source_location": f"L{line}", - "weight": 1.0, - }) - - for declaration in facts.declarations: - ensure_symbol_node(declaration.file_path, declaration.name, declaration.line) - - local_aliases_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} - for import_fact in facts.imports: - file_path = import_fact.file_path.resolve() - local_aliases_by_file.setdefault(file_path, {})[import_fact.local_name] = ( - import_fact.target_path.resolve(), - import_fact.imported_name, - ) - - pending_aliases_by_file: dict[Path, list[_SymbolAliasFact]] = {} - for alias_fact in facts.aliases: - pending_aliases_by_file.setdefault(alias_fact.file_path.resolve(), []).append(alias_fact) - - for file_path, aliases in pending_aliases_by_file.items(): - local_aliases = local_aliases_by_file.setdefault(file_path, {}) - changed = True - while changed: - changed = False - for alias_fact in aliases: - if alias_fact.alias in local_aliases: - continue - origin = local_aliases.get(alias_fact.target_name) - if origin is not None: - local_aliases[alias_fact.alias] = origin - changed = True - - named_exports_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} - star_exports_by_file: dict[Path, list[Path]] = {} - - for star_fact in facts.star_exports: - source_path = star_fact.file_path.resolve() - target_path = star_fact.target_path.resolve() - star_exports_by_file.setdefault(source_path, []).append(target_path) - source_id = source_file_id.get(source_path) - if source_id is not None: - add_edge( - source_id, - _make_id(str(path_by_resolved.get(target_path, target_path))), - "re_exports", - "export", - star_fact.line, - star_fact.file_path, - ) - - for namespace_fact in facts.namespace_exports: - source_path = namespace_fact.file_path.resolve() - target_path = namespace_fact.target_path.resolve() - namespace_id = ensure_symbol_node( - namespace_fact.file_path, - namespace_fact.exported_name, - namespace_fact.line, - ) - named_exports_by_file.setdefault(source_path, {})[ - namespace_fact.exported_name - ] = (source_path, namespace_fact.exported_name) - source_id = source_file_id.get(source_path) - if source_id is not None: - add_edge( - source_id, - namespace_id, - "contains", - "namespace_export", - namespace_fact.line, - namespace_fact.file_path, - ) - add_edge( - source_id, - _make_id(str(path_by_resolved.get(target_path, target_path))), - "re_exports", - "export", - namespace_fact.line, - namespace_fact.file_path, - ) - - for export_fact in facts.exports: - file_path = export_fact.file_path.resolve() - origin: tuple[Path, str] | None = None - if export_fact.target_path is not None and export_fact.target_name is not None: - origin = (export_fact.target_path.resolve(), export_fact.target_name) - elif export_fact.local_name is not None: - origin = local_aliases_by_file.get(file_path, {}).get(export_fact.local_name) - if origin is None and (file_path, export_fact.local_name) in symbol_nodes: - origin = (file_path, export_fact.local_name) - if origin is None: - continue - named_exports_by_file.setdefault(file_path, {})[export_fact.exported_name] = origin - if origin[0] != file_path: - source_id = source_file_id.get(file_path) - if source_id is not None: - add_edge( - source_id, - _make_id(str(path_by_resolved.get(origin[0], origin[0]))), - "re_exports", - "export", - export_fact.line, - export_fact.file_path, - ) - - def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tuple[Path, str]] | None = None) -> tuple[Path, str]: - target_path = target_path.resolve() - key = (target_path, imported_name) - if seen is None: - seen = set() - if key in seen: - return key - seen.add(key) - origin = named_exports_by_file.get(target_path, {}).get(imported_name) - if origin is not None: - return resolve_exported_origin(origin[0], origin[1], seen) - for star_target in star_exports_by_file.get(target_path, []): - star_key = (star_target, imported_name) - if star_key in symbol_nodes: - return star_key - resolved = resolve_exported_origin(star_target, imported_name, seen) - if resolved in symbol_nodes: - return resolved - return key - - for import_fact in facts.imports: - source_id = source_file_id.get(import_fact.file_path.resolve()) - if source_id is None: - continue - origin_path, origin_symbol = resolve_exported_origin( - import_fact.target_path, - import_fact.imported_name, - ) - target_id = symbol_nodes.get((origin_path, origin_symbol)) - if target_id is None: - continue - add_edge( - source_id, - target_id, - "imports", - "import", - import_fact.line, - import_fact.file_path, - ) - - # #1146: emit file-to-file imports_from edges for package-form submodule imports. - for from_path, to_path, line in facts.module_imports: - try: - from_rel = from_path.relative_to(root) - to_rel = to_path.relative_to(root) - except ValueError: - continue - source_id = _make_id(_file_stem(from_rel)) - target_id = _make_id(_file_stem(to_rel)) - add_edge(source_id, target_id, "imports_from", "submodule_import", line, from_path) - - for use_fact in facts.uses: - file_path = use_fact.file_path.resolve() - target_id = None - unresolved_origin = local_aliases_by_file.get(file_path, {}).get(use_fact.local_name) - if unresolved_origin is not None: - origin_path, origin_symbol = resolve_exported_origin(*unresolved_origin) - target_id = symbol_nodes.get((origin_path, origin_symbol)) - if target_id is None and use_fact.relation in ("inherits", "implements"): - # Same-file fallback for HERITAGE only: a base declared in the same - # file (`class X extends Y`, `interface A extends B`) has no import - # alias, so resolve it directly against the file's own symbol nodes. - # Scoped to heritage because same-file calls/uses already resolve via - # the dedicated call-graph pass; widening this would duplicate those - # edges. Import resolution still takes precedence (#1095). - target_id = symbol_nodes.get((file_path, use_fact.local_name)) - if target_id is None: - continue - add_edge( - use_fact.source_id, - target_id, - use_fact.relation, - use_fact.context, - use_fact.line, - use_fact.file_path, - ) - - -def _parse_js_tree(path: Path): - try: - from tree_sitter import Language, Parser - # .vue embeds the script in non-JS markup; mask it out and parse the - # close tag + pos = m.end() + if lang is None: + lang_m = _VUE_SCRIPT_LANG_RE.search(m.group(1)) + if lang_m: + lang = lang_m.group(1).lower() + out.append(_blank(src[pos:])) + return "".join(out), lang + +def _source_key(source_file: str, root: Path) -> str: + if not source_file: + return "" + source_path = Path(source_file) + try: + return str(source_path.resolve().relative_to(root)) + except Exception: + return str(source_path) + +def _node_disambiguation_source_key(node: dict, root: Path) -> str: + source_file = str(node.get("source_file", "")) + if source_file: + return _source_key(source_file, root) + return _source_key(str(node.get("origin_file", "")), root) + +def _disambiguate_colliding_node_ids( + nodes: list[dict], + edges: list[dict], + raw_calls: list[dict], + root: Path, +) -> None: + """Rewrite only colliding node IDs, using source path as the disambiguator. + + Module anchor nodes (#1327) are exempt: ``import CoreKit`` from three files + yields three ``type=module`` nodes with the same id but different + source_files. Those are the *same* module, not distinct same-named symbols, + so they must collapse to one shared node — disambiguating them by path would + scatter a single module across N file-qualified duplicates. + """ + by_id: dict[str, list[dict]] = {} + for node in nodes: + if node.get("type") in ("module", "namespace"): + continue + nid = node.get("id") + if isinstance(nid, str) and nid: + by_id.setdefault(nid, []).append(node) + + remap: dict[tuple[str, str], str] = {} + ambiguous_ids: set[str] = set() + for old_id, group in by_id.items(): + source_keys = {_node_disambiguation_source_key(node, root) for node in group} + if len(group) < 2 or len(source_keys) < 2: + continue + ambiguous_ids.add(old_id) + # Salt the colliding id with the *path* it came from. The naive salt is + # ``_make_id(source_key, old_id)`` — source_key is the raw repo-relative + # path. But _make_id collapses every separator, so two DISTINCT paths + # whose only difference is a separator-vs-inner-punctuation swap + # (``a/b/c.md`` vs ``a.b/c.md``, ``foo/bar_baz.md`` vs ``foo_bar/baz.md``) + # normalize to the SAME salted id and still collide (#1522 — the residual + # of #1504 the 0.9.0 full-path stem didn't reach). When that happens, + # append a short stable hash of the *raw* source_key, which IS injective + # over distinct paths, so the colliders separate. Computed in code from + # source_file (never trusted from the LLM), so AST↔semantic parity holds. + naive: dict[str, str] = {} # source_key -> _make_id(source_key, old_id) + for source_key in source_keys: + if source_key: + naive[source_key] = _make_id(source_key, old_id) + # source_keys that, after normalization, are not unique among themselves. + seen: dict[str, int] = {} + for nid in naive.values(): + seen[nid] = seen.get(nid, 0) + 1 + needs_hash = {sk for sk, nid in naive.items() if seen.get(nid, 0) > 1} + for node in group: + source_key = _node_disambiguation_source_key(node, root) + if not source_key: + continue + if source_key in needs_hash: + salt = hashlib.sha1(source_key.encode("utf-8")).hexdigest()[:6] + new_id = _make_id(source_key, old_id, salt) + else: + new_id = naive.get(source_key) or _make_id(source_key, old_id) + remap[(old_id, source_key)] = new_id + if new_id != old_id: + node["id"] = new_id + + if not remap: + return + + unambiguous_remaps: dict[str, str] = {} + for old_id, group in by_id.items(): + if old_id in ambiguous_ids: + continue + candidates = { + node["id"] for node in group + if isinstance(node.get("id"), str) and node["id"] != old_id + } + if len(candidates) == 1: + unambiguous_remaps[old_id] = next(iter(candidates)) + + # A C/ObjC/C++ `#include "foo.h"` / `#import "foo.h"` resolves to the header's + # file node, but `foo.h` and its sibling `foo.c`/`foo.m`/`foo.cpp` collapse to + # the same `foo` file id, so disambiguation salts them apart by path. A + # cross-file import edge from a THIRD file carries neither salt's source_key, so + # the (target, edge_source_key) lookup misses and the edge dangles on the now + # dead `foo` id. Repoint those import edges to the HEADER variant (the include + # always targeted the header), keyed by the original colliding id (#1475). + _HEADER_SUFFIXES = (".h", ".hpp", ".hh", ".hxx") + header_remaps: dict[str, str] = {} + for old_id in ambiguous_ids: + for node in by_id.get(old_id, []): + sk = _node_disambiguation_source_key(node, root) + if sk and Path(sk).suffix.lower() in _HEADER_SUFFIXES: + new_id = remap.get((old_id, sk)) + if new_id: + header_remaps[old_id] = new_id + break + + for edge in edges: + edge_source_key = _source_key(str(edge.get("source_file", "")), root) + source_key = (edge.get("source", ""), edge_source_key) + target_key = (edge.get("target", ""), edge_source_key) + if source_key in remap: + edge["source"] = remap[source_key] + elif edge.get("source") in unambiguous_remaps: + edge["source"] = unambiguous_remaps[str(edge["source"])] + # imports/imports_from always target a header file, so they must resolve to + # the header variant BEFORE the same-source-file salt is considered. Keying + # the import target by the importer's own source file mis-points a `.m` + # importing its own `.h` back at itself (self-loop), and is wrong for any + # cross-file import whose importer shares the colliding id (#1475). + if (edge.get("relation") in ("imports", "imports_from") + and edge.get("target") in header_remaps): + edge["target"] = header_remaps[str(edge["target"])] + elif target_key in remap: + edge["target"] = remap[target_key] + elif edge.get("target") in unambiguous_remaps: + edge["target"] = unambiguous_remaps[str(edge["target"])] + + for raw_call in raw_calls: + call_source_key = _source_key(str(raw_call.get("source_file", "")), root) + caller_key = (raw_call.get("caller_nid", ""), call_source_key) + if caller_key in remap: + raw_call["caller_nid"] = remap[caller_key] + elif raw_call.get("caller_nid") in unambiguous_remaps: + raw_call["caller_nid"] = unambiguous_remaps[str(raw_call["caller_nid"])] + +def _is_type_like_definition(node: dict) -> bool: + if node.get("type") == "namespace": + return False + label = str(node.get("label", "")).strip() + if not label: + return False + if label.endswith(")") or label.startswith("."): + return False + if "." in label: + return False + return node.get("file_type") == "code" + +def _js_source_path(source_file: str, root: Path) -> Path | None: + if not source_file: + return None + path = Path(source_file) + if not path.is_absolute(): + path = root / path + try: + return path.resolve() + except Exception: + return path + +def _apply_symbol_resolution_facts( + paths: list[Path], + nodes: list[dict], + edges: list[dict], + root: Path, + facts: _SymbolResolutionFacts, +) -> None: + """Apply language-provided import/export/use facts to graph edges.""" + if not ( + facts.declarations + or facts.imports + or facts.aliases + or facts.exports + or facts.star_exports + or facts.namespace_exports + or facts.uses + or facts.module_imports + ): + return + + path_by_resolved = {path.resolve(): path for path in paths} + source_file_id = {path.resolve(): _make_id(str(path)) for path in paths} + symbol_nodes: dict[tuple[Path, str], str] = {} + for node in nodes: + source_path = _js_source_path(str(node.get("source_file", "")), root) + if source_path is None: + continue + label = str(node.get("label", "")).strip().strip("()").lstrip(".") + if label and node.get("id"): + symbol_nodes[(source_path, label)] = str(node["id"]) + + def ensure_symbol_node(path: Path, name: str, line: int) -> str: + resolved_path = path.resolve() + existing = symbol_nodes.get((resolved_path, name)) + if existing is not None: + return existing + node_id = _make_id(_file_stem(path), name) + symbol_nodes[(resolved_path, name)] = node_id + nodes.append({ + "id": node_id, + "label": name, + "file_type": "code", + "source_file": str(path), + "source_location": f"L{line}", + }) + return node_id + + existing_edges = { + ( + str(edge.get("source")), + str(edge.get("target")), + str(edge.get("relation")), + str(edge.get("context") or ""), + ) + for edge in edges + } + + def add_edge(source: str, target: str, relation: str, context: str, line: int, source_path: Path) -> None: + key = (source, target, relation, context or "") + if key in existing_edges: + return + existing_edges.add(key) + edges.append({ + "source": source, + "target": target, + "relation": relation, + "context": context, + "confidence": "EXTRACTED", + "source_file": str(source_path), + "source_location": f"L{line}", + "weight": 1.0, + }) + + for declaration in facts.declarations: + ensure_symbol_node(declaration.file_path, declaration.name, declaration.line) + + local_aliases_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} + for import_fact in facts.imports: + file_path = import_fact.file_path.resolve() + local_aliases_by_file.setdefault(file_path, {})[import_fact.local_name] = ( + import_fact.target_path.resolve(), + import_fact.imported_name, + ) + + pending_aliases_by_file: dict[Path, list[_SymbolAliasFact]] = {} + for alias_fact in facts.aliases: + pending_aliases_by_file.setdefault(alias_fact.file_path.resolve(), []).append(alias_fact) + + for file_path, aliases in pending_aliases_by_file.items(): + local_aliases = local_aliases_by_file.setdefault(file_path, {}) + changed = True + while changed: + changed = False + for alias_fact in aliases: + if alias_fact.alias in local_aliases: + continue + origin = local_aliases.get(alias_fact.target_name) + if origin is not None: + local_aliases[alias_fact.alias] = origin + changed = True + + named_exports_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} + star_exports_by_file: dict[Path, list[Path]] = {} + + for star_fact in facts.star_exports: + source_path = star_fact.file_path.resolve() + target_path = star_fact.target_path.resolve() + star_exports_by_file.setdefault(source_path, []).append(target_path) + source_id = source_file_id.get(source_path) + if source_id is not None: + add_edge( + source_id, + _make_id(str(path_by_resolved.get(target_path, target_path))), + "re_exports", + "export", + star_fact.line, + star_fact.file_path, + ) + + for namespace_fact in facts.namespace_exports: + source_path = namespace_fact.file_path.resolve() + target_path = namespace_fact.target_path.resolve() + namespace_id = ensure_symbol_node( + namespace_fact.file_path, + namespace_fact.exported_name, + namespace_fact.line, + ) + named_exports_by_file.setdefault(source_path, {})[ + namespace_fact.exported_name + ] = (source_path, namespace_fact.exported_name) + source_id = source_file_id.get(source_path) + if source_id is not None: + add_edge( + source_id, + namespace_id, + "contains", + "namespace_export", + namespace_fact.line, + namespace_fact.file_path, + ) + add_edge( + source_id, + _make_id(str(path_by_resolved.get(target_path, target_path))), + "re_exports", + "export", + namespace_fact.line, + namespace_fact.file_path, + ) + + for export_fact in facts.exports: + file_path = export_fact.file_path.resolve() + origin: tuple[Path, str] | None = None + if export_fact.target_path is not None and export_fact.target_name is not None: + origin = (export_fact.target_path.resolve(), export_fact.target_name) + elif export_fact.local_name is not None: + origin = local_aliases_by_file.get(file_path, {}).get(export_fact.local_name) + if origin is None and (file_path, export_fact.local_name) in symbol_nodes: + origin = (file_path, export_fact.local_name) + if origin is None: + continue + named_exports_by_file.setdefault(file_path, {})[export_fact.exported_name] = origin + if origin[0] != file_path: + source_id = source_file_id.get(file_path) + if source_id is not None: + add_edge( + source_id, + _make_id(str(path_by_resolved.get(origin[0], origin[0]))), + "re_exports", + "export", + export_fact.line, + export_fact.file_path, + ) + + def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tuple[Path, str]] | None = None) -> tuple[Path, str]: + target_path = target_path.resolve() + key = (target_path, imported_name) + if seen is None: + seen = set() + if key in seen: + return key + seen.add(key) + origin = named_exports_by_file.get(target_path, {}).get(imported_name) + if origin is not None: + return resolve_exported_origin(origin[0], origin[1], seen) + for star_target in star_exports_by_file.get(target_path, []): + star_key = (star_target, imported_name) + if star_key in symbol_nodes: + return star_key + resolved = resolve_exported_origin(star_target, imported_name, seen) + if resolved in symbol_nodes: + return resolved + return key + + for import_fact in facts.imports: + source_id = source_file_id.get(import_fact.file_path.resolve()) + if source_id is None: + continue + origin_path, origin_symbol = resolve_exported_origin( + import_fact.target_path, + import_fact.imported_name, + ) + target_id = symbol_nodes.get((origin_path, origin_symbol)) + if target_id is None: + continue + add_edge( + source_id, + target_id, + "imports", + "import", + import_fact.line, + import_fact.file_path, + ) + + # #1146: emit file-to-file imports_from edges for package-form submodule imports. + for from_path, to_path, line in facts.module_imports: + try: + from_rel = from_path.relative_to(root) + to_rel = to_path.relative_to(root) + except ValueError: + continue + source_id = _make_id(_file_stem(from_rel)) + target_id = _make_id(_file_stem(to_rel)) + add_edge(source_id, target_id, "imports_from", "submodule_import", line, from_path) + + for use_fact in facts.uses: + file_path = use_fact.file_path.resolve() + target_id = None + unresolved_origin = local_aliases_by_file.get(file_path, {}).get(use_fact.local_name) + if unresolved_origin is not None: + origin_path, origin_symbol = resolve_exported_origin(*unresolved_origin) + target_id = symbol_nodes.get((origin_path, origin_symbol)) + if target_id is None and use_fact.relation in ("inherits", "implements"): + # Same-file fallback for HERITAGE only: a base declared in the same + # file (`class X extends Y`, `interface A extends B`) has no import + # alias, so resolve it directly against the file's own symbol nodes. + # Scoped to heritage because same-file calls/uses already resolve via + # the dedicated call-graph pass; widening this would duplicate those + # edges. Import resolution still takes precedence (#1095). + target_id = symbol_nodes.get((file_path, use_fact.local_name)) + if target_id is None: + continue + add_edge( + use_fact.source_id, + target_id, + use_fact.relation, + use_fact.context, + use_fact.line, + use_fact.file_path, + ) + +def _parse_js_tree(path: Path): + try: + from tree_sitter import Language, Parser + # .vue embeds the script in non-JS markup; mask it out and parse the + #