Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 66 additions & 2 deletions src/chunk.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@ pub fn chunk_file(
if source.len() > MAX_FILE_BYTES {
return Vec::new();
}
// Detect binary content by scanning the first 512 bytes for null bytes.
if source.as_bytes().iter().take(512).any(|&b| b == 0) {
// Cheap early-out before we pay for a parse. The authoritative guard lives in `chunk_text`
// (and in the walk, before the graph sees the bytes) so it cannot be bypassed — see `is_binary`.
if is_binary(source) {
return Vec::new();
}

Expand Down Expand Up @@ -84,9 +85,27 @@ pub fn chunk_tree(
/// Chunk free text (e.g. PDF-extracted text) through the windowed path text files already take.
#[must_use]
pub fn chunk_text(file_path: &str, text: &str, language: &str, tuning: IndexTuning) -> Vec<Chunk> {
// The guard belongs HERE, at the point a chunk is produced — not only at `chunk_file`'s door.
// `chunk_file` used to be the sole holder of the binary check, and the graph-enabled walk path
// never calls it (it parses the tree itself and falls back straight to windowing), so raw NUL
// bytes reached `Chunk::content` on the one path production actually runs.
if is_binary(text) {
return Vec::new();
}
window_chunks(file_path, text, language, tuning)
}

/// Binary-content sniff: a NUL byte within the first 512 bytes.
///
/// NUL is a perfectly legal Unicode scalar, so a blob can be valid UTF-8 — passing every
/// `read_to_string` check — and still be binary. It has to be caught by content, not by encoding.
/// This matters downstream and not just aesthetically: PostgreSQL's `text` type rejects the NUL
/// codepoint outright, so a contaminated chunk fails at persist time, far from its cause.
#[must_use]
pub(crate) fn is_binary(source: &str) -> bool {
source.as_bytes().iter().take(512).any(|&b| b == 0)
}

/// Recursively collect interesting nodes. We walk the full tree (not just top-level children) so that
/// methods inside `impl` blocks, nested functions, and inner classes are captured.
fn collect_items(
Expand Down Expand Up @@ -154,6 +173,14 @@ pub(crate) fn interesting_node(
let (kind, name_field) = match node.kind() {
// Rust
"function_item" => ("function", Some("name")),
// A trait method declared WITHOUT a default body (`fn greet(&self);`) parses as
// `function_signature_item`, not `function_item`. Omitting it made trait interfaces —
// arguably the most important symbols in a Rust codebase — invisible to both the chunker
// and the graph. Same `kind` as a bodied function on purpose: it is the same thing to a
// reader searching for it, and a new `chunk_type` value would leak into every consumer's
// stored data. The graph draws the one distinction that matters (it is not a call target)
// from the tree-sitter node kind instead — see `Classifier::is_call_target`.
"function_signature_item" => ("function", Some("name")),
"impl_item" => ("impl", None),
"struct_item" => ("struct", Some("name")),
"enum_item" => ("enum", Some("name")),
Expand Down Expand Up @@ -306,4 +333,41 @@ mod tests {
);
assert!(chunks.iter().all(|c| c.chunk_type == "window"));
}

#[test]
fn signature_only_trait_method_is_chunked_like_a_bodied_one() {
// REGRESSION: `fn greet(&self);` parses as `function_signature_item`, which was classified
// as nothing — so a trait interface produced no chunk and was invisible to semantic search.
let src = "pub trait Greeter {\n fn greet(&self) -> String;\n fn shout(&self) -> String { String::new() }\n}\n";
let chunks = chunk_file("g.rs", src, "rust", IndexTuning::default());
let names: Vec<&str> = chunks
.iter()
.filter_map(|c| c.symbol_name.as_deref())
.collect();
assert!(
names.contains(&"greet"),
"the bodiless declaration must be chunked; got {names:?}"
);
assert!(
names.contains(&"shout"),
"the default-bodied method must still be chunked; got {names:?}"
);
// Same chunk_type as a bodied function on purpose — a new value would leak into every
// consumer's stored data for no reader-visible benefit.
let greet = chunks
.iter()
.find(|c| c.symbol_name.as_deref() == Some("greet"))
.unwrap();
assert_eq!(greet.chunk_type, "function");
}

#[test]
fn is_binary_detects_a_nul_only_within_the_sniff_window() {
assert!(is_binary("a\0b"));
assert!(!is_binary("plain text"));
// NUL beyond the 512-byte sniff window is deliberately not detected — the guard is a cheap
// prefix sniff, not a full scan. Documented so the bound is a decision, not an accident.
let late = format!("{}\0", "x".repeat(512));
assert!(!is_binary(&late));
}
}
106 changes: 101 additions & 5 deletions src/graph/emit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,13 @@ fn walk(
source_file: source_file.to_string(),
start_line,
});
// Functions/methods are callable — record for resolution, tagged with their type scope.
if is_callable && let Some(n) = name.clone() {
// Record for resolution, tagged with type scope. Note this is `is_call_target`, NOT
// `is_callable`: a bodiless trait method is callable-shaped (it earned the `()` label
// and the `method` edge above) but must not compete with its own implementations for
// the name — see `Classifier::is_call_target`.
if classifier.is_call_target(&child, kind)
&& let Some(n) = name.clone()
{
facts.callables.push(Callable {
name: n,
node_id: node_id.clone(),
Expand Down Expand Up @@ -199,6 +204,33 @@ impl Classifier<'_> {
}
}

/// Whether a definition is a **call target** — something a call site may resolve to.
///
/// This is deliberately narrower than "is callable-shaped". A callable-shaped def gets the `()`
/// label suffix and a `method` edge; a *call target* additionally competes for a name in
/// [`super::resolve::resolve`]. A trait method DECLARATION is the first but not the second: a
/// call dispatches to an implementation, never to the declaration.
///
/// Keeping declarations out of the candidate set is load-bearing, not tidiness. The resolver is
/// precision-favouring — several same-named candidates with no disambiguating qualifier are
/// dropped, not fanned out. A trait with exactly one impl would otherwise go from one candidate
/// to two the moment declarations were indexed, and every call to that method would stop
/// resolving. Fixing the missing-symbol bug would then have silently deleted `calls` edges.
fn is_call_target(self, node: &Node<'_>, kind: &str) -> bool {
if !matches!(kind, "function" | "method") {
return false;
}
match self {
// The one Rust node kind that is callable-shaped but bodiless.
Classifier::Rust => node.kind() != "function_signature_item",
// Tags-driven languages capture definitions, and an abstract/interface method that the
// grammar's `tags.scm` reports is still reported as a definition; leave them as targets
// rather than guess per-grammar. Revisit if a language shows the same duplicate-name
// regression Rust would have had.
Classifier::Tagged(_) => true,
}
}

/// The type name a container def introduces for its children — used only to disambiguate several
/// same-named callables (e.g. two classes each with `run`). `None` for non-containers.
fn container_scope(
Expand Down Expand Up @@ -329,9 +361,7 @@ mod tests {

#[test]
fn trait_container_emits_a_method_edge_for_its_fn() {
// A default-bodied trait method: interesting_node only classifies `function_item`, which is
// what a body-bearing method parses as. (A signature-only `fn f(&self);` parses as
// `function_signature_item` and is NOT classified at all — see the crate-level bug report.)
// A default-bodied trait method (`function_item`).
let src = "trait T {\n fn f(&self) {}\n}\n";
let facts = facts_for("rust", "src/t.rs", src);
let trait_node = facts.nodes.iter().find(|n| n.label == "T").unwrap();
Expand All @@ -345,6 +375,72 @@ mod tests {
);
}

#[test]
fn signature_only_trait_method_is_a_node_with_a_method_edge() {
// REGRESSION: a trait method with no default body parses as `function_signature_item`, not
// `function_item`, and used to be classified as nothing at all — so a trait interface
// contributed zero callable symbols to the graph or to semantic search.
let src = "trait T {\n fn f(&self);\n}\n";
let facts = facts_for("rust", "src/t.rs", src);
let trait_node = facts.nodes.iter().find(|n| n.label == "T").unwrap();
let f = facts
.nodes
.iter()
.find(|n| n.label == "f()")
.expect("signature-only trait method must produce a node");
assert_eq!(f.start_line, 2);
assert!(
facts.contains.iter().any(|e| e.relation == "method"
&& e.source == trait_node.node_id
&& e.target == f.node_id),
"trait → f must be a `method` edge; got {:?}",
facts.contains
);
}

#[test]
fn signature_only_trait_method_is_not_a_call_target() {
// It is callable-SHAPED (it earned the `()` label and the `method` edge above) but must not
// enter the resolver's candidate set: a call dispatches to an implementation, never to the
// declaration. See `Classifier::is_call_target`.
let src = "trait T {\n fn f(&self);\n}\n";
let facts = facts_for("rust", "src/t.rs", src);
assert!(
facts.callables.is_empty(),
"a bodiless declaration must not be a call target; got {:?}",
facts.callables
);
}

#[test]
fn a_bodied_method_next_to_its_declaration_is_still_the_only_call_target() {
// The whole reason declarations are excluded from the candidate set. A trait with exactly
// ONE impl would otherwise go from one candidate to two the moment declarations were
// indexed, and the precision-favouring resolver would drop every call to that method —
// so fixing the missing-symbol bug would have silently deleted `calls` edges.
let src = "trait T {\n fn f(&self);\n}\n\
struct S;\n\
impl T for S {\n fn f(&self) {}\n}\n";
let facts = facts_for("rust", "src/t.rs", src);
let targets: Vec<_> = facts.callables.iter().map(|c| c.node_id.as_str()).collect();
assert_eq!(
targets.len(),
1,
"exactly one call target (the impl) must survive; got {targets:?}"
);
assert!(
targets[0].ends_with(":f"),
"the surviving target must be the bodied impl method; got {targets:?}"
);
// Both defs still exist as nodes — the declaration is indexed, just not dispatched to.
assert_eq!(
facts.nodes.iter().filter(|n| n.label == "f()").count(),
2,
"declaration and implementation must BOTH be nodes; got {:?}",
facts.nodes
);
}

#[test]
fn tagged_class_container_emits_a_method_edge_for_its_method() {
let src = "class C:\n def m(self):\n pass\n";
Expand Down
50 changes: 50 additions & 0 deletions src/graph/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -651,3 +651,53 @@ class Builder { Object build() { return Circle.make(); } }
"→ Circle.make only"
);
}

#[test]
fn a_call_to_a_single_impl_trait_method_still_resolves_to_the_impl() {
// The end-to-end guard for the decision in `Classifier::is_call_target`. Indexing trait method
// DECLARATIONS as nodes (so trait interfaces are searchable) must not make them compete with
// their own implementations for the name: the resolver is precision-favouring and drops a bare
// name matching several candidates. If declarations were call targets, this call would go from
// one candidate to two and the `calls` edge below would silently disappear.
let g = graph_of(&[
(
"src/shape.rs",
"trait Shape {\n fn describe(&self) -> f64;\n}\n",
),
(
"src/circle.rs",
"struct Circle;\nimpl Shape for Circle {\n fn describe(&self) -> f64 {\n 1.0\n }\n}\n",
),
(
"src/main.rs",
"fn run(c: &Circle) -> f64 {\n c.describe()\n}\n",
),
]);

// Both the declaration and the implementation are indexed as nodes.
let describes: Vec<&GraphNode> = g.nodes.iter().filter(|n| n.label == "describe()").collect();
assert_eq!(
describes.len(),
2,
"declaration and implementation must both be nodes; got {describes:?}"
);
assert!(describes.iter().any(|n| n.source_file == "src/shape.rs"));

// ...but the call resolves, and resolves to the IMPLEMENTATION, not the declaration.
let caller = node(&g, "run()").node_id.clone();
let targets: Vec<&str> = g
.edges
.iter()
.filter(|e| e.relation == "calls" && e.source == caller)
.map(|e| e.target.as_str())
.collect();
assert_eq!(
targets.len(),
1,
"the call must still resolve to exactly one target; got {targets:?}"
);
assert!(
targets[0].starts_with("src/circle.rs"),
"must resolve to the impl in circle.rs, not the declaration; got {targets:?}"
);
}
16 changes: 16 additions & 0 deletions src/walk.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,13 @@ pub struct WalkStats {
pub paths_ignored: usize,
pub pdfs_extracted: usize,
pub pdfs_skipped: usize,
/// Files rejected as binary by the content sniff (a NUL byte in the first 512 bytes), despite
/// decoding as valid UTF-8 and carrying an indexable extension.
///
/// Counted rather than silently dropped: without it, "this repo has fewer indexable files than
/// I expected" and "a generated/binary asset is misnamed with a source extension" look exactly
/// the same from the summary line — and the second is an actionable repo problem.
pub files_skipped_binary: usize,
}

/// The output of a walk: chunks to embed and the resolved structural graph.
Expand Down Expand Up @@ -157,6 +164,14 @@ pub fn walk_checkout(root: &Path, options: &WalkOptions) -> anyhow::Result<WalkO
if source.len() > MAX_FILE_BYTES {
continue; // over the byte cap
}
// Binary content, caught by CONTENT rather than encoding: NUL is a legal Unicode scalar, so
// a binary blob can decode as valid UTF-8 above and still be junk. Rejected here — before
// either consumer — so the graph never ingests it either, not just the chunker.
if chunk::is_binary(&source) {
tracing::debug!(path = %rel_path, "codegraph: binary content, skipped");
stats.files_skipped_binary += 1;
continue;
}

let file_chunks = if options.build_graph && lang::has_graph(language) {
// Parse ONCE and feed both the chunker and the graph builder (ADR-0086 "parse once").
Expand Down Expand Up @@ -193,6 +208,7 @@ pub fn walk_checkout(root: &Path, options: &WalkOptions) -> anyhow::Result<WalkO
graph_nodes = graph.nodes.len(),
graph_edges = graph.edges.len(),
paths_ignored = stats.paths_ignored,
files_skipped_binary = stats.files_skipped_binary,
pdfs_extracted = stats.pdfs_extracted,
pdfs_skipped = stats.pdfs_skipped,
"codegraph: walk complete"
Expand Down
11 changes: 11 additions & 0 deletions tests/golden/sample-repo.graph.json
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,12 @@
"source_file": "src/shapes.rs",
"start_line": 30
},
{
"node_id": "src/shapes.rs#31:describe",
"label": "describe()",
"source_file": "src/shapes.rs",
"start_line": 31
},
{
"node_id": "src/shapes.rs#34:impl",
"label": "impl",
Expand Down Expand Up @@ -196,6 +202,11 @@
"target": "src/shapes.rs#25:new",
"relation": "method"
},
{
"source": "src/shapes.rs#30:Shape",
"target": "src/shapes.rs#31:describe",
"relation": "method"
},
{
"source": "src/shapes.rs#34:impl",
"target": "src/shapes.rs#35:describe",
Expand Down
Loading