diff --git a/apps/cli/src/commands/environment.ts b/apps/cli/src/commands/environment.ts index 17d949c945..a47d7b44e3 100644 --- a/apps/cli/src/commands/environment.ts +++ b/apps/cli/src/commands/environment.ts @@ -39,6 +39,8 @@ interface EnvironmentBranchesCommandOptions { interface EnvironmentPathsCommandOptions { directories?: boolean; files?: boolean; + /** Commander `--no-hidden`: true unless the flag is passed. */ + hidden?: boolean; json?: boolean; limit?: string; query?: string; @@ -417,6 +419,7 @@ export function registerEnvironmentCommands( .option("--limit ", "Maximum paths") .option("--files", "Include files") .option("--directories", "Include directories") + .option("--no-hidden", "Exclude dot-prefixed paths") .option("--json", "Print machine-readable JSON output") .action( action(async (id: string, opts: EnvironmentPathsCommandOptions) => { @@ -428,6 +431,7 @@ export function registerEnvironmentCommands( environmentId: id, includeFiles: booleanQueryValue(includeFiles), includeDirectories: booleanQueryValue(includeDirectories), + ...(opts.hidden === false ? { includeHidden: "false" } : {}), ...(opts.query !== undefined ? { query: opts.query } : {}), ...(opts.limit !== undefined ? { limit: opts.limit } : {}), }); diff --git a/apps/cli/src/commands/file.ts b/apps/cli/src/commands/file.ts index 17edee22cc..421df96a58 100644 --- a/apps/cli/src/commands/file.ts +++ b/apps/cli/src/commands/file.ts @@ -12,6 +12,8 @@ interface FileTargetOptions { interface FileListOptions extends FileTargetOptions { directories?: boolean; files?: boolean; + /** Commander `--no-hidden`: true unless the flag is passed. */ + hidden?: boolean; limit?: string; query?: string; } @@ -143,6 +145,7 @@ export function registerFileCommands( .option("--limit ", "Maximum entries") .option("--files", "Include files") .option("--directories", "Include directories") + .option("--no-hidden", "Exclude dot-prefixed paths") .option("--host ", "Machine ID") .option("--json", "Print machine-readable JSON output") .action( @@ -154,6 +157,7 @@ export function registerFileCommands( path, includeFiles, includeDirectories, + ...(opts.hidden === false ? { includeHidden: false } : {}), ...(opts.host ? { hostId: opts.host } : {}), ...(opts.query ? { query: opts.query } : {}), ...(limit ? { limit } : {}), diff --git a/apps/cli/src/commands/project.ts b/apps/cli/src/commands/project.ts index f0c9ddbc67..af1570cda7 100644 --- a/apps/cli/src/commands/project.ts +++ b/apps/cli/src/commands/project.ts @@ -49,6 +49,8 @@ interface ProjectReorderCommandOptions { interface ProjectDiscoveryCommandOptions { environment?: string; host?: string; + /** Commander `--no-hidden` on `paths`: true unless the flag is passed. */ + hidden?: boolean; machine?: string; json?: boolean; limit?: string; @@ -408,6 +410,7 @@ export function registerProjectCommands( .description("Search project workspace files and directories") .option("--query ", "Fuzzy path query") .option("--limit ", "Maximum paths") + .option("--no-hidden", "Exclude dot-prefixed paths") .option("--json", "Print machine-readable JSON output") .action( action(async (id: string, opts: ProjectDiscoveryCommandOptions) => { @@ -417,6 +420,7 @@ export function registerProjectCommands( ...(await resolveMachineEnvironmentRouting(opts, serverUrl)), includeFiles: "true", includeDirectories: "true", + ...(opts.hidden === false ? { includeHidden: "false" } : {}), ...(opts.query ? { query: opts.query } : {}), ...(opts.limit ? { limit: opts.limit } : {}), }); diff --git a/apps/host-daemon/src/command-handlers/file-list.test.ts b/apps/host-daemon/src/command-handlers/file-list.test.ts index 5ea132ca90..ec8f0d2dd0 100644 --- a/apps/host-daemon/src/command-handlers/file-list.test.ts +++ b/apps/host-daemon/src/command-handlers/file-list.test.ts @@ -1,14 +1,38 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { finalizeListedFiles, finalizeListedPaths, listPathsRecursively, + listRootPaths, normalizeListedPath, } from "./file-list.js"; +const execFileAsync = promisify(execFile); + +async function runGit(args: string[], cwd: string): Promise { + await execFileAsync("git", args, { cwd }); +} + +async function initGitRepo(root: string): Promise { + await runGit(["init", "-q", "-b", "main"], root); + await runGit(["config", "user.name", "BB Tests"], root); + await runGit(["config", "user.email", "bb@example.com"], root); + await runGit(["config", "commit.gpgsign", "false"], root); +} + +const WALK_ALL = { + includeFiles: true, + includeDirectories: true, + includeHidden: true, + excludeNames: new Set(), + maxEntries: 50_000, +}; + describe("finalizeListedFiles", () => { it("preserves walk order for an empty query", () => { const result = finalizeListedFiles({ @@ -200,61 +224,94 @@ describe("finalizeListedPaths", () => { }); describe("listPathsRecursively", () => { + let root: string; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), "bb-file-list-")); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + it("returns slash-separated relative paths for nested entries", async () => { - const root = await fs.mkdtemp(path.join(os.tmpdir(), "bb-file-list-")); - try { - await fs.mkdir(path.join(root, "src", "components"), { - recursive: true, - }); - await fs.writeFile( - path.join(root, "src", "components", "Button.tsx"), - "", - ); - - const result = await listPathsRecursively({ - dir: root, - root, - includeFiles: true, - includeDirectories: true, - }); - - expect(result).toEqual([ - { kind: "directory", path: "src", name: "src" }, - { - kind: "directory", - path: "src/components", - name: "components", - }, - { - kind: "file", - path: "src/components/Button.tsx", - name: "Button.tsx", - }, - ]); - } finally { - await fs.rm(root, { recursive: true, force: true }); - } + await fs.mkdir(path.join(root, "src", "components"), { recursive: true }); + await fs.writeFile(path.join(root, "src", "components", "Button.tsx"), ""); + + const result = await listPathsRecursively({ ...WALK_ALL, dir: root, root }); + + expect(result.paths).toEqual([ + { kind: "directory", path: "src", name: "src" }, + { kind: "directory", path: "src/components", name: "components" }, + { kind: "file", path: "src/components/Button.tsx", name: "Button.tsx" }, + ]); + expect(result.truncated).toBe(false); }); it("does not return symlinked files as regular path entries", async () => { - const root = await fs.mkdtemp(path.join(os.tmpdir(), "bb-file-list-")); - try { - await fs.writeFile(path.join(root, "state.json"), "{}"); - await fs.symlink(path.join(root, "state.json"), path.join(root, "logo.svg")); - - const result = await listPathsRecursively({ - dir: root, - root, - includeFiles: true, - includeDirectories: false, - }); - - expect(result).toEqual([ - { kind: "file", path: "state.json", name: "state.json" }, - ]); - } finally { - await fs.rm(root, { recursive: true, force: true }); + await fs.writeFile(path.join(root, "state.json"), "{}"); + await fs.symlink(path.join(root, "state.json"), path.join(root, "logo.svg")); + + const result = await listPathsRecursively({ + ...WALK_ALL, + dir: root, + root, + includeDirectories: false, + }); + + expect(result.paths).toEqual([ + { kind: "file", path: "state.json", name: "state.json" }, + ]); + }); + + it("walks dot paths when hidden entries are included but never .git (#2093)", async () => { + await fs.mkdir(path.join(root, ".github", "workflows"), { recursive: true }); + await fs.writeFile(path.join(root, ".github", "workflows", "ci.yml"), ""); + await fs.mkdir(path.join(root, ".git"), { recursive: true }); + await fs.writeFile(path.join(root, ".git", "config"), "[core]\n"); + await fs.writeFile(path.join(root, "AGENTS.md"), ""); + + const result = await listPathsRecursively({ ...WALK_ALL, dir: root, root }); + const paths = result.paths.map((entry) => entry.path); + + expect(paths).toContain("AGENTS.md"); + expect(paths).toContain(".github/workflows/ci.yml"); + expect(paths).not.toContain(".git"); + expect(paths).not.toContain(".git/config"); + }); + + it("applies includeHidden and excludeNames at the entry, pruning the subtree", async () => { + await fs.mkdir(path.join(root, ".github"), { recursive: true }); + await fs.writeFile(path.join(root, ".github", "ci.yml"), ""); + await fs.mkdir(path.join(root, "node_modules", "pkg"), { recursive: true }); + await fs.writeFile(path.join(root, "node_modules", "pkg", "index.js"), ""); + await fs.writeFile(path.join(root, "index.ts"), ""); + + const result = await listPathsRecursively({ + ...WALK_ALL, + dir: root, + root, + includeHidden: false, + excludeNames: new Set(["node_modules"]), + }); + + expect(result.paths.map((entry) => entry.path)).toEqual(["index.ts"]); + }); + + it("stops at the entry cap and reports truncation", async () => { + for (let index = 0; index < 6; index += 1) { + await fs.writeFile(path.join(root, `file-${index}.txt`), ""); } + + const result = await listPathsRecursively({ + ...WALK_ALL, + dir: root, + root, + maxEntries: 4, + }); + + expect(result.paths).toHaveLength(4); + expect(result.truncated).toBe(true); }); it("normalizes Windows separators before returning paths", () => { @@ -263,3 +320,160 @@ describe("listPathsRecursively", () => { ); }); }); + +describe("listRootPaths", () => { + let root: string; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), "bb-file-list-git-")); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + async function seedRepo(): Promise { + await initGitRepo(root); + await fs.mkdir(path.join(root, ".github", "workflows"), { recursive: true }); + await fs.writeFile(path.join(root, ".github", "workflows", "ci.yml"), ""); + await fs.mkdir(path.join(root, "src"), { recursive: true }); + await fs.writeFile(path.join(root, "src", "index.ts"), ""); + await fs.writeFile(path.join(root, ".gitignore"), ".venv/\n.env\n"); + await fs.mkdir(path.join(root, ".venv", "lib", "site-packages"), { + recursive: true, + }); + await fs.writeFile( + path.join(root, ".venv", "lib", "site-packages", "config.py"), + "", + ); + await fs.writeFile(path.join(root, ".env"), "SECRET=1\n"); + await fs.mkdir(path.join(root, "node_modules", "pkg"), { recursive: true }); + await fs.writeFile(path.join(root, "node_modules", "pkg", "index.js"), ""); + } + + it("lists tracked and untracked-not-ignored paths from git, synthesising directories", async () => { + await seedRepo(); + await runGit(["add", ".github", "src", ".gitignore"], root); + await runGit(["commit", "-q", "-m", "init"], root); + await fs.writeFile(path.join(root, "untracked.md"), ""); + + const result = await listRootPaths({ + root, + includeFiles: true, + includeDirectories: true, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, + }); + + expect(result.paths).toEqual([ + { kind: "directory", path: ".github", name: ".github" }, + { kind: "directory", path: ".github/workflows", name: "workflows" }, + { kind: "file", path: ".github/workflows/ci.yml", name: "ci.yml" }, + { kind: "file", path: ".gitignore", name: ".gitignore" }, + { kind: "directory", path: "src", name: "src" }, + { kind: "file", path: "src/index.ts", name: "index.ts" }, + { kind: "file", path: "untracked.md", name: "untracked.md" }, + ]); + expect(result.truncated).toBe(false); + }); + + it("keeps gitignored trees, node_modules and .git out of the listing", async () => { + await seedRepo(); + + const result = await listRootPaths({ + root, + includeFiles: true, + includeDirectories: true, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, + }); + const paths = result.paths.map((entry) => entry.path); + + expect(paths).toContain(".github/workflows/ci.yml"); + expect(paths.filter((entry) => entry.startsWith(".venv"))).toEqual([]); + expect(paths).not.toContain(".env"); + expect(paths.filter((entry) => entry.startsWith("node_modules"))).toEqual( + [], + ); + expect(paths.filter((entry) => entry.startsWith(".git/"))).toEqual([]); + expect(paths).not.toContain(".git"); + }); + + it("hides dot paths from the git listing when includeHidden is false", async () => { + await seedRepo(); + + const result = await listRootPaths({ + root, + includeFiles: true, + includeDirectories: false, + includeHidden: false, + excludeNames: [], + respectGitignore: true, + }); + + expect(result.paths.map((entry) => entry.path)).toEqual([ + "node_modules/pkg/index.js", + "src/index.ts", + ]); + }); + + it("walks the disk when gitignore is not to be respected", async () => { + await seedRepo(); + + const result = await listRootPaths({ + root, + includeFiles: true, + includeDirectories: false, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: false, + }); + const paths = result.paths.map((entry) => entry.path); + + expect(paths).toContain(".venv/lib/site-packages/config.py"); + expect(paths).toContain(".env"); + expect(paths).toContain(".github/workflows/ci.yml"); + expect(paths.filter((entry) => entry.startsWith(".git/"))).toEqual([]); + }); + + it("falls back to the disk walk outside a git worktree", async () => { + await fs.mkdir(path.join(root, ".github"), { recursive: true }); + await fs.writeFile(path.join(root, ".github", "ci.yml"), ""); + await fs.writeFile(path.join(root, "notes.md"), ""); + + const result = await listRootPaths({ + root, + includeFiles: true, + includeDirectories: false, + includeHidden: true, + excludeNames: [], + respectGitignore: true, + }); + + expect(result.paths.map((entry) => entry.path).sort()).toEqual([ + ".github/ci.yml", + "notes.md", + ]); + }); + + it("falls back to the disk walk when the root itself is gitignored", async () => { + await initGitRepo(root); + await fs.writeFile(path.join(root, ".gitignore"), "scratch/\n"); + await fs.mkdir(path.join(root, "scratch"), { recursive: true }); + await fs.writeFile(path.join(root, "scratch", "notes.md"), ""); + + const result = await listRootPaths({ + root: path.join(root, "scratch"), + includeFiles: true, + includeDirectories: false, + includeHidden: true, + excludeNames: [], + respectGitignore: true, + }); + + expect(result.paths.map((entry) => entry.path)).toEqual(["notes.md"]); + }); +}); + diff --git a/apps/host-daemon/src/command-handlers/file-list.ts b/apps/host-daemon/src/command-handlers/file-list.ts index 43120eee98..8a24445394 100644 --- a/apps/host-daemon/src/command-handlers/file-list.ts +++ b/apps/host-daemon/src/command-handlers/file-list.ts @@ -4,7 +4,18 @@ import { fuzzyMatchPaths } from "@bb/fuzzy-match"; import type { HostPathEntry, HostPathEntryKind, + PathListPolicy, } from "@bb/host-daemon-contract"; +import { WorkspaceError, runGitWithNullRecordLimit } from "@bb/host-workspace"; + +/** + * Hard ceiling on the entries one listing call may produce, whichever source + * feeds it. A bare host folder under `$HOME` or a non-git tree with a `.venv` + * must not pin the daemon; callers see `truncated: true` instead. + */ +export const PATH_LIST_ENTRY_LIMIT = 50_000; + +const GIT_LS_FILES_TIMEOUT_MS = 15_000; interface FinalizeListedFilesArgs { filePaths: string[]; @@ -44,9 +55,22 @@ interface FinalizedPathList { truncated: boolean; } +interface ListRootPathsArgs extends PathListInclusion, PathListPolicy { + root: string; +} + interface ListPathsRecursivelyArgs extends PathListInclusion { dir: string; root: string; + includeHidden: boolean; + excludeNames: ReadonlySet; + maxEntries: number; +} + +interface ListedPathList { + paths: ListedPath[]; + /** True when the walk stopped at `maxEntries` before the tree was exhausted. */ + truncated: boolean; } function shouldIncludePath( @@ -134,57 +158,201 @@ export function finalizeListedPaths( }; } +function isHiddenName(name: string): boolean { + return name.startsWith("."); +} + +/** + * Readdir walk. Applies the caller's policy at each directory entry, never + * descends `.git` (that is not a product decision: nothing should list the + * object store), skips symlinks so a link cycle cannot loop the walk, and + * stops at `maxEntries`. + */ export async function listPathsRecursively( args: ListPathsRecursivelyArgs, -): Promise { - const entries = await fs.readdir(args.dir, { withFileTypes: true }); +): Promise { const results: ListedPath[] = []; - for (const entry of entries) { - if (entry.name.startsWith(".")) continue; - if (entry.name === "node_modules") continue; - if (entry.isSymbolicLink()) continue; - - const fullPath = path.join(args.dir, entry.name); - const relativePath = normalizeListedPath( - path.relative(args.root, fullPath), - ); - if (entry.isDirectory()) { - if (args.includeDirectories) { + // Counts every entry the walk visits, listed or not, so the bound is on + // the work done rather than on what the caller asked to see. + let visited = 0; + let truncated = false; + + async function walk(dir: string): Promise { + const entries = await fs.readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + if (truncated) return; + if (entry.name === ".git") continue; + if (!args.includeHidden && isHiddenName(entry.name)) continue; + if (args.excludeNames.has(entry.name)) continue; + if (entry.isSymbolicLink()) continue; + visited += 1; + if (visited > args.maxEntries) { + truncated = true; + return; + } + + const fullPath = path.join(dir, entry.name); + const relativePath = normalizeListedPath( + path.relative(args.root, fullPath), + ); + if (entry.isDirectory()) { + if (args.includeDirectories) { + results.push({ + kind: "directory", + path: relativePath, + name: entry.name, + }); + } + await walk(fullPath); + continue; + } + + if (args.includeFiles) { results.push({ - kind: "directory", + kind: "file", path: relativePath, name: entry.name, }); } - results.push( - ...(await listPathsRecursively({ - ...args, - dir: fullPath, - })), - ); - continue; + } + } + + await walk(args.dir); + return { paths: results, truncated }; +} + +interface ListGitWorktreePathsArgs extends PathListInclusion { + root: string; + includeHidden: boolean; + excludeNames: ReadonlySet; + maxEntries: number; +} + +function isExcludedGitPath( + segments: string[], + args: Pick, +): boolean { + return segments.some( + (segment) => + (!args.includeHidden && isHiddenName(segment)) || + args.excludeNames.has(segment), + ); +} + +/** + * Candidate list for a root inside a git worktree: tracked plus + * untracked-not-ignored files from `git ls-files`, relative to `root` (git + * scopes the listing to the cwd, so a project rooted in a repo subdirectory + * lists only its own subtree). Directory entries are synthesised from the + * file paths because git does not track directories; an empty directory + * therefore does not appear, and a submodule shows as one file-kind entry. + * + * Returns `null` when `root` is not inside a git worktree, or when git lists + * nothing — either the repository is empty (the readdir fallback then finds + * nothing either) or the root itself is ignored, in which case the caller + * asked for a listing of an ignored tree and should get the readdir walk. + */ +async function listGitWorktreePaths( + args: ListGitWorktreePathsArgs, +): Promise { + let result; + try { + result = await runGitWithNullRecordLimit( + ["ls-files", "-z", "--cached", "--others", "--exclude-standard"], + { + cwd: args.root, + allowFailure: true, + timeoutMs: GIT_LS_FILES_TIMEOUT_MS, + }, + "single", + args.maxEntries + 1, + ); + } catch (error) { + // No git on this host, or git stalled: the readdir walk still works. + if (error instanceof WorkspaceError) return null; + throw error; + } + if (result.exitCode !== 0 || result.recordCount === 0) { + return null; + } + + const paths: ListedPath[] = []; + const seenDirectories = new Set(); + let truncated = result.recordLimitReached; + + // git prints the cached and the untracked sets as two separately sorted + // runs; one sorted pass keeps the unqueried listing stable and every + // synthesised directory ahead of its children. + const records = result.stdout + .split("\0") + .filter((record) => record.length > 0) + .sort(); + for (const record of records) { + if (paths.length >= args.maxEntries) { + truncated = true; + break; + } + const segments = record.split("/"); + if (isExcludedGitPath(segments, args)) continue; + + if (args.includeDirectories) { + for (let depth = 1; depth < segments.length; depth += 1) { + const directoryPath = segments.slice(0, depth).join("/"); + if (seenDirectories.has(directoryPath)) continue; + seenDirectories.add(directoryPath); + if (paths.length >= args.maxEntries) { + truncated = true; + break; + } + paths.push({ + kind: "directory", + path: directoryPath, + name: segments[depth - 1] ?? directoryPath, + }); + } + if (truncated) break; } if (args.includeFiles) { - results.push({ + paths.push({ kind: "file", - path: relativePath, - name: entry.name, + path: record, + name: segments[segments.length - 1] ?? record, }); } } - return results; + + return { paths, truncated }; } -export async function listFilesRecursively( - dir: string, - root: string, -): Promise { - const paths = await listPathsRecursively({ - dir, - root, - includeFiles: true, - includeDirectories: false, +/** + * List every path under `root` according to the server-supplied policy. + * `respectGitignore` prefers the git candidate list (see + * `listGitWorktreePaths`); everything else, and every non-git root, takes the + * capped readdir walk. + */ +export async function listRootPaths( + args: ListRootPathsArgs, +): Promise { + const excludeNames = new Set(args.excludeNames); + if (args.respectGitignore) { + const gitListed = await listGitWorktreePaths({ + root: args.root, + includeFiles: args.includeFiles, + includeDirectories: args.includeDirectories, + includeHidden: args.includeHidden, + excludeNames, + maxEntries: PATH_LIST_ENTRY_LIMIT, + }); + if (gitListed !== null) return gitListed; + } + return listPathsRecursively({ + dir: args.root, + root: args.root, + includeFiles: args.includeFiles, + includeDirectories: args.includeDirectories, + includeHidden: args.includeHidden, + excludeNames, + maxEntries: PATH_LIST_ENTRY_LIMIT, }); - return paths.map((pathEntry) => pathEntry.path); } diff --git a/apps/host-daemon/src/command-handlers/host-files.test.ts b/apps/host-daemon/src/command-handlers/host-files.test.ts index d6dd10a6e3..00f1479ee7 100644 --- a/apps/host-daemon/src/command-handlers/host-files.test.ts +++ b/apps/host-daemon/src/command-handlers/host-files.test.ts @@ -11,6 +11,8 @@ import { } from "../command-dispatch-support.js"; import { browseHostDirectory, + listHostFiles, + listHostPaths, readHostFile, readHostFileMetadata, readHostRelativeFile, @@ -239,6 +241,88 @@ describe("readHostFileMetadata", () => { }); }); +describe("listHostPaths / listHostFiles (#2093)", () => { + const workspacePolicy = { + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, + }; + + async function seedWorkspace(): Promise { + const root = await initRepo(); + await fs.mkdir(path.join(root, ".github", "workflows"), { recursive: true }); + await fs.writeFile(path.join(root, ".github", "workflows", "ci.yml"), "name: ci\n"); + await fs.writeFile(path.join(root, "AGENTS.md"), "# agents\n"); + await fs.writeFile(path.join(root, ".gitignore"), ".venv/\n"); + await fs.mkdir(path.join(root, ".venv", "lib"), { recursive: true }); + await fs.writeFile(path.join(root, ".venv", "lib", "ci.yml"), ""); + await runGit(["add", "-A"], { cwd: root }); + await runGit(["commit", "-q", "-m", "init"], { cwd: root }); + return root; + } + + it("finds the same .github/workflows/ci.yml that host.read_file serves", async () => { + const root = await seedWorkspace(); + + const search = await listHostPaths({ + type: "host.list_paths", + path: root, + query: "ci.yml", + limit: 5, + includeFiles: true, + includeDirectories: false, + ...workspacePolicy, + }); + const read = await readHostFile({ + type: "host.read_file", + path: path.join(root, ".github", "workflows", "ci.yml"), + }); + + expect(read.content).toBe("name: ci\n"); + expect(search.paths.map((entry) => entry.path)).toEqual([ + ".github/workflows/ci.yml", + ]); + expect(search.truncated).toBe(false); + }); + + it("keeps gitignored trees out of the full listing while listing dot paths", async () => { + const root = await seedWorkspace(); + + const result = await listHostPaths({ + type: "host.list_paths", + path: root, + limit: 100, + includeFiles: true, + includeDirectories: true, + ...workspacePolicy, + }); + + expect(result.paths.map((entry) => entry.path)).toEqual([ + ".github", + ".github/workflows", + ".github/workflows/ci.yml", + ".gitignore", + "AGENTS.md", + ]); + }); + + it("hides dot entries from host.list_files when told to", async () => { + const root = await seedWorkspace(); + await fs.writeFile(path.join(root, ".DS_Store"), ""); + + const result = await listHostFiles({ + type: "host.list_files", + path: root, + limit: 100, + includeHidden: false, + excludeNames: [], + respectGitignore: false, + }); + + expect(result.files.map((entry) => entry.path)).toEqual(["AGENTS.md"]); + }); +}); + describe("browseHostDirectory", () => { it("lists immediate children sorted directories-first, hiding noise", async () => { const root = await makeTempDir("bb-browse-"); diff --git a/apps/host-daemon/src/command-handlers/host-files.ts b/apps/host-daemon/src/command-handlers/host-files.ts index 1efcb9c7bf..b684f07d72 100644 --- a/apps/host-daemon/src/command-handlers/host-files.ts +++ b/apps/host-daemon/src/command-handlers/host-files.ts @@ -12,8 +12,7 @@ import { isFsErrorWithCode } from "../fs-errors.js"; import { finalizeListedFiles, finalizeListedPaths, - listFilesRecursively, - listPathsRecursively, + listRootPaths, } from "./file-list.js"; import { readFileForTransport, @@ -72,11 +71,23 @@ export async function listHostFiles( path: command.path, }); - return finalizeListedFiles({ - filePaths: await listFilesRecursively(realRootPath, realRootPath), + const listed = await listRootPaths({ + root: realRootPath, + includeFiles: true, + includeDirectories: false, + includeHidden: command.includeHidden, + excludeNames: command.excludeNames, + respectGitignore: command.respectGitignore, + }); + const result = finalizeListedFiles({ + filePaths: listed.paths.map((pathEntry) => pathEntry.path), limit: command.limit, ...(command.query ? { query: command.query } : {}), }); + return { + files: result.files, + truncated: result.truncated || listed.truncated, + }; } catch (error) { if (isFsErrorWithCode(error, "ENOENT")) { return { files: [], truncated: false }; @@ -98,18 +109,25 @@ export async function listHostPaths( path: command.path, }); - return finalizeListedPaths({ - paths: await listPathsRecursively({ - dir: realRootPath, - root: realRootPath, - includeFiles: command.includeFiles, - includeDirectories: command.includeDirectories, - }), + const listed = await listRootPaths({ + root: realRootPath, + includeFiles: command.includeFiles, + includeDirectories: command.includeDirectories, + includeHidden: command.includeHidden, + excludeNames: command.excludeNames, + respectGitignore: command.respectGitignore, + }); + const result = finalizeListedPaths({ + paths: listed.paths, limit: command.limit, includeFiles: command.includeFiles, includeDirectories: command.includeDirectories, ...(command.query ? { query: command.query } : {}), }); + return { + paths: result.paths, + truncated: result.truncated || listed.truncated, + }; } catch (error) { if (isFsErrorWithCode(error, "ENOENT")) { return { paths: [], truncated: false }; diff --git a/apps/host-daemon/test/command/dispatch-helpers.ts b/apps/host-daemon/test/command/dispatch-helpers.ts index 3f8038ff29..6dc4f75193 100644 --- a/apps/host-daemon/test/command/dispatch-helpers.ts +++ b/apps/host-daemon/test/command/dispatch-helpers.ts @@ -26,7 +26,7 @@ import type { PullRequestActionOptions, } from "@bb/host-workspace"; import { RuntimeManager } from "../../src/runtime-manager.js"; -import { listFilesRecursively } from "../../src/command-handlers/file-list.js"; +import { listRootPaths } from "../../src/command-handlers/file-list.js"; import { noopEventSink } from "../../src/command-dispatch-support.js"; import type { CommandDispatchOptions } from "../../src/command-dispatch-support.js"; import type { FetchProjectAttachment } from "../../src/project-attachments.js"; @@ -220,7 +220,15 @@ export function createFakeWorkspace(pathname: string) { state.lastPullRequestAction = action; }, async listFiles() { - return listFilesRecursively(pathname, pathname); + const listed = await listRootPaths({ + root: pathname, + includeFiles: true, + includeDirectories: false, + includeHidden: false, + excludeNames: ["node_modules"], + respectGitignore: false, + }); + return listed.paths.map((pathEntry) => pathEntry.path); }, async commit(options: { message: string; noVerify: boolean }) { state.lastCommitMessage = options.message; diff --git a/apps/host-daemon/test/command/workspace-dispatch.test.ts b/apps/host-daemon/test/command/workspace-dispatch.test.ts index dfb2c1e06c..aaec73c71a 100644 --- a/apps/host-daemon/test/command/workspace-dispatch.test.ts +++ b/apps/host-daemon/test/command/workspace-dispatch.test.ts @@ -401,6 +401,9 @@ describe("workspace command dispatch", () => { type: "host.list_files", path: tempDir, limit: 1000, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, }, harness.dispatchOptions(), ); @@ -423,6 +426,9 @@ describe("workspace command dispatch", () => { limit: 1000, includeFiles: true, includeDirectories: true, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, }, harness.dispatchOptions(), ); @@ -452,6 +458,9 @@ describe("workspace command dispatch", () => { type: "host.list_files", path: missingPath, limit: 1000, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, }, harness.dispatchOptions(), ); @@ -472,6 +481,9 @@ describe("workspace command dispatch", () => { limit: 1000, includeFiles: true, includeDirectories: true, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, }, harness.dispatchOptions(), ); @@ -496,6 +508,9 @@ describe("workspace command dispatch", () => { type: "host.list_files", path: symlinkRoot, limit: 1000, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, }, harness.dispatchOptions(), ), diff --git a/apps/server/src/routes/environments.ts b/apps/server/src/routes/environments.ts index f5a53b85cc..6bb0f96693 100644 --- a/apps/server/src/routes/environments.ts +++ b/apps/server/src/routes/environments.ts @@ -38,6 +38,10 @@ import { } from "./branch-list-query.js"; import { parseFileListLimit } from "./file-list-query.js"; import { parsePathKindInclusion } from "./path-list-inclusion.js"; +import { + parseIncludeHiddenQueryValue, + workspacePathListPolicy, +} from "./path-list-policy.js"; import { requireWorkspaceCommandTarget, type WorkspaceCommandTarget, @@ -629,6 +633,9 @@ export function registerEnvironmentRoutes(app: Hono, deps: AppDeps): void { limit, includeFiles: inclusion.includeFiles, includeDirectories: inclusion.includeDirectories, + ...workspacePathListPolicy({ + includeHidden: parseIncludeHiddenQueryValue(query.includeHidden), + }), }, }); return context.json({ diff --git a/apps/server/src/routes/files.ts b/apps/server/src/routes/files.ts index 0281a33616..54632e7cb6 100644 --- a/apps/server/src/routes/files.ts +++ b/apps/server/src/routes/files.ts @@ -25,6 +25,7 @@ import { requirePrimaryHostId, } from "../services/hosts/primary-host.js"; import { requirePublicThreadEnvironment } from "../services/lib/entity-lookup.js"; +import { workspacePathListPolicy } from "./path-list-policy.js"; const HOST_FILE_LIST_LIMIT_DEFAULT = 1000; @@ -274,6 +275,7 @@ export function registerFileRoutes(app: Hono, deps: AppDeps): void { path: payload.path, limit: payload.limit ?? HOST_FILE_LIST_LIMIT_DEFAULT, ...(payload.query !== undefined ? { query: payload.query } : {}), + ...workspacePathListPolicy({ includeHidden: undefined }), }, }); return context.json(result); @@ -295,6 +297,7 @@ export function registerFileRoutes(app: Hono, deps: AppDeps): void { includeFiles: payload.includeFiles, includeDirectories: payload.includeDirectories, ...(payload.query !== undefined ? { query: payload.query } : {}), + ...workspacePathListPolicy({ includeHidden: payload.includeHidden }), }, }); return context.json(result); diff --git a/apps/server/src/routes/path-list-policy.ts b/apps/server/src/routes/path-list-policy.ts new file mode 100644 index 0000000000..20ecad523e --- /dev/null +++ b/apps/server/src/routes/path-list-policy.ts @@ -0,0 +1,45 @@ +import type { PathListPolicy } from "@bb/host-daemon-contract"; +import type { PathListIncludeQueryValue } from "@bb/server-contract"; + +/** + * Product policy for listing a workspace (quick-open, @-mentions, file trees, + * `bb file paths`): dot paths are real files the user can open, so they are + * listed; `node_modules` never is; and a git worktree is listed through its + * own ignore rules so `.venv`, `.next`, `.turbo` and friends stay out of the + * walk. Filled in once here and sent explicitly to the daemon. + */ +const WORKSPACE_PATH_LIST_EXCLUDE_NAMES: readonly string[] = ["node_modules"]; + +interface WorkspacePathListPolicyArgs { + /** Caller override; omitted means the product default (hidden paths shown). */ + includeHidden: boolean | undefined; +} + +export function workspacePathListPolicy( + args: WorkspacePathListPolicyArgs, +): PathListPolicy { + return { + includeHidden: args.includeHidden ?? true, + excludeNames: [...WORKSPACE_PATH_LIST_EXCLUDE_NAMES], + respectGitignore: true, + }; +} + +/** + * Thread storage is a bb-owned directory under the server data dir, not a + * workspace: gitignore semantics do not apply to it, so it always takes the + * capped disk walk. + */ +export function threadStoragePathListPolicy(): PathListPolicy { + return { + includeHidden: true, + excludeNames: [...WORKSPACE_PATH_LIST_EXCLUDE_NAMES], + respectGitignore: false, + }; +} + +export function parseIncludeHiddenQueryValue( + value: PathListIncludeQueryValue | undefined, +): boolean | undefined { + return value === undefined ? undefined : value === "true"; +} diff --git a/apps/server/src/routes/projects.ts b/apps/server/src/routes/projects.ts index 5bdbf8c201..6441e10da0 100644 --- a/apps/server/src/routes/projects.ts +++ b/apps/server/src/routes/projects.ts @@ -75,6 +75,10 @@ import { import { resolveDefaultWorktreeBaseBranch } from "../services/projects/worktree-base-branch.js"; import { listProjectPromptHistory } from "../services/prompt-history.js"; import { parsePathKindInclusion } from "./path-list-inclusion.js"; +import { + parseIncludeHiddenQueryValue, + workspacePathListPolicy, +} from "./path-list-policy.js"; import { normalizeBranchQuery, parseBranchListLimit, @@ -621,6 +625,7 @@ export function registerProjectRoutes(app: Hono, deps: AppDeps): void { path: target.path, ...(query.query ? { query: query.query } : {}), limit, + ...workspacePathListPolicy({ includeHidden: undefined }), }, }); return context.json({ files: result.files, truncated: result.truncated }); @@ -684,6 +689,9 @@ export function registerProjectRoutes(app: Hono, deps: AppDeps): void { limit, includeFiles: inclusion.includeFiles, includeDirectories: inclusion.includeDirectories, + ...workspacePathListPolicy({ + includeHidden: parseIncludeHiddenQueryValue(query.includeHidden), + }), }, }); return context.json({ paths: result.paths, truncated: result.truncated }); diff --git a/apps/server/src/routes/threads/data.ts b/apps/server/src/routes/threads/data.ts index f8a741ade4..f858f1374f 100644 --- a/apps/server/src/routes/threads/data.ts +++ b/apps/server/src/routes/threads/data.ts @@ -81,6 +81,7 @@ import { } from "../../services/lib/validation.js"; import { resolveProviderPlanCommand } from "../../services/providers/provider-plan-command.js"; import { parsePathKindInclusion } from "../path-list-inclusion.js"; +import { threadStoragePathListPolicy } from "../path-list-policy.js"; import { parseFileListLimit } from "../file-list-query.js"; import { parseSafeRelativeRoutePath } from "../relative-route-path.js"; @@ -602,6 +603,7 @@ export function registerThreadDataRoutes(app: Hono, deps: AppDeps): void { path: target.storagePath, ...(query.query ? { query: query.query } : {}), limit, + ...threadStoragePathListPolicy(), }, }); return context.json({ @@ -660,6 +662,7 @@ export function registerThreadDataRoutes(app: Hono, deps: AppDeps): void { limit, includeFiles: inclusion.includeFiles, includeDirectories: inclusion.includeDirectories, + ...threadStoragePathListPolicy(), }, }); return context.json({ diff --git a/apps/server/src/services/skills/builtin-skills/bb-cli/SKILL.md b/apps/server/src/services/skills/builtin-skills/bb-cli/SKILL.md index 45c87d9e28..267b9f86ae 100644 --- a/apps/server/src/services/skills/builtin-skills/bb-cli/SKILL.md +++ b/apps/server/src/services/skills/builtin-skills/bb-cli/SKILL.md @@ -295,6 +295,10 @@ status|install` to inspect or install provider CLIs on a selected machine. project source; omitting both intentionally uses the primary machine source. `bb project content --json` returns UTF-8 text or base64 binary content with an explicit `contentEncoding`. +- `bb project paths`, `bb environment paths` and `bb file paths` list + dot-prefixed paths (`.github/workflows/ci.yml`, `.env`, ...) by default; + pass `--no-hidden` to exclude them. `node_modules` and, in a git worktree, + gitignored paths (`.venv`, `.next`, `.turbo`, ...) are never listed. - Use `bb project attachment upload --client-file ` when the bytes live on the CLI machine, including when the CLI and bb server are on different hosts. It reads locally and sends multipart bytes through the diff --git a/apps/server/src/services/skills/injected-skills.ts b/apps/server/src/services/skills/injected-skills.ts index 1423e5d3c9..16829ff14d 100644 --- a/apps/server/src/services/skills/injected-skills.ts +++ b/apps/server/src/services/skills/injected-skills.ts @@ -4,13 +4,29 @@ import { createHash } from "node:crypto"; import path from "node:path"; import matter from "gray-matter"; import { resolveDataDirSkillsRootPath } from "@bb/config/skill-storage-paths"; -import type { HostDaemonInjectedSkillSource } from "@bb/host-daemon-contract"; +import type { + HostDaemonInjectedSkillSource, + PathListPolicy, +} from "@bb/host-daemon-contract"; import { z } from "zod"; import type { ServerLogger } from "../../types.js"; import { isFsErrorWithCode } from "../lib/fs-errors.js"; import { REGISTRY_SKILL_PROVENANCE_FILE_NAME } from "./registry-skill-provenance.js"; const SKILL_FILE_NAME = "SKILL.md"; + +/** + * Listing policy for a host skill directory (`.bb/skills`, or one skill's + * root). Dot entries stay hidden because the matching read path + * (`host.read_file_relative` with `dotfiles: "deny"`) refuses them, so a listed + * `.DS_Store` would be a dead link; skills are a product convention, not a + * git one, so gitignore does not decide what a skill contains. + */ +export const SKILL_DIRECTORY_LIST_POLICY: PathListPolicy = { + includeHidden: false, + excludeNames: ["node_modules"], + respectGitignore: false, +}; const SKILL_NAME_PATTERN = /^(?!.*--)[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$/u; const SKILL_FRONTMATTER_DELIMITER = "---"; diff --git a/apps/server/src/services/skills/skill-listing.ts b/apps/server/src/services/skills/skill-listing.ts index 12afab0c3d..512c0095b7 100644 --- a/apps/server/src/services/skills/skill-listing.ts +++ b/apps/server/src/services/skills/skill-listing.ts @@ -18,7 +18,10 @@ import { callHostRetryableOnlineRpc, } from "../hosts/online-rpc.js"; import type { ProjectCommandWorkspace as CommandWorkspace } from "../projects/project-workspace.js"; -import { resolveServerOwnedSkillCatalogEntries } from "./injected-skills.js"; +import { + SKILL_DIRECTORY_LIST_POLICY, + resolveServerOwnedSkillCatalogEntries, +} from "./injected-skills.js"; import { resolveSkillCatalog } from "./skill-catalog.js"; import { readRegistrySkillProvenance } from "./registry-skill-provenance.js"; import { hostPathDirname, resolveSharedSkills } from "./shared-skills.js"; @@ -413,7 +416,12 @@ export async function listProjectSkillFiles( const result = await callHostRetryableOnlineRpc(deps, { hostId: args.workspace.hostId, timeoutMs: COMMAND_TIMEOUT_MS, - command: { type: "host.list_files", path: rootPath, limit: 200 }, + command: { + type: "host.list_files", + path: rootPath, + limit: 200, + ...SKILL_DIRECTORY_LIST_POLICY, + }, }); const files = result.files .map((file) => file.path) diff --git a/apps/server/src/services/skills/workspace-skills.ts b/apps/server/src/services/skills/workspace-skills.ts index 7f278b75dc..24381324a8 100644 --- a/apps/server/src/services/skills/workspace-skills.ts +++ b/apps/server/src/services/skills/workspace-skills.ts @@ -5,6 +5,7 @@ import { ApiError } from "../../errors.js"; import type { LoggedWorkSessionDeps } from "../../types.js"; import { callHostRetryableOnlineRpc } from "../hosts/online-rpc.js"; import { + SKILL_DIRECTORY_LIST_POLICY, resolveProjectSkillSourceFromContent, type ProjectInjectedSkillSource, } from "./injected-skills.js"; @@ -83,6 +84,7 @@ export async function resolveWorkspaceProjectSkills( path: skillsRootPath, query: SKILL_FILE_NAME, limit: MAX_PROJECT_SKILLS, + ...SKILL_DIRECTORY_LIST_POLICY, }, }); if (result.truncated) { diff --git a/apps/server/test/files/host-file-routes.test.ts b/apps/server/test/files/host-file-routes.test.ts index ae9fe6b35c..9a9b611686 100644 --- a/apps/server/test/files/host-file-routes.test.ts +++ b/apps/server/test/files/host-file-routes.test.ts @@ -191,6 +191,9 @@ describe("host file routes", () => { limit: 1000, includeFiles: true, includeDirectories: true, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, }, { type: "host.mkdir", @@ -214,6 +217,45 @@ describe("host file routes", () => { }); }); + it("passes an explicit includeHidden override through to the listing policy", async () => { + await withTestHarness(async (harness) => { + const { host, session } = seedHostSession(harness.deps); + seedPrimaryHost(harness.deps, host.id); + const commands: HostDaemonOnlineRpcRequestMessage["command"][] = []; + registerHostRpcResponder(harness, { + hostId: host.id, + sessionId: session.id, + handle: (request) => { + commands.push(request.command); + return { ok: true, result: { paths: [], truncated: false } }; + }, + }); + + const response = await harness.app.request( + ...postJson("/api/v1/files/paths", { + path: "/notes", + includeFiles: true, + includeDirectories: false, + includeHidden: false, + }), + ); + expect(response.status).toBe(200); + + expect(commands).toEqual([ + { + type: "host.list_paths", + path: "/notes", + limit: 1000, + includeFiles: true, + includeDirectories: false, + includeHidden: false, + excludeNames: ["node_modules"], + respectGitignore: true, + }, + ]); + }); + }); + it("fills write defaults and resolves the primary host at the boundary", async () => { await withTestHarness(async (harness) => { const { host, session } = seedHostSession(harness.deps); diff --git a/apps/server/test/internal/internal-environment-change.test.ts b/apps/server/test/internal/internal-environment-change.test.ts index 798444bbea..b04646a2ec 100644 --- a/apps/server/test/internal/internal-environment-change.test.ts +++ b/apps/server/test/internal/internal-environment-change.test.ts @@ -70,6 +70,9 @@ describe("internal environment change websocket hints", () => { type: "host.list_files", path: "/tmp/session-scope-test", limit: 10, + includeHidden: true, + excludeNames: [], + respectGitignore: false, }, }, }); diff --git a/apps/server/test/public/public-environments.test.ts b/apps/server/test/public/public-environments.test.ts index 1507c95062..621a64137d 100644 --- a/apps/server/test/public/public-environments.test.ts +++ b/apps/server/test/public/public-environments.test.ts @@ -328,11 +328,17 @@ describe("public environments", () => { command.type === "host.list_paths" && command.path === "/tmp/personal-workspace", ); + // Workspace search policy is filled in once here: dot paths such as + // .github/workflows/ci.yml are listed (#2093), node_modules is not, and + // gitignored trees stay out of the walk. expect(pathsCommand.command).toMatchObject({ path: "/tmp/personal-workspace", query: "app", includeFiles: true, includeDirectories: false, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, }); await reportQueuedCommandSuccess(harness, pathsCommand, { paths: [ @@ -364,6 +370,42 @@ describe("public environments", () => { }); }); + it("passes includeHidden=false through the environment paths route", async () => { + await withTestHarness(async (harness) => { + const { host } = seedHostSession(harness.deps, { + id: "host-environment-paths-hidden", + }); + const { project } = seedProjectWithSource(harness.deps, { + hostId: host.id, + }); + const environment = seedEnvironment(harness.deps, { + hostId: host.id, + projectId: project.id, + path: "/tmp/hidden-workspace", + }); + + const pathsPromise = harness.app.request( + `/api/v1/environments/${environment.id}/paths?includeFiles=true&includeDirectories=true&includeHidden=false`, + ); + const pathsCommand = await waitForQueuedCommand( + harness, + ({ command }) => + command.type === "host.list_paths" && + command.path === "/tmp/hidden-workspace", + ); + expect(pathsCommand.command).toMatchObject({ + includeHidden: false, + excludeNames: ["node_modules"], + respectGitignore: true, + }); + await reportQueuedCommandSuccess(harness, pathsCommand, { + paths: [], + truncated: false, + }); + expect((await pathsPromise).status).toBe(200); + }); + }); + it("returns not-ready for workspace path search on an unprovisioned environment", async () => { await withTestHarness(async (harness) => { const { host } = seedHostSession(harness.deps, { diff --git a/apps/server/test/public/public-project-skills.test.ts b/apps/server/test/public/public-project-skills.test.ts index 9a17093666..aab4008949 100644 --- a/apps/server/test/public/public-project-skills.test.ts +++ b/apps/server/test/public/public-project-skills.test.ts @@ -1554,6 +1554,15 @@ describe("public project skills route", () => { path: "references/layout.md", dotfiles: "deny", }); + // The read path denies dotfiles, so the listing must never offer one. + expect(stub.requests.map((request) => request.command)).toContainEqual({ + type: "host.list_files", + path: "/home/.codex/skills/documents", + limit: 200, + includeHidden: false, + excludeNames: ["node_modules"], + respectGitignore: false, + }); }); }); diff --git a/apps/server/test/threads/thread-runtime-config.test.ts b/apps/server/test/threads/thread-runtime-config.test.ts index 8871774a34..bcd951a9b8 100644 --- a/apps/server/test/threads/thread-runtime-config.test.ts +++ b/apps/server/test/threads/thread-runtime-config.test.ts @@ -1422,6 +1422,9 @@ describe("thread runtime config", () => { command: expect.objectContaining({ type: "host.list_files", path: path.join(workspacePath, ".bb", "skills"), + // The read side denies dotfiles, so the listing must too. + includeHidden: false, + respectGitignore: false, }), }), expect.objectContaining({ diff --git a/packages/domain/src/plugin-sdk-version.ts b/packages/domain/src/plugin-sdk-version.ts index 1cb75dabe4..d3d061645d 100644 --- a/packages/domain/src/plugin-sdk-version.ts +++ b/packages/domain/src/plugin-sdk-version.ts @@ -16,7 +16,7 @@ // PLUGIN_SDK_MAJOR is 0, so the major-only artifact gate cannot distinguish // 0.x releases and is intentionally vacuous for them until a future 1.0. // Rebuildable artifacts still rebuild on the exact sdkVersion-differs trigger. -export const PLUGIN_SDK_VERSION = "0.4.11"; +export const PLUGIN_SDK_VERSION = "0.4.12"; /** Major of {@link PLUGIN_SDK_VERSION} — the plugin API compatibility number. */ export const PLUGIN_SDK_MAJOR = Number(PLUGIN_SDK_VERSION.split(".", 1)[0]); diff --git a/packages/host-daemon-contract/src/commands.ts b/packages/host-daemon-contract/src/commands.ts index f4cf39c643..c78dc6bd57 100644 --- a/packages/host-daemon-contract/src/commands.ts +++ b/packages/host-daemon-contract/src/commands.ts @@ -623,7 +623,29 @@ const hostWriteFileCommandSchema = z }) .strict(); -const hostListFilesCommandSchema = z.object({ +/** + * Listing policy for the recursive walkers (`host.list_files`, + * `host.list_paths`). The server owns these decisions and fills them in once + * at its boundary; the daemon applies exactly what it is told. Its only + * literal is that `.git` is never listed or descended. + * + * - `includeHidden`: list dot-prefixed entries (`.github`, `.env`, ...). + * - `excludeNames`: basenames that are neither listed nor descended + * (`node_modules`, ...). + * - `respectGitignore`: when the root is inside a git worktree, take the + * candidates from `git ls-files --cached --others --exclude-standard` + * (tracked + untracked-not-ignored) instead of a readdir walk, so ignored + * trees such as `.venv`, `.next` or `.turbo` never enter the walk. Roots + * outside git fall back to the readdir walk. + */ +const pathListPolicySchema = z.object({ + includeHidden: z.boolean(), + excludeNames: z.array(z.string().min(1)), + respectGitignore: z.boolean(), +}); +export type PathListPolicy = z.infer; + +const hostListFilesCommandSchema = pathListPolicySchema.extend({ type: z.literal("host.list_files"), path: z.string().min(1), query: z.string().max(FILE_LIST_QUERY_MAX_LENGTH).optional(), @@ -642,8 +664,8 @@ const hostPathEntrySchema = z.object({ }); export type HostPathEntry = z.infer; -const hostListPathsCommandSchema = z - .object({ +const hostListPathsCommandSchema = pathListPolicySchema + .extend({ type: z.literal("host.list_paths"), path: z.string().min(1), query: z.string().max(FILE_LIST_QUERY_MAX_LENGTH).optional(), diff --git a/packages/host-daemon-contract/src/protocol.ts b/packages/host-daemon-contract/src/protocol.ts index ee0275bb1c..32c4cd5cfa 100644 --- a/packages/host-daemon-contract/src/protocol.ts +++ b/packages/host-daemon-contract/src/protocol.ts @@ -1,3 +1,11 @@ +// Version 151 moves path-listing policy to the server. `host.list_files` and +// `host.list_paths` gain REQUIRED `includeHidden`, `excludeNames` and +// `respectGitignore` fields; the daemon applies exactly what it is told +// instead of its old hardcoded dot-entry / node_modules skip, lists a git +// worktree through `git ls-files --exclude-standard` when asked to respect +// gitignore, and caps the readdir walk. An older daemon rejects the new +// fields; a newer daemon would reject an older server's commands. +// // Version 150 adds an OPTIONAL `presentation` to each bb-injected tool // definition (`dynamicTools[]` on thread.start, turn.submit and the resume // contexts): how a call to the tool reads as a timeline row (grammar v3), @@ -154,7 +162,7 @@ // // The version mismatch is what triggers the enrolled daemon's automatic update // instead of an `invalid-message` reconnect loop. -export const HOST_DAEMON_PROTOCOL_VERSION = 150 as const; +export const HOST_DAEMON_PROTOCOL_VERSION = 151 as const; /** * Absolute ceiling for any executable artifact delivered to a host daemon — diff --git a/packages/host-daemon-contract/test/contract.test.ts b/packages/host-daemon-contract/test/contract.test.ts index 429806005f..c8c66aa5be 100644 --- a/packages/host-daemon-contract/test/contract.test.ts +++ b/packages/host-daemon-contract/test/contract.test.ts @@ -1128,7 +1128,7 @@ describe("host-daemon command schemas", () => { // mixed version. Version 113 carried the Devin Desktop open target rename // and remains part of the protocol lineage. it("uses the current host-daemon protocol version", () => { - expect(HOST_DAEMON_PROTOCOL_VERSION).toBe(150); + expect(HOST_DAEMON_PROTOCOL_VERSION).toBe(151); expect(HOST_ARTIFACT_MAX_BYTES).toBe(256 * 1024 * 1024); }); @@ -1452,13 +1452,29 @@ describe("host-daemon command schemas", () => { type: "host.list_files", path: "/tmp/workspace", limit: 1000, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, }), ).toMatchObject({ type: "host.list_files", path: "/tmp/workspace", limit: 1000, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, }); + // Listing policy is the server's to fill in; a command without it is + // an older server and must be rejected rather than defaulted. + expect(() => + hostDaemonOnlineRpcCommandSchema.parse({ + type: "host.list_files", + path: "/tmp/workspace", + limit: 1000, + }), + ).toThrow(); + expect( hostDaemonOnlineRpcCommandSchema.parse({ type: "host.list_paths", @@ -1466,6 +1482,9 @@ describe("host-daemon command schemas", () => { limit: 1000, includeFiles: true, includeDirectories: true, + includeHidden: false, + excludeNames: [], + respectGitignore: false, }), ).toMatchObject({ type: "host.list_paths", @@ -1473,6 +1492,9 @@ describe("host-daemon command schemas", () => { limit: 1000, includeFiles: true, includeDirectories: true, + includeHidden: false, + excludeNames: [], + respectGitignore: false, }); expect( @@ -1615,6 +1637,9 @@ describe("host-daemon command schemas", () => { type: "host.list_files", path: "/tmp/bb-data/thread-storage/thread-123", limit: 100, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: false, }), ).toMatchObject({ type: "host.list_files", @@ -1722,13 +1747,23 @@ describe("host-daemon command schemas", () => { it("rejects online-RPC-only read commands from the settled command schema", () => { const onlineReadCommands = [ - { type: "host.list_files", path: "/tmp/workspace", limit: 100 }, + { + type: "host.list_files", + path: "/tmp/workspace", + limit: 100, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, + }, { type: "host.list_paths", path: "/tmp/workspace", limit: 100, includeFiles: true, includeDirectories: true, + includeHidden: true, + excludeNames: ["node_modules"], + respectGitignore: true, }, { type: "host.list_branch_options", diff --git a/packages/host-workspace/src/index.ts b/packages/host-workspace/src/index.ts index 3b5a2a693e..e91621c00c 100644 --- a/packages/host-workspace/src/index.ts +++ b/packages/host-workspace/src/index.ts @@ -23,4 +23,5 @@ export { readDefaultBranchRefs, readGitBlob, runGit, + runGitWithNullRecordLimit, } from "./git.js"; diff --git a/packages/plugin-sdk/package.json b/packages/plugin-sdk/package.json index dec2ece2ca..6314583303 100644 --- a/packages/plugin-sdk/package.json +++ b/packages/plugin-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@get-bb/plugin-sdk", - "version": "0.4.11", + "version": "0.4.12", "homepage": "https://github.com/get-bb/bb#readme", "bugs": { "url": "https://github.com/get-bb/bb/issues" diff --git a/packages/sdk/src/areas/environments.ts b/packages/sdk/src/areas/environments.ts index 393875cb6c..e29e6e222a 100644 --- a/packages/sdk/src/areas/environments.ts +++ b/packages/sdk/src/areas/environments.ts @@ -244,6 +244,7 @@ function environmentPathsQuery( return { includeDirectories: args.includeDirectories, includeFiles: args.includeFiles, + includeHidden: args.includeHidden, limit: args.limit, query: args.query, }; diff --git a/packages/sdk/src/areas/files.ts b/packages/sdk/src/areas/files.ts index 17441d6ab4..fb2e4f888f 100644 --- a/packages/sdk/src/areas/files.ts +++ b/packages/sdk/src/areas/files.ts @@ -52,6 +52,8 @@ export interface FileListArgs { export interface PathListArgs extends FileListArgs { includeFiles: boolean; includeDirectories: boolean; + /** Dot-prefixed paths. Defaults to true; `node_modules` and gitignored paths are never listed. */ + includeHidden?: boolean; } export interface FileMkdirArgs { @@ -147,6 +149,7 @@ export function createFilesArea(args: CreateSdkAreaArgs): FilesArea { hostId: input.hostId, includeDirectories: input.includeDirectories, includeFiles: input.includeFiles, + includeHidden: input.includeHidden, limit: input.limit, path: input.path, query: input.query, diff --git a/packages/server-contract/src/api/environments.ts b/packages/server-contract/src/api/environments.ts index 71f406edc0..a6e354c9ae 100644 --- a/packages/server-contract/src/api/environments.ts +++ b/packages/server-contract/src/api/environments.ts @@ -43,6 +43,8 @@ export const environmentPathsQuerySchema = z.object({ limit: z.string().regex(/^\d+$/).optional(), includeFiles: pathListIncludeQueryValueSchema, includeDirectories: pathListIncludeQueryValueSchema, + /** Dot-prefixed paths; omitted means the server default (shown). */ + includeHidden: pathListIncludeQueryValueSchema.optional(), }); export type EnvironmentPathsQuery = z.infer; diff --git a/packages/server-contract/src/api/files.ts b/packages/server-contract/src/api/files.ts index efe51318d2..ab1b0d1aea 100644 --- a/packages/server-contract/src/api/files.ts +++ b/packages/server-contract/src/api/files.ts @@ -54,6 +54,8 @@ export const hostPathListRequestSchema = z limit: z.number().int().positive().max(FILE_LIST_LIMIT_MAX).optional(), includeFiles: z.boolean(), includeDirectories: z.boolean(), + /** Dot-prefixed paths; omitted means the server default (shown). */ + includeHidden: z.boolean().optional(), }) .strict(); export type HostPathListRequest = z.infer; diff --git a/packages/server-contract/src/api/projects.ts b/packages/server-contract/src/api/projects.ts index 17ab5cec77..40c35456d2 100644 --- a/packages/server-contract/src/api/projects.ts +++ b/packages/server-contract/src/api/projects.ts @@ -193,12 +193,15 @@ export const projectPathsQuerySchema = z limit: z.string().regex(/^\d+$/).optional(), includeFiles: pathListIncludeQueryValueSchema, includeDirectories: pathListIncludeQueryValueSchema, + /** Dot-prefixed paths; omitted means the server default (shown). */ + includeHidden: pathListIncludeQueryValueSchema, }) .partial({ hostId: true, environmentId: true, query: true, limit: true, + includeHidden: true, }) .superRefine(rejectMultipleProjectWorkspaceSelectors); export type ProjectPathsQuery = z.infer; diff --git a/packages/templates/src/templates/bb-guide-customization.md b/packages/templates/src/templates/bb-guide-customization.md index 5d6a8d2d13..61e945b5fe 100644 --- a/packages/templates/src/templates/bb-guide-customization.md +++ b/packages/templates/src/templates/bb-guide-customization.md @@ -158,7 +158,10 @@ Voice transcription uses the `BB_TRANSCRIPTION` model, which defaults to `bb file` supports `--host` for remote machines and `--root` on mutating commands to confine access beneath an absolute directory. Use `--json` for -metadata and machine-readable results. +metadata and machine-readable results. `bb file list` and `bb file paths` +include dot-prefixed paths by default (`bb file paths --no-hidden` excludes +them); node_modules and, inside a git worktree, gitignored paths are never +listed. Client-local UI preferences diff --git a/packages/templates/src/templates/bb-guide-environments.md b/packages/templates/src/templates/bb-guide-environments.md index 0bccfca842..c84d0dafa5 100644 --- a/packages/templates/src/templates/bb-guide-environments.md +++ b/packages/templates/src/templates/bb-guide-environments.md @@ -74,6 +74,10 @@ Making your repo work with bb: --limit Maximum results --files Include only files unless combined with --directories --directories Include only directories unless combined with --files + --no-hidden Exclude dot-prefixed paths (.github, .env, ...) + + Path search lists dot-prefixed paths by default. node_modules and, in a git + worktree, gitignored paths (.venv, .next, .turbo, ...) are never listed. bb environment diff Show file summary and full git diff bb environment diff-files List changed-file metadata diff --git a/packages/templates/src/templates/bb-guide-projects.md b/packages/templates/src/templates/bb-guide-projects.md index e8b892a2d9..a25ac26990 100644 --- a/packages/templates/src/templates/bb-guide-projects.md +++ b/packages/templates/src/templates/bb-guide-projects.md @@ -35,6 +35,8 @@ Discovery: bb project branches --host List branches for a machine source bb project paths Search workspace paths + --query Fuzzy path query + --no-hidden Exclude dot-prefixed paths (.github, .env, ...) bb project files List workspace files bb project content Read file content (binary is base64) bb project commands --provider @@ -48,6 +50,10 @@ Discovery: machine selects that machine's project source. Omitting both intentionally falls back to the primary machine's project source. + Path and file listings include dot-prefixed paths by default. node_modules + and, in a git worktree, gitignored paths (.venv, .next, .turbo, ...) are + never listed. + Attachments: bb project attachment upload Upload bytes from the CLI machine diff --git a/plugins/docs/server.ts b/plugins/docs/server.ts index d581edf3ee..3790092b1c 100644 --- a/plugins/docs/server.ts +++ b/plugins/docs/server.ts @@ -809,11 +809,15 @@ export default async function plugin( async function listEntries( vault: Vault, ): Promise<{ entries: VaultEntry[]; truncated: boolean }> { + // Vault paths reject dot segments (`requireVaultPath`), and the sync state + // file sits at the root as a dotfile, so every vault listing excludes + // hidden paths. const result = await bb.sdk.files.listPaths({ ...hostArgs(vault), path: vault.rootPath, includeFiles: true, includeDirectories: true, + includeHidden: false, limit: MAX_TREE_ENTRIES, }); return { @@ -1146,6 +1150,7 @@ export default async function plugin( path: vault.rootPath, includeFiles: true, includeDirectories: true, + includeHidden: false, limit: MAX_TREE_ENTRIES, }); if (result.truncated) { @@ -1300,6 +1305,7 @@ export default async function plugin( path: vault.rootPath, includeFiles: true, includeDirectories: true, + includeHidden: false, limit: MAX_TREE_ENTRIES, }); if (currentListing.truncated) { @@ -2242,6 +2248,7 @@ export default async function plugin( path: rootPath, includeFiles: true, includeDirectories: true, + includeHidden: false, limit: MAX_TREE_ENTRIES, }); if (listing.truncated) {