diff --git a/sdk/typescript/_bundled_plugin/scripts/generate_in_scope_files.py b/sdk/typescript/_bundled_plugin/scripts/generate_in_scope_files.py index a2eeb6ca..3badbbf4 100644 --- a/sdk/typescript/_bundled_plugin/scripts/generate_in_scope_files.py +++ b/sdk/typescript/_bundled_plugin/scripts/generate_in_scope_files.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import os import subprocess import sys import tempfile @@ -67,8 +68,23 @@ def resolve_output(value: str) -> Path: def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int: - """Atomically write the exact ripgrep inventory sorted as ``LC_ALL=C``.""" - command = ["rg", "--files", "--hidden", "--no-ignore", "--glob", "!.git/**", "--", scope] + """Atomically inventory visible files and ignored files tracked by Git.""" + command = [ + "rg", + "--no-config", + "--files", + "--hidden", + "--no-require-git", + "--no-ignore-parent", + "--no-ignore-global", + "--glob", + "!.git/**", + ] + for name in (".gitignore", ".ignore", ".rgignore"): + ignore = repository / name + if ignore.is_file() and not ignore.is_symlink(): + command.extend(["--ignore-file", str(ignore)]) + command.extend(["--", scope]) with tempfile.TemporaryFile(mode="w+b") as inventory: try: result = subprocess.run( @@ -89,7 +105,138 @@ def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int: raise InventoryError(message) inventory.seek(0) - rows = sorted(inventory) + rows = set(inventory) + + environment = os.environ.copy() + for name in ( + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_CEILING_DIRECTORIES", + "GIT_COMMON_DIR", + "GIT_DIR", + "GIT_DISCOVERY_ACROSS_FILESYSTEM", + "GIT_INDEX_FILE", + "GIT_NAMESPACE", + "GIT_OBJECT_DIRECTORY", + "GIT_WORK_TREE", + ): + environment.pop(name, None) + environment["GIT_LITERAL_PATHSPECS"] = "1" + environment["LC_ALL"] = "C" + git = [ + "git", + "-c", + "core.fsmonitor=false", + "-c", + f"core.excludesFile={os.devnull}", + "--literal-pathspecs", + ] + try: + worktree = subprocess.run( + [*git, "rev-parse", "--is-inside-work-tree"], + cwd=repository, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + check=False, + ) + except OSError as error: + if (repository / ".git").exists(): + raise InventoryError(f"could not inspect Git worktree: {error}") from error + worktree = None + + if worktree is not None and worktree.returncode: + detail = worktree.stderr.decode("utf-8", errors="replace").strip() + if worktree.returncode == 128 and "not a git repository" in detail.lower(): + worktree = None + else: + message = f"git rev-parse exited with status {worktree.returncode}" + if detail: + message = f"{message}: {detail}" + raise InventoryError(message) + + if worktree is not None and worktree.stdout.strip() == b"true": + prefix = b"./" if scope == "." or scope.startswith("./") else b"" + listed: list[bytes] = [] + for arguments in (["--cached"], ["--others", "--exclude-standard"]): + try: + result = subprocess.run( + [*git, "ls-files", *arguments, "-z", "--", scope], + cwd=repository, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + check=False, + ) + except OSError as error: + raise InventoryError(f"could not list repository files: {error}") from error + if result.returncode: + detail = result.stderr.decode("utf-8", errors="replace").strip() + message = f"git ls-files exited with status {result.returncode}" + if detail: + message = f"{message}: {detail}" + raise InventoryError(message) + listed.append(result.stdout) + + def normalized(path: bytes) -> bytes: + return path.replace(b"\\", b"/") if os.name == "nt" else path + + allowed = { + normalized(prefix + relative) + for collection in listed + for relative in collection.split(b"\0") + if relative + } + nested_worktrees = tuple(path for path in allowed if path.endswith(b"/")) + explicitly_ignored = False + if scope not in (".", "./"): + ignored_environment = environment.copy() + ignored_environment.pop("GIT_LITERAL_PATHSPECS", None) + explicit_path = scope if scope.startswith("./") else f"./{scope}" + try: + ignored = subprocess.run( + [*git[:-1], "check-ignore", "--quiet", "--no-index", "--", explicit_path], + cwd=repository, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=ignored_environment, + check=False, + ) + except OSError as error: + raise InventoryError(f"could not inspect scoped Git ignores: {error}") from error + if ignored.returncode not in (0, 1): + detail = ignored.stderr.decode("utf-8", errors="replace").strip() + message = f"git check-ignore exited with status {ignored.returncode}" + if detail: + message = f"{message}: {detail}" + raise InventoryError(message) + explicitly_ignored = ignored.returncode == 0 + + if not explicitly_ignored: + rows = { + row + for row in rows + if (path := normalized(row.rstrip(b"\r\n"))) in allowed + or any(path.startswith(worktree) for worktree in nested_worktrees) + } + recorded = {normalized(row.rstrip(b"\r\n")) for row in rows} + + for relative in listed[0].split(b"\0"): + if not relative: + continue + candidate = repository / os.fsdecode(relative) + if candidate.is_symlink() or not candidate.is_file(): + continue + try: + candidate.resolve(strict=True).relative_to(repository) + except (OSError, ValueError): + continue + relative_path = prefix + relative + key = normalized(relative_path) + if key not in recorded: + rows.add(relative_path + b"\n") + recorded.add(key) + + rows = sorted(rows) output.parent.mkdir(parents=True, exist_ok=True) temporary: Path | None = None diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index 65a26b24..f45d49f2 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -229,7 +229,8 @@ describe("plugin runtime preparation", () => { join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"), "utf8", ); - expect(generator).toContain('"--no-ignore"'); + expect(generator).not.toContain('"--no-ignore"'); + expect(generator).toContain('"--cached"'); return; } diff --git a/sdk/typescript/tests-ts/scan-inventory.test.ts b/sdk/typescript/tests-ts/scan-inventory.test.ts new file mode 100644 index 00000000..ebffd428 --- /dev/null +++ b/sdk/typescript/tests-ts/scan-inventory.test.ts @@ -0,0 +1,266 @@ +import { execFileSync } from "node:child_process"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test } from "bun:test"; +import { PLUGIN_ROOT } from "./plugin-root.js"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +describe("security scan file inventory", () => { + test("includes hidden source files without exposing ignored repository files", async () => { + if (Bun.which("rg") === null) { + const generator = await readFile( + join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"), + "utf8", + ); + expect(generator).not.toContain('"--no-ignore"'); + expect(generator).toContain('"--cached"'); + expect(generator).toContain('"--no-config"'); + expect(generator).toContain('"--no-ignore-parent"'); + expect(generator).toContain('"--no-require-git"'); + expect(generator).toContain('"--literal-pathspecs"'); + expect(generator).toContain('"core.fsmonitor=false"'); + return; + } + + const root = await realpath( + await mkdtemp(join(tmpdir(), "codex-security-scan-inventory-")), + ); + temporaryDirectories.push(root); + + const repository = join(root, "repository"); + const output = join(root, "in-scope-files.txt"); + const globalIgnore = join(root, "global-ignore"); + await mkdir(join(repository, "src"), { recursive: true }); + await mkdir(join(repository, "ignored")); + execFileSync("git", ["init", "-q"], { cwd: repository }); + + await Promise.all([ + writeFile( + join(repository, ".gitignore"), + "ignored/\n.env\ntracked.env\ntracked-link\n", + ), + writeFile(join(repository, ".env"), "SECRET=private\n"), + writeFile(join(repository, ".visible-config"), "visible=true\n"), + writeFile(join(repository, "ignored", "secret.ts"), "private data\n"), + writeFile(join(repository, "src", "handler.ts"), "export {};\n"), + writeFile(join(repository, "tracked.env"), "checked in intentionally\n"), + writeFile(join(repository, ".ignore"), "hidden-by-rg.ts\n"), + writeFile(join(repository, "hidden-by-rg.ts"), "tracked source\n"), + writeFile( + join(repository, "info-secret.ts"), + "local Git-excluded data\n", + ), + writeFile(globalIgnore, "*.ts\n"), + ]); + await writeFile( + join(repository, ".git", "info", "exclude"), + "info-secret.ts\n", + ); + execFileSync( + "git", + ["add", "--force", "--", "tracked.env", "hidden-by-rg.ts"], + { + cwd: repository, + }, + ); + if (process.platform !== "win32") { + const external = join(root, "external.txt"); + await writeFile(external, "private external file\n"); + await symlink(external, join(repository, "tracked-link")); + execFileSync("git", ["add", "--force", "--", "tracked-link"], { + cwd: repository, + }); + } + + const python = + Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); + expect(python).not.toBeNull(); + if (python === null) throw new Error("A Python interpreter is required."); + + execFileSync( + python, + [ + "-B", + join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"), + "--repo", + repository, + "--scope", + ".", + "--out", + output, + ], + { + cwd: repository, + stdio: "pipe", + env: { + ...process.env, + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "core.excludesFile", + GIT_CONFIG_VALUE_0: globalIgnore, + }, + }, + ); + + expect( + (await readFile(output, "utf8")) + .trimEnd() + .split("\n") + .map((path) => path.replaceAll("\\", "/")), + ).toEqual([ + "./.gitignore", + "./.ignore", + "./.visible-config", + "./hidden-by-rg.ts", + "./src/handler.ts", + "./tracked.env", + ]); + }); + + test("respects ignore files in non-Git directory snapshots", async () => { + if (Bun.which("rg") === null) return; + + const root = await realpath( + await mkdtemp(join(tmpdir(), "codex-security-directory-inventory-")), + ); + temporaryDirectories.push(root); + const repository = join(root, "snapshot"); + const output = join(root, "in-scope-files.txt"); + await mkdir(repository); + await writeFile(join(root, ".gitignore"), "snapshot/source.ts\n"); + await Promise.all([ + writeFile(join(repository, ".gitignore"), ".env\n"), + writeFile(join(repository, ".env"), "SECRET=private\n"), + writeFile(join(repository, "source.ts"), "export {};\n"), + ]); + + const python = + Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); + expect(python).not.toBeNull(); + if (python === null) throw new Error("A Python interpreter is required."); + execFileSync( + python, + [ + "-B", + join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"), + "--repo", + repository, + "--scope", + ".", + "--out", + output, + ], + { cwd: repository, stdio: "pipe" }, + ); + + const rows = (await readFile(output, "utf8")) + .trimEnd() + .split("\n") + .map((path) => path.replaceAll("\\", "/")); + expect(rows).toEqual(["./.gitignore", "./source.ts"]); + }); + + test("retains visible files inside nested Git worktrees", async () => { + if (Bun.which("rg") === null) return; + + const root = await realpath( + await mkdtemp(join(tmpdir(), "codex-security-nested-inventory-")), + ); + temporaryDirectories.push(root); + const repository = join(root, "repository"); + const nested = join(repository, "nested"); + const output = join(root, "in-scope-files.txt"); + await mkdir(nested, { recursive: true }); + execFileSync("git", ["init", "-q"], { cwd: repository }); + execFileSync("git", ["init", "-q"], { cwd: nested }); + await Promise.all([ + writeFile(join(nested, ".gitignore"), ".env\n"), + writeFile(join(nested, ".env"), "SECRET=private\n"), + writeFile(join(nested, "tracked.py"), "print('tracked')\n"), + writeFile(join(nested, "local.py"), "print('local')\n"), + ]); + execFileSync("git", ["add", "--", "tracked.py"], { cwd: nested }); + + const python = + Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); + expect(python).not.toBeNull(); + if (python === null) throw new Error("A Python interpreter is required."); + execFileSync( + python, + [ + "-B", + join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"), + "--repo", + repository, + "--scope", + ".", + "--out", + output, + ], + { cwd: repository, stdio: "pipe" }, + ); + + const rows = (await readFile(output, "utf8")) + .trimEnd() + .split("\n") + .map((path) => path.replaceAll("\\", "/")); + expect(rows).toContain("./nested/tracked.py"); + expect(rows).toContain("./nested/local.py"); + expect(rows).not.toContain("./nested/.env"); + }); + + test("retains an explicitly scoped Git-ignored file", async () => { + if (Bun.which("rg") === null) return; + + const root = await realpath( + await mkdtemp(join(tmpdir(), "codex-security-explicit-inventory-")), + ); + temporaryDirectories.push(root); + const repository = join(root, "repository"); + const output = join(root, "in-scope-files.txt"); + await mkdir(repository); + execFileSync("git", ["init", "-q"], { cwd: repository }); + await Promise.all([ + writeFile(join(repository, ".gitignore"), "*.skip\n"), + writeFile(join(repository, "selected.skip"), "explicit source\n"), + ]); + + const python = + Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); + expect(python).not.toBeNull(); + if (python === null) throw new Error("A Python interpreter is required."); + execFileSync( + python, + [ + "-B", + join(PLUGIN_ROOT, "scripts", "generate_in_scope_files.py"), + "--repo", + repository, + "--scope", + "selected.skip", + "--out", + output, + ], + { cwd: repository, stdio: "pipe" }, + ); + + expect((await readFile(output, "utf8")).trim()).toBe("selected.skip"); + }); +});