diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index ef97b474..e96ac153 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -21,6 +21,7 @@ Complete reference for defining evaluation tasks in Coder Eval. - [Template Sources](#template-sources) - [Success Criteria](#success-criteria) - [Continuous Scoring](#continuous-scoring) + - [Glob patterns in path](#glob-patterns-in-path) - [file_exists](#file_exists) - [file_contains](#file_contains) - [file_check](#file_check) @@ -670,6 +671,30 @@ score mattered. **Weighted score:** `weighted_score = sum(score * weight) / sum(weight)` — calculated regardless for quality assessment. +### Glob patterns in `path` + +Every sandbox-relative path field accepts a glob — `path` on `file_exists`, `file_contains`, `file_matches_regex`, `file_check`, `json_check` and `classification_match`, `json_schema` on `json_check`, and `agent_file` on `reference_comparison`. Use one when the prompt does not pin where the file lands — a scaffolding tool that creates a wrapper directory the agent names itself, for example. + +```yaml +- type: "file_contains" + path: "**/*.flow" # matches any depth under the sandbox root + includes: ['"core.logic.decision"'] + description: "flow wires a Decision node" +``` + +Rules: + +- **A path that exists is never treated as a pattern.** A literal `path` behaves exactly as before, including one containing `*`, `?`, or `[` — a real file named `report[2024].json` is graded as itself, not as a character class that would match `report2.json`. Globbing only kicks in when the literal path does not exist. +- **Glob matches skip ignored directories.** Expansion runs over the live sandbox root, which also holds harness-created content the agent never wrote (`.venv` for any task with a `python:` block, `node_modules`, `dist`, `build`, `__pycache__`, …), so matches are filtered through the same [`ignore_patterns`](#sandbox-configuration) set used for template copying. A segment your pattern names *literally* is an opt-in and survives, so `dist/**/*.js` still grades `dist`; to un-ignore a directory a wildcard has to discover, use the negation escape hatch — `ignore_patterns: ["!dist"]`. +- Matches are sorted, and directories are skipped. +- `file_exists` passes when the glob matches **at least one** file. +- Content checks require the glob to match **exactly one** file. An ambiguous glob scores 0.0 and reports the matches (first 10, then `+N more`) rather than silently grading one of them — narrow the pattern. +- When a glob resolves, the file that was actually graded is echoed in the criterion's `details` as `resolved: `. + +Prefer a glob over a hardcoded path whose leading directory the task prompt never specifies: a correct artifact in an unexpected directory otherwise scores 0.0 on the path alone. Glob away only the segment the prompt leaves free, though — if the free part is an unknown wrapper directory, `**/.flow` stays unique where a blanket `**/*.flow` turns exactly-one into a hard 0.0 the moment a second flow file exists. + +> **Dataset note:** `${row.}` substitution runs over `success_criteria` string leaves, so a row value containing `*`, `?`, or `[` lands inside `path`. Literal-first resolution means such a path still grades the real file when it exists; it falls back to glob expansion only when it does not. + ### `file_exists` Checks if a file exists. **Binary scoring.** diff --git a/src/coder_eval/criteria/file_check.py b/src/coder_eval/criteria/file_check.py index 7a9922d7..82020ef8 100644 --- a/src/coder_eval/criteria/file_check.py +++ b/src/coder_eval/criteria/file_check.py @@ -47,14 +47,18 @@ def _check_impl( has_includes = len(criterion.includes) > 0 has_excludes = len(criterion.excludes) > 0 has_patterns = len(criterion.patterns) > 0 + resolved = sandbox.resolved_path_label(criterion.path) # 2. Pure existence check (no sub-checks specified) if not has_includes and not has_excludes and not has_patterns: + details = f"File '{criterion.path}' exists" + if resolved: + details += f" (resolved: {resolved})" return CriterionResult( criterion_type=criterion.type, description=criterion.description, score=1.0, - details=f"File '{criterion.path}' exists", + details=details, ) # 3. Read file content @@ -62,6 +66,8 @@ def _check_impl( scores: list[float] = [] details_parts: list[str] = [] + if resolved: + details_parts.append(f"Resolved: {resolved}") # 4a. Includes score if has_includes: diff --git a/src/coder_eval/criteria/file_contains.py b/src/coder_eval/criteria/file_contains.py index 8cb2deef..561a7522 100644 --- a/src/coder_eval/criteria/file_contains.py +++ b/src/coder_eval/criteria/file_contains.py @@ -71,6 +71,9 @@ def _check_impl( # Build details details_parts = [] + resolved = sandbox.resolved_path_label(criterion.path) + if resolved: + details_parts.append(f"Resolved: {resolved}") details_parts.append(f"Includes: {includes_found}/{includes_total} found") if criterion.excludes: excludes_absent = len(criterion.excludes) - sum(1 for exc in criterion.excludes if exc in content) diff --git a/src/coder_eval/criteria/file_exists.py b/src/coder_eval/criteria/file_exists.py index e5667717..bf05fb3f 100644 --- a/src/coder_eval/criteria/file_exists.py +++ b/src/coder_eval/criteria/file_exists.py @@ -37,9 +37,14 @@ def _check_impl( exists = sandbox.file_exists(criterion.path) score = 1.0 if exists else 0.0 + details = f"File '{criterion.path}' {'exists' if exists else 'does not exist'}" + resolved = sandbox.resolved_path_label(criterion.path) + if resolved: + details += f" (resolved: {resolved})" + return CriterionResult( criterion_type=criterion.type, description=criterion.description, score=score, - details=f"File '{criterion.path}' {'exists' if exists else 'does not exist'}", + details=details, ) diff --git a/src/coder_eval/criteria/file_matches_regex.py b/src/coder_eval/criteria/file_matches_regex.py index 9fd2f41d..4ff31eab 100644 --- a/src/coder_eval/criteria/file_matches_regex.py +++ b/src/coder_eval/criteria/file_matches_regex.py @@ -78,6 +78,10 @@ def _check_impl( matched_text = match.group()[:100] details = f"Pattern '{criterion.pattern}' found but should not be present (matched: '{matched_text}')" + resolved = sandbox.resolved_path_label(criterion.path) + if resolved: + details += f" (resolved: {resolved})" + return CriterionResult( criterion_type=criterion.type, description=criterion.description, diff --git a/src/coder_eval/criteria/json_check.py b/src/coder_eval/criteria/json_check.py index d69898f7..a49bd2c6 100644 --- a/src/coder_eval/criteria/json_check.py +++ b/src/coder_eval/criteria/json_check.py @@ -91,18 +91,24 @@ def _check_impl( has_schema = criterion.json_schema is not None has_assertions = len(criterion.assertions) > 0 + resolved = sandbox.resolved_path_label(criterion.path) # 3. Pure validity check if not has_schema and not has_assertions: + details = f"'{criterion.path}' is valid JSON" + if resolved: + details += f" (resolved: {resolved})" return CriterionResult( criterion_type=criterion.type, description=criterion.description, score=1.0, - details=f"'{criterion.path}' is valid JSON", + details=details, ) scores: list[float] = [] details_parts: list[str] = [] + if resolved: + details_parts.append(f"Resolved: {resolved}") # 4. Schema validation (gates assertions — if schema fails, skip assertions) if has_schema: diff --git a/src/coder_eval/criteria/reference_comparison.py b/src/coder_eval/criteria/reference_comparison.py index 9c845f6f..c75fa14b 100644 --- a/src/coder_eval/criteria/reference_comparison.py +++ b/src/coder_eval/criteria/reference_comparison.py @@ -62,18 +62,18 @@ def _check_impl( error="Sandbox not initialized", ) - # Load agent code - agent_path = sandbox.sandbox_dir / criterion.agent_file - if not agent_path.exists(): + # Load agent code through the shared path seam, so `agent_file` resolves + # (glob expansion, ignore filtering, exactly-one) like every other + # sandbox-relative criterion path. + try: + agent_code = sandbox.get_file_content(criterion.agent_file) + except FileNotFoundError: return CriterionResult( criterion_type="reference_comparison", description=criterion.description, score=0.0, error=f"Agent file not found: {criterion.agent_file}", ) - - try: - agent_code = agent_path.read_text(encoding="utf-8") except Exception as e: return CriterionResult( criterion_type="reference_comparison", diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index 9795a74e..e1bf0fcc 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -321,7 +321,9 @@ class FileExistsCriterion(BaseSuccessCriterion): """ type: Literal["file_exists"] = "file_exists" - path: str = Field(description="Path to the file that must exist") + path: str = Field( + description="Path to the file that must exist; a glob pattern passes when it matches at least one file" + ) class FileContainsCriterion(BaseSuccessCriterion): @@ -331,7 +333,7 @@ class FileContainsCriterion(BaseSuccessCriterion): """ type: Literal["file_contains"] = "file_contains" - path: str = Field(description="Path to the file to check") + path: str = Field(description="Path to the file to check; may be a glob matching exactly one file") includes: list[str] = Field(description="List of strings that must be present in the file") excludes: list[str] | None = Field(default=None, description="List of strings that must NOT be present in the file") @@ -404,7 +406,7 @@ class FileMatchesRegexCriterion(BaseSuccessCriterion): """ type: Literal["file_matches_regex"] = "file_matches_regex" - path: str = Field(description="Path to the file to check") + path: str = Field(description="Path to the file to check; may be a glob matching exactly one file") pattern: str = Field(description="Regex pattern that must match somewhere in the file") must_match: bool = Field(default=True, description="If True, pattern must match; if False, pattern must NOT match") flags: int = Field(default=0, description="Regex flags (e.g., re.IGNORECASE=2, re.MULTILINE=8, re.DOTALL=16)") @@ -777,8 +779,13 @@ class JsonCheckCriterion(BaseSuccessCriterion): """ type: Literal["json_check"] = "json_check" - path: str = Field(description="Path to the JSON file (relative to sandbox root)") - json_schema: str | None = Field(default=None, description="Path to JSON Schema file (relative to sandbox root)") + path: str = Field( + description="Path to the JSON file (relative to sandbox root); may be a glob matching exactly one file" + ) + json_schema: str | None = Field( + default=None, + description="Path to JSON Schema file (relative to sandbox root); may be a glob matching exactly one file", + ) assertions: list[JMESPathAssertion] = Field( default_factory=list, description="JMESPath assertions to evaluate against the parsed JSON" ) @@ -809,7 +816,9 @@ class FileCheckCriterion(BaseSuccessCriterion): """ type: Literal["file_check"] = "file_check" - path: str = Field(description="Path to the file to check (relative to sandbox root)") + path: str = Field( + description="Path to the file to check (relative to sandbox root); may be a glob matching exactly one file" + ) includes: list[str] = Field(default_factory=list, description="Strings that must be present in the file") excludes: list[str] = Field(default_factory=list, description="Strings that must NOT be present in the file") patterns: list[RegexPattern] = Field( @@ -841,7 +850,9 @@ class ReferenceComparisonCriterion(BaseSuccessCriterion): type: Literal["reference_comparison"] = "reference_comparison" # Required fields - agent_file: str = Field(description="Path to agent's generated file (relative to sandbox root)") + agent_file: str = Field( + description="Path to agent's generated file (relative to sandbox root); may be a glob matching exactly one file" + ) comparison_method: Literal["ast", "token", "complexity"] = Field( default="ast", @@ -1033,7 +1044,12 @@ class ClassificationMatchCriterion(BaseSuccessCriterion): """ type: Literal["classification_match"] = "classification_match" - path: str = Field(description="Path to the file (relative to sandbox) containing the agent's predicted label") + path: str = Field( + description=( + "Path to the file (relative to sandbox) containing the agent's predicted label; " + "may be a glob matching exactly one file" + ) + ) expected_label: str = Field(description="Ground-truth label for this row") allowed_labels: list[str] = Field( min_length=1, diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 2748443f..6791b407 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -66,6 +66,27 @@ ".wget-hsts", ) +# Characters that make a criterion `path` eligible for glob expansion. Eligible, +# not automatic: `Sandbox.resolve_files` tries the literal path first. +_GLOB_METACHARACTERS = "*?[" + +# Cap on how many matches an ambiguity error enumerates. The message is +# persisted to task.json and injected into judge prompts, so an unbounded +# listing over a wide pattern is a real payload. +_MAX_LISTED_MATCHES = 10 + + +def _is_glob(path: str) -> bool: + """Return whether ``path`` contains a glob metacharacter.""" + return any(c in path for c in _GLOB_METACHARACTERS) + + +def _format_matches(matches: list[Path], root: Path) -> str: + """Render matches as sandbox-relative paths, truncated to a bounded list.""" + listed = ", ".join(str(p.relative_to(root)) for p in matches[:_MAX_LISTED_MATCHES]) + remaining = len(matches) - _MAX_LISTED_MATCHES + return f"{listed}, +{remaining} more" if remaining > 0 else listed + def _grant_read_traverse(root: Path) -> None: """Recursively apply ``chmod a+rX`` semantics under ``root``. @@ -1093,38 +1114,121 @@ def run_command(self, command: str, timeout: float | int | None = None) -> tuple # needs filesystem access beyond the sandbox root (e.g., reading installed packages, # system headers). Path traversal protection is handled at the agent permission level. + def resolve_files(self, path: str) -> list[Path]: + """Resolve a criterion ``path`` to the sandbox files it addresses. + + A path that names an existing file or directory resolves to itself, + **even when it contains a glob metacharacter** — a real file called + ``report[2024].json`` is graded as itself rather than reinterpreted as + a character class that would silently match ``report2.json``. Only when + the literal does not exist is a path containing ``*``, ``?`` or ``[`` + expanded against the sandbox root, so a criterion can address a file + whose exact location the task prompt does not pin — e.g. ``**/*.flow`` + matches a scaffolded wrapper directory the agent was free to name. + + Glob matches are filtered through the sandbox's ignore patterns + (``.venv``, ``node_modules``, ``dist``, … — see + :func:`~coder_eval.resources.get_ignore_patterns`), because the sandbox + root holds harness-created content the agent never authored and + grading off it is neither fair nor deterministic. Only path segments + the glob *discovered* are filtered: a segment the pattern names + literally (``dist/**/*.js``) is an explicit opt-in and survives. + Matches are sorted so grading is deterministic, and directories are + dropped so a glob cannot resolve to something unreadable. + + Args: + path: Relative path or glob pattern + + Returns: + Sorted matching files; empty when nothing matches + """ + if not self.sandbox_dir: + return [] + + # Literal first: an existing path is never reinterpreted as a pattern. + candidate = self.sandbox_dir / path + if candidate.exists(): + return [candidate] + + if not _is_glob(path): + return [] + + patterns = get_ignore_patterns(self.config.ignore_patterns) + pinned = {segment for segment in path.split("/") if segment and not _is_glob(segment)} + + matches: list[Path] = [] + for match in self.sandbox_dir.glob(path): + if not match.is_file(): + continue + discovered = [part for part in match.relative_to(self.sandbox_dir).parts if part not in pinned] + if discovered and should_ignore_path(Path(*discovered), patterns): + continue + matches.append(match) + + return sorted(matches) + + def resolved_path_label(self, path: str) -> str | None: + """Sandbox-relative path a glob resolved to, for grading transparency. + + With exactly-one-match semantics on content reads, *which* file was + graded is most of the signal. Returns ``None`` for a literal path + (nothing was inferred) and for a pattern that did not resolve to + exactly one file. + + Args: + path: Relative path or glob pattern + + Returns: + Sandbox-relative path of the single match, or ``None`` + """ + if not self.sandbox_dir or not _is_glob(path): + return None + + matches = self.resolve_files(path) + if len(matches) != 1: + return None + + return str(matches[0].relative_to(self.sandbox_dir)) + def get_file_content(self, path: str) -> str: """Read the content of a file in the sandbox. Args: - path: Relative path to the file + path: Relative path to the file, or a glob pattern matching exactly + one file Returns: File content as string Raises: RuntimeError: If sandbox is not set up - FileNotFoundError: If file doesn't exist + FileNotFoundError: If nothing matches ``path`` + ValueError: If a glob matches more than one file """ if not self.sandbox_dir: raise RuntimeError("Sandbox not set up") - file_path = self.sandbox_dir / path - return file_path.read_text(encoding="utf-8") + matches = self.resolve_files(path) + if not matches: + raise FileNotFoundError(f"No file matches '{path}' in the sandbox") + if len(matches) > 1: + raise ValueError( + f"Pattern '{path}' matches {len(matches)} files — refusing to guess which to grade: " + + _format_matches(matches, self.sandbox_dir) + ) + + return matches[0].read_text(encoding="utf-8") def file_exists(self, path: str) -> bool: """Check if a file exists in the sandbox. Args: - path: Relative path to the file + path: Relative path to the file, or a glob pattern Returns: - True if file exists, False otherwise + True if at least one file matches, False otherwise """ - if not self.sandbox_dir: - return False - - return (self.sandbox_dir / path).exists() + return bool(self.resolve_files(path)) def list_files(self, path: str = ".") -> list[str]: """List files in a directory within the sandbox. diff --git a/tests/lint/rules/ce032_criteria_path_seam.py b/tests/lint/rules/ce032_criteria_path_seam.py new file mode 100644 index 00000000..60a78ca1 --- /dev/null +++ b/tests/lint/rules/ce032_criteria_path_seam.py @@ -0,0 +1,50 @@ +"""CE032: Criterion checkers must resolve sandbox paths through the Sandbox seam. + +`Sandbox.resolve_files` is the single place criterion `path` semantics live: +literal-first resolution (so a real file named `report[2024].json` is not +reinterpreted as a character class), glob expansion for artifacts whose location +the prompt does not pin, ignore-pattern filtering (so `.venv` / `node_modules` / +`dist` cannot be graded as agent output), and exactly-one enforcement on content +reads. A checker that builds its own path with `sandbox.sandbox_dir / ` +and reads it directly silently opts out of all of that, so path semantics differ +per criterion — which is exactly how `reference_comparison.agent_file` drifted +from every other path field. + +Use `sandbox.file_exists` / `sandbox.get_file_content` / `sandbox.resolve_files` +instead. Only files under `coder_eval/criteria/` are checked; reading +`sandbox.sandbox_dir` on its own (an initialization guard, or passing the root +to a sub-agent) is fine — the rule fires on joining a path onto it. + +Use `# noqa: CE032` for a checker that genuinely needs the raw root (e.g. it +walks a directory tree rather than addressing a file). +""" + +import ast +import re + +from tests.lint.rules.base import BaseRule + + +class CriteriaPathSeam(BaseRule): + id = "CE032" + + _CRITERIA_PATH = re.compile(r"[/\\]coder_eval[/\\]criteria[/\\]") + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(self._CRITERIA_PATH.search(filepath)) + + def visit_BinOp(self, node: ast.BinOp) -> None: + if ( + self._in_scope + and isinstance(node.op, ast.Div) + and isinstance(node.left, ast.Attribute) + and node.left.attr == "sandbox_dir" + ): + message = ( + "criterion checker joins a path onto 'sandbox_dir', bypassing the path seam; use " + "sandbox.file_exists / sandbox.get_file_content / sandbox.resolve_files so the field " + "inherits literal-first resolution, glob expansion and ignore filtering" + ) + self.violation(node, message) + self.generic_visit(node) diff --git a/tests/lint/runner.py b/tests/lint/runner.py index e360b8ec..c3b4b570 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -20,6 +20,7 @@ from tests.lint.rules.ce022_dialog_loop_statement_cap import SimulationDialogLoopStatementCap from tests.lint.rules.ce023_no_proxy_shim_import import NoProxyShimImports from tests.lint.rules.ce024_discriminated_unions import DiscriminatedUnions +from tests.lint.rules.ce032_criteria_path_seam import CriteriaPathSeam from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -63,6 +64,7 @@ SimulationDialogLoopStatementCap, NoProxyShimImports, DiscriminatedUnions, + CriteriaPathSeam, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index fe9ab6cc..26a46b4c 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -1371,3 +1371,43 @@ def test_shields_label_must_decode_to_the_listing_name(self, tmp_path: Path): findings = find_slug_mismatches([page], "coder_eval") assert len(findings) == 1 assert "displays as 'coder eval'" in findings[0].message + + +@pytest.mark.lint +class TestCE032CriteriaPathSeam: + """CE032 fires on a criterion checker joining a path onto sandbox_dir.""" + + CRITERION_FILE = "/repo/src/coder_eval/criteria/file_check.py" + + @staticmethod + def _run(src: str, filepath: str): + import ast + + from tests.lint.rules.ce032_criteria_path_seam import CriteriaPathSeam + + return CriteriaPathSeam(filepath).check(ast.parse(src)) + + def test_flags_direct_join(self): + violations = self._run("agent_path = sandbox.sandbox_dir / criterion.agent_file", self.CRITERION_FILE) + assert len(violations) == 1 + assert "bypassing the path seam" in violations[0].message + + def test_flags_join_with_literal(self): + assert self._run("p = self.sandbox.sandbox_dir / 'solution.py'", self.CRITERION_FILE) + + def test_allows_reading_the_root_without_joining(self): + assert not self._run("if not sandbox.sandbox_dir:\n return None", self.CRITERION_FILE) + + def test_allows_the_seam(self): + assert not self._run("content = sandbox.get_file_content(criterion.path)", self.CRITERION_FILE) + + def test_scoped_to_the_criteria_package(self): + # Sandbox itself, and non-criteria consumers, legitimately join onto the root. + assert not self._run( + "target = self.sandbox_dir / path", + "/repo/src/coder_eval/sandbox.py", + ) + assert not self._run( + "target = sandbox.sandbox_dir / path", + "/repo/src/coder_eval/orchestrator.py", + ) diff --git a/tests/test_glob_paths_in_file_criteria.py b/tests/test_glob_paths_in_file_criteria.py new file mode 100644 index 00000000..3b3705da --- /dev/null +++ b/tests/test_glob_paths_in_file_criteria.py @@ -0,0 +1,253 @@ +"""Tests for glob patterns in criterion ``path`` fields. + +A criterion can address a file whose exact location the task prompt does not +pin — e.g. a scaffolded wrapper directory the agent was free to name. Path +resolution lives in ``Sandbox.resolve_files``, so every path-based criterion +type inherits the behavior. +""" + +from pathlib import Path + +import pytest + +from coder_eval.evaluation.checker import SuccessChecker +from coder_eval.models import ( + FileCheckCriterion, + FileContainsCriterion, + FileExistsCriterion, + SandboxConfig, +) +from coder_eval.sandbox import Sandbox + + +FLOW_BODY = '{"nodes": [{"type": "uipath.human-in-the-loop.quick-form"}]}' + + +@pytest.fixture +def sandbox(): + sb = Sandbox(SandboxConfig(driver="tempdir", python=None), task_id="test_glob_paths") + sb.setup() + yield sb + sb.cleanup(preserve=False) + + +def _write(sandbox, relpath: str, body: str = FLOW_BODY): + target = sandbox.sandbox_dir / relpath + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body, encoding="utf-8") + return target + + +class TestResolveFiles: + def test_plain_path_unchanged(self, sandbox): + _write(sandbox, "app.py", "print('hi')") + + assert sandbox.resolve_files("app.py") == [sandbox.sandbox_dir / "app.py"] + assert sandbox.file_exists("app.py") is True + + def test_plain_path_missing(self, sandbox): + assert sandbox.resolve_files("nope.py") == [] + assert sandbox.file_exists("nope.py") is False + + def test_glob_finds_file_under_unpinned_directory(self, sandbox): + _write(sandbox, "InvoiceApprovalSolution/InvoiceApproval/InvoiceApproval.flow") + + assert sandbox.file_exists("**/*.flow") is True + assert sandbox.get_file_content("**/*.flow") == FLOW_BODY + assert sandbox.file_exists("InvoiceApproval/InvoiceApproval/InvoiceApproval.flow") is False + + def test_glob_no_match(self, sandbox): + assert sandbox.resolve_files("**/*.flow") == [] + assert sandbox.file_exists("**/*.flow") is False + with pytest.raises(FileNotFoundError, match="No file matches"): + sandbox.get_file_content("**/*.flow") + + def test_glob_skips_directories(self, sandbox): + (sandbox.sandbox_dir / "build.flow").mkdir() + _write(sandbox, "proj/real.flow") + + assert sandbox.resolve_files("**/*.flow") == [sandbox.sandbox_dir / "proj" / "real.flow"] + + def test_ambiguous_glob_refuses_to_guess(self, sandbox): + _write(sandbox, "a/one.flow") + _write(sandbox, "b/two.flow") + + assert sandbox.file_exists("**/*.flow") is True + with pytest.raises(ValueError, match="refusing to guess"): + sandbox.get_file_content("**/*.flow") + + def test_matches_are_sorted(self, sandbox): + _write(sandbox, "z/last.flow") + _write(sandbox, "a/first.flow") + _write(sandbox, "m/middle.flow") + + assert sandbox.resolve_files("**/*.flow") == [ + sandbox.sandbox_dir / "a" / "first.flow", + sandbox.sandbox_dir / "m" / "middle.flow", + sandbox.sandbox_dir / "z" / "last.flow", + ] + + def test_no_sandbox_dir(self): + sb = Sandbox(SandboxConfig(driver="tempdir", python=None), task_id="test_glob_unset") + + assert sb.resolve_files("**/*.flow") == [] + assert sb.file_exists("**/*.flow") is False + assert sb.resolved_path_label("**/*.flow") is None + with pytest.raises(RuntimeError, match="not set up"): + sb.get_file_content("**/*.flow") + + +class TestLiteralPathsWinOverGlobInterpretation: + """A path that exists is never reinterpreted as a pattern. + + ``Path.glob`` turns ``[...]`` into a character class, so without a + literal-first probe a plain filename carrying a metacharacter would grade a + different file — the exact silent-wrong-file failure globbing is meant to + prevent. Dataset fan-out substitutes ``${row.}`` into criterion + paths, so such filenames are not only hand-written. + """ + + def test_bracketed_literal_beats_character_class_decoy(self, sandbox): + _write(sandbox, "report[2024].json", "literal") + _write(sandbox, "report2.json", "decoy") + + assert sandbox.resolve_files("report[2024].json") == [sandbox.sandbox_dir / "report[2024].json"] + assert sandbox.get_file_content("report[2024].json") == "literal" + + def test_bracketed_literal_still_exists(self, sandbox): + _write(sandbox, "logs[1]", "entries") + + assert sandbox.file_exists("logs[1]") is True + assert sandbox.get_file_content("logs[1]") == "entries" + + def test_glob_still_expands_when_literal_is_absent(self, sandbox): + _write(sandbox, "runs/report2.json", "matched") + + assert sandbox.get_file_content("runs/report[123].json") == "matched" + + +class TestIgnoredDirectoriesAreNotGraded: + """The sandbox root holds harness-created content the agent never authored. + + ``.venv`` is created inside the root for any task with a ``python:`` block, + and templates copy vendored trees in before the agent runs; ``Path.glob`` + descends into dotdirs. Grading off those files is neither fair (a pass with + no agent output) nor deterministic (ambiguity that depends on the harness). + """ + + def test_venv_and_node_modules_are_skipped(self, sandbox): + _write(sandbox, ".venv/lib/python3.13/site-packages/dep/config.json", "vendored") + _write(sandbox, "node_modules/left-pad/package.json", "vendored") + _write(sandbox, "src/config.json", "authored") + + assert sandbox.resolve_files("**/*.json") == [sandbox.sandbox_dir / "src" / "config.json"] + assert sandbox.get_file_content("**/*.json") == "authored" + + def test_ignored_tree_alone_does_not_satisfy_file_exists(self, sandbox): + _write(sandbox, ".venv/lib/site-packages/dep/__init__.py", "vendored") + + assert sandbox.file_exists("**/*.py") is False + + def test_literally_named_segment_is_an_opt_in(self, sandbox): + _write(sandbox, "dist/bundle.js", "built") + + assert sandbox.resolve_files("dist/**/*.js") == [sandbox.sandbox_dir / "dist" / "bundle.js"] + + def test_ignore_pattern_negation_un_ignores_a_discovered_segment(self): + sb = Sandbox(SandboxConfig(driver="tempdir", python=None, ignore_patterns=["!dist"]), task_id="test_glob_neg") + sb.setup() + try: + _write(sb, "dist/bundle.js", "built") + + assert sb.resolve_files("**/*.js") == [sb.sandbox_dir / "dist" / "bundle.js"] + finally: + sb.cleanup(preserve=False) + + +class TestResolvedPathIsReported: + def test_label_is_none_for_a_literal_path(self, sandbox): + _write(sandbox, "app.py", "print('hi')") + + assert sandbox.resolved_path_label("app.py") is None + + def test_label_is_none_when_ambiguous(self, sandbox): + _write(sandbox, "a/one.flow") + _write(sandbox, "b/two.flow") + + assert sandbox.resolved_path_label("**/*.flow") is None + + def test_label_names_the_single_match(self, sandbox): + _write(sandbox, "Wrapper/Proj/Proj.flow") + + assert sandbox.resolved_path_label("**/*.flow") == str(Path("Wrapper") / "Proj" / "Proj.flow") + + def test_ambiguity_message_is_capped(self, sandbox): + for i in range(14): + _write(sandbox, f"d{i:02d}/f.flow") + + with pytest.raises(ValueError, match=r"\+4 more"): + sandbox.get_file_content("**/*.flow") + + +class TestGlobThroughCriteria: + def test_file_exists_criterion(self, sandbox): + _write(sandbox, "WrapperSolution/Proj/Proj.flow") + + result = SuccessChecker(sandbox).check(FileExistsCriterion(description="flow exists", path="**/*.flow")) + + assert result.score == 1.0 + + def test_file_contains_criterion(self, sandbox): + _write(sandbox, "WrapperSolution/Proj/Proj.flow") + + result = SuccessChecker(sandbox).check( + FileContainsCriterion( + description="has HITL node", + path="**/*.flow", + includes=['"uipath.human-in-the-loop.quick-form"'], + ) + ) + + assert result.score == 1.0 + + def test_file_check_criterion(self, sandbox): + _write(sandbox, "WrapperSolution/Proj/Proj.flow") + + result = SuccessChecker(sandbox).check( + FileCheckCriterion( + description="has HITL node, no manual trigger", + path="**/*.flow", + includes=['"uipath.human-in-the-loop.quick-form"'], + excludes=['"core.trigger.manual"'], + ) + ) + + assert result.score == 1.0 + + def test_ambiguous_glob_scores_zero_with_message(self, sandbox): + _write(sandbox, "a/one.flow") + _write(sandbox, "b/two.flow") + + result = SuccessChecker(sandbox).check( + FileContainsCriterion(description="ambiguous", path="**/*.flow", includes=["nodes"]) + ) + + assert result.score == 0.0 + assert "refusing to guess" in (result.error or "") + + def test_details_name_the_graded_file(self, sandbox): + _write(sandbox, "WrapperSolution/Proj/Proj.flow") + expected = str(Path("WrapperSolution") / "Proj" / "Proj.flow") + + exists = SuccessChecker(sandbox).check(FileExistsCriterion(description="flow exists", path="**/*.flow")) + contains = SuccessChecker(sandbox).check( + FileContainsCriterion(description="has node", path="**/*.flow", includes=["nodes"]) + ) + + assert expected in (exists.details or "") + assert expected in (contains.details or "") + + def test_missing_glob_scores_zero(self, sandbox): + result = SuccessChecker(sandbox).check(FileExistsCriterion(description="no flow", path="**/*.flow")) + + assert result.score == 0.0 diff --git a/tests/test_reference_comparison_scoring.py b/tests/test_reference_comparison_scoring.py index d4689d40..3833d781 100644 --- a/tests/test_reference_comparison_scoring.py +++ b/tests/test_reference_comparison_scoring.py @@ -24,6 +24,9 @@ def test_complexity_comparison_uses_reference_code(self, tmp_path): agent_file = tmp_path / "solution.py" agent_file.write_text(agent_code) sandbox.file_exists.return_value = True + # agent_file resolves through the shared path seam (glob expansion, + # ignore filtering, exactly-one), not a direct sandbox_dir read. + sandbox.get_file_content.return_value = agent_code criterion = ReferenceComparisonCriterion( description="Compare complexity",